From 271474d3e685e5232ac2034bb563d079d506abdf Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 12:46:46 -0400 Subject: [PATCH 001/141] feat(migrate): add migration command suite --- .gitignore | 2 + bun.lock | 6 +- packages/cli-core/package.json | 4 +- packages/cli-core/src/cli-program.ts | 2 + .../cli-core/src/commands/migrate/README.md | 742 +++++++++++++++++ .../src/commands/migrate/delete.test.ts | 422 ++++++++++ .../cli-core/src/commands/migrate/delete.ts | 336 ++++++++ .../src/commands/migrate/export/auth0.test.ts | 308 +++++++ .../src/commands/migrate/export/auth0.ts | 346 ++++++++ .../src/commands/migrate/export/authjs.ts | 141 ++++ .../src/commands/migrate/export/betterauth.ts | 191 +++++ .../src/commands/migrate/export/clerk.test.ts | 281 +++++++ .../src/commands/migrate/export/clerk.ts | 263 ++++++ .../migrate/export/db-exports.test.ts | 358 +++++++++ .../src/commands/migrate/export/db-options.ts | 111 +++ .../commands/migrate/export/firebase.test.ts | 451 +++++++++++ .../src/commands/migrate/export/firebase.ts | 458 +++++++++++ .../src/commands/migrate/export/index.ts | 181 +++++ .../commands/migrate/export/registry.test.ts | 39 + .../src/commands/migrate/export/registry.ts | 80 ++ .../src/commands/migrate/export/shared.ts | 85 ++ .../src/commands/migrate/export/supabase.ts | 138 ++++ .../src/commands/migrate/import-users.test.ts | 335 ++++++++ .../src/commands/migrate/import-users.ts | 347 ++++++++ .../src/commands/migrate/index.test.ts | 208 +++++ .../cli-core/src/commands/migrate/index.ts | 134 ++++ .../src/commands/migrate/lib/analysis.test.ts | 97 +++ .../src/commands/migrate/lib/analysis.ts | 80 ++ .../commands/migrate/lib/clerk-config.test.ts | 85 ++ .../src/commands/migrate/lib/clerk-config.ts | 119 +++ .../src/commands/migrate/lib/db.test.ts | 237 ++++++ .../cli-core/src/commands/migrate/lib/db.ts | 227 ++++++ .../src/commands/migrate/lib/instance.test.ts | 83 ++ .../src/commands/migrate/lib/instance.ts | 93 +++ .../commands/migrate/lib/log-files.test.ts | 200 +++++ .../src/commands/migrate/lib/log-files.ts | 141 ++++ .../src/commands/migrate/lib/logger.test.ts | 110 +++ .../src/commands/migrate/lib/logger.ts | 114 +++ .../commands/migrate/lib/readiness.test.ts | 322 ++++++++ .../src/commands/migrate/lib/readiness.ts | 238 ++++++ .../src/commands/migrate/lib/retry.test.ts | 156 ++++ .../src/commands/migrate/lib/retry.ts | 66 ++ .../commands/migrate/lib/scheduler.test.ts | 69 ++ .../src/commands/migrate/lib/scheduler.ts | 51 ++ .../src/commands/migrate/lib/settings.test.ts | 42 + .../src/commands/migrate/lib/settings.ts | 43 + .../migrate/lib/supabase-providers.test.ts | 143 ++++ .../migrate/lib/supabase-providers.ts | 145 ++++ .../commands/migrate/lib/transform.test.ts | 218 +++++ .../src/commands/migrate/lib/transform.ts | 469 +++++++++++ .../src/commands/migrate/logs/clean.ts | 69 ++ .../src/commands/migrate/logs/convert.ts | 121 +++ .../src/commands/migrate/logs/index.ts | 72 ++ .../src/commands/migrate/logs/list.ts | 64 ++ .../migrate/logs/logs-interactive.test.ts | 177 +++++ .../src/commands/migrate/logs/logs.test.ts | 219 +++++ .../src/commands/migrate/readme.test.ts | 124 +++ .../commands/migrate/run-interactive.test.ts | 301 +++++++ .../cli-core/src/commands/migrate/run.test.ts | 749 ++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 519 ++++++++++++ .../commands/migrate/transformers/auth0.ts | 43 + .../commands/migrate/transformers/authjs.ts | 38 + .../migrate/transformers/betterauth.ts | 46 ++ .../commands/migrate/transformers/clerk.ts | 45 ++ .../commands/migrate/transformers/firebase.ts | 121 +++ .../migrate/transformers/list.test.ts | 116 +++ .../src/commands/migrate/transformers/list.ts | 67 ++ .../migrate/transformers/load-custom.test.ts | 222 ++++++ .../migrate/transformers/load-custom.ts | 164 ++++ .../commands/migrate/transformers/registry.ts | 74 ++ .../commands/migrate/transformers/shared.ts | 93 +++ .../commands/migrate/transformers/supabase.ts | 70 ++ .../migrate/transformers/transformers.test.ts | 405 ++++++++++ .../cli-core/src/commands/migrate/types.ts | 193 +++++ .../src/commands/migrate/validator.test.ts | 80 ++ .../src/commands/migrate/validator.ts | 99 +++ .../src/commands/migrate/wizard.test.ts | 267 +++++++ .../cli-core/src/commands/migrate/wizard.ts | 167 ++++ 78 files changed, 14240 insertions(+), 2 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/README.md create mode 100644 packages/cli-core/src/commands/migrate/delete.test.ts create mode 100644 packages/cli-core/src/commands/migrate/delete.ts create mode 100644 packages/cli-core/src/commands/migrate/export/auth0.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/auth0.ts create mode 100644 packages/cli-core/src/commands/migrate/export/authjs.ts create mode 100644 packages/cli-core/src/commands/migrate/export/betterauth.ts create mode 100644 packages/cli-core/src/commands/migrate/export/clerk.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/clerk.ts create mode 100644 packages/cli-core/src/commands/migrate/export/db-exports.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/db-options.ts create mode 100644 packages/cli-core/src/commands/migrate/export/firebase.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/firebase.ts create mode 100644 packages/cli-core/src/commands/migrate/export/index.ts create mode 100644 packages/cli-core/src/commands/migrate/export/registry.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/registry.ts create mode 100644 packages/cli-core/src/commands/migrate/export/shared.ts create mode 100644 packages/cli-core/src/commands/migrate/export/supabase.ts create mode 100644 packages/cli-core/src/commands/migrate/import-users.test.ts create mode 100644 packages/cli-core/src/commands/migrate/import-users.ts create mode 100644 packages/cli-core/src/commands/migrate/index.test.ts create mode 100644 packages/cli-core/src/commands/migrate/index.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/analysis.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/analysis.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/clerk-config.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/db.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/db.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/instance.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/instance.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/log-files.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/log-files.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/logger.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/logger.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/readiness.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/readiness.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/retry.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/retry.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/scheduler.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/scheduler.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/settings.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/settings.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/supabase-providers.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/transform.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/transform.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/clean.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/convert.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/index.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/list.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/logs.test.ts create mode 100644 packages/cli-core/src/commands/migrate/readme.test.ts create mode 100644 packages/cli-core/src/commands/migrate/run-interactive.test.ts create mode 100644 packages/cli-core/src/commands/migrate/run.test.ts create mode 100644 packages/cli-core/src/commands/migrate/run.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/auth0.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/authjs.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/betterauth.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/clerk.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/firebase.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/list.test.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/list.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/load-custom.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/registry.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/shared.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/supabase.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/transformers.test.ts create mode 100644 packages/cli-core/src/commands/migrate/types.ts create mode 100644 packages/cli-core/src/commands/migrate/validator.test.ts create mode 100644 packages/cli-core/src/commands/migrate/validator.ts create mode 100644 packages/cli-core/src/commands/migrate/wizard.test.ts create mode 100644 packages/cli-core/src/commands/migrate/wizard.ts diff --git a/.gitignore b/.gitignore index 91df74696..c5f69f393 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ coverage # logs logs +!packages/cli-core/src/commands/migrate/logs/ +!packages/cli-core/src/commands/migrate/logs/** _.log report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json diff --git a/bun.lock b/bun.lock index 6dd7ef10c..40357bdac 100644 --- a/bun.lock +++ b/bun.lock @@ -19,7 +19,7 @@ }, "packages/cli": { "name": "clerk", - "version": "2.3.0", + "version": "3.0.0", "bin": { "clerk": "./bin/clerk", }, @@ -36,11 +36,13 @@ "@commander-js/extra-typings": "^15.0.0", "@napi-rs/keyring": "^1.3.0", "commander": "^15.0.0", + "csv-parser": "^3.2.1", "env-paths": "^4.0.0", "external-editor": "^3.1.0", "magicast": "^0.5.3", "semver": "^7.8.5", "yaml": "^2.9.0", + "zod": "^4.4.3", }, "devDependencies": { "@clerk/shared": "^4.13.1", @@ -335,6 +337,8 @@ "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "csv-parser": ["csv-parser@3.2.1", "", { "bin": { "csv-parser": "bin/csv-parser" } }, "sha512-v8RPMSglouR9od735SnwSxLBbCJqEPSbgm1R5qfr8yIiMUCEFjox56kRZid0SvgHJEkxeIEu3+a9QS3YRh7CuA=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], diff --git a/packages/cli-core/package.json b/packages/cli-core/package.json index f6be6ee93..ffddc6db2 100644 --- a/packages/cli-core/package.json +++ b/packages/cli-core/package.json @@ -22,11 +22,13 @@ "@commander-js/extra-typings": "^15.0.0", "@napi-rs/keyring": "^1.3.0", "commander": "^15.0.0", + "csv-parser": "^3.2.1", "env-paths": "^4.0.0", "external-editor": "^3.1.0", "magicast": "^0.5.3", "semver": "^7.8.5", - "yaml": "^2.9.0" + "yaml": "^2.9.0", + "zod": "^4.4.3" }, "devDependencies": { "@clerk/shared": "^4.13.1", diff --git a/packages/cli-core/src/cli-program.ts b/packages/cli-core/src/cli-program.ts index 9ea89cb54..ca63b2212 100644 --- a/packages/cli-core/src/cli-program.ts +++ b/packages/cli-core/src/cli-program.ts @@ -22,6 +22,7 @@ import { registerCompletion } from "./commands/completion/index.ts"; import { registerUpdate } from "./commands/update/index.ts"; import { registerDeploy } from "./commands/deploy/index.ts"; import { registerWebhooks } from "./commands/webhooks/index.ts"; +import { registerMigrate } from "./commands/migrate/index.ts"; import { getEnvironment } from "./lib/config.ts"; import { setCurrentEnv, @@ -75,6 +76,7 @@ const registrants: CommandRegistrant[] = [ registerUpdate, registerDeploy, registerWebhooks, + registerMigrate, registerExtras, ]; diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md new file mode 100644 index 000000000..f18e38223 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/README.md @@ -0,0 +1,742 @@ +# `clerk migrate` + +Migrate users into a Clerk instance from another auth provider, or from another +Clerk instance. + +## Targeting And Auth + +`migrate run` resolves its Backend API key through the CLI's standard chain: + +| Flag | Description | +| ------------------------ | ---------------------------------------------------------------- | +| `--secret-key ` | Use a specific Backend API secret key directly | +| `--clerk-secret-key ` | **Deprecated** alias for `--secret-key`; warns and keeps working | +| `--app ` | Target an application directly, even outside a linked project | +| `--instance ` | Target `dev`, `prod`, or a full instance ID | + +Resolution order: `--secret-key` → `--app` + Platform API lookup → +`CLERK_SECRET_KEY` → the keyless project's own key → a linked project profile +from `clerk link`. + +The **instance type is read from the key**: `sk_live_…` is treated as +production, anything else as development. That choice drives the throughput +defaults and the hard development-instance cap below. + +## Commands + +### `clerk migrate` (interactive) + +Bare `clerk migrate` dispatches to `migrate run`, which walks a human through +the migration instead of demanding flags — mirroring how bare `clerk deploy` +dispatches to `deploy run`. + +```sh +clerk migrate +``` + +It picks the transformer from a list built off the registry, asks for the file, +collects Firebase's hash parameters when they are needed, and pre-fills every +answer from the last run's `.settings` so a repeat migration is mostly pressing +enter. Anything already passed as a flag is not asked for. + +Then it prints the [Migration Readiness report](#migration-readiness-report) +and waits for confirmation. Declining writes nothing to Clerk. + +**Agent mode never prompts.** `clerk migrate` with no flags exits with a usage +error naming exactly what to pass: + +``` +`clerk migrate` is interactive and cannot prompt in agent mode. +Pass --transformer and --file . +``` + +### `clerk migrate run` + +Reads an exported user file, maps it onto Clerk's user schema, validates every +record, and creates the users through the Backend API. + +```sh +clerk migrate run -y --transformer clerk --file users.json +``` + +| Flag | Description | +| --------------------------------------- | --------------------------------------------------------------- | +| `-t, --transformer ` | Source platform the file came from (see below) | +| `--transformer-file ` | A transformer you wrote, for a platform with no built-in | +| `-f, --file ` | Path to the export. `.json` or `.csv` | +| `-r, --resume-after ` | Skip every user up to and including this **source** ID | +| `--require-password` | Import only users that carry a password digest | +| `--skip-unsupported-providers` | Supabase: skip users whose only social provider is off in Clerk | +| `--firebase-signer-key ` | Firebase base64 signer key | +| `--firebase-salt-separator ` | Firebase base64 salt separator | +| `--firebase-rounds ` | Firebase scrypt rounds | +| `--firebase-mem-cost ` | Firebase scrypt memory cost | +| `-y, --yes` | Skip the confirmation prompt | + +Plus the targeting flags from the table above: `--secret-key`, +`--clerk-secret-key`, `--app` and `--instance`. + +`--transformer` and `--file` are required. Omitting either fails with a usage +error that names the valid values. + +Failures do not stop the run: each user's outcome is written to the log and the +import continues. A `429` backs off — honouring `Retry-After` when the response +carries it — and retries up to 5 times before the user is recorded as failed. +The command exits non-zero if any user failed. + +Two failures do abort the whole run, because continuing would produce a +corrupt instance: + +- An **unrecognized password hasher**, which would import credentials nobody can + sign in with. +- A **`--resume-after` ID that is not in the file**, which would otherwise + re-import every user the previous run already created. + +#### Additional identifiers + +Only the first verified email and phone go on `POST /v1/users`. Every +additional verified identifier, and every unverified one, is attached +afterwards with its own request. A failure there is logged and the user still +counts as imported — a duplicate secondary email should not undo an otherwise +successful user. + +#### Throughput + +Defaults follow Clerk's documented `POST /v1/users` limits: 100 req/s for +production instances, 10 req/s for development. Concurrency defaults to ~95% of +that, assuming ~100ms of API latency. Both are overridable: + +| Variable | Effect | +| --------------------------------- | ----------------------------- | +| `CLERK_MIGRATE_RATE_LIMIT` | Requests per second | +| `CLERK_MIGRATE_CONCURRENCY_LIMIT` | Concurrent in-flight requests | + +A non-numeric or non-positive value is ignored in favour of the default. + +**Development instances refuse imports over 500 users**, matching Clerk's own +limit — the run fails before any request is sent. + +### `clerk migrate export` + +Gets users **out** of a source platform, so there is something to feed +`migrate run`. + +```sh +clerk migrate export # pick a platform +clerk migrate export clerk --output users.json +clerk migrate export auth0 --domain my-tenant.us.auth0.com \ + --client-id … --client-secret … +``` + +The platform is an optional positional. Omitted, you get a picker built from +the registry; given, it runs directly. Each platform resolves its own flags — +what Auth0 needs (a tenant domain and M2M credentials) has nothing in common +with what a database export needs. + +| Platform | Source | Feeds | +| ------------ | -------------------------------- | -------------------------- | +| `clerk` | Clerk Backend API | `--transformer clerk` | +| `auth0` | Auth0 Management API | `--transformer auth0` | +| `supabase` | Supabase Postgres (`auth.users`) | `--transformer supabase` | +| `authjs` | Auth.js database | `--transformer authjs` | +| `betterauth` | Better Auth database | `--transformer betterauth` | +| `firebase` | Firebase Identity Toolkit | `--transformer firebase` | + +Exports land at `./exports/-export.json` unless `--output` says +otherwise. `--output` resolves against the **current directory**, like every +other path flag here. + +| Flag | Platforms | Description | +| -------------------------- | ---------------------------------- | -------------------------------------------- | +| `-o, --output ` | all | Where to write the export | +| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL or SQLite connection string | +| `--service-account ` | `firebase` | Path to a service account key JSON file | +| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | +| `--client-id ` | `auth0` | Machine-to-machine application client ID | +| `--client-secret ` | `auth0` | Machine-to-machine application client secret | + +`export clerk` also takes the targeting flags — it reads from a Clerk instance, +so it resolves a key exactly the way `migrate run` does. + +After each export you get a field-coverage table — which Clerk-relevant fields +were present on how many users — so you know the data is thin _before_ you +import it, not after: + +``` +Field coverage + ● 3/3 have an email address + ○ 0/3 have a phone number + ○ 1/3 have a username + ○ 2/3 have a password (not exportable — see below) + +Exported 3 user(s) to /project/exports/clerk-export.json +Next: clerk migrate run --transformer clerk --file exports/clerk-export.json +``` + +Every export also writes `logs/export-.log`, so `migrate logs list` +sees it alongside imports and deletions. + +#### Neither platform exports passwords + +- **Clerk** never returns password digests, TOTP secrets or backup codes over + the API — only the `*_enabled` booleans. Migrated users must reset their + password in the destination instance. +- **Auth0**'s Management API does not return password hashes either; they come + only from a support request. Add a `passwordHash` field to each user before + importing, or migrate without passwords. + +Both say so on every run. The coverage row counts users who _have_ a password, +so the size of the gap is visible up front. + +#### Database-backed exports (`supabase`, `authjs`, `betterauth`) + +These three read the database directly, over **`--db-url`**: + +```sh +clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" +clerk migrate export authjs --db-url "mysql://user:...@127.0.0.1:3306/authjs" +clerk migrate export betterauth --db-url "./db.sqlite" +``` + +Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`. Both are +built into the runtime, so nothing native ships in the binary — that is the +whole reason the `engines.bun` floor exists. Resolution is `--db-url`, then +`SUPABASE_DB_URL` / `AUTHJS_DB_URL` / `BETTERAUTH_DB_URL`, then a masked prompt, +since a connection string carries the password inline. + +**Connection strings are redacted everywhere.** Errors show +`postgres://***@host/db`, including when the password itself contains an +unencoded `@` — the most common mistake, and exactly when the string ends up in +an error message. + +Connection failures get a hint rather than a driver error. Bun reports both an +unreachable host and a closed port as "Connection closed", so: + +| Situation | What you are told | +| ------------------------ | ------------------------------------------------------------------------------------------------- | +| Host or port unreachable | Check the host and port. On Supabase: use the pooler connection string, or enable the IPv4 add-on | +| Credentials rejected | Check the user and password | +| Table missing | Check the database name and SELECT permission. On Supabase: enable Auth, connect as `postgres` | +| SQLite file missing | Check the path and that the file is readable | + +**`supabase` reads the database rather than the Admin API** because +`encrypted_password` exists only there. An API-based export would force every +user to reset their password; this one carries the bcrypt digests across. It +also keeps `raw_app_meta_data`, which is what `--skip-unsupported-providers` +reads at import time. + +**`authjs` tries `User`, then `user`, then `users`.** Auth.js has no single +schema — Prisma capitalizes the table, Drizzle does not, and Postgres treats +the difference as significant once quoted. The run reports which one it found. +Auth.js core stores no passwords, so its users arrive without credentials. + +**`betterauth` detects its plugin columns from the schema.** The username +plugin adds `username`, admin adds `banned`, phone-number adds `phoneNumber`, +and so on; selecting a column that is not there fails the whole query, and the +database answers the question better than the user can. Passwords come from a +`LEFT JOIN` onto the credential `account` row — left, not inner, so a user who +only ever signed in with OAuth is still exported. + +#### `firebase` + +```sh +clerk migrate export firebase --service-account ./service-account.json +``` + +Needs a service account key from **Project settings → Service accounts → +Generate new private key**, with the Firebase Authentication Admin role. The +file is validated before anything reaches the network, so downloading the web +app config by mistake fails in a second with the right console page named +rather than after an auth round-trip. Key material never appears in output. + +Firebase's scrypt is a modified variant, so a digest is worthless without the +project's four hash parameters. The export **reads them from the project** and +prints the exact import command: + +``` +Password hash parameters +Read from the project. Import with: + clerk migrate run -y --transformer firebase --file exports/firebase-export.json \ + --firebase-signer-key "…" --firebase-salt-separator "…" \ + --firebase-rounds 8 --firebase-mem-cost 14 +``` + +Reading the config needs a broader role than listing users, so if it is denied +the export still succeeds and points at **Authentication → Users → (⋮) → +Password hash parameters** instead. An export with no password hashes says so +and asks for nothing. + +A user whose hash is present but whose salt is not (or the reverse) has both +dropped: half a credential produces a user nobody can sign in as. + +`FIREBASE_AUTH_EMULATOR_HOST` is honoured, so this works against the local +Firebase emulator as well as production. + +**No `firebase-admin`.** The spike the plan called for was run and _passed_ — a +compiled binary can import the SDK and complete `listUsers`, so the known +Firestore-under-compile bug does not reach the Auth Admin surface. It was still +not adopted: the SDK is 74 MB across 158 packages, including Firestore and +Cloud Storage, which would roughly double the ~62 MB binary every user +downloads, to serve one subcommand. What it does here is two REST calls and an +RS256 JWT, and Bun's Web Crypto signs RS256 with no dependency at all. + +#### Auth0 credentials + +Needs a machine-to-machine application with the `read:users` scope +(Applications → APIs → Auth0 Management API → Machine to Machine +Applications). Resolved from flags, then `AUTH0_DOMAIN` / `AUTH0_CLIENT_ID` / +`AUTH0_CLIENT_SECRET`, then a prompt. In agent mode a prompt is impossible, so +it exits naming **every** missing credential at once rather than one per run. + +Auth0 pages this endpoint only through the first **1000** users. Past that the +export stops and says so, pointing at Auth0's bulk export job — silently +returning the first thousand would read as "that is everyone". + +### `clerk migrate delete` + +The undo for a bad migration. Deletes the users a previous `clerk migrate run` +created in this directory, matched by the `external_id` the import stamped on +each one. + +```sh +clerk migrate delete # confirms first +clerk migrate delete -y # non-interactive +``` + +Takes the same targeting flags as `migrate run` (`--secret-key`, `--app`, +`--instance`). + +Flat rather than under a noun group: it is the one command in this tree that +destroys data **in Clerk**, and is worth keeping short and prominent. (Contrast +`migrate logs clean`, which only removes local files.) + +#### What it will and will not touch + +`.settings` is the only record of what a run created, so that is what +identifies the migration being undone. Without it the command fails and +explains — deleting nothing silently would look like a successful undo. + +Users are found with `GET /v1/users?external_id=…`, 100 IDs per request. Only a +user Clerk itself reports as carrying one of _this_ migration's external IDs is +ever deleted; anything else in the instance is out of scope. IDs with no +matching user are skipped and reported, which is the normal case for a partial +migration or one already partly undone. + +It confirms before acting — defaulting to **no** — and requires `-y` in +non-interactive or agent mode. + +#### Failures + +Rate limiting and 429 retries are literally the same code path as the import +(`lib/retry.ts`), not a second implementation that drifts. + +A failure on one user is logged and the rest continue: a half-undone migration +with no record of which half is far worse than a reported failure. Every +attempt lands in a timestamped `logs/user-deletion-.log`, carrying +both the source ID and the Clerk ID. The command exits non-zero if any deletion +failed. + +### `clerk migrate logs` + +Everything that touches the local `./logs/` directory. Noun-verb like every +other group in the CLI (`config pull`, `users list`), rather than the standalone +tool's `clean-logs`/`convert-logs`, which were npm script names. + +Grouping also disambiguates the two deletes in this tree: `migrate logs clean` +removes **local files**, `migrate delete` removes **users from a Clerk +instance**. + +```sh +clerk migrate logs # defaults to list +clerk migrate logs list --json +clerk migrate logs clean -y +clerk migrate logs convert --all +clerk migrate logs convert migration-2026-01-01T12-00-00.log +``` + +| Subcommand | Takes | Description | +| -------------- | ------------------ | ----------------------------------------------- | +| `logs list` | `--json` | Type, timestamp, size and entry count per file | +| `logs clean` | `-y, --yes` | Delete the `.log` files in `./logs/` | +| `logs convert` | `[file…]`, `--all` | NDJSON → a JSON array, written as `.json` | + +All three read the directory through one shared enumerator, which is what makes +`logs list` nearly free. + +#### `logs list` + +The default, because listing is read-only and therefore safe to run by +accident. Reports each file's type, timestamp, size and entry count, newest +first; `--json` gives an agent the same data without parsing NDJSON. + +``` +TYPE TIMESTAMP SIZE ENTRIES +migration 2026-02-01T09-14-22 4.1 KB 120 +deletion 2026-01-30T17-02-51 612 B 18 +``` + +Says so plainly when `./logs/` is empty or absent. + +#### `logs clean` + +Destructive, so the confirmation is not optional: interactive runs prompt +(defaulting to **no**), and non-interactive or agent runs must pass `-y` rather +than being allowed to assume. Deletes `.log` files only — converted `.json` +output is left alone. + +#### `logs convert` + +Turns NDJSON into a JSON array for spreadsheet or database analysis, written +alongside the original as `.json`. The original is left in place. + +Takes file positionals or `--all`; given neither, an interactive terminal +offers a multiselect and an agent gets a usage error naming both alternatives. + +A malformed line is reported with its line number and skipped, and the +remaining entries still convert: + +``` +migration-2026-01-01T12-00-00.log:2 is not valid JSON and was skipped — … +1 malformed line skipped. +``` + +That beats failing the whole file: a run killed mid-write leaves one truncated +final line, and the hundreds of complete entries before it are still worth +having. It also beats dropping the line silently, which would leave a JSON +array that looks complete. + +## Transformers + +A transformer maps one platform's export onto Clerk's user schema. Adding a +platform is one file in `transformers/` plus one line in `transformers/registry.ts` — +`--transformer`'s accepted values and its tab-completion both read from that array. + +| Key | Source | Passwords | Notes | +| ------------ | ----------------------------- | ----------------- | ---------------------------------------------------------------- | +| `clerk` | Clerk Dashboard export | as exported | Instance to instance, e.g. development → production | +| `auth0` | Auth0 Export Users API | `bcrypt` | Hashes need a support request to Auth0; not in a standard export | +| `authjs` | Auth.js / NextAuth user table | none | Assumes `SELECT id, name, email, email_verified, created_at` | +| `betterauth` | Better Auth export | `bcrypt` | Reads the credential account's `password_hash` | +| `firebase` | `firebase auth:export` | `scrypt_firebase` | CSV or JSON; needs the four hash parameters below | +| `supabase` | Supabase `auth.users` export | `bcrypt` | Supports `--skip-unsupported-providers` | + +### `clerk migrate transformers list` + +Which mappings are available. New in the CLI: the standalone tool's interactive +picker was the only place these appeared, which was fine when the user had the +source tree to grep. A compiled binary's users have neither. + +```sh +clerk migrate transformers list +clerk migrate transformers list --json +clerk migrate transformers list --transformer-file ./my-transformer.ts +``` + +| Flag | Description | +| --------------------------- | --------------------------------- | +| `--json` | Output as JSON | +| `--transformer-file ` | Also list a transformer you wrote | + +`--json` gives an agent the same data, including which source field each +transformer maps to `userId`. + +### Custom transformers (`--transformer-file`) + +Migrating from a platform with no built-in, without recompiling the CLI: + +```sh +clerk migrate run --transformer-file ./my-platform.ts --file users.json +``` + +The file lives in **your** project, not in the CLI, and is imported at runtime. +It exports the same shape the built-ins use — plain data, no imports, since +there is nothing in a compiled binary for your file to import from: + +```ts +export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform's admin console.", + transformer: { + account_ref: "userId", // required: becomes the Clerk user's external_id + contact_email: "email", + given: "firstName", + family: "lastName", + pw_bcrypt: "password", + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { + if (!user.firstName) delete user.firstName; + }, +}; +``` + +TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, +`satisfies` and `as const` all work in a file the compiled binary imports. +Plain `.js` works too. + +`--transformer-file` and `--transformer` together is an error: both name a +transformer and there is no sensible precedence between the one you wrote and +the one we ship. + +#### Validation + +The file is code the CLI executes, so its shape is checked before use and +rejected with the specific problem rather than crashing mid-pipeline: + +| Problem | Message | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------- | +| Path does not exist | `No transformer file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in transformer` | +| A hook is not a function | `postTransform must be a function when present` | + +The `userId` check is the load-bearing one: without it the import would run to +completion and create every user with no `external_id`, which is what makes a +migration re-runnable and what `migrate delete` matches on. + +### Verified vs unverified identifiers + +Every platform records verification differently, and each transformer declares +which style it uses. An identifier the source never confirmed is routed to +`unverifiedEmailAddresses` / `unverifiedPhoneNumbers` rather than the primary +field, because Clerk creates primary identifiers **verified** — sending an +unconfirmed address there would silently promote it. + +- **Boolean** (`auth0`, `betterauth`, `firebase`): `true`/`false`. A CSV export + stringifies these, so `"false"` is read as false, not as a non-empty string. +- **Timestamp** (`authjs`, `supabase`): a nullable confirmation time. Any real + value means verified; `""`, `null` and `\N` do not. + +### Firebase hash parameters + +Firebase uses a modified scrypt, so Clerk needs the project's four parameters +alongside each digest. Find them in the Firebase console under +**Authentication → Users → (⋮) → Password hash parameters**. + +```sh +clerk migrate run -y -t firebase -f users.json \ + --firebase-signer-key --firebase-salt-separator \ + --firebase-rounds 8 --firebase-mem-cost 14 +``` + +All four are **required as a set** — supplying some but not all is a usage error +naming what is missing. A partial set produces a well-formed digest that +verifies against nothing, so users would import successfully and then be unable +to sign in. They are saved to `.settings` and reused on the next run. + +An export with no password hashes needs no parameters at all. + +### `--skip-unsupported-providers` (Supabase) + +Reads each user's `raw_app_meta_data.providers` and cross-references it against +the social providers the destination instance has enabled (via BAPI +`/v1/domains` → the instance's Frontend API `/v1/environment`). + +A user is skipped **only when every one of their providers is disabled**. Anyone +who can still sign in another way — email, phone, or an enabled social provider +— is imported. The number skipped is reported, broken down by provider. + +If the instance configuration cannot be read, nobody is skipped and a warning is +printed: a failed lookup must not be mistaken for "no providers are enabled". + +## Schema fields + +What a transformer maps _onto_. Every user is validated against this schema +before any request is made, so a field a transformer produces that is not listed +here is silently dropped — Zod strips unknown keys — and never reaches Clerk. +Writing a custom transformer means targeting these names exactly. + +The schema lives in `validator.ts`; adding a source platform means adding a +transformer, not editing it. + +**Required:** `userId` (`string`). It becomes the Clerk user's `external_id`, +which is what makes a migration re-runnable and what `migrate delete` matches on. + +**Identifiers.** At least one of these must be present, or the user is logged as +a validation failure and skipped. Each accepts a single value or an array. + +| Field | Type | Description | +| -------------------------- | -------------------- | ---------------------------------- | +| `email` | `string \| string[]` | Primary verified email address(es) | +| `emailAddresses` | `string \| string[]` | Additional verified emails | +| `unverifiedEmailAddresses` | `string \| string[]` | Unverified emails | +| `phone` | `string \| string[]` | Primary verified phone number(s) | +| `phoneNumbers` | `string \| string[]` | Additional verified phones | +| `unverifiedPhoneNumbers` | `string \| string[]` | Unverified phones | +| `username` | `string` | Username | + +**Profile, password and 2FA.** + +| Field | Type | Description | +| -------------------- | ---------- | --------------------------------------------------- | +| `firstName` | `string` | First name | +| `lastName` | `string` | Last name | +| `password` | `string` | The hashed password from the source platform | +| `passwordHasher` | `enum` | **Required whenever `password` is set** (see below) | +| `totpSecret` | `string` | TOTP secret | +| `backupCodesEnabled` | `boolean` | Whether backup codes are enabled | +| `backupCodes` | `string[]` | Backup codes | + +Clerk verifies the digest as-is, so `passwordHasher` must name the algorithm the +source actually used: + +`argon2i`, `argon2id`, `awscognito`, `bcrypt`, `bcrypt_peppered`, +`bcrypt_sha256_django`, `hmac_sha256_utf16_b64`, `ldap_ssha`, `md5`, +`md5_phpass`, `md5_salted`, `pbkdf2_sha1`, `pbkdf2_sha256`, +`pbkdf2_sha256_django`, `pbkdf2_sha512`, `pbkdf2_sha512_hex`, `scrypt_firebase`, +`scrypt_werkzeug`, `sha256`, `sha256_salted`, `sha512_symfony` + +An unrecognized hasher aborts the run rather than importing credentials nobody +can sign in with. + +**Metadata.** + +| Field | Type | Description | +| ----------------- | -------- | -------------------------------------------------------- | +| `unsafeMetadata` | `object` | Readable **and writable** by the client — never trust it | +| `publicMetadata` | `object` | Readable by the client, writable only server-side | +| `privateMetadata` | `object` | Server-side only | + +**Account state.** These are passed straight through to `POST /v1/users`, and +are how a Clerk-to-Clerk migration keeps original signup dates instead of +stamping every user with today's. + +| Field | Type | Description | +| --------------------------- | --------- | ----------------------------------------- | +| `createdAt` | `string` | Original creation timestamp | +| `legalAcceptedAt` | `string` | When legal terms were accepted | +| `banned` | `boolean` | Whether the user is banned | +| `bypassClientTrust` | `boolean` | Skip client trust verification | +| `createOrganizationEnabled` | `boolean` | Whether the user can create orgs | +| `createOrganizationsLimit` | `number` | Maximum orgs the user can create | +| `deleteSelfEnabled` | `boolean` | Whether the user can delete their account | +| `skipLegalChecks` | `boolean` | Skip legal acceptance checks | +| `skipPasswordChecks` | `boolean` | Skip password requirements on import | + +## Migration Readiness report + +Printed immediately before the confirmation prompt, so declining aborts with +nothing written to Clerk. Skipped only for `-y`, which says "don't ask, don't +lecture" and should not pay for the two extra round-trips. Agent runs without +`-y` still get it — an agent can act on it exactly as a human would. + +It cross-references the file against the destination instance's live settings +(BAPI `/v1/domains` → that instance's Frontend API `/v1/environment`) and +flags the two failure modes a migration otherwise discovers halfway through: + +- **Required in Clerk, missing from the file.** Those users fail one at a time, + mid-import, after earlier users already exist. +- **Present in the file, disabled in Clerk.** Social providers users actually + signed up with, or an identifier the instance has switched off. + +``` +Migration readiness + 120 users ready to import + 3 failed validation and will be skipped + +Identifiers + ⚠ Email — required in Clerk, but 12 users lack it — 108/120 users + ✓ Username — enabled in Clerk — all users + +Social connections + ✓ Google — enabled in Clerk — 40/120 users + ⚠ Discord — not enabled in Clerk — 12/120 users + +⚠ 2 settings need attention +``` + +If the instance settings cannot be read — the secret key is rejected, or FAPI +is unreachable — the report degrades to a coverage-only listing with a note. +Nothing is flagged in that case: "could not read" is not the same as "switched +off", and treating it as such would raise alarms about settings that are +perfectly fine. + +## Artifacts + +Both are written relative to the **current working directory**, not to the +CLI's config directory, because they describe "which file am I migrating" +rather than "which project is linked here". + +| Path | Contents | +| -------------------------------------- | --------------------------------------------------------------------- | +| `./logs/migration-.log` | NDJSON: one line per user, plus validation failures and retry notices | +| `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | +| `./logs/export-.log` | NDJSON: one line per exported user | +| `./exports/-export.json` | The export itself, unless `--output` says otherwise | +| `./.settings` | The transformer key and file path of the last run | + +`.settings` is what `migrate delete` reads to know which migration to undo, so +it is load-bearing rather than a convenience. + +Log writes are synchronous appends, so a run interrupted with Ctrl-C still +leaves a complete record of everything already processed. Use the last +successful `userId` in that log with `--resume-after` to continue. + +### Why the logs are NDJSON + +One JSON object per line, rather than one JSON array per file. A migration is a +long append-only stream, and that format is the one that survives it: + +- **Appendable.** Each entry is written as it happens, without rewriting the + file. A JSON array would have to be re-serialized on every user. +- **Crash-safe.** Kill the process at any point and every line already written + is still valid. A truncated array is not parseable at all. +- **Streamable.** `tail -f` shows a long import progressing live, and analysis + reads line by line instead of loading a million-user log into memory. + +Which is also why it greps usefully without any tooling: + +```sh +grep '"status":"success"' logs/migration-2026-01-01T12-00-00.log | wc -l +grep '"userId":"user_123"' logs/migration-2026-01-01T12-00-00.log +``` + +The trade-off is that spreadsheets, databases and most JSON tooling want an +array. That is what `clerk migrate logs convert` is for — convert when you need +to open a log in Excel or hand it to someone who should not have to know what +NDJSON is. The original `.log` stays put. + +## API Endpoints + +| Method | Path | Used by | +| -------- | -------------------------- | ------------------------------------------------------------------------------------ | +| `POST` | `/v1/users` | `migrate run` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate run` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate run` — attaches additional phones | +| `GET` | `/v1/users?external_id=…` | `migrate delete` — finds this migration's users, 100 IDs a call | +| `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | +| `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | +| `GET` | `/v1/domains` | Readiness report and `--skip-unsupported-providers` — resolves the Frontend API host | + +Two exports talk to their own platform rather than to Clerk: + +| Method | Path | Used by | +| ------ | ---------------------------------------------- | ---------------------------------------------------- | +| `POST` | `https:///oauth/token` | `export auth0` — Management API access token | +| `GET` | `https:///api/v2/users` | `export auth0` — 100 per page, 1000 users maximum | +| `POST` | `https://oauth2.googleapis.com/token` | `export firebase` — RS256 assertion → access token | +| `GET` | `…/v1/projects/{project_id}/accounts:batchGet` | `export firebase` — pages users, 1000 at a time | +| `GET` | `…/admin/v2/projects/{project_id}/config` | `export firebase` — reads the scrypt hash parameters | + +The two Identity Toolkit paths are on `identitytoolkit.googleapis.com`, or on +`FIREBASE_AUTH_EMULATOR_HOST` when that is set. + +The three database exports (`supabase`, `authjs`, `betterauth`) make no HTTP +calls at all — they connect over `--db-url`. + +The readiness report and `--skip-unsupported-providers` additionally read the +instance's Frontend API `GET /v1/environment` for its attributes and enabled +social providers. + +## Notes + +- `userId` in the source file becomes the Clerk user's `external_id`. That is + what makes a migration re-runnable and reversible. +- CSV input is coerced before validation: `a@x.dev,b@x.dev` and `["a@x.dev"]` + both become arrays, `"true"`/`1` become booleans, and JSON metadata columns + are parsed. An empty column is dropped rather than sent as null. +- A user must end up with at least one identifier (email, phone or username). + Users that do not are logged as validation failures and skipped. diff --git a/packages/cli-core/src/commands/migrate/delete.test.ts b/packages/cli-core/src/commands/migrate/delete.test.ts new file mode 100644 index 000000000..5336e2312 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/delete.test.ts @@ -0,0 +1,422 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../lib/errors.ts"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { + batch, + deleteMigration, + deleteMigratedUsers, + findMigratedUsers, + readMigratedExternalIds, + resolveMigrationToUndo, +} from "./delete.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import { getLogDir } from "./lib/logger.ts"; +import { saveSettings } from "./lib/settings.ts"; + +const captured = useCaptureLog(); + +const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurrencyLimit: 8 }; +const DATE_TIME = "2026-01-01T00:00:00"; + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { method: string; url: string }[]; + +const EXPORT = [ + { id: "legacy_a", primary_email_address: "a@x.dev" }, + { id: "legacy_b", primary_email_address: "b@x.dev" }, +]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-delete-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + process.exitCode = 0; +}); + +/** + * Stubs BAPI: `GET /v1/users` answers with whichever of `present` the request + * asked for, mirroring how Clerk ignores external IDs it does not find. + */ +function stubBapi(present: Record, onDelete?: (id: string) => Response) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ method: init?.method ?? "GET", url }); + + if (url.includes("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/v1/users?")) { + const asked = new URL(url).searchParams.getAll("external_id"); + return Response.json( + asked + .filter((externalId) => externalId in present) + .map((externalId) => ({ id: present[externalId], external_id: externalId })), + ); + } + + const match = /\/v1\/users\/([^/?]+)/.exec(url); + if (init?.method === "DELETE" && match) { + return onDelete ? onDelete(match[1] as string) : Response.json({ deleted: true }); + } + return Response.json({}); + }) as unknown as typeof fetch; +} + +const logEntries = () => + fs + .readdirSync(getLogDir()) + .flatMap((name) => fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n")) + .map((line) => JSON.parse(line) as Record); + +const deleteCalls = () => requests.filter((r) => r.method === "DELETE").map((r) => r.url); + +describe("resolveMigrationToUndo", () => { + test("reads the file and transformer from .settings", () => { + saveSettings({ key: "clerk", file: "export.json" }); + expect(resolveMigrationToUndo()).toEqual({ file: "export.json", key: "clerk" }); + }); + + // Deleting nothing silently would look like a successful undo. + test("explains when there is no .settings at all", () => { + expect(() => resolveMigrationToUndo()).toThrow(/no `.settings` from a previous/); + }); + + test.each([ + ["no file", { key: "clerk" }], + ["no transformer", { file: "export.json" }], + ])("explains when .settings has %s", (_label, settings) => { + saveSettings(settings); + expect(() => resolveMigrationToUndo()).toThrow(CliError); + }); + + test("explains when the migration file has since been removed", () => { + saveSettings({ key: "clerk", file: "gone.json" }); + expect(() => resolveMigrationToUndo()).toThrow(/no longer there/); + }); +}); + +describe("readMigratedExternalIds", () => { + test("returns the source IDs the import stamped as external_id", async () => { + expect(await readMigratedExternalIds("export.json", "clerk")).toEqual(["legacy_a", "legacy_b"]); + }); + + test("uses each transformer's own id field", async () => { + fs.writeFileSync( + path.join(workDir, "auth0.json"), + JSON.stringify([{ user_id: "auth0|1", email: "a@x.dev" }]), + ); + expect(await readMigratedExternalIds("auth0.json", "auth0")).toEqual(["auth0|1"]); + }); + + // Firebase's postTransform demands the project's hash parameters; deleting + // must not require them, so only the field mapping runs. + test("reads a firebase export without needing its password hash parameters", async () => { + fs.writeFileSync( + path.join(workDir, "firebase.json"), + JSON.stringify({ + users: [{ localId: "fb1", email: "a@x.dev", passwordHash: "H", salt: "S" }], + }), + ); + expect(await readMigratedExternalIds("firebase.json", "firebase")).toEqual(["fb1"]); + }); + + test("dedupes repeated IDs", async () => { + fs.writeFileSync( + path.join(workDir, "dupes.json"), + JSON.stringify([{ id: "legacy_a" }, { id: "legacy_a" }]), + ); + expect(await readMigratedExternalIds("dupes.json", "clerk")).toEqual(["legacy_a"]); + }); + + test("skips rows with no ID rather than matching on an empty string", async () => { + fs.writeFileSync( + path.join(workDir, "partial.json"), + JSON.stringify([{ id: "legacy_a" }, { primary_email_address: "b@x.dev" }, { id: "" }]), + ); + expect(await readMigratedExternalIds("partial.json", "clerk")).toEqual(["legacy_a"]); + }); +}); + +describe("batch", () => { + test.each([ + [0, 0], + [1, 1], + [100, 1], + [101, 2], + [250, 3], + ])("%i ids become %i request(s)", (count, expected) => { + const ids = Array.from({ length: count }, (_, i) => `u${i}`); + expect(batch(ids, 100)).toHaveLength(expected); + }); + + test("keeps every item, in order", () => { + expect(batch([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]); + }); +}); + +describe("findMigratedUsers", () => { + test("queries by external_id instead of listing the instance", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + const found = await findMigratedUsers({ + externalIds: ["legacy_a", "legacy_b"], + secretKey: "sk_test_x", + }); + + expect(found).toEqual([ + { id: "user_1", externalId: "legacy_a" }, + { id: "user_2", externalId: "legacy_b" }, + ]); + expect(requests).toHaveLength(1); + expect(requests[0]?.url).toContain("external_id=legacy_a"); + }); + + test("omits IDs the instance does not have", async () => { + stubBapi({ legacy_a: "user_1" }); + + const found = await findMigratedUsers({ + externalIds: ["legacy_a", "legacy_b"], + secretKey: "sk_test_x", + }); + + expect(found).toEqual([{ id: "user_1", externalId: "legacy_a" }]); + }); + + test("pages in batches of 100, the BAPI limit", async () => { + const ids = Array.from({ length: 150 }, (_, i) => `legacy_${i}`); + stubBapi(Object.fromEntries(ids.map((id, i) => [id, `user_${i}`]))); + + const found = await findMigratedUsers({ externalIds: ids, secretKey: "sk_test_x" }); + + expect(found).toHaveLength(150); + expect(requests).toHaveLength(2); + }); + + test("asks for a page large enough to hold the whole batch", async () => { + stubBapi({ legacy_a: "user_1" }); + await findMigratedUsers({ externalIds: ["legacy_a"], secretKey: "sk_test_x" }); + expect(requests[0]?.url).toContain("limit=100"); + }); + + // The guard that keeps this command from touching anything it did not create. + test("ignores a user whose external_id was not asked for", async () => { + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push({ method: "GET", url: input.toString() }); + return Response.json([ + { id: "user_1", external_id: "legacy_a" }, + { id: "user_999", external_id: "somebody_else" }, + { id: "user_888" }, + ]); + }) as unknown as typeof fetch; + + const found = await findMigratedUsers({ externalIds: ["legacy_a"], secretKey: "sk_test_x" }); + + expect(found).toEqual([{ id: "user_1", externalId: "legacy_a" }]); + }); +}); + +describe("deleteMigratedUsers", () => { + const users = [ + { id: "user_1", externalId: "legacy_a" }, + { id: "user_2", externalId: "legacy_b" }, + ]; + + test("deletes each user and logs the outcome", async () => { + stubBapi({}); + + const summary = await deleteMigratedUsers({ + users, + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ deleted: 2, failed: 0 }); + expect(deleteCalls()).toHaveLength(2); + expect(logEntries().filter((e) => e.status === "success")).toHaveLength(2); + }); + + test("records the source ID alongside the Clerk ID in the log", async () => { + stubBapi({}); + + await deleteMigratedUsers({ + users: [users[0] as (typeof users)[0]], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(logEntries()[0]).toMatchObject({ + userId: "legacy_a", + clerkUserId: "user_1", + status: "success", + }); + }); + + // A half-undone migration with no record of which half is worse than a + // reported failure. + test("keeps going after one user fails", async () => { + stubBapi({}, (id) => + id === "user_1" + ? new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { + status: 422, + }) + : Response.json({ deleted: true }), + ); + + const summary = await deleteMigratedUsers({ + users, + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ deleted: 1, failed: 1 }); + expect(deleteCalls()).toHaveLength(2); + expect(logEntries().some((e) => e.status === "error" && e.code === "422")).toBe(true); + }); + + test("retries a 429 and logs the attempt", async () => { + const attempts = new Map(); + stubBapi({}, (id) => { + const attempt = (attempts.get(id) ?? 0) + 1; + attempts.set(id, attempt); + return attempt === 1 + ? new Response(JSON.stringify({ errors: [{ code: "e", message: "slow down" }] }), { + status: 429, + headers: { "retry-after": "1" }, + }) + : Response.json({ deleted: true }); + }); + + const summary = await deleteMigratedUsers({ + users: [users[0] as (typeof users)[0]], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ deleted: 1, failed: 0 }); + expect(deleteCalls()).toHaveLength(2); + expect(logEntries().some((e) => e.status === "429_retry")).toBe(true); + }); + + test("groups identical failures in the breakdown", async () => { + stubBapi( + {}, + () => + new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { + status: 422, + }), + ); + + const summary = await deleteMigratedUsers({ + users, + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect([...summary.errorBreakdown.values()]).toEqual([2]); + }); +}); + +describe("deleteMigration", () => { + const baseOptions = { yes: true, secretKey: "sk_test_x" }; + + beforeEach(() => { + saveSettings({ key: "clerk", file: "export.json" }); + }); + + test("deletes the users the last run created", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + await deleteMigration(baseOptions); + + expect(deleteCalls()).toEqual([ + expect.stringContaining("/v1/users/user_1"), + expect.stringContaining("/v1/users/user_2"), + ]); + expect(captured.err).toContain("Deleted:"); + }); + + test("writes a timestamped NDJSON deletion log", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + await deleteMigration(baseOptions); + + const logs = fs.readdirSync(getLogDir()); + expect(logs).toHaveLength(1); + expect(logs[0]).toMatch(/^user-deletion-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + }); + + test("leaves users the migration did not create alone", async () => { + stubBapi({ legacy_a: "user_1" }); + + await deleteMigration(baseOptions); + + expect(deleteCalls()).toEqual([expect.stringContaining("/v1/users/user_1")]); + expect(captured.err).toContain("1 of the file's user(s) are not in this instance"); + }); + + test("does nothing when none of the migration's users are present", async () => { + stubBapi({}); + + await deleteMigration(baseOptions); + + expect(deleteCalls()).toHaveLength(0); + expect(captured.err).toContain("Nothing to delete"); + }); + + // Tests run non-TTY, which is the same signal an agent gives. + test("refuses without -y when it cannot prompt, and says how many are at stake", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + await expect(deleteMigration({ secretKey: "sk_test_x" })).rejects.toThrow( + /permanently deletes 2 user\(s\) and cannot prompt here/, + ); + expect(deleteCalls()).toHaveLength(0); + }); + + test("fails before any API call when there is no .settings", async () => { + fs.rmSync(path.join(workDir, ".settings"), { force: true }); + stubBapi({ legacy_a: "user_1" }); + + await expect(deleteMigration(baseOptions)).rejects.toThrow(CliError); + expect(requests).toHaveLength(0); + }); + + test("exits non-zero when a deletion failed", async () => { + stubBapi( + { legacy_a: "user_1" }, + () => + new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { + status: 422, + }), + ); + + await deleteMigration(baseOptions); + + expect(process.exitCode).toBe(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts new file mode 100644 index 000000000..5cc78dbef --- /dev/null +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -0,0 +1,336 @@ +/** + * `clerk migrate delete` — undo a migration. + * + * Ported from the standalone migration-tool's `src/delete/index.ts`, with two + * substantive changes: + * + * - **Users are looked up by `external_id`, not by downloading the instance.** + * The original paged through every user in the instance 500 at a time and + * intersected client-side, which on a large instance means fetching hundreds + * of thousands of users to delete a few hundred. `GET /v1/users` filters on + * up to 100 `external_id`s per call and ignores IDs it does not find, so the + * work is proportional to the migration rather than to the instance. + * - **IDs come from the existing transform pipeline.** The original + * re-implemented per-format ID extraction with its own Firebase CSV header + * list and a chain of `userId`/`user_id`/`localId`/`id` fallbacks. The + * transformer already declares which source field becomes `userId`. + * + * This is the one command in the `migrate` tree that destroys data in Clerk, so + * it stays flat and prominent rather than buried under a noun group, and it + * confirms before acting. + */ + +import { bapiRequest } from "../../lib/bapi.ts"; +import { bold, dim, green, red } from "../../lib/color.ts"; +import { + BapiError, + CliError, + ERROR_CODE, + throwUsageError, + throwUserAbort, +} from "../../lib/errors.ts"; +import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; +import { log } from "../../lib/log.ts"; +import { confirm } from "../../lib/prompts.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../mode.ts"; +import { normalizeErrorMessage } from "./import-users.ts"; +import { resolveLimits, type ResolvedLimits } from "./lib/instance.ts"; +import { deleteErrorLogger, deleteLogger, getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; +import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; +import { createApiScheduler } from "./lib/scheduler.ts"; +import { loadSettings } from "./lib/settings.ts"; +import { fileExists, readRawUsers, transformKeys } from "./lib/transform.ts"; +import { getTransformer } from "./transformers/registry.ts"; + +/** BAPI accepts at most 100 `external_id` values per `GET /v1/users` call. */ +const EXTERNAL_ID_BATCH = 100; + +export type MigrateDeleteOptions = { + yes?: boolean; + secretKey?: string; + clerkSecretKey?: string; + app?: string; + instance?: string; +}; + +export type MigratedUser = { + /** The Clerk user ID to delete. */ + id: string; + /** The source platform's ID, stamped on the user as `external_id`. */ + externalId: string; +}; + +/** + * Resolves which migration is being undone. + * + * `.settings` is the only record of that — this command has no independent way + * to know what a previous run created, which is why it is coupled to `run`. + */ +export function resolveMigrationToUndo(): { file: string; key: string } { + const settings = loadSettings(); + + if (!settings.file || !settings.key) { + throw new CliError( + "No migration to undo: this directory has no `.settings` from a previous `clerk migrate run`.\n" + + "Run `clerk migrate delete` from the directory you migrated from.", + { code: ERROR_CODE.FILE_NOT_FOUND }, + ); + } + + if (!fileExists(settings.file)) { + throw new CliError( + `The migration file ${settings.file} named in .settings is no longer there, so the users it created cannot be identified.`, + { code: ERROR_CODE.FILE_NOT_FOUND }, + ); + } + + return { file: settings.file, key: settings.key }; +} + +/** + * The source IDs a migration stamped onto Clerk users as `external_id`. + * + * Runs the transformer's field mapping but not its `postTransform`: only the + * ID matters here, and Firebase's post-transform would demand the project's + * password hash parameters to rebuild digests nobody is importing. + */ +export async function readMigratedExternalIds(file: string, key: string): Promise { + const transformer = getTransformer(key); + const rows = await readRawUsers(file, key); + + const ids = new Set(); + for (const row of rows) { + const userId = transformKeys(row, transformer).userId; + if (typeof userId === "string" && userId.length > 0) ids.add(userId); + } + return [...ids]; +} + +/** Splits `items` into chunks of at most `size`. */ +export function batch(items: T[], size: number): T[][] { + const batches: T[][] = []; + for (let i = 0; i < items.length; i += size) batches.push(items.slice(i, i + size)); + return batches; +} + +/** + * Finds the Clerk users a migration created, by `external_id`. + * + * IDs with no matching user are simply absent from the result — a partial + * migration, or one already partly undone, is the normal case. + */ +export async function findMigratedUsers(options: { + externalIds: string[]; + secretKey: string; + spinner?: SpinnerControls; +}): Promise { + const found: MigratedUser[] = []; + const batches = batch(options.externalIds, EXTERNAL_ID_BATCH); + + for (const [index, ids] of batches.entries()) { + options.spinner?.update(`Finding migrated users: batch ${index + 1}/${batches.length}`); + + const params = new URLSearchParams(); + params.set("limit", String(EXTERNAL_ID_BATCH)); + for (const id of ids) params.append("external_id", id); + + const response = await retryOn429(() => + bapiRequest({ + method: "GET", + path: `/v1/users?${params.toString()}`, + secretKey: options.secretKey, + }), + ); + + const users = (response.body ?? []) as { id?: string; external_id?: string }[]; + for (const user of Array.isArray(users) ? users : []) { + // Never delete on a partial match: only a user Clerk itself reports as + // carrying one of this migration's external IDs is in scope. + if (user.id && user.external_id && ids.includes(user.external_id)) { + found.push({ id: user.id, externalId: user.external_id }); + } + } + } + + return found; +} + +export type DeleteSummary = { + deleted: number; + failed: number; + errorBreakdown: Map; +}; + +/** Deletes each user, rate-limited and 429-retried exactly as the import is. */ +export async function deleteMigratedUsers(options: { + users: MigratedUser[]; + secretKey: string; + limits: ResolvedLimits; + dateTime: string; + spinner?: SpinnerControls; +}): Promise { + const { users, secretKey, limits, dateTime, spinner } = options; + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + const errorBreakdown = new Map(); + + let processed = 0; + let deleted = 0; + let failed = 0; + + const progress = () => + spinner?.update( + `Deleting users: [${processed}/${users.length}] (${deleted} deleted, ${failed} failed)`, + ); + + // A failure on one user must not abort the rest: a half-undone migration + // with no record of which half is far worse than a reported failure. + const recordFailure = (user: MigratedUser, message: string, code: string) => { + failed++; + processed++; + const normalized = normalizeErrorMessage(message); + errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); + deleteLogger( + { userId: user.externalId, clerkUserId: user.id, status: "error", error: message, code }, + dateTime, + ); + progress(); + }; + + const deleteOne = async (user: MigratedUser): Promise => { + try { + await retryOn429( + () => + schedule(() => + bapiRequest({ method: "DELETE", path: `/v1/users/${user.id}`, secretKey }), + ), + { + onRetry: ({ message }) => + deleteErrorLogger( + { + userId: user.externalId, + status: "429_retry", + errors: [{ code: "rate_limit_retry", message, longMessage: message }], + }, + dateTime, + ), + }, + ); + + deleted++; + processed++; + deleteLogger({ userId: user.externalId, clerkUserId: user.id, status: "success" }, dateTime); + progress(); + } catch (error) { + if (error instanceof RateLimitExceededError) { + recordFailure(user, error.message, "429"); + return; + } + const apiError = error as BapiError; + const message = apiError.longMessage ?? apiError.message ?? "Unknown error"; + recordFailure(user, message, String(apiError.status ?? "unknown")); + } + }; + + progress(); + await Promise.all(users.map(deleteOne)); + + return { deleted, failed, errorBreakdown }; +} + +function formatSummary(summary: DeleteSummary, logFile: string): string { + const lines = [ + `${bold("Deleted:")} ${green(String(summary.deleted))}`, + `${bold("Failed:")} ${red(String(summary.failed))}`, + ]; + + if (summary.errorBreakdown.size > 0) { + lines.push("", bold("Error breakdown:")); + for (const [error, count] of summary.errorBreakdown) { + lines.push(` ${count} user${count === 1 ? "" : "s"}: ${error}`); + } + } + lines.push("", dim(`Log: ${logFile}`)); + + return lines.join("\n"); +} + +export async function deleteMigration(options: MigrateDeleteOptions): Promise { + if (options.clerkSecretKey) { + log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); + } + const secretKeyOption = options.secretKey ?? options.clerkSecretKey; + + const { file, key } = resolveMigrationToUndo(); + + await withGutter("Undoing a migration", async () => { + const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const limits = resolveLimits(secretKey); + const dateTime = getDateTimeStamp(); + const logFile = getLogFilePath("user-deletion", dateTime); + + const externalIds = await readMigratedExternalIds(file, key); + if (externalIds.length === 0) { + log.warn(`No user IDs found in ${file}; nothing to undo.`); + return; + } + + const users = await withSpinner( + "Finding migrated users", + (spinner) => findMigratedUsers({ externalIds, secretKey, spinner }), + "Search complete", + ); + + if (users.length === 0) { + log.info( + `None of the ${externalIds.length} user(s) in ${file} are in ${target ?? "this instance"}. Nothing to delete.`, + ); + return; + } + + log.warn( + `About to delete ${users.length} user${users.length === 1 ? "" : "s"} from ` + + `${target ?? "the resolved instance"}, matched to ${file} by external ID.`, + ); + if (users.length < externalIds.length) { + log.info( + dim( + `${externalIds.length - users.length} of the file's user(s) are not in this instance and will be left alone.`, + ), + ); + } + + if (!options.yes) { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate delete\` permanently deletes ${users.length} user(s) and cannot prompt here. Pass -y to confirm.`, + undefined, + undefined, + [ + { + command: "clerk migrate delete -y", + description: "Delete the migrated users without prompting", + }, + ], + ); + } + + const proceed = await confirm({ + message: `Permanently delete ${users.length} user${users.length === 1 ? "" : "s"}?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + const summary = await withSpinner( + `Deleting users: [0/${users.length}]`, + (spinner) => deleteMigratedUsers({ users, secretKey, limits, dateTime, spinner }), + "Deletion complete", + ); + + log.raw(formatSummary(summary, logFile)); + + if (summary.failed > 0) process.exitCode = 1; + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts new file mode 100644 index 000000000..b48b89819 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -0,0 +1,308 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildAuth0Export, + exportAuth0, + fetchAllAuth0Users, + fetchAuth0Token, + mapAuth0UserToExport, + normalizeAuth0Domain, + resolveAuth0Credentials, +} from "./auth0.ts"; + +const captured = useCaptureLog(); + +const CREDENTIALS = { domain: "t.auth0.com", clientId: "cid", clientSecret: "csec" }; + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { url: string; body: unknown }[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expauth0-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +const auth0User = (i: number, overrides: Record = {}) => ({ + user_id: `auth0|a${i}`, + email: `a${i}@x.dev`, + email_verified: true, + given_name: `Given${i}`, + family_name: `Family${i}`, + ...overrides, +}); + +/** Stubs the token exchange plus one page of users per entry in `pages`. */ +function stubAuth0(pages: Record[][], token: Response | null = null) { + let page = 0; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ url, body: init?.body ? JSON.parse(init.body as string) : null }); + + if (url.includes("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/oauth/token")) { + return token ?? Response.json({ access_token: "tok" }); + } + return Response.json({ + users: pages[page++] ?? [], + total: pages.reduce((sum, p) => sum + p.length, 0), + }); + }) as unknown as typeof fetch; +} + +describe("normalizeAuth0Domain", () => { + test.each([ + ["t.auth0.com", "t.auth0.com"], + ["https://t.auth0.com", "t.auth0.com"], + ["http://t.auth0.com/", "t.auth0.com"], + [" t.auth0.com ", "t.auth0.com"], + ])("%s -> %s", (input, expected) => { + expect(normalizeAuth0Domain(input)).toBe(expected); + }); +}); + +describe("resolveAuth0Credentials", () => { + test("prefers flags", async () => { + const resolved = await resolveAuth0Credentials( + { domain: "flag.auth0.com", clientId: "f", clientSecret: "s" }, + { AUTH0_DOMAIN: "env.auth0.com" }, + ); + expect(resolved.domain).toBe("flag.auth0.com"); + }); + + test("falls back to the environment", async () => { + const resolved = await resolveAuth0Credentials( + {}, + { AUTH0_DOMAIN: "env.auth0.com", AUTH0_CLIENT_ID: "e", AUTH0_CLIENT_SECRET: "s" }, + ); + expect(resolved).toEqual({ domain: "env.auth0.com", clientId: "e", clientSecret: "s" }); + }); + + test("normalizes a domain that came with a scheme", async () => { + const resolved = await resolveAuth0Credentials( + { domain: "https://t.auth0.com/", clientId: "c", clientSecret: "s" }, + {}, + ); + expect(resolved.domain).toBe("t.auth0.com"); + }); + + // Tests run non-TTY, the same signal an agent gives. + test("names every missing credential at once rather than one at a time", async () => { + await expect(resolveAuth0Credentials({}, {})).rejects.toThrow( + /--domain \(or AUTH0_DOMAIN\), --client-id \(or AUTH0_CLIENT_ID\), --client-secret \(or AUTH0_CLIENT_SECRET\)/, + ); + }); + + test("names only what is actually missing", async () => { + await expect( + resolveAuth0Credentials({ domain: "t.auth0.com", clientId: "c" }, {}), + ).rejects.toThrow(/Missing: --client-secret \(or AUTH0_CLIENT_SECRET\)\./); + }); +}); + +describe("fetchAuth0Token", () => { + test("exchanges client credentials for the Management API audience", async () => { + stubAuth0([[]]); + + expect(await fetchAuth0Token(CREDENTIALS)).toBe("tok"); + expect(requests[0]?.url).toBe("https://t.auth0.com/oauth/token"); + expect(requests[0]?.body).toEqual({ + grant_type: "client_credentials", + client_id: "cid", + client_secret: "csec", + audience: "https://t.auth0.com/api/v2/", + }); + }); + + test("explains a rejection instead of surfacing a raw status", async () => { + stubAuth0( + [[]], + new Response(JSON.stringify({ error_description: "Wrong client secret" }), { status: 401 }), + ); + + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow( + /Auth0 rejected the credentials \(401\): Wrong client secret/, + ); + }); + + test("mentions the read:users scope, the usual cause", async () => { + stubAuth0([[]], new Response("{}", { status: 403 })); + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(/read:users/); + }); + + test("fails when a 200 carries no token", async () => { + stubAuth0([[]], Response.json({})); + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(CliError); + }); +}); + +describe("fetchAllAuth0Users", () => { + test("pages until a short page arrives", async () => { + stubAuth0([ + Array.from({ length: 100 }, (_, i) => auth0User(i)), + Array.from({ length: 4 }, (_, i) => auth0User(100 + i)), + ]); + + const all = await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + + expect(all).toHaveLength(104); + expect(requests[0]?.url).toContain("page=0"); + expect(requests[1]?.url).toContain("page=1"); + expect(requests).toHaveLength(2); + }); + + test("asks for totals and the documented page size", async () => { + stubAuth0([[]]); + await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + expect(requests[0]?.url).toContain("per_page=100"); + expect(requests[0]?.url).toContain("include_totals=true"); + }); + + // Auth0 caps offset pagination at 1000. Returning the first thousand quietly + // would read as "that is everyone". + test("stops at Auth0's 1000-record ceiling and says so", async () => { + stubAuth0( + Array.from({ length: 12 }, () => Array.from({ length: 100 }, (_, i) => auth0User(i))), + ); + + const all = await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + + expect(all).toHaveLength(1000); + expect(captured.err).toContain("only pages through the first 1000 users"); + expect(captured.err).toContain("bulk user export job"); + }); + + test("raises a clear error on a failed page request", async () => { + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + + await expect(fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" })).rejects.toThrow( + /Auth0 returned 500 listing users/, + ); + }); +}); + +describe("mapAuth0UserToExport", () => { + test("keeps the fields the auth0 transformer maps from", () => { + expect( + mapAuth0UserToExport(auth0User(0, { phone_number: "+1555", created_at: "2025-01-01" })), + ).toEqual({ + user_id: "auth0|a0", + email: "a0@x.dev", + given_name: "Given0", + family_name: "Family0", + phone_number: "+1555", + created_at: "2025-01-01", + email_verified: true, + }); + }); + + // Dropping a false flag would import an unconfirmed address as verified. + test.each([ + ["email_verified", false], + ["phone_verified", false], + ])("keeps %s when it is %p", (field, value) => { + const mapped = mapAuth0UserToExport(auth0User(0, { [field]: value })); + expect(mapped[field]).toBe(value); + }); + + test("drops tenant internals the import has no use for", () => { + const mapped = mapAuth0UserToExport( + auth0User(0, { + identities: [{ provider: "auth0" }], + logins_count: 42, + last_login: "2026-01-01", + multifactor: ["guardian"], + }), + ); + for (const noise of ["identities", "logins_count", "last_login", "multifactor"]) { + expect(noise in mapped).toBe(false); + } + }); + + test("omits empty metadata", () => { + const mapped = mapAuth0UserToExport( + auth0User(0, { user_metadata: {}, app_metadata: { plan: "pro" } }), + ); + expect("user_metadata" in mapped).toBe(false); + expect(mapped.app_metadata).toEqual({ plan: "pro" }); + }); +}); + +describe("buildAuth0Export", () => { + test("counts coverage and logs each user", () => { + const { users, coverage } = buildAuth0Export( + [auth0User(0), auth0User(1, { given_name: undefined })], + "2026-01-01T00:00:00", + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + + const logged = fs.readdirSync(getLogDir()); + expect(logged[0]).toMatch(/^export-/); + }); +}); + +describe("exportAuth0", () => { + test("writes the default path and reports coverage", async () => { + stubAuth0([[auth0User(0)], []]); + + await exportAuth0({ ...CREDENTIALS }); + + const written = JSON.parse( + fs.readFileSync(path.join(workDir, "exports", "auth0-export.json"), "utf-8"), + ) as Record[]; + expect(written[0]?.user_id).toBe("auth0|a0"); + expect(captured.err).toContain("Field coverage"); + }); + + test("names the command that consumes the file", async () => { + stubAuth0([[auth0User(0)], []]); + await exportAuth0({ ...CREDENTIALS }); + expect(captured.err).toContain( + "migrate run --transformer auth0 --file exports/auth0-export.json", + ); + }); + + test("--output controls the destination", async () => { + stubAuth0([[auth0User(0)], []]); + + await exportAuth0({ ...CREDENTIALS, output: "tenant.json" }); + + expect(fs.existsSync(path.join(workDir, "tenant.json"))).toBe(true); + }); + + // Auth0 only releases hashes through a support request; finding that out + // after the import means nobody can sign in. + test("says plainly that password hashes are not in the file", async () => { + stubAuth0([[auth0User(0)], []]); + await exportAuth0({ ...CREDENTIALS }); + expect(captured.err).toContain("does not return password hashes"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts new file mode 100644 index 000000000..0e94933b5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -0,0 +1,346 @@ +/** + * `clerk migrate export auth0` — pull users out of an Auth0 tenant. + * + * Ported from the standalone migration-tool's `src/export/auth0.ts`, but + * **without the `auth0` SDK**. The SDK is 28 MB across five transitive + * dependencies — including a bundled legacy copy of itself — to make two REST + * calls, and it does its own HTTP, so nothing it sends would appear under + * `--verbose`. `.claude/rules/debug-logging.md` requires library HTTP to go + * through `loggedFetch`; two direct calls satisfy that and ship nothing extra + * inside the compiled binary. + * + * **Passwords do not come out of the Management API.** Auth0 exports password + * hashes only via a support request. The coverage report says so rather than + * leaving it to be discovered when nobody can sign in. + */ + +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt, text } from "../../../lib/prompts.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; + +const PAGE_SIZE = 100; + +/** + * Auth0 caps offset pagination on `GET /api/v2/users` at 1000 records. + * Past that the tenant needs a bulk export job, so the run says so instead of + * quietly returning the first thousand as though that were everyone. + */ +const AUTH0_PAGINATION_CEILING = 1000; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/auth0"; + +export type ExportAuth0Options = { + domain?: string; + clientId?: string; + clientSecret?: string; + output?: string; +}; + +export type Auth0Credentials = { + domain: string; + clientId: string; + clientSecret: string; +}; + +/** Strips a scheme and trailing slash, so both forms of `--domain` work. */ +export function normalizeAuth0Domain(domain: string): string { + return domain + .trim() + .replace(/^https?:\/\//, "") + .replace(/\/+$/, ""); +} + +/** + * Resolves the tenant credentials: flags, then environment, then a prompt. + * + * @throws CliError in agent mode when anything is still missing, naming each + * absent flag rather than failing on the first one. + */ +export async function resolveAuth0Credentials( + options: ExportAuth0Options, + env: Record = process.env, +): Promise { + const resolved = { + domain: options.domain ?? env.AUTH0_DOMAIN, + clientId: options.clientId ?? env.AUTH0_CLIENT_ID, + clientSecret: options.clientSecret ?? env.AUTH0_CLIENT_SECRET, + }; + + const missing = ( + [ + ["domain", "--domain", "AUTH0_DOMAIN"], + ["clientId", "--client-id", "AUTH0_CLIENT_ID"], + ["clientSecret", "--client-secret", "AUTH0_CLIENT_SECRET"], + ] as const + ).filter(([key]) => !resolved[key]); + + if (missing.length === 0) { + return { + domain: normalizeAuth0Domain(resolved.domain as string), + clientId: resolved.clientId as string, + clientSecret: resolved.clientSecret as string, + }; + } + + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export auth0\` needs credentials for a machine-to-machine application and cannot prompt here.\n` + + `Missing: ${missing.map(([, flag, variable]) => `${flag} (or ${variable})`).join(", ")}.`, + DOCS_URL, + undefined, + [ + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export with explicit credentials", + }, + ], + ); + } + + log.info( + "Auth0 needs a machine-to-machine application with the `read:users` scope. Create one under Applications → APIs → Auth0 Management API → Machine to Machine Applications.", + ); + + const domain = + resolved.domain ?? + (await text({ + message: "Auth0 tenant domain (e.g. my-tenant.us.auth0.com)", + validate: (value) => (value?.trim() ? undefined : "A domain is required"), + })); + const clientId = + resolved.clientId ?? + (await text({ + message: "Machine-to-machine client ID", + validate: (value) => (value?.trim() ? undefined : "A client ID is required"), + })); + const clientSecret = + resolved.clientSecret ?? + (await passwordPrompt({ + message: "Machine-to-machine client secret", + validate: (value) => (value?.trim() ? undefined : "A client secret is required"), + })); + + return { + domain: normalizeAuth0Domain(domain), + clientId: clientId.trim(), + clientSecret: clientSecret.trim(), + }; +} + +/** Exchanges the client credentials for a Management API access token. */ +export async function fetchAuth0Token(credentials: Auth0Credentials): Promise { + const url = new URL(`https://${credentials.domain}/oauth/token`); + + const response = await loggedFetch(url, { + tag: "auth0", + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + grant_type: "client_credentials", + client_id: credentials.clientId, + client_secret: credentials.clientSecret, + audience: `https://${credentials.domain}/api/v2/`, + }), + }); + + const body = (await response.json().catch(() => ({}))) as { + access_token?: string; + error_description?: string; + error?: string; + }; + + if (!response.ok || !body.access_token) { + throw new CliError( + `Auth0 rejected the credentials (${response.status}): ${body.error_description ?? body.error ?? "no access token returned"}\n` + + "Check the domain, client ID and secret, and that the application is authorized for the Management API with the `read:users` scope.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + return body.access_token; +} + +type Auth0User = Record & { user_id?: string }; + +/** Fetches one page of users from the Management API. */ +async function fetchAuth0Page( + credentials: Auth0Credentials, + token: string, + page: number, +): Promise<{ users: Auth0User[]; total: number }> { + const url = new URL(`https://${credentials.domain}/api/v2/users`); + url.searchParams.set("page", String(page)); + url.searchParams.set("per_page", String(PAGE_SIZE)); + url.searchParams.set("include_totals", "true"); + + const response = await loggedFetch(url, { + tag: "auth0", + method: "GET", + headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, + }); + + if (!response.ok) { + const body = await response.text(); + throw new CliError(`Auth0 returned ${response.status} listing users: ${body}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + } + + const body = (await response.json()) as { users?: Auth0User[]; total?: number }; + return { users: body.users ?? [], total: body.total ?? 0 }; +} + +/** + * Pages through the tenant's users. + * + * Stops at Auth0's 1000-record ceiling with a warning naming the bulk export + * job — silently truncating would read as "that is everyone". + */ +export async function fetchAllAuth0Users(options: { + credentials: Auth0Credentials; + token: string; + spinner?: SpinnerControls; +}): Promise { + const all: Auth0User[] = []; + + for (let page = 0; ; page++) { + const { users, total } = await fetchAuth0Page(options.credentials, options.token, page); + all.push(...users); + options.spinner?.update(`Fetching users from Auth0: ${all.length} so far`); + + if (users.length < PAGE_SIZE) break; + + if (all.length >= AUTH0_PAGINATION_CEILING) { + log.warn( + `Auth0 only pages through the first ${AUTH0_PAGINATION_CEILING} users on this endpoint` + + (total > AUTH0_PAGINATION_CEILING ? `, and this tenant reports ${total}` : "") + + ". Exported what is reachable; use Auth0's bulk user export job for the rest.", + ); + break; + } + } + + return all; +} + +/** + * Keeps the fields the `auth0` transformer maps from. + * + * Deliberately a copy rather than the raw record: an Auth0 user carries + * identities, session counts and tenant internals that would bloat the export + * and mean nothing to the import. + */ +export function mapAuth0UserToExport(user: Auth0User): Record { + const exported: Record = {}; + + for (const field of [ + "user_id", + "email", + "username", + "given_name", + "family_name", + "phone_number", + "created_at", + ] as const) { + if (user[field]) exported[field] = user[field]; + } + + // Verification flags are meaningful when false, so they are copied on + // presence rather than on truthiness. + for (const field of ["email_verified", "phone_verified"] as const) { + if (user[field] !== undefined) exported[field] = user[field]; + } + + for (const field of ["user_metadata", "app_metadata"] as const) { + const value = user[field]; + if (value && typeof value === "object" && Object.keys(value).length > 0) { + exported[field] = value; + } + } + + return exported; +} + +export type Auth0ExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0ExportResult { + const exported: Record[] = []; + const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0 }; + + for (const user of users) { + const userId = String(user.user_id ?? ""); + try { + const mapped = mapAuth0UserToExport(user); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.username) counts.username++; + if (mapped.given_name) counts.firstName++; + if (mapped.family_name) counts.lastName++; + if (mapped.phone_number) counts.phone++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a phone number", count: counts.phone }, + { label: "have a username", count: counts.username }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + ], + }; +} + +export async function exportAuth0(options: ExportAuth0Options): Promise { + const credentials = await resolveAuth0Credentials(options); + + await withGutter("Exporting users from Auth0", async () => { + const dateTime = getDateTimeStamp(); + log.info(`Exporting from ${credentials.domain}.`); + + const token = await withSpinner("Authenticating with Auth0", () => + fetchAuth0Token(credentials), + ); + + const users = await withSpinner( + "Fetching users from Auth0", + (spinner) => fetchAllAuth0Users({ credentials, token, spinner }), + "Users fetched", + ); + + const { users: exported, coverage } = buildAuth0Export(users, dateTime); + const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("auth0")); + + reportExport({ + platform: "auth0", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "auth0", + }); + + if (exported.length > 0) { + log.warn( + "Auth0's Management API does not return password hashes. Request a password hash export from Auth0 support and add a `passwordHash` field to each user before importing, or migrate without passwords.", + ); + log.info(dim(`See ${DOCS_URL}`)); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts new file mode 100644 index 000000000..d8183d981 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -0,0 +1,141 @@ +/** + * `clerk migrate export authjs` — read users out of an Auth.js database. + * + * Ported from the standalone migration-tool's `src/export/authjs.ts`, on the + * `Bun.sql`/`bun:sqlite` client. + * + * Auth.js has no export tool and no single schema: the adapter decides the + * table name, and Prisma's `User` differs from Drizzle's `user` only in + * casing — which Postgres and SQLite treat as significant once quoted. The + * export tries the documented casing first and falls back rather than making + * the user find out from a driver error. + */ + +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import { log } from "../../../lib/log.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; + +/** Table names to try, in order. Prisma capitalizes; Drizzle does not. */ +const TABLE_CANDIDATES = ["User", "user", "users"] as const; + +type AuthJsRow = Record & { + id?: unknown; + name?: string | null; + email?: string | null; + email_verified?: unknown; +}; + +export function buildAuthJsQuery(client: DbClient, table: string): string { + const q = (identifier: string) => client.quote(identifier); + return ( + `SELECT ${q("id")}, ${q("name")}, ${q("email")}, ${q("emailVerified")} AS ${q("email_verified")} ` + + `FROM ${q(table)} ORDER BY ${q("id")} ASC` + ); +} + +/** True for an error that means "wrong table name", not "broken connection". */ +function isMissingTable(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /does not exist|no such table|doesn't exist|unknown table/i.test(message); +} + +/** + * Reads the user table, trying each casing until one answers. + * + * @returns The rows and the table they came from, so the run can say which. + */ +export async function fetchAuthJsUsers( + client: DbClient, +): Promise<{ rows: AuthJsRow[]; table: string }> { + let lastError: unknown; + + for (const table of TABLE_CANDIDATES) { + try { + return { rows: await client.query(buildAuthJsQuery(client, table)), table }; + } catch (error) { + if (!isMissingTable(error)) throw error; + lastError = error; + } + } + + throw lastError instanceof Error + ? new Error( + `No Auth.js user table found. Tried ${TABLE_CANDIDATES.join(", ")}. ${lastError.message}`, + ) + : new Error(`No Auth.js user table found. Tried ${TABLE_CANDIDATES.join(", ")}.`); +} + +export function buildAuthJsExport(rows: AuthJsRow[], dateTime: string) { + const users: Record[] = []; + const counts = { email: 0, emailVerified: 0, name: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + const user: Record = { id: userId }; + + if (row.name) { + user.name = row.name; + counts.name++; + } + if (row.email) { + user.email = row.email; + counts.email++; + } + // A nullable timestamp, not a boolean: the transformer reads presence. + if (row.email_verified) { + user.email_verified = + row.email_verified instanceof Date ? row.email_verified.toISOString() : row.email_verified; + counts.emailVerified++; + } + + users.push(user); + exportLogger({ userId, status: "success" }, dateTime); + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.emailVerified }, + { label: "have a name", count: counts.name }, + ], + }; +} + +export async function exportAuthJs(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", + hint: "Postgres, MySQL or a SQLite file — whichever your Auth.js adapter uses.", + }); + + await withGutter("Exporting users from Auth.js", async () => { + const dateTime = getDateTimeStamp(); + + const { rows, table } = await withSpinner("Reading the user table", () => + withDbClient(dbUrl, "authjs", fetchAuthJsUsers), + ); + log.info(`Read ${rows.length} row(s) from ${table}.`); + + const { users, coverage } = buildAuthJsExport(rows, dateTime); + const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("authjs")); + + reportExport({ + platform: "authjs", + userCount: users.length, + outputPath, + coverage, + transformerKey: "authjs", + }); + + if (users.length > 0) { + log.warn( + "Auth.js core stores no passwords — its users sign in with OAuth or email links, so they arrive without credentials and will use the same providers in Clerk.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts new file mode 100644 index 000000000..e2dcad93e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -0,0 +1,191 @@ +/** + * `clerk migrate export betterauth` — read users out of a Better Auth database. + * + * Ported from the standalone migration-tool's `src/export/betterauth.ts`, on + * the `Bun.sql`/`bun:sqlite` client. + * + * Better Auth's schema depends on which plugins are enabled, so the columns + * are **detected from the schema** rather than asked for: the username plugin + * adds `username`, admin adds `banned`, phone-number adds `phoneNumber`, and + * so on. Selecting a column that is not there fails the whole query, and + * asking the user which plugins they run is a question their database can + * already answer. + * + * Passwords live on the `account` row for the credential provider, not on the + * user, which is why the export joins. + */ + +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; + +/** Columns a Better Auth plugin adds to the user table. */ +export const PLUGIN_COLUMNS = [ + "username", + "displayUsername", + "phoneNumber", + "phoneNumberVerified", + "role", + "banned", + "banReason", + "banExpires", + "twoFactorEnabled", +] as const; + +export type PluginColumn = (typeof PLUGIN_COLUMNS)[number]; + +/** Columns every Better Auth install has. */ +const CORE_COLUMNS = ["id", "email", "emailVerified", "name", "createdAt", "updatedAt"] as const; + +/** + * Asks the schema which plugin columns exist. + * + * SQLite has no `information_schema`, so it goes through `PRAGMA` — and the + * PRAGMA takes the table name inline rather than as a bind parameter. + */ +export async function detectPluginColumns(client: DbClient): Promise> { + const present = new Set(); + + if (client.dbType === "sqlite") { + const rows = await client.query<{ name: string }>(`PRAGMA table_info(${client.quote("user")})`); + const columns = new Set(rows.map((row) => row.name)); + for (const column of PLUGIN_COLUMNS) { + if (columns.has(column)) present.add(column); + } + return present; + } + + const scope = client.dbType === "mysql" ? "DATABASE()" : "current_schema()"; + const placeholders = PLUGIN_COLUMNS.map((_, index) => client.placeholder(index + 1)).join(", "); + + const rows = await client.query<{ column_name?: string; COLUMN_NAME?: string }>( + `SELECT column_name FROM information_schema.columns + WHERE table_name = 'user' AND table_schema = ${scope} + AND column_name IN (${placeholders})`, + [...PLUGIN_COLUMNS], + ); + + for (const row of rows) { + // MySQL 8 answers with an upper-case column label. + const name = (row.column_name ?? row.COLUMN_NAME) as PluginColumn | undefined; + if (name && (PLUGIN_COLUMNS as readonly string[]).includes(name)) present.add(name); + } + + return present; +} + +/** + * Builds the SELECT, including only the plugin columns that exist. + * + * @param pluginColumns - From {@link detectPluginColumns}. + */ +export function buildBetterAuthQuery(client: DbClient, pluginColumns: Set): string { + const q = (identifier: string) => client.quote(identifier); + const selected = [ + ...CORE_COLUMNS.map((column) => `u.${q(column)}`), + ...PLUGIN_COLUMNS.filter((column) => pluginColumns.has(column)).map( + (column) => `u.${q(column)}`, + ), + ]; + + // LEFT JOIN, not INNER: a user who only ever signed in with OAuth has no + // credential account, and dropping them would silently shrink the export. + return ( + `SELECT ${selected.join(", ")}, a.${q("password")} AS ${q("password_hash")} ` + + `FROM ${q("user")} u ` + + `LEFT JOIN ${q("account")} a ON a.${q("userId")} = u.${q("id")} ` + + `AND a.${q("providerId")} = 'credential' ` + + `ORDER BY u.${q("id")} ASC` + ); +} + +type BetterAuthRow = Record & { id?: unknown }; + +/** Renames the schema's camelCase onto what the betterauth transformer reads. */ +const FIELD_ALIASES: Record = { + id: "user_id", + emailVerified: "email_verified", + phoneNumber: "phone_number", + phoneNumberVerified: "phone_number_verified", + displayUsername: "display_username", + createdAt: "created_at", + updatedAt: "updated_at", +}; + +export function buildBetterAuthExport(rows: BetterAuthRow[], dateTime: string) { + const users: Record[] = []; + const counts = { email: 0, emailVerified: 0, password: 0, name: 0, username: 0, phone: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + const user: Record = {}; + + for (const [key, value] of Object.entries(row)) { + if (value === null || value === undefined) continue; + user[FIELD_ALIASES[key] ?? key] = value instanceof Date ? value.toISOString() : value; + } + + if (row.email) counts.email++; + if (row.emailVerified) counts.emailVerified++; + if (row.password_hash) counts.password++; + if (row.name) counts.name++; + if (row.username) counts.username++; + if (row.phoneNumber) counts.phone++; + + users.push(user); + exportLogger({ userId, status: "success" }, dateTime); + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.emailVerified }, + { label: "have a password hash", count: counts.password }, + { label: "have a name", count: counts.name }, + { label: "have a username", count: counts.username }, + { label: "have a phone number", count: counts.phone }, + ], + }; +} + +export async function exportBetterAuth(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, { + platform: "betterauth", + envVar: "BETTERAUTH_DB_URL", + prompt: "Better Auth database connection string", + hint: "Postgres, MySQL or a SQLite file — whichever your Better Auth install uses.", + }); + + await withGutter("Exporting users from Better Auth", async () => { + const dateTime = getDateTimeStamp(); + + const { rows, plugins } = await withSpinner("Reading the user table", () => + withDbClient(dbUrl, "betterauth", async (client) => { + const plugins = await detectPluginColumns(client); + const rows = await client.query(buildBetterAuthQuery(client, plugins)); + return { rows, plugins }; + }), + ); + + log.info( + plugins.size > 0 + ? `Detected plugin columns: ${[...plugins].join(", ")}.` + : "No plugin columns detected; exporting the core user fields.", + ); + + const { users, coverage } = buildBetterAuthExport(rows, dateTime); + const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("betterauth")); + + reportExport({ + platform: "betterauth", + userCount: users.length, + outputPath, + coverage, + transformerKey: "betterauth", + }); + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts new file mode 100644 index 000000000..12a3062db --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -0,0 +1,281 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildClerkExport, + exportClerk, + fetchAllClerkUsers, + mapClerkUserToExport, +} from "./clerk.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: string[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expclerk-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +/** Answers `GET /v1/users` from `pages`, one page per call. */ +function stubPages(pages: unknown[][]) { + let call = 0; + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push(input.toString()); + return Response.json(pages[call++] ?? []); + }) as unknown as typeof fetch; +} + +const user = (overrides: Record = {}) => ({ + id: "user_1", + primary_email_address_id: "idn_1", + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + ], + phone_numbers: [], + ...overrides, +}); + +describe("mapClerkUserToExport", () => { + test("writes the field names the clerk transformer reads", () => { + expect( + mapClerkUserToExport(user({ first_name: "Ada", last_name: "L", username: "ada" })), + ).toMatchObject({ + id: "user_1", + primary_email_address: "a@x.dev", + first_name: "Ada", + last_name: "L", + username: "ada", + }); + }); + + // `migrate run` puts the first entry on POST /v1/users and attaches the rest + // afterwards, so a reordered list would change which address signs the user in. + test("keeps the primary identifier out of the additional list", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("a@x.dev"); + expect(mapped.verified_email_addresses).toEqual(["b@x.dev"]); + }); + + test("separates unverified identifiers", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "c@x.dev", verification: { status: "unverified" } }, + ], + }), + ); + expect(mapped.unverified_email_addresses).toEqual(["c@x.dev"]); + expect(mapped.verified_email_addresses).toBeUndefined(); + }); + + test("promotes the first verified address when none is flagged primary", () => { + const mapped = mapClerkUserToExport( + user({ + primary_email_address_id: null, + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("a@x.dev"); + expect(mapped.verified_email_addresses).toEqual(["b@x.dev"]); + }); + + test("maps phone numbers the same way", () => { + const mapped = mapClerkUserToExport( + user({ + primary_phone_number_id: "pn_1", + phone_numbers: [ + { id: "pn_1", phone_number: "+15555550100", verification: { status: "verified" } }, + { id: "pn_2", phone_number: "+15555550101", verification: { status: "unverified" } }, + ], + }), + ); + expect(mapped.primary_phone_number).toBe("+15555550100"); + expect(mapped.unverified_phone_numbers).toEqual(["+15555550101"]); + }); + + test("converts BAPI's Unix-millisecond timestamps to RFC3339", () => { + const mapped = mapClerkUserToExport(user({ created_at: 1704067200000 })); + expect(mapped.created_at).toBe("2024-01-01T00:00:00.000Z"); + }); + + test("omits empty metadata rather than writing empty objects", () => { + const mapped = mapClerkUserToExport( + user({ public_metadata: {}, private_metadata: { plan: "pro" } }), + ); + expect("public_metadata" in mapped).toBe(false); + expect(mapped.private_metadata).toEqual({ plan: "pro" }); + }); + + test("carries the account-state fields the import accepts", () => { + const mapped = mapClerkUserToExport( + user({ + banned: true, + create_organization_enabled: false, + create_organizations_limit: 3, + delete_self_enabled: true, + }), + ); + expect(mapped).toMatchObject({ + banned: true, + create_organization_enabled: false, + create_organizations_limit: 3, + delete_self_enabled: true, + }); + }); +}); + +describe("fetchAllClerkUsers", () => { + test("pages until a short page arrives", async () => { + stubPages([ + Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), + Array.from({ length: 12 }, (_, i) => user({ id: `v${i}` })), + ]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(512); + expect(requests).toHaveLength(2); + expect(requests[1]).toContain("offset=500"); + }); + + // A full final page must still trigger one more request, or an instance whose + // size is an exact multiple of the page size would look short by one page. + test("makes one more request when the last page is exactly full", async () => { + stubPages([Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), []]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(500); + expect(requests).toHaveLength(2); + }); + + test("asks for BAPI's maximum page size", async () => { + stubPages([[]]); + await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + expect(requests[0]).toContain("limit=500"); + }); + + test("copes with an instance that has no users", async () => { + stubPages([[]]); + expect(await fetchAllClerkUsers({ secretKey: "sk_test_x" })).toEqual([]); + }); +}); + +describe("buildClerkExport", () => { + test("counts coverage per field", () => { + const { coverage } = buildClerkExport( + [user({ id: "u1", first_name: "Ada", password_enabled: true }), user({ id: "u2" })], + "2026-01-01T00:00:00", + ); + + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + expect(byLabel["have a password (not exportable — see below)"]).toBe(1); + }); + + test("logs one NDJSON line per exported user", () => { + buildClerkExport([user({ id: "u1" }), user({ id: "u2" })], "2026-01-01T00:00:00"); + + const entries = fs + .readdirSync(getLogDir()) + .flatMap((name) => fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n")) + .map((line) => JSON.parse(line) as Record); + + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ userId: "u1", status: "success" }); + }); + + test("writes the export log where `logs list` will find it", () => { + buildClerkExport([user()], "2026-01-01T12:00:00"); + expect(fs.readdirSync(getLogDir())[0]).toBe("export-2026-01-01T12-00-00.log"); + }); +}); + +describe("exportClerk", () => { + test("writes the default path and reports coverage", async () => { + stubPages([[user({ id: "u1", first_name: "Ada" })], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + const written = JSON.parse( + fs.readFileSync(path.join(workDir, "exports", "clerk-export.json"), "utf-8"), + ) as Record[]; + expect(written).toHaveLength(1); + expect(written[0]?.id).toBe("u1"); + expect(captured.err).toContain("Field coverage"); + expect(captured.err).toContain("Exported 1 user(s)"); + }); + + test("names the command that consumes the file", async () => { + stubPages([[user()], []]); + await exportClerk({ secretKey: "sk_test_x" }); + expect(captured.err).toContain( + "migrate run --transformer clerk --file exports/clerk-export.json", + ); + }); + + test("--output controls the destination, relative to the working directory", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x", output: "somewhere/mine.json" }); + + expect(fs.existsSync(path.join(workDir, "somewhere", "mine.json"))).toBe(true); + expect(fs.existsSync(path.join(workDir, "exports", "clerk-export.json"))).toBe(false); + }); + + // Silence here would be the worst outcome: the operator finds out when + // nobody can sign in to the destination instance. + test("says plainly that passwords are not in the file", async () => { + stubPages([[user({ password_enabled: true })], []]); + await exportClerk({ secretKey: "sk_test_x" }); + expect(captured.err).toContain("never returns password digests"); + }); + + test("writes an empty file and says so when the instance has no users", async () => { + stubPages([[]]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("No users found to export"); + expect( + JSON.parse(fs.readFileSync(path.join(workDir, "exports", "clerk-export.json"), "utf-8")), + ).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts new file mode 100644 index 000000000..37c10d802 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -0,0 +1,263 @@ +/** + * `clerk migrate export clerk` — pull users out of a Clerk instance. + * + * Ported from the standalone migration-tool's `src/export/clerk.ts`, rewritten + * onto `bapiRequest` instead of `@clerk/backend` so it shares the CLI's auth + * resolution, `--verbose` request tracing and error taxonomy. + * + * The output feeds `clerk migrate run --transformer clerk` unedited, which is + * what makes development → production a two-command operation. + * + * **Passwords do not come out of this endpoint.** Clerk never returns password + * digests, TOTP secrets or backup codes over the API; only the `*_enabled` + * booleans. The coverage report says how many users *have* a password so the + * gap is visible before the import, not after. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { retryOn429 } from "../lib/retry.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; + +/** BAPI's maximum page size for `GET /v1/users`. */ +const PAGE_SIZE = 500; + +export type ExportClerkOptions = { + output?: string; + secretKey?: string; + clerkSecretKey?: string; + app?: string; + instance?: string; +}; + +type BapiIdentifier = { + email_address?: string; + phone_number?: string; + verification?: { status?: string } | null; +}; + +type BapiUser = { + id: string; + external_id?: string | null; + username?: string | null; + first_name?: string | null; + last_name?: string | null; + email_addresses?: BapiIdentifier[]; + phone_numbers?: BapiIdentifier[]; + primary_email_address_id?: string | null; + primary_phone_number_id?: string | null; + public_metadata?: Record; + private_metadata?: Record; + unsafe_metadata?: Record; + password_enabled?: boolean; + totp_enabled?: boolean; + banned?: boolean; + create_organization_enabled?: boolean; + create_organizations_limit?: number | null; + delete_self_enabled?: boolean; + created_at?: number; + legal_accepted_at?: number | null; +}; + +type IdentifierWithId = BapiIdentifier & { id?: string }; + +/** + * Splits identifiers into verified and unverified, primary first. + * + * The primary has to lead: `migrate run` puts the first entry on + * `POST /v1/users` and attaches the rest afterwards, so a reordered list would + * silently change which address the user signs in with. + */ +function splitIdentifiers( + entries: IdentifierWithId[] | undefined, + primaryId: string | null | undefined, + read: (entry: BapiIdentifier) => string | undefined, +): { primary?: string; verified: string[]; unverified: string[] } { + const verified: string[] = []; + const unverified: string[] = []; + let primary: string | undefined; + + for (const entry of entries ?? []) { + const value = read(entry); + if (!value) continue; + + if (entry.id && entry.id === primaryId) { + primary = value; + continue; + } + if (entry.verification?.status === "verified") verified.push(value); + else unverified.push(value); + } + + // No primary flagged: promote the first verified one so the export still has + // an identifier the import can lead with. + if (!primary && verified.length > 0) primary = verified.shift(); + + return { primary, verified, unverified }; +} + +/** Maps a BAPI user onto the shape the `clerk` transformer reads. */ +export function mapClerkUserToExport(user: BapiUser): Record { + const exported: Record = { id: user.id }; + + const emails = splitIdentifiers( + user.email_addresses, + user.primary_email_address_id, + (entry) => entry.email_address, + ); + if (emails.primary) exported.primary_email_address = emails.primary; + if (emails.verified.length > 0) exported.verified_email_addresses = emails.verified; + if (emails.unverified.length > 0) exported.unverified_email_addresses = emails.unverified; + + const phones = splitIdentifiers( + user.phone_numbers, + user.primary_phone_number_id, + (entry) => entry.phone_number, + ); + if (phones.primary) exported.primary_phone_number = phones.primary; + if (phones.verified.length > 0) exported.verified_phone_numbers = phones.verified; + if (phones.unverified.length > 0) exported.unverified_phone_numbers = phones.unverified; + + if (user.username) exported.username = user.username; + if (user.first_name) exported.first_name = user.first_name; + if (user.last_name) exported.last_name = user.last_name; + + for (const [source, target] of [ + ["public_metadata", "public_metadata"], + ["private_metadata", "private_metadata"], + ["unsafe_metadata", "unsafe_metadata"], + ] as const) { + const value = user[source]; + if (value && Object.keys(value).length > 0) exported[target] = value; + } + + if (user.banned) exported.banned = true; + if (user.create_organization_enabled !== undefined) { + exported.create_organization_enabled = user.create_organization_enabled; + } + if (user.create_organizations_limit !== null && user.create_organizations_limit !== undefined) { + exported.create_organizations_limit = user.create_organizations_limit; + } + if (user.delete_self_enabled !== undefined) { + exported.delete_self_enabled = user.delete_self_enabled; + } + + // BAPI reports timestamps as Unix milliseconds; the schema wants RFC3339. + if (user.created_at) exported.created_at = new Date(user.created_at).toISOString(); + if (user.legal_accepted_at) { + exported.legal_accepted_at = new Date(user.legal_accepted_at).toISOString(); + } + + return exported; +} + +/** Pages through every user in the instance. */ +export async function fetchAllClerkUsers(options: { + secretKey: string; + spinner?: SpinnerControls; +}): Promise { + const all: BapiUser[] = []; + + for (let offset = 0; ; offset += PAGE_SIZE) { + const response = await retryOn429(() => + bapiRequest({ + method: "GET", + path: `/v1/users?limit=${PAGE_SIZE}&offset=${offset}`, + secretKey: options.secretKey, + }), + ); + + const page = Array.isArray(response.body) ? (response.body as BapiUser[]) : []; + all.push(...page); + options.spinner?.update(`Fetching users from Clerk: ${all.length} so far`); + + // A short page means the end; anything else would loop forever on an + // instance whose size happens to be a multiple of the page size. + if (page.length < PAGE_SIZE) break; + } + + return all; +} + +export type ClerkExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +/** Maps every user and counts what the export actually contains. */ +export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExportResult { + const exported: Record[] = []; + const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0, password: 0 }; + + for (const user of users) { + try { + const mapped = mapClerkUserToExport(user); + exported.push(mapped); + + if (mapped.primary_email_address) counts.email++; + if (mapped.username) counts.username++; + if (mapped.first_name) counts.firstName++; + if (mapped.last_name) counts.lastName++; + if (mapped.primary_phone_number) counts.phone++; + if (user.password_enabled) counts.password++; + + exportLogger({ userId: user.id, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId: user.id, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a phone number", count: counts.phone }, + { label: "have a username", count: counts.username }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + { label: "have a password (not exportable — see below)", count: counts.password }, + ], + }; +} + +export async function exportClerk(options: ExportClerkOptions): Promise { + if (options.clerkSecretKey) { + log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); + } + const secretKeyOption = options.secretKey ?? options.clerkSecretKey; + + await withGutter("Exporting users from Clerk", async () => { + const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const dateTime = getDateTimeStamp(); + + log.info(`Exporting from ${target ?? "the resolved instance"}.`); + + const users = await withSpinner( + "Fetching users from Clerk", + (spinner) => fetchAllClerkUsers({ secretKey, spinner }), + "Users fetched", + ); + + const { users: exported, coverage } = buildClerkExport(users, dateTime); + const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("clerk")); + + reportExport({ + platform: "clerk", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "clerk", + }); + + if (exported.length > 0) { + log.warn( + "Clerk's API never returns password digests, TOTP secrets or backup codes, so they are not in this file. " + + "Users will need to reset their password in the destination instance.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts new file mode 100644 index 000000000..e7601df39 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -0,0 +1,358 @@ +/** + * The three database-backed exports, driven against a real SQLite database. + * + * SQLite because it is the one engine that needs no container, and it + * exercises the same client, the same query building and the same plugin + * detection path (via `PRAGMA` rather than `information_schema`). Postgres and + * MySQL are covered by the manual matrix run recorded in the ticket. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { createDbClient, type DbClient } from "../lib/db.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { buildAuthJsExport, buildAuthJsQuery, exportAuthJs, fetchAuthJsUsers } from "./authjs.ts"; +import { + buildBetterAuthExport, + buildBetterAuthQuery, + detectPluginColumns, + exportBetterAuth, + PLUGIN_COLUMNS, +} from "./betterauth.ts"; +import { buildSupabaseExport } from "./supabase.ts"; +import { looksLikeConnectionString, resolveDbUrl } from "./db-options.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-dbexp-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +/** Builds a fresh SQLite file so each test starts from a known schema. */ +function makeDb(build: (db: Database) => void): string { + const file = path.join(workDir, `db-${counter++}.sqlite`); + const db = new Database(file, { create: true }); + build(db); + db.close(); + return file; +} + +function betterAuthDb(pluginColumns: string[], rows: Record[] = []): string { + return makeDb((db) => { + const extra = pluginColumns.map((column) => `, "${column}" TEXT`).join(""); + db.run( + `CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER, name TEXT, + "createdAt" TEXT, "updatedAt" TEXT${extra})`, + ); + db.run(`CREATE TABLE "account" (id TEXT, "userId" TEXT, "providerId" TEXT, password TEXT)`); + for (const row of rows) { + const keys = Object.keys(row); + db.run( + `INSERT INTO "user" (${keys.map((k) => `"${k}"`).join(",")}) VALUES (${keys.map(() => "?").join(",")})`, + keys.map((k) => row[k]) as never[], + ); + } + }); +} + +async function withClient(file: string, work: (client: DbClient) => Promise): Promise { + const client = await createDbClient(file); + try { + return await work(client); + } finally { + await client.close(); + } +} + +describe("looksLikeConnectionString", () => { + test.each([ + ["postgres://u:p@h:5432/db", true], + ["mysql://u:p@h:3306/db", true], + ["./db.sqlite", true], + ["file:./db.sqlite", true], + ["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/abs/app.db", true], + ["", false], + [" ", false], + ["just some words", false], + ["postgres://", false], + ])("%p -> %p", (input, expected) => { + expect(looksLikeConnectionString(input)).toBe(expected); + }); +}); + +describe("resolveDbUrl", () => { + const config = { platform: "authjs" as const, envVar: "AUTHJS_DB_URL", prompt: "url" }; + + test("prefers the flag", async () => { + const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, { + AUTHJS_DB_URL: "mysql://u:p@h/db", + }); + expect(url).toBe("postgres://u:p@h/db"); + }); + + test("falls back to the environment variable", async () => { + expect(await resolveDbUrl({}, config, { AUTHJS_DB_URL: "mysql://u:p@h/db" })).toBe( + "mysql://u:p@h/db", + ); + }); + + test("rejects a flag that is not a connection string, naming the encoding trap", async () => { + await expect(resolveDbUrl({ dbUrl: "not a url" }, config, {})).rejects.toThrow(/URL-encode it/); + }); + + test("warns and moves on when the environment variable is unusable", async () => { + // Tests run non-TTY, so it then hits the agent-mode branch. + await expect(resolveDbUrl({}, config, { AUTHJS_DB_URL: "garbage" })).rejects.toThrow( + /cannot prompt here/, + ); + expect(captured.err).toContain("AUTHJS_DB_URL is not a valid connection string"); + }); + + test("names both the flag and the variable when it cannot prompt", async () => { + await expect(resolveDbUrl({}, config, {})).rejects.toThrow(/--db-url.*AUTHJS_DB_URL/s); + }); +}); + +describe("authjs export", () => { + const authJsDb = (table: string) => + makeDb((db) => { + db.run( + `CREATE TABLE "${table}" (id TEXT PRIMARY KEY, name TEXT, email TEXT, "emailVerified" TEXT)`, + ); + db.run(`INSERT INTO "${table}" VALUES (?,?,?,?)`, [ + "aj1", + "Jane Doe", + "jane@x.dev", + "2024-01-15", + ]); + db.run(`INSERT INTO "${table}" VALUES (?,?,?,?)`, ["aj2", "John Smith", "john@x.dev", null]); + }); + + test("quotes identifiers for the dialect", async () => { + await withClient(authJsDb("User"), async (client) => { + expect(buildAuthJsQuery(client, "User")).toContain('"User"'); + expect(buildAuthJsQuery(client, "User")).toContain('"emailVerified" AS "email_verified"'); + }); + }); + + // Prisma capitalizes the table, Drizzle does not, and Auth.js has no single + // schema — so the export tries rather than making the user guess. + test.each([["User"], ["user"], ["users"]])("finds the %s table", async (table) => { + const { rows } = await withClient(authJsDb(table), fetchAuthJsUsers); + expect(rows).toHaveLength(2); + }); + + test("fails clearly when no candidate table exists", async () => { + const file = makeDb((db) => db.run(`CREATE TABLE unrelated (id TEXT)`)); + await expect(withClient(file, fetchAuthJsUsers)).rejects.toThrow( + /No Auth.js user table found. Tried User, user, users/, + ); + }); + + test("treats email_verified as a nullable timestamp, not a boolean", () => { + const { users } = buildAuthJsExport( + [ + { id: "a", email: "a@x.dev", email_verified: "2024-01-15" }, + { id: "b", email: "b@x.dev", email_verified: null }, + ], + "2026-01-01T00:00:00", + ); + expect(users[0]?.email_verified).toBe("2024-01-15"); + expect("email_verified" in (users[1] ?? {})).toBe(false); + }); + + test("counts coverage", () => { + const { coverage } = buildAuthJsExport( + [{ id: "a", email: "a@x.dev", name: "A", email_verified: "2024-01-01" }, { id: "b" }], + "2026-01-01T00:00:00", + ); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(1); + expect(byLabel["have a verified email"]).toBe(1); + }); + + test("exports end to end and says which table it read", async () => { + await exportAuthJs({ dbUrl: authJsDb("User"), output: "authjs.json" }); + + const written = JSON.parse(fs.readFileSync(path.join(workDir, "authjs.json"), "utf-8")); + expect(written).toHaveLength(2); + expect(captured.err).toContain("Read 2 row(s) from"); + expect(captured.err).toContain("stores no passwords"); + }); +}); + +describe("betterauth export", () => { + test("detects only the plugin columns that exist", async () => { + await withClient(betterAuthDb(["username", "banned"]), async (client) => { + expect([...(await detectPluginColumns(client))].sort()).toEqual(["banned", "username"]); + }); + }); + + test("detects nothing on a core-only schema", async () => { + await withClient(betterAuthDb([]), async (client) => { + expect((await detectPluginColumns(client)).size).toBe(0); + }); + }); + + test("detects every plugin column when all are present", async () => { + await withClient(betterAuthDb([...PLUGIN_COLUMNS]), async (client) => { + expect((await detectPluginColumns(client)).size).toBe(PLUGIN_COLUMNS.length); + }); + }); + + // Selecting a column that is not there fails the whole query, which is why + // the columns are detected rather than assumed. + test("selects only detected columns", async () => { + await withClient(betterAuthDb(["username"]), async (client) => { + const query = buildBetterAuthQuery(client, await detectPluginColumns(client)); + expect(query).toContain('"username"'); + expect(query).not.toContain('"twoFactorEnabled"'); + }); + }); + + test("the built query actually runs against the schema it was built for", async () => { + const file = betterAuthDb( + ["username", "role"], + [{ id: "u1", email: "a@x.dev", username: "a" }], + ); + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, await detectPluginColumns(client))), + ); + expect(rows).toHaveLength(1); + }); + + // A user who only ever signed in with OAuth has no credential account; + // an INNER JOIN would drop them and silently shrink the export. + test("keeps a user with no credential account", async () => { + const file = betterAuthDb( + [], + [ + { id: "u1", email: "a@x.dev" }, + { id: "u2", email: "b@x.dev" }, + ], + ); + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, new Set())), + ); + expect(rows).toHaveLength(2); + }); + + test("renames camelCase columns onto what the transformer reads", () => { + const { users } = buildBetterAuthExport( + [{ id: "u1", emailVerified: 1, phoneNumber: "+1555", createdAt: "2025-01-01" }], + "2026-01-01T00:00:00", + ); + expect(users[0]).toMatchObject({ + user_id: "u1", + email_verified: 1, + phone_number: "+1555", + created_at: "2025-01-01", + }); + }); + + test("exports end to end and reports the detected plugins", async () => { + const file = betterAuthDb(["username"], [{ id: "u1", email: "a@x.dev", username: "ada" }]); + + await exportBetterAuth({ dbUrl: file, output: "ba.json" }); + + expect(captured.err).toContain("Detected plugin columns: username"); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "ba.json"), "utf-8"))).toHaveLength(1); + }); + + test("says so plainly when no plugins are in use", async () => { + await exportBetterAuth({ dbUrl: betterAuthDb([]), output: "ba2.json" }); + expect(captured.err).toContain("No plugin columns detected"); + }); +}); + +describe("supabase export", () => { + test("serializes timestamps the transformer can parse", () => { + const { users } = buildSupabaseExport( + [{ id: "u1", email: "a@x.dev", created_at: new Date("2024-01-01T00:00:00Z") }], + "2026-01-01T00:00:00", + ); + expect(users[0]?.created_at).toBe("2024-01-01T00:00:00.000Z"); + }); + + test("omits null columns rather than exporting them", () => { + const { users } = buildSupabaseExport( + [{ id: "u1", email: "a@x.dev", phone: null, last_name: null }], + "2026-01-01T00:00:00", + ); + expect("phone" in (users[0] ?? {})).toBe(false); + expect("last_name" in (users[0] ?? {})).toBe(false); + }); + + test("counts the password hashes, the reason this reads the database", () => { + const { coverage } = buildSupabaseExport( + [ + { id: "u1", email: "a@x.dev", encrypted_password: "$2b$10$x" }, + { id: "u2", email: "b@x.dev" }, + ], + "2026-01-01T00:00:00", + ); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have a password hash"]).toBe(1); + }); + + test("keeps raw_app_meta_data, which --skip-unsupported-providers reads", () => { + const { users } = buildSupabaseExport( + [{ id: "u1", email: "a@x.dev", raw_app_meta_data: { providers: ["discord"] } }], + "2026-01-01T00:00:00", + ); + expect(users[0]?.raw_app_meta_data).toEqual({ providers: ["discord"] }); + }); + + test("logs one NDJSON line per exported user", () => { + buildSupabaseExport([{ id: "u1" }, { id: "u2" }], "2026-01-01T12:00:00"); + + const written = fs.readdirSync(getLogDir()); + expect(written[0]).toBe("export-2026-01-01T12-00-00.log"); + expect( + fs + .readFileSync(path.join(getLogDir(), written[0] as string), "utf-8") + .trim() + .split("\n"), + ).toHaveLength(2); + }); +}); + +describe("connection failures", () => { + afterEach(() => { + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); + }); + + test("a missing SQLite file fails before anything is written", async () => { + await expect(exportAuthJs({ dbUrl: "./definitely-not-here.sqlite" })).rejects.toThrow(CliError); + expect(fs.existsSync(path.join(workDir, "exports"))).toBe(false); + }); + + test("the failure never contains the password", async () => { + await expect( + exportBetterAuth({ dbUrl: "postgres://user:hunter2@127.0.0.1:1/db" }), + ).rejects.toThrow( + expect.objectContaining({ message: expect.not.stringContaining("hunter2") }) as Error, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts new file mode 100644 index 000000000..880e177da --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -0,0 +1,111 @@ +/** + * Resolving a `--db-url` for the database-backed exports. + * + * Shared by supabase, authjs and betterauth: all three take one connection + * string, from a flag, an environment variable, or a prompt. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { detectDbType, redactConnectionString, type DbPlatform } from "../lib/db.ts"; + +export type DbExportOptions = { + dbUrl?: string; + output?: string; +}; + +type ResolveConfig = { + platform: DbPlatform; + /** Environment variable checked when `--db-url` is absent. */ + envVar: string; + prompt: string; + /** Extra guidance shown before prompting. */ + hint?: string; +}; + +/** True for something that could plausibly be a connection string. */ +export function looksLikeConnectionString(value: string): boolean { + const trimmed = value.trim(); + if (!trimmed) return false; + + if (/^(postgresql|postgres|mysql|mysql2):\/\//i.test(trimmed)) { + try { + // A hostname is required: `postgres://` alone parses as a valid URL, and + // accepting it only defers the failure into the driver. + return new URL(trimmed).hostname.length > 0; + } catch { + // A password with an unencoded `@` or `#` is the usual cause, and it is + // worth saying so rather than failing later inside the driver. + return false; + } + } + + return ( + trimmed.startsWith("file:") || /\.(sqlite3?|db)$/i.test(trimmed) || trimmed.startsWith("./") + ); +} + +/** + * Resolves the connection string: flag, then environment, then a prompt. + * + * Prompted as a password so it is not echoed — a connection string carries the + * database password inline. + */ +export async function resolveDbUrl( + options: DbExportOptions, + config: ResolveConfig, + env: Record = process.env, +): Promise { + const fromFlag = options.dbUrl?.trim(); + if (fromFlag) { + if (!looksLikeConnectionString(fromFlag)) { + throwUsageError( + `--db-url does not look like a connection string. Expected postgres://…, mysql://… or a SQLite file path.\n` + + "If the password contains @, # or /, URL-encode it.", + ); + } + return fromFlag; + } + + const fromEnv = env[config.envVar]?.trim(); + if (fromEnv) { + if (looksLikeConnectionString(fromEnv)) return fromEnv; + // Falling through silently would make the prompt look unexplained. + log.warn(`${config.envVar} is not a valid connection string; ignoring it.`); + } + + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export ${config.platform}\` needs a database connection and cannot prompt here.\n` + + `Pass --db-url, or set ${config.envVar}.`, + undefined, + undefined, + [ + { + command: `clerk migrate export ${config.platform} --db-url "postgres://user:password@host:5432/db"`, + description: "Export from Postgres", + }, + ], + ); + } + + if (config.hint) log.info(dim(config.hint)); + + const answer = await passwordPrompt({ + message: config.prompt, + validate: (value) => + looksLikeConnectionString(value ?? "") + ? undefined + : "Expected postgres://…, mysql://… or a SQLite file path", + }); + + return answer.trim(); +} + +/** Describes the target for the run's opening line, credentials removed. */ +export function describeTarget(connectionString: string): string { + return `${detectDbType(connectionString)} at ${redactConnectionString(connectionString)}`; +} diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts new file mode 100644 index 000000000..3772e35de --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -0,0 +1,451 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildFirebaseExport, + exportFirebase, + fetchAccessToken, + fetchAllFirebaseUsers, + fetchHashConfig, + formatHashConfigGuidance, + mapFirebaseUserToExport, + readServiceAccount, + signServiceAccountJwt, + type ServiceAccount, +} from "./firebase.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { url: string; body: unknown }[]; +let account: ServiceAccount; + +beforeAll(async () => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-fb-"))); + process.chdir(workDir); + + // A real RSA key, so the signing path is genuinely exercised. + const pair = await crypto.subtle.generateKey( + { + name: "RSASSA-PKCS1-v1_5", + modulusLength: 2048, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, + true, + ["sign", "verify"], + ); + const pkcs8 = await crypto.subtle.exportKey("pkcs8", pair.privateKey); + const body = btoa(String.fromCharCode(...new Uint8Array(pkcs8))).replace(/(.{64})/g, "$1\n"); + + account = { + project_id: "demo-fb", + client_email: "exp@demo-fb.iam.gserviceaccount.com", + private_key: `-----BEGIN PRIVATE KEY-----\n${body}\n-----END PRIVATE KEY-----\n`, + }; + fs.writeFileSync( + path.join(workDir, "sa.json"), + JSON.stringify({ type: "service_account", ...account }), + ); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; +}); + +/** Answers the token exchange, then one page per entry in `pages`. */ +function stubFirebase(pages: Record[][], hashConfig?: unknown) { + let page = 0; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ url, body: init?.body ?? null }); + + if (url.includes("oauth2.googleapis.com/token")) { + return Response.json({ access_token: "tok" }); + } + if (url.includes("/config")) { + return hashConfig === undefined + ? new Response("forbidden", { status: 403 }) + : Response.json(hashConfig); + } + const current = pages[page++] ?? []; + const hasMore = page < pages.length; + return Response.json({ users: current, ...(hasMore ? { nextPageToken: `p${page}` } : {}) }); + }) as unknown as typeof fetch; +} + +const fbUser = (i: number, overrides: Record = {}) => ({ + localId: `fb${i}`, + email: `u${i}@fb.dev`, + emailVerified: true, + displayName: `User ${i}`, + passwordHash: `SGFzaA${i}`, + salt: `U2FsdA${i}`, + createdAt: "1704067200000", + ...overrides, +}); + +describe("readServiceAccount", () => { + test("reads a valid key file", () => { + expect(readServiceAccount("./sa.json").project_id).toBe("demo-fb"); + }); + + test("reports a path that is not there", () => { + expect(() => readServiceAccount("./nope.json")).toThrow(/No service account file at/); + }); + + test("reports a file that is not JSON", () => { + fs.writeFileSync(path.join(workDir, "bad.json"), "not json"); + expect(() => readServiceAccount("./bad.json")).toThrow(/is not valid JSON/); + }); + + // Downloading the web app config instead of a service account key is the + // usual mistake, and the two files look similar at a glance. + test("points at the right console page for a web app config", () => { + fs.writeFileSync(path.join(workDir, "web.json"), JSON.stringify({ apiKey: "x" })); + expect(() => readServiceAccount("./web.json")).toThrow(/"project_id" is missing/); + }); + + test("names a wrong type explicitly", () => { + fs.writeFileSync(path.join(workDir, "wrong.json"), JSON.stringify({ type: "authorized_user" })); + expect(() => readServiceAccount("./wrong.json")).toThrow( + /"type" is "authorized_user".*Generate new private key/s, + ); + }); + + test.each([["project_id"], ["client_email"], ["private_key"]])( + "reports a missing %s", + (field) => { + const partial: Record = { type: "service_account", ...account }; + delete partial[field]; + fs.writeFileSync(path.join(workDir, `no-${field}.json`), JSON.stringify(partial)); + expect(() => readServiceAccount(`./no-${field}.json`)).toThrow( + new RegExp(`"${field}" is missing`), + ); + }, + ); + + // Pasting a key through a form that eats newlines is common, and the failure + // would otherwise surface as an opaque crypto error. + test("catches a private key whose newlines were mangled", () => { + fs.writeFileSync( + path.join(workDir, "mangled.json"), + JSON.stringify({ type: "service_account", ...account, private_key: "mangled" }), + ); + expect(() => readServiceAccount("./mangled.json")).toThrow(/newlines survived copying/); + }); + + test("raises CliError so the global handler formats it", () => { + expect(() => readServiceAccount("./nope.json")).toThrow(CliError); + }); +}); + +describe("signServiceAccountJwt", () => { + test("produces a three-segment RS256 JWT", async () => { + const jwt = await signServiceAccountJwt(account); + expect(jwt.split(".")).toHaveLength(3); + }); + + test("claims the right issuer, audience and scopes", async () => { + const jwt = await signServiceAccountJwt(account, 1_700_000_000); + const claims = JSON.parse( + atob((jwt.split(".")[1] as string).replace(/-/g, "+").replace(/_/g, "/")), + ); + + expect(claims).toMatchObject({ + iss: "exp@demo-fb.iam.gserviceaccount.com", + aud: "https://oauth2.googleapis.com/token", + iat: 1_700_000_000, + exp: 1_700_003_600, + }); + expect(claims.scope).toContain("cloud-platform"); + }); + + test("declares RS256 in the header", async () => { + const jwt = await signServiceAccountJwt(account); + const header = JSON.parse( + atob((jwt.split(".")[0] as string).replace(/-/g, "+").replace(/_/g, "/")), + ); + expect(header).toEqual({ alg: "RS256", typ: "JWT" }); + }); + + test("rejects a private key that is not valid base64", async () => { + await expect( + signServiceAccountJwt({ + ...account, + private_key: "-----BEGIN PRIVATE KEY-----\n!!!\n-----END PRIVATE KEY-----", + }), + ).rejects.toThrow(CliError); + }); +}); + +describe("fetchAccessToken", () => { + test("exchanges the assertion for a token", async () => { + stubFirebase([[]]); + + expect(await fetchAccessToken(account)).toBe("tok"); + expect(String(requests[0]?.body)).toContain("grant-type%3Ajwt-bearer"); + }); + + test("explains a rejection rather than surfacing a raw status", async () => { + globalThis.fetch = (async () => + new Response(JSON.stringify({ error_description: "Invalid JWT Signature" }), { + status: 400, + })) as unknown as typeof fetch; + + await expect(fetchAccessToken(account)).rejects.toThrow( + /Google rejected the service account \(400\): Invalid JWT Signature/, + ); + }); + + test("names the role the service account usually lacks", async () => { + globalThis.fetch = (async () => new Response("{}", { status: 403 })) as unknown as typeof fetch; + await expect(fetchAccessToken(account)).rejects.toThrow(/Firebase Authentication Admin/); + }); + + // The emulator has no token endpoint; `firebase-admin` uses the same bearer. + test("skips the exchange entirely against the emulator", async () => { + process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; + globalThis.fetch = (async () => { + throw new Error("should not have been called"); + }) as unknown as typeof fetch; + + expect(await fetchAccessToken(account)).toBe("owner"); + }); +}); + +describe("fetchAllFirebaseUsers", () => { + test("follows nextPageToken until it stops coming", async () => { + stubFirebase([ + Array.from({ length: 1000 }, (_, i) => fbUser(i)), + Array.from({ length: 7 }, (_, i) => fbUser(1000 + i)), + ]); + + const all = await fetchAllFirebaseUsers({ account, token: "tok" }); + + expect(all).toHaveLength(1007); + expect(requests[1]?.url).toContain("nextPageToken=p1"); + }); + + test("asks for the endpoint's maximum page size", async () => { + stubFirebase([[]]); + await fetchAllFirebaseUsers({ account, token: "tok" }); + expect(requests[0]?.url).toContain("maxResults=1000"); + }); + + test("targets the project named in the key", async () => { + stubFirebase([[]]); + await fetchAllFirebaseUsers({ account, token: "tok" }); + expect(requests[0]?.url).toContain("/projects/demo-fb/accounts:batchGet"); + }); + + test("routes through the emulator when one is configured", async () => { + process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; + stubFirebase([[]]); + + await fetchAllFirebaseUsers({ account, token: "owner" }); + + expect(requests[0]?.url).toStartWith("http://127.0.0.1:9099/"); + }); + + test("raises a clear error on a failed page", async () => { + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + + await expect(fetchAllFirebaseUsers({ account, token: "tok" })).rejects.toThrow( + /Firebase returned 500 listing users/, + ); + }); +}); + +describe("mapFirebaseUserToExport", () => { + test("keeps the fields the firebase transformer maps from", () => { + expect(mapFirebaseUserToExport(fbUser(0))).toEqual({ + localId: "fb0", + email: "u0@fb.dev", + displayName: "User 0", + createdAt: "1704067200000", + emailVerified: true, + passwordHash: "SGFzaA0", + salt: "U2FsdA0", + }); + }); + + test("drops project internals the import has no use for", () => { + const mapped = mapFirebaseUserToExport( + fbUser(0, { + providerUserInfo: [{ providerId: "password" }], + lastLoginAt: "1704153600000", + customAttributes: '{"role":"x"}', + validSince: "1704067200", + }), + ); + for (const noise of ["providerUserInfo", "lastLoginAt", "customAttributes", "validSince"]) { + expect(noise in mapped).toBe(false); + } + }); + + // A digest without its salt cannot be verified, so exporting one alone would + // produce a user nobody can sign in as. + test.each([ + ["hash without salt", { passwordHash: "H", salt: undefined }], + ["salt without hash", { passwordHash: undefined, salt: "S" }], + ])("drops a %s", (_label, overrides) => { + const mapped = mapFirebaseUserToExport(fbUser(0, overrides)); + expect("passwordHash" in mapped).toBe(false); + expect("salt" in mapped).toBe(false); + }); + + test("keeps emailVerified when it is false", () => { + expect(mapFirebaseUserToExport(fbUser(0, { emailVerified: false })).emailVerified).toBe(false); + }); + + test("copes with a phone-only user", () => { + const mapped = mapFirebaseUserToExport({ localId: "fb9", phoneNumber: "+15555550100" }); + expect(mapped).toEqual({ localId: "fb9", phoneNumber: "+15555550100" }); + }); +}); + +describe("buildFirebaseExport", () => { + test("counts coverage and logs each user", () => { + const { users, coverage } = buildFirebaseExport( + [fbUser(0), { localId: "fb1", phoneNumber: "+1555" }], + "2026-01-01T12:00:00", + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have a password hash"]).toBe(1); + expect(byLabel["have a phone number"]).toBe(1); + expect(fs.readdirSync(getLogDir())[0]).toBe("export-2026-01-01T12-00-00.log"); + }); +}); + +describe("fetchHashConfig", () => { + test("reads the project's scrypt parameters", async () => { + stubFirebase([[]], { + signIn: { + hashConfig: { signerKey: "KEY==", saltSeparator: "Bw==", rounds: 8, memoryCost: 14 }, + }, + }); + + expect(await fetchHashConfig(account, "tok")).toEqual({ + signerKey: "KEY==", + saltSeparator: "Bw==", + rounds: 8, + memoryCost: 14, + }); + }); + + // Reading the config needs a broader role than listing users, so a project + // where it is denied must still export. + test("returns null rather than failing when the call is not permitted", async () => { + stubFirebase([[]]); + expect(await fetchHashConfig(account, "tok")).toBeNull(); + }); + + test("returns null when the response carries no hash config", async () => { + stubFirebase([[]], { signIn: {} }); + expect(await fetchHashConfig(account, "tok")).toBeNull(); + }); +}); + +describe("formatHashConfigGuidance", () => { + const config = { signerKey: "KEY==", saltSeparator: "Bw==", rounds: 8, memoryCost: 14 }; + + test("prints the exact import command when the parameters are known", () => { + const text = formatHashConfigGuidance(config, "exports/firebase-export.json", 3).join("\n"); + expect(text).toContain('--firebase-signer-key "KEY=="'); + expect(text).toContain('--firebase-salt-separator "Bw=="'); + expect(text).toContain("--firebase-rounds 8 --firebase-mem-cost 14"); + }); + + test("says where to find them when the project would not say", () => { + const text = formatHashConfigGuidance(null, "out.json", 3).join("\n"); + expect(text).toContain("Password hash parameters"); + expect(text).toContain("Authentication → Users"); + }); + + // Nothing to configure, so nothing to tell them to configure. + test("says nothing is needed when the export has no hashes", () => { + expect(formatHashConfigGuidance(null, "out.json", 0).join("\n")).toContain( + "no hash parameters are needed", + ); + }); +}); + +describe("exportFirebase", () => { + test("exports end to end and reports coverage", async () => { + stubFirebase([[fbUser(0), fbUser(1)]], { + signIn: { hashConfig: { signerKey: "K", saltSeparator: "S", rounds: 8, memoryCost: 14 } }, + }); + + await exportFirebase({ serviceAccount: "./sa.json" }); + + const written = JSON.parse( + fs.readFileSync(path.join(workDir, "exports", "firebase-export.json"), "utf-8"), + ) as Record[]; + expect(written).toHaveLength(2); + expect(captured.err).toContain("Field coverage"); + expect(captured.err).toContain("demo-fb project"); + }); + + test("names the command that consumes the file", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(captured.err).toContain( + "migrate run --transformer firebase --file exports/firebase-export.json", + ); + }); + + test("--output controls the destination", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json", output: "fb.json" }); + expect(fs.existsSync(path.join(workDir, "fb.json"))).toBe(true); + }); + + test("requires --service-account, before anything is read", async () => { + await expect(exportFirebase({})).rejects.toThrow(/needs a service account key file/); + }); + + test("validates the key file before making any request", async () => { + stubFirebase([[fbUser(0)]]); + await expect(exportFirebase({ serviceAccount: "./nope.json" })).rejects.toThrow(CliError); + expect(requests).toHaveLength(0); + }); + + test("never puts key material in the output", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(captured.err).not.toContain("BEGIN PRIVATE KEY"); + expect(captured.err).not.toContain(account.private_key.slice(40, 80)); + }); + + test("skips the hash-parameter section when nothing has a password", async () => { + stubFirebase([[{ localId: "fb9", phoneNumber: "+1555" }]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(captured.err).toContain("no hash parameters are needed"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts new file mode 100644 index 000000000..0491227ae --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -0,0 +1,458 @@ +/** + * `clerk migrate export firebase` — pull users out of Firebase Authentication. + * + * Ported from the standalone migration-tool's `src/export/firebase.ts`, but + * **without `firebase-admin`**. + * + * The spike the ticket asked for was run first, and it passed: a + * `bun build --compile` binary imports `firebase-admin`, initializes it, and + * completes `listUsers` against Identity Toolkit. The known Firestore-under- + * compile bug does not reach the Auth Admin surface. + * + * The SDK was still not adopted, on the second measurement: it is **74 MB + * across 158 packages**, including `@google-cloud/firestore` and + * `@google-cloud/storage`, neither of which this command touches. The compiled + * `clerk` binary is ~65 MB today, so that roughly doubles the artifact every + * user downloads — to serve one subcommand. + * + * What the SDK actually does here is two REST calls and an RS256 JWT, and Bun's + * Web Crypto signs RS256 with no dependency at all (verified compiled). So this + * adds **zero** packages, and its HTTP goes through `loggedFetch`, so a + * `--verbose` run shows the requests — which an SDK doing its own fetch would + * not. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { bold, dim } from "../../../lib/color.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; + +/** Identity Toolkit's maximum for `accounts:batchGet`. */ +const PAGE_SIZE = 1000; + +const TOKEN_URL = "https://oauth2.googleapis.com/token"; +const SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/firebase", +].join(" "); + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/firebase"; + +export type ExportFirebaseOptions = { + serviceAccount?: string; + output?: string; +}; + +export type ServiceAccount = { + project_id: string; + client_email: string; + private_key: string; +}; + +/** + * Reads and validates a service-account key file. + * + * Every failure names the field, because the usual causes are downloading the + * wrong JSON from the console (a web app config rather than a service account) + * or pasting a key with its newlines mangled. + */ +export function readServiceAccount(file: string): ServiceAccount { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No service account file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + + let parsed: unknown; + try { + parsed = JSON.parse(fs.readFileSync(resolved, "utf-8")); + } catch (error) { + throw new CliError(`${file} is not valid JSON: ${(error as Error).message}`, { + code: ERROR_CODE.INVALID_JSON, + docsUrl: DOCS_URL, + }); + } + + const account = parsed as Partial & { type?: string }; + const invalid = (problem: string): never => { + throw new CliError(`${file} is not a usable service account key: ${problem}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + }; + + if (account.type && account.type !== "service_account") { + invalid( + `its "type" is "${account.type}", not "service_account". Download a private key from ` + + "Project settings → Service accounts → Generate new private key.", + ); + } + for (const field of ["project_id", "client_email", "private_key"] as const) { + if (typeof account[field] !== "string" || account[field].length === 0) { + invalid(`"${field}" is missing`); + } + } + if (!account.private_key?.includes("PRIVATE KEY")) { + invalid('"private_key" does not look like a PEM key — check its newlines survived copying'); + } + + return account as ServiceAccount; +} + +function base64Url(input: string | Uint8Array): string { + const binary = + typeof input === "string" ? input : String.fromCharCode(...(input as unknown as number[])); + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +/** Imports the PEM private key for RS256 signing. */ +async function importPrivateKey(pem: string): Promise { + const body = pem.replace(/-----[^-]+-----/g, "").replace(/\s+/g, ""); + let der: Uint8Array; + try { + der = Uint8Array.from(atob(body), (character) => character.charCodeAt(0)); + } catch { + throw new CliError("The service account's private_key is not valid base64.", { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + } + + try { + return await crypto.subtle.importKey( + "pkcs8", + der, + { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, + false, + ["sign"], + ); + } catch (error) { + throw new CliError( + `The service account's private_key could not be read: ${(error as Error).message}`, + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } +} + +/** + * Signs the assertion Google exchanges for an access token. + * + * @param now - Seconds since the epoch; injectable so tests are not clock-bound. + */ +export async function signServiceAccountJwt( + account: ServiceAccount, + now: number = Math.floor(Date.now() / 1000), +): Promise { + const key = await importPrivateKey(account.private_key); + const claims = { + iss: account.client_email, + scope: SCOPES, + aud: TOKEN_URL, + iat: now, + exp: now + 3600, + }; + const body = `${base64Url(JSON.stringify({ alg: "RS256", typ: "JWT" }))}.${base64Url(JSON.stringify(claims))}`; + + const signature = await crypto.subtle.sign( + "RSASSA-PKCS1-v1_5", + key, + new TextEncoder().encode(body), + ); + + return `${body}.${base64Url(new Uint8Array(signature))}`; +} + +/** + * Exchanges the signed assertion for an Identity Toolkit access token. + * + * Against the emulator there is nothing to exchange with — Google's token + * endpoint is not part of it — so the run uses the `owner` bearer the emulator + * accepts, matching what `firebase-admin` does. + */ +export async function fetchAccessToken(account: ServiceAccount): Promise { + if (process.env.FIREBASE_AUTH_EMULATOR_HOST) return "owner"; + + const assertion = await signServiceAccountJwt(account); + + const response = await loggedFetch(new URL(TOKEN_URL), { + tag: "firebase", + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", + assertion, + }).toString(), + }); + + const body = (await response.json().catch(() => ({}))) as { + access_token?: string; + error_description?: string; + error?: string; + }; + + if (!response.ok || !body.access_token) { + throw new CliError( + `Google rejected the service account (${response.status}): ${body.error_description ?? body.error ?? "no access token returned"}\n` + + "Check the key has not been revoked, and that the service account has the Firebase Authentication Admin role.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + return body.access_token; +} + +/** + * Base URL for Identity Toolkit. + * + * Honours `FIREBASE_AUTH_EMULATOR_HOST`, the variable Firebase's own tooling + * uses, so this works against the local emulator as well as production. + */ +function identityToolkitBase(): string { + const emulator = process.env.FIREBASE_AUTH_EMULATOR_HOST; + return emulator + ? `http://${emulator}/identitytoolkit.googleapis.com` + : "https://identitytoolkit.googleapis.com"; +} + +export type FirebaseUser = Record & { localId?: string }; + +/** Pages through every user in the project. */ +export async function fetchAllFirebaseUsers(options: { + account: ServiceAccount; + token: string; + spinner?: SpinnerControls; +}): Promise { + const all: FirebaseUser[] = []; + let pageToken: string | undefined; + + do { + const url = new URL( + `${identityToolkitBase()}/v1/projects/${options.account.project_id}/accounts:batchGet`, + ); + url.searchParams.set("maxResults", String(PAGE_SIZE)); + if (pageToken) url.searchParams.set("nextPageToken", pageToken); + + const response = await loggedFetch(url, { + tag: "firebase", + method: "GET", + headers: { Authorization: `Bearer ${options.token}`, Accept: "application/json" }, + }); + + if (!response.ok) { + throw new CliError( + `Firebase returned ${response.status} listing users: ${await response.text()}`, + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + const body = (await response.json()) as { users?: FirebaseUser[]; nextPageToken?: string }; + all.push(...(body.users ?? [])); + options.spinner?.update(`Fetching users from Firebase: ${all.length} so far`); + pageToken = body.nextPageToken; + } while (pageToken); + + return all; +} + +export type HashConfig = { + signerKey: string; + saltSeparator: string; + rounds: number; + memoryCost: number; +}; + +/** + * Reads the project's scrypt parameters. + * + * These are the whole reason a Firebase migration keeps its passwords: without + * them Clerk cannot verify a single digest. Fetching them here saves the user + * hunting through the console — and if the call is not permitted, the run says + * exactly where to look instead. + * + * @returns `null` when the config could not be read. + */ +export async function fetchHashConfig( + account: ServiceAccount, + token: string, +): Promise { + try { + const url = new URL(`${identityToolkitBase()}/admin/v2/projects/${account.project_id}/config`); + const response = await loggedFetch(url, { + tag: "firebase", + method: "GET", + headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, + }); + if (!response.ok) { + log.debug(`firebase: ${response.status} reading the project config`); + return null; + } + + const body = (await response.json()) as { + signIn?: { hashConfig?: Partial & { algorithm?: string } }; + }; + const config = body.signIn?.hashConfig; + if (!config?.signerKey || !config.saltSeparator) return null; + + return { + signerKey: config.signerKey, + saltSeparator: config.saltSeparator, + rounds: Number(config.rounds ?? 8), + memoryCost: Number(config.memoryCost ?? 14), + }; + } catch (error) { + log.debug(`firebase: could not read the project config: ${String(error)}`); + return null; + } +} + +/** + * Keeps the fields the `firebase` transformer maps from. + * + * A Firebase user also carries provider records, custom claims and sign-in + * timestamps that would bloat the export and mean nothing to the import. + */ +export function mapFirebaseUserToExport(user: FirebaseUser): Record { + const exported: Record = {}; + + for (const field of ["localId", "email", "displayName", "phoneNumber", "createdAt"] as const) { + if (user[field]) exported[field] = user[field]; + } + // Meaningful when false, so copied on presence rather than truthiness. + if (user.emailVerified !== undefined) exported.emailVerified = user.emailVerified; + + // Both halves or neither: a digest without its salt cannot be verified. + if (user.passwordHash && user.salt) { + exported.passwordHash = user.passwordHash; + exported.salt = user.salt; + } + + return exported; +} + +export function buildFirebaseExport(users: FirebaseUser[], dateTime: string) { + const exported: Record[] = []; + const counts = { email: 0, verified: 0, password: 0, name: 0, phone: 0 }; + + for (const user of users) { + const userId = String(user.localId ?? ""); + try { + const mapped = mapFirebaseUserToExport(user); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.emailVerified) counts.verified++; + if (mapped.passwordHash) counts.password++; + if (mapped.displayName) counts.name++; + if (mapped.phoneNumber) counts.phone++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.verified }, + { label: "have a password hash", count: counts.password }, + { label: "have a display name", count: counts.name }, + { label: "have a phone number", count: counts.phone }, + ], + }; +} + +/** The exact `migrate run` invocation, with the project's own parameters. */ +export function formatHashConfigGuidance( + config: HashConfig | null, + outputPath: string, + passwordCount: number, +): string[] { + if (passwordCount === 0) { + return [dim("No password hashes in this export, so no hash parameters are needed.")]; + } + + if (!config) { + return [ + bold("Password hash parameters"), + "This export carries password hashes, which Clerk can only verify with the project's", + "scrypt parameters. Find them in the Firebase console under", + "Authentication → Users → (⋮) → Password hash parameters, then pass:", + dim( + " --firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost", + ), + ]; + } + + return [ + bold("Password hash parameters"), + "Read from the project. Import with:", + dim( + ` clerk migrate run -y --transformer firebase --file ${outputPath} \\\n` + + ` --firebase-signer-key "${config.signerKey}" \\\n` + + ` --firebase-salt-separator "${config.saltSeparator}" \\\n` + + ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, + ), + ]; +} + +export async function exportFirebase(options: ExportFirebaseOptions): Promise { + if (!options.serviceAccount) { + throwUsageError( + "`clerk migrate export firebase` needs a service account key file. Pass --service-account .", + DOCS_URL, + undefined, + [ + { + command: "clerk migrate export firebase --service-account ./service-account.json", + description: "Export using a downloaded service account key", + }, + ], + ); + } + + // Read and validate before anything reaches the network, so a wrong file + // fails in a second rather than after an auth round-trip. + const account = readServiceAccount(options.serviceAccount); + + await withGutter("Exporting users from Firebase", async () => { + const dateTime = getDateTimeStamp(); + log.info(`Exporting from the ${account.project_id} project.`); + + const token = await withSpinner("Authenticating with Google", () => fetchAccessToken(account)); + + const users = await withSpinner( + "Fetching users from Firebase", + (spinner) => fetchAllFirebaseUsers({ account, token, spinner }), + "Users fetched", + ); + + const { users: exported, coverage } = buildFirebaseExport(users, dateTime); + const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("firebase")); + + reportExport({ + platform: "firebase", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "firebase", + }); + + const passwordCount = coverage.find((entry) => entry.label.includes("password"))?.count ?? 0; + const hashConfig = passwordCount > 0 ? await fetchHashConfig(account, token) : null; + + log.blank(); + for (const line of formatHashConfigGuidance(hashConfig, outputPath, passwordCount)) { + log.info(line); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts new file mode 100644 index 000000000..2ac69b5b3 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -0,0 +1,181 @@ +import type { Command } from "@commander-js/extra-typings"; +import { throwUsageError } from "../../../lib/errors.ts"; +import { select } from "../../../lib/listage.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { exportAuth0 } from "./auth0.ts"; +import { exportAuthJs } from "./authjs.ts"; +import { exportBetterAuth } from "./betterauth.ts"; +import { exportClerk } from "./clerk.ts"; +import { exportFirebase } from "./firebase.ts"; +import { exportSupabase } from "./supabase.ts"; +import type { DbExportOptions } from "./db-options.ts"; +import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; + +/** + * Bare `clerk migrate export` — pick a platform, then run its export. + * + * The picker is built from the registry, so a new platform appears without a + * second place to update. Whatever the chosen platform needs beyond the + * platform name, it prompts for itself. + */ +export async function exportPicker(options: Record = {}): Promise { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export\` needs a platform and cannot prompt here. Name one: ${exportPlatformKeys().join(", ")}.`, + undefined, + undefined, + exportPlatforms.map((entry) => ({ + command: `clerk migrate export ${entry.key}`, + description: entry.description, + })), + ); + } + + const platform = await select({ + message: "Which platform are you exporting from?", + choices: exportPlatforms.map((entry) => ({ + name: entry.label, + value: entry.key, + description: entry.description, + })), + }); + + const entry = getExportPlatform(platform); + // Unreachable via the picker; a guard so a registry edit cannot silently + // produce a choice with nothing behind it. + if (!entry) throwUsageError(`Unknown export platform "${platform}".`); + + await entry.run(options); +} + +const handlers = { + picker: exportPicker, + clerk: exportClerk, + auth0: exportAuth0, + supabase: exportSupabase, + authjs: exportAuthJs, + betterauth: exportBetterAuth, + firebase: exportFirebase, +}; + +/** The three platforms that read a database, which share `--db-url`. */ +const DB_PLATFORMS = [ + { + key: "supabase", + summary: "Export users from a Supabase Postgres database", + envVar: "SUPABASE_DB_URL", + example: "postgres://postgres:password@db.xxx.supabase.co:5432/postgres", + }, + { + key: "authjs", + summary: "Export users from an Auth.js database", + envVar: "AUTHJS_DB_URL", + example: "mysql://user:password@127.0.0.1:3306/authjs", + }, + { + key: "betterauth", + summary: "Export users from a Better Auth database", + envVar: "BETTERAUTH_DB_URL", + example: "./db.sqlite", + }, +] as const; + +/** Registers `export [platform]` under the `migrate` group. */ +export function registerMigrateExport(migrateCommand: Command<[], Record>): void { + const exportCommand = migrateCommand + .command("export") + .description("Export users from a source platform, ready for `migrate run`") + .setExamples([ + { command: "clerk migrate export", description: "Pick a platform interactively" }, + { + command: "clerk migrate export clerk --output users.json", + description: "Export from a Clerk instance", + }, + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export from an Auth0 tenant", + }, + ]) + .action((_opts, cmd) => handlers.picker(cmd.optsWithGlobals() as Record)); + + exportCommand + .command("clerk") + .description("Export users from a Clerk instance (default: ./exports/clerk-export.json)") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("--secret-key ", "Backend API secret key to use") + .option("--clerk-secret-key ", "Deprecated alias for --secret-key") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate export clerk", + description: "Export to ./exports/clerk-export.json", + }, + { + command: "clerk migrate export clerk --instance prod --output prod-users.json", + description: "Export a specific instance to a chosen path", + }, + ]) + .action((_opts, cmd) => + handlers.clerk(cmd.optsWithGlobals() as Parameters[0]), + ); + + exportCommand + .command("auth0") + .description("Export users from an Auth0 tenant (default: ./exports/auth0-export.json)") + .option("--domain ", "Auth0 tenant domain, e.g. my-tenant.us.auth0.com") + .option("--client-id ", "Machine-to-machine application client ID") + .option("--client-secret ", "Machine-to-machine application client secret") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .setExamples([ + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export with explicit credentials", + }, + { + command: "clerk migrate export auth0", + description: "Read AUTH0_DOMAIN, AUTH0_CLIENT_ID and AUTH0_CLIENT_SECRET, or prompt", + }, + ]) + .action((_opts, cmd) => + handlers.auth0(cmd.optsWithGlobals() as Parameters[0]), + ); + + exportCommand + .command("firebase") + .description("Export users from a Firebase project (default: ./exports/firebase-export.json)") + .option("--service-account ", "Path to a service account key JSON file") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .setExamples([ + { + command: "clerk migrate export firebase --service-account ./service-account.json", + description: "Export using a downloaded service account key", + }, + ]) + .action((_opts, cmd) => + handlers.firebase(cmd.optsWithGlobals() as Parameters[0]), + ); + + // All three take exactly one connection string, so they are registered from + // a table rather than three near-identical blocks. + for (const platform of DB_PLATFORMS) { + exportCommand + .command(platform.key) + .description(`${platform.summary} (default: ./exports/${platform.key}-export.json)`) + .option("--db-url ", "Postgres, MySQL or SQLite connection string") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .setExamples([ + { + command: `clerk migrate export ${platform.key} --db-url "${platform.example}"`, + description: "Export from an explicit database", + }, + { + command: `clerk migrate export ${platform.key}`, + description: `Read ${platform.envVar}, or prompt`, + }, + ]) + .action((_opts, cmd) => handlers[platform.key](cmd.optsWithGlobals() as DbExportOptions)); + } +} diff --git a/packages/cli-core/src/commands/migrate/export/registry.test.ts b/packages/cli-core/src/commands/migrate/export/registry.test.ts new file mode 100644 index 000000000..1acded24f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/registry.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, test } from "bun:test"; +import { transformerKeys } from "../transformers/registry.ts"; +import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; + +describe("export registry", () => { + test("registers every source platform", () => { + expect(exportPlatformKeys()).toEqual([ + "clerk", + "auth0", + "supabase", + "authjs", + "firebase", + "betterauth", + ]); + }); + + test.each([...exportPlatforms])("$key carries a label and description", (entry) => { + expect(entry.label.length).toBeGreaterThan(0); + expect(entry.description.length).toBeGreaterThan(0); + }); + + // The picker, the docs and the "what next" line all read this, so a typo + // would send someone to a transformer that does not exist. + test.each([...exportPlatforms])("$key names a real transformer", (entry) => { + expect(transformerKeys()).toContain(entry.transformerKey); + }); + + test.each([...exportPlatforms])("$key has something to run", (entry) => { + expect(typeof entry.run).toBe("function"); + }); + + test("looks a platform up by key", () => { + expect(getExportPlatform("auth0")?.label).toBe("Auth0"); + }); + + test("returns nothing for a platform that is not registered", () => { + expect(getExportPlatform("okta")).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/registry.ts b/packages/cli-core/src/commands/migrate/export/registry.ts new file mode 100644 index 000000000..93c4f2605 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/registry.ts @@ -0,0 +1,80 @@ +/** + * Export registry. + * + * The picker behind a bare `clerk migrate export` is built from this array, so + * adding a platform is one file plus one entry — the same shape the transformer + * registry uses. + * + * `run` takes no arguments on purpose: each platform resolves its own flags, + * environment variables and prompts, because what Auth0 needs (a tenant domain + * and M2M credentials) has nothing in common with what a database export needs. + */ + +import { exportAuth0 } from "./auth0.ts"; +import { exportAuthJs } from "./authjs.ts"; +import { exportBetterAuth } from "./betterauth.ts"; +import { exportClerk } from "./clerk.ts"; +import { exportFirebase } from "./firebase.ts"; +import { exportSupabase } from "./supabase.ts"; + +export type ExportRegistryEntry = { + key: string; + label: string; + description: string; + /** Which `--transformer` reads the file this export writes. */ + transformerKey: string; + run: (options: Record) => Promise; +}; + +export const exportPlatforms: ExportRegistryEntry[] = [ + { + key: "clerk", + label: "Clerk", + description: "Another Clerk instance, e.g. development → production", + transformerKey: "clerk", + run: (options) => exportClerk(options), + }, + { + key: "auth0", + label: "Auth0", + description: "An Auth0 tenant, via the Management API", + transformerKey: "auth0", + run: (options) => exportAuth0(options), + }, + { + key: "supabase", + label: "Supabase", + description: "A Supabase Postgres database — includes password hashes", + transformerKey: "supabase", + run: (options) => exportSupabase(options), + }, + { + key: "authjs", + label: "Auth.js (NextAuth)", + description: "An Auth.js database — Postgres, MySQL or SQLite", + transformerKey: "authjs", + run: (options) => exportAuthJs(options), + }, + { + key: "firebase", + label: "Firebase", + description: "A Firebase project, via Identity Toolkit", + transformerKey: "firebase", + run: (options) => exportFirebase(options), + }, + { + key: "betterauth", + label: "Better Auth", + description: "A Better Auth database — plugin columns detected automatically", + transformerKey: "betterauth", + run: (options) => exportBetterAuth(options), + }, +]; + +export function exportPlatformKeys(): string[] { + return exportPlatforms.map((entry) => entry.key); +} + +export function getExportPlatform(key: string): ExportRegistryEntry | undefined { + return exportPlatforms.find((entry) => entry.key === key); +} diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts new file mode 100644 index 000000000..aa25ea50f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -0,0 +1,85 @@ +/** + * Shared plumbing for the export modules: where the file lands, and what the + * user is told about it. + * + * Ported from the standalone migration-tool's `src/lib/export.ts`, with one + * behavioural change: `--output` resolves against the **current working + * directory**, the way every other path flag in this CLI does. The original + * resolved a relative `--output` inside `exports/`, so `--output ./here.json` + * silently wrote to `exports/here.json`. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { dim, green, yellow } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; + +/** Where an export lands when `--output` is not given. */ +export function defaultOutputPath(platform: string): string { + return path.join("exports", `${platform}-export.json`); +} + +/** + * Writes the export, creating any missing parent directories. + * + * @returns The absolute path written, for reporting. + */ +export function writeExportOutput(users: unknown[], outputFile: string): string { + const resolved = path.resolve(process.cwd(), outputFile); + fs.mkdirSync(path.dirname(resolved), { recursive: true }); + fs.writeFileSync(resolved, JSON.stringify(users, null, 2)); + return resolved; +} + +export type CoverageField = { label: string; count: number }; + +/** + * How complete an export is, per field. + * + * ● every user, ○ some, dim ○ none. The point is to see *before* importing + * that, say, only 3 of 400 users have a password — which changes what the + * migration means. + */ +export function formatFieldCoverage(fields: CoverageField[], total: number): string[] { + return fields.map(({ label, count }) => { + const icon = count === total ? green("●") : count > 0 ? yellow("○") : dim("○"); + return ` ${icon} ${dim(`${count}/${total} ${label}`)}`; + }); +} + +export type ExportSummary = { + platform: string; + userCount: number; + outputPath: string; + coverage: CoverageField[]; + /** The transformer that reads this file, for the "what next" line. */ + transformerKey: string; +}; + +/** Reports the coverage table and the exact command that consumes the file. */ +export function reportExport(summary: ExportSummary): void { + log.blank(); + if (summary.userCount === 0) { + log.warn(`No users found to export. Wrote an empty file to ${summary.outputPath}.`); + return; + } + + log.info("Field coverage"); + for (const line of formatFieldCoverage(summary.coverage, summary.userCount)) { + log.info(line); + } + + log.blank(); + log.success(`Exported ${summary.userCount} user(s) to ${summary.outputPath}`); + log.info( + dim( + `Next: clerk migrate run --transformer ${summary.transformerKey} --file ${relativeIfInside(summary.outputPath)}`, + ), + ); +} + +/** Shortens a path for display when it sits under the working directory. */ +function relativeIfInside(absolute: string): string { + const relative = path.relative(process.cwd(), absolute); + return relative.startsWith("..") ? absolute : relative; +} diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts new file mode 100644 index 000000000..585ebf9d9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -0,0 +1,138 @@ +/** + * `clerk migrate export supabase` — read users straight out of `auth.users`. + * + * Ported from the standalone migration-tool's `src/export/supabase.ts`, on + * `Bun.sql` instead of `pg`. + * + * The database rather than the Admin API because **`encrypted_password` only + * exists here**. Supabase's API does not return password hashes, so an + * API-based export forces every user to reset their password; this one carries + * the bcrypt digests across. + */ + +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; + +/** + * `display_name` is coalesced into `first_name` here rather than in the + * transformer so a user who writes their own SQL sees the shape the + * transformer expects. + */ +const EXPORT_QUERY = ` + SELECT + id, + email, + email_confirmed_at, + encrypted_password, + phone, + phone_confirmed_at, + COALESCE( + raw_user_meta_data->>'display_name', + raw_user_meta_data->>'first_name', + raw_user_meta_data->>'name' + ) AS first_name, + raw_user_meta_data->>'last_name' AS last_name, + raw_user_meta_data, + raw_app_meta_data, + created_at + FROM auth.users + ORDER BY created_at +`; + +type SupabaseRow = Record & { + id?: unknown; + email?: string | null; + encrypted_password?: string | null; + raw_app_meta_data?: unknown; +}; + +/** Serializes values the JSON export cannot carry as-is. */ +function normalizeRow(row: SupabaseRow): Record { + const normalized: Record = {}; + + for (const [key, value] of Object.entries(row)) { + if (value === null || value === undefined) continue; + // Postgres returns timestamps as Date objects; the transformer parses + // strings, and JSON.stringify would otherwise bury the format difference. + normalized[key] = value instanceof Date ? value.toISOString() : value; + } + + return normalized; +} + +export async function fetchSupabaseUsers(client: DbClient): Promise { + return client.query(EXPORT_QUERY); +} + +export function buildSupabaseExport(rows: SupabaseRow[], dateTime: string) { + const users: Record[] = []; + const counts = { email: 0, emailConfirmed: 0, password: 0, phone: 0, firstName: 0, lastName: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + try { + users.push(normalizeRow(row)); + + if (row.email) counts.email++; + if (row.email_confirmed_at) counts.emailConfirmed++; + if (row.encrypted_password) counts.password++; + if (row.phone) counts.phone++; + if (row.first_name) counts.firstName++; + if (row.last_name) counts.lastName++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a confirmed email", count: counts.emailConfirmed }, + { label: "have a password hash", count: counts.password }, + { label: "have a phone number", count: counts.phone }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + ], + }; +} + +export async function exportSupabase(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, { + platform: "supabase", + envVar: "SUPABASE_DB_URL", + prompt: "Supabase Postgres connection string", + hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", + }); + + await withGutter("Exporting users from Supabase", async () => { + const dateTime = getDateTimeStamp(); + + const rows = await withSpinner("Reading auth.users", () => + withDbClient(dbUrl, "supabase", fetchSupabaseUsers), + ); + + const { users, coverage } = buildSupabaseExport(rows, dateTime); + const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("supabase")); + + reportExport({ + platform: "supabase", + userCount: users.length, + outputPath, + coverage, + transformerKey: "supabase", + }); + + if (users.length > 0) { + log.info( + "Password hashes are included — this is why the export reads the database rather than the Admin API.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts new file mode 100644 index 000000000..739833322 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -0,0 +1,335 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { BapiError } from "../../lib/errors.ts"; +import { + buildCreateUserBody, + importUsers, + normalizeErrorMessage, + readRetryAfter, + splitIdentifiers, +} from "./import-users.ts"; +import { getLogFilePath } from "./lib/logger.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import type { User } from "./types.ts"; + +const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurrencyLimit: 8 }; +const DATE_TIME = "2026-01-01T00:00:00"; + +const user = (overrides: Partial = {}): User => + ({ userId: "u1", email: "a@x.dev", ...overrides }) as User; + +describe("splitIdentifiers", () => { + test("promotes the first verified email and phone to primary", () => { + const result = splitIdentifiers( + user({ email: ["a@x.dev", "b@x.dev"], phone: ["+15555550100", "+15555550101"] }), + ); + expect(result.primaryEmail).toBe("a@x.dev"); + expect(result.additionalEmails).toEqual(["b@x.dev"]); + expect(result.primaryPhone).toBe("+15555550100"); + expect(result.additionalPhones).toEqual(["+15555550101"]); + }); + + test("merges the email and emailAddresses fields, deduping", () => { + const result = splitIdentifiers( + user({ email: "a@x.dev", emailAddresses: ["a@x.dev", "b@x.dev"] }), + ); + expect(result.primaryEmail).toBe("a@x.dev"); + expect(result.additionalEmails).toEqual(["b@x.dev"]); + }); + + test("drops an unverified identifier that is already verified", () => { + const result = splitIdentifiers( + user({ email: ["a@x.dev"], unverifiedEmailAddresses: ["a@x.dev", "c@x.dev"] }), + ); + expect(result.unverifiedEmails).toEqual(["c@x.dev"]); + }); + + test("copes with a user identified only by username", () => { + const result = splitIdentifiers({ userId: "u1", username: "alice" } as User); + expect(result.primaryEmail).toBeUndefined(); + expect(result.additionalEmails).toEqual([]); + }); +}); + +describe("buildCreateUserBody", () => { + test("maps the schema onto BAPI's snake_case body", () => { + const target = user({ + firstName: "Alice", + lastName: "Smith", + username: "alice", + createdAt: "2024-01-01T00:00:00.000Z", + publicMetadata: { plan: "pro" }, + createOrganizationsLimit: 3, + banned: true, + }); + const body = buildCreateUserBody(target, splitIdentifiers(target), true); + + expect(body).toMatchObject({ + external_id: "u1", + email_address: ["a@x.dev"], + first_name: "Alice", + last_name: "Smith", + username: "alice", + created_at: "2024-01-01T00:00:00.000Z", + public_metadata: { plan: "pro" }, + create_organizations_limit: 3, + banned: true, + }); + }); + + test("sends only the primary identifier; the rest are attached separately", () => { + const target = user({ email: ["a@x.dev", "b@x.dev"] }); + expect(buildCreateUserBody(target, splitIdentifiers(target), true).email_address).toEqual([ + "a@x.dev", + ]); + }); + + test("omits fields the source platform never recorded", () => { + const body = buildCreateUserBody(user(), splitIdentifiers(user()), true); + expect("first_name" in body).toBe(false); + expect("banned" in body).toBe(false); + expect("created_at" in body).toBe(false); + }); + + test("sends the password digest and hasher together", () => { + const target = user({ password: "digest", passwordHasher: "bcrypt" }); + const body = buildCreateUserBody(target, splitIdentifiers(target), true); + expect(body).toMatchObject({ password_digest: "digest", password_hasher: "bcrypt" }); + expect("skip_password_requirement" in body).toBe(false); + }); + + test.each([ + [true, true], + [false, false], + ])("skipPasswordRequirement=%p on a passwordless user -> flag present: %p", (skip, present) => { + const body = buildCreateUserBody(user(), splitIdentifiers(user()), skip); + expect("skip_password_requirement" in body).toBe(present); + }); +}); + +describe("readRetryAfter", () => { + const withHeader = (value: string) => + new BapiError(429, "{}", new Headers({ "retry-after": value })); + + test.each([ + ["12", 12], + ["0", undefined], + ["soon", undefined], + ])("Retry-After: %s -> %p", (header, expected) => { + expect(readRetryAfter(withHeader(header))).toBe(expected as number | undefined); + }); + + test("falls back to the error body's retryAfter meta", () => { + const error = new BapiError( + 429, + JSON.stringify({ + errors: [{ code: "rate_limit", message: "slow down", meta: { retryAfter: 7 } }], + }), + new Headers(), + ); + expect(readRetryAfter(error)).toBe(7); + }); + + test("returns undefined when neither source carries a value", () => { + expect(readRetryAfter(new BapiError(429, "{}", new Headers()))).toBeUndefined(); + }); +}); + +describe("normalizeErrorMessage", () => { + test("sorts field arrays so equivalent errors group together", () => { + const a = normalizeErrorMessage('["last_name" "first_name"] data does not match'); + const b = normalizeErrorMessage('["first_name" "last_name"] data does not match'); + expect(a).toBe(b); + expect(a).toBe('["first_name" "last_name"] data does not match'); + }); + + test("leaves messages without field arrays untouched", () => { + expect(normalizeErrorMessage("that email is taken")).toBe("that email is taken"); + }); +}); + +describe("importUsers", () => { + let workDir: string; + let originalCwd: string; + let originalFetch: typeof globalThis.fetch; + let requests: { method: string; url: string; body: unknown }[]; + + beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-import-")); + process.chdir(workDir); + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + }); + + beforeEach(() => { + requests = []; + fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + /** Installs a fetch that records every request and replies per `respond`. */ + function stub(respond: (url: string, attempt: number) => Response): void { + const attempts = new Map(); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + const attempt = (attempts.get(url) ?? 0) + 1; + attempts.set(url, attempt); + return respond(url, attempt); + }) as typeof fetch; + } + + const ok = (id: string) => new Response(JSON.stringify({ id }), { status: 200 }); + + const clerkError = (status: number, message: string, headers?: Record) => + new Response(JSON.stringify({ errors: [{ code: "err", message, long_message: message }] }), { + status, + headers, + }); + + const logEntries = () => + fs + .readFileSync(getLogFilePath("migration", DATE_TIME), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + + test("creates each user and reports them as successful", async () => { + stub(() => ok("user_created")); + + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ totalProcessed: 2, successful: 2, failed: 0 }); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + expect(logEntries().filter((e) => e.status === "success")).toHaveLength(2); + }); + + test("attaches additional and unverified identifiers after the user exists", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [ + user({ + email: ["a@x.dev", "b@x.dev"], + unverifiedEmailAddresses: ["c@x.dev"], + phone: ["+15555550100", "+15555550101"], + }), + ], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + const emails = requests.filter((r) => r.url.endsWith("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/v1/email_addresses")); + expect(emails.map((r) => r.body)).toEqual([ + { user_id: "user_created", email_address: "b@x.dev", primary: false, verified: true }, + { user_id: "user_created", email_address: "c@x.dev", primary: false, verified: false }, + ]); + expect(requests.filter((r) => r.url.endsWith("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/v1/phone_numbers"))).toHaveLength(1); + }); + + test("logs a failed additional identifier without failing the user", async () => { + stub((url) => + url.endsWith("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/v1/email_addresses") + ? clerkError(422, "that email is taken") + : ok("user_created"), + ); + + const summary = await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + expect(logEntries().some((e) => e.status === "additional_email_error")).toBe(true); + }); + + test("records a failed user and keeps going", async () => { + stub((_url, attempt) => + attempt === 1 ? clerkError(422, "that email is taken") : ok("user_ok"), + ); + + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary.successful + summary.failed).toBe(2); + expect(summary.failed).toBe(1); + expect([...summary.errorBreakdown.values()]).toEqual([1]); + expect(logEntries().some((e) => e.status === "error" && e.code === "422")).toBe(true); + }); + + test("retries a 429 after the interval the server asked for", async () => { + stub((_url, attempt) => + attempt === 1 ? clerkError(429, "slow down", { "retry-after": "1" }) : ok("user_ok"), + ); + + const started = performance.now(); + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + expect(performance.now() - started).toBeGreaterThanOrEqual(900); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + expect(logEntries().some((e) => e.status === "429_retry")).toBe(true); + }); + + test("gives up after the retry ceiling and records the user as failed", async () => { + stub(() => clerkError(429, "slow down", { "retry-after": "1" })); + + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ successful: 0, failed: 1 }); + // One initial attempt plus MAX_RETRIES retries. + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(6); + expect(logEntries().some((e) => e.code === "429")).toBe(true); + }, 20_000); + + test("carries the validation failure count into the summary", async () => { + stub(() => ok("user_ok")); + + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + validationFailed: 4, + }); + + expect(summary.validationFailed).toBe(4); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts new file mode 100644 index 000000000..749142cf4 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -0,0 +1,347 @@ +/** + * Creates users in Clerk from a validated batch. + * + * Ported from the standalone migration-tool's `src/migrate/import-users.ts`, + * rewritten onto `bapiRequest` instead of `@clerk/backend`. Two things fall out + * of that move: + * + * - The request body is BAPI's snake_case shape directly, so `created_at` and + * `legal_accepted_at` stay RFC3339 strings rather than round-tripping + * through `Date`. + * - `banned`, `delete_self_enabled` and the organization limits are all + * accepted by `POST /v1/users`, so the follow-up `updateUser`/`banUser` + * calls the SDK version needed are gone. + * + * Run state is local to {@link importUsers} rather than module-level, so two + * runs in one process (or one test file) cannot see each other's counters. + */ + +import { bapiRequest } from "../../lib/bapi.ts"; +import { BapiError } from "../../lib/errors.ts"; +import type { SpinnerControls } from "../../lib/spinner.ts"; +import { errorLogger, importLogger } from "./lib/logger.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; +import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; +import type { ImportSummary, User } from "./types.ts"; + +// Re-exported for the tests and callers that grew up against this module. +export { readRetryAfter } from "./lib/retry.ts"; + +/** + * Groups error messages that differ only in field ordering, so the summary + * reports "12 users: [\"first_name\" \"last_name\"] ..." once instead of twice. + */ +export function normalizeErrorMessage(errorMessage: string): string { + let normalized = ""; + let lastCopiedIndex = 0; + let arrayStartIndex = -1; + + for (let i = 0; i < errorMessage.length; i++) { + const char = errorMessage[i]; + + if (arrayStartIndex === -1) { + if (char === "[") arrayStartIndex = i; + continue; + } + if (char !== "]") continue; + + normalized += errorMessage.slice(lastCopiedIndex, arrayStartIndex); + normalized += normalizeFieldArray(errorMessage.slice(arrayStartIndex + 1, i)); + lastCopiedIndex = i + 1; + arrayStartIndex = -1; + } + + return normalized + errorMessage.slice(lastCopiedIndex); +} + +function normalizeFieldArray(fields: string): string { + const fieldNames: string[] = []; + let current = ""; + + for (const char of fields) { + if (char === '"' || char === "'" || char.trim() === "") { + if (current.length > 0) { + fieldNames.push(current); + current = ""; + } + continue; + } + current += char; + } + if (current.length > 0) fieldNames.push(current); + + fieldNames.sort(); + return `[${fieldNames.map((name) => `"${name}"`).join(" ")}]`; +} + +function toArray(value: string | string[] | undefined): string[] { + if (!value) return []; + return Array.isArray(value) ? value : [value]; +} + +function dedupe(values: string[]): string[] { + const seen: string[] = []; + for (const value of values) { + if (value && !seen.includes(value)) seen.push(value); + } + return seen; +} + +type Identifiers = { + primaryEmail: string | undefined; + additionalEmails: string[]; + unverifiedEmails: string[]; + primaryPhone: string | undefined; + additionalPhones: string[]; + unverifiedPhones: string[]; +}; + +/** + * Splits a user's identifiers into the one that goes on `POST /v1/users` and + * the rest, which are attached afterwards. + */ +export function splitIdentifiers(user: User): Identifiers { + const verifiedEmails = dedupe([...toArray(user.email), ...toArray(user.emailAddresses)]); + const verifiedPhones = dedupe([...toArray(user.phone), ...toArray(user.phoneNumbers)]); + + return { + primaryEmail: verifiedEmails[0], + additionalEmails: verifiedEmails.slice(1), + unverifiedEmails: dedupe( + toArray(user.unverifiedEmailAddresses).filter((email) => !verifiedEmails.includes(email)), + ), + primaryPhone: verifiedPhones[0], + additionalPhones: verifiedPhones.slice(1), + unverifiedPhones: dedupe( + toArray(user.unverifiedPhoneNumbers).filter((phone) => !verifiedPhones.includes(phone)), + ), + }; +} + +/** + * Builds the `POST /v1/users` request body. + * + * Optional fields are omitted rather than sent as null so Clerk applies its own + * defaults for anything the source platform did not record. + */ +export function buildCreateUserBody( + user: User, + identifiers: Identifiers, + skipPasswordRequirement: boolean, +): Record { + const body: Record = { external_id: user.userId }; + + if (identifiers.primaryEmail) body.email_address = [identifiers.primaryEmail]; + if (identifiers.primaryPhone) body.phone_number = [identifiers.primaryPhone]; + if (user.firstName) body.first_name = user.firstName; + if (user.lastName) body.last_name = user.lastName; + if (user.username) body.username = user.username; + if (user.totpSecret) body.totp_secret = user.totpSecret; + if (user.backupCodes) body.backup_codes = user.backupCodes; + if (user.unsafeMetadata) body.unsafe_metadata = user.unsafeMetadata; + if (user.privateMetadata) body.private_metadata = user.privateMetadata; + if (user.publicMetadata) body.public_metadata = user.publicMetadata; + if (user.createdAt) body.created_at = user.createdAt; + if (user.legalAcceptedAt) body.legal_accepted_at = user.legalAcceptedAt; + if (user.skipLegalChecks !== undefined) body.skip_legal_checks = user.skipLegalChecks; + if (user.skipPasswordChecks !== undefined) body.skip_password_checks = user.skipPasswordChecks; + if (user.banned !== undefined) body.banned = user.banned; + if (user.bypassClientTrust !== undefined) body.bypass_client_trust = user.bypassClientTrust; + if (user.deleteSelfEnabled !== undefined) body.delete_self_enabled = user.deleteSelfEnabled; + if (user.createOrganizationEnabled !== undefined) { + body.create_organization_enabled = user.createOrganizationEnabled; + } + if (user.createOrganizationsLimit !== undefined) { + body.create_organizations_limit = user.createOrganizationsLimit; + } + + if (user.password && user.passwordHasher) { + body.password_digest = user.password; + body.password_hasher = user.passwordHasher; + } else if (skipPasswordRequirement) { + body.skip_password_requirement = true; + } + // Without a password and without skipPasswordRequirement, Clerk rejects the + // user — which is exactly what --require-password is asking for. + + return body; +} + +type CreateContext = { + secretKey: string; + schedule: ApiScheduler; + dateTime: string; +}; + +/** Attaches one extra identifier, logging (but not rethrowing) any failure. */ +async function attachIdentifier( + ctx: CreateContext, + userId: string, + clerkUserId: string, + kind: "email" | "phone", + value: string, + verified: boolean, +): Promise { + const path = kind === "email" ? "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/v1/email_addresses" : "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/v1/phone_numbers"; + const body = + kind === "email" + ? { user_id: clerkUserId, email_address: value, primary: false, verified } + : { user_id: clerkUserId, phone_number: value, primary: false, verified }; + + try { + await ctx.schedule(() => + bapiRequest({ + method: "POST", + path, + secretKey: ctx.secretKey, + body: JSON.stringify(body), + }), + ); + } catch (error) { + const label = `${verified ? "additional" : "unverified"} ${kind} ${value}`; + errorLogger( + { + userId, + status: `additional_${kind}_error`, + errors: [ + { + code: `additional_${kind}_failed`, + message: `Failed to add ${label}`, + longMessage: `Failed to add ${label}: ${(error as Error).message}`, + }, + ], + }, + ctx.dateTime, + ); + } +} + +/** Creates one user, then attaches any additional identifiers it carries. */ +async function createUser( + ctx: CreateContext, + user: User, + skipPasswordRequirement: boolean, +): Promise { + const identifiers = splitIdentifiers(user); + + const response = await ctx.schedule(() => + bapiRequest({ + method: "POST", + path: "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/v1/users", + secretKey: ctx.secretKey, + body: JSON.stringify(buildCreateUserBody(user, identifiers, skipPasswordRequirement)), + }), + ); + + const clerkUserId = (response.body as { id?: string })?.id ?? ""; + + // Extra identifiers are best-effort: a duplicate secondary email should not + // undo a user who was otherwise imported successfully. + await Promise.all([ + ...identifiers.additionalEmails.map((email) => + attachIdentifier(ctx, user.userId, clerkUserId, "email", email, true), + ), + ...identifiers.unverifiedEmails.map((email) => + attachIdentifier(ctx, user.userId, clerkUserId, "email", email, false), + ), + ...identifiers.additionalPhones.map((phone) => + attachIdentifier(ctx, user.userId, clerkUserId, "phone", phone, true), + ), + ...identifiers.unverifiedPhones.map((phone) => + attachIdentifier(ctx, user.userId, clerkUserId, "phone", phone, false), + ), + ]); + + return clerkUserId; +} + +export type ImportUsersOptions = { + users: User[]; + secretKey: string; + limits: ResolvedLimits; + dateTime: string; + /** Allow users that carry no password. */ + skipPasswordRequirement?: boolean; + /** Carried into the summary so the report covers the whole file. */ + validationFailed?: number; + spinner?: SpinnerControls; +}; + +/** + * Imports every user, concurrently and within the instance's rate limit. + * + * A failed user is recorded and the run continues; a 429 backs off (honouring + * `Retry-After`) and retries up to {@link MAX_RETRIES} times. + */ +export async function importUsers(options: ImportUsersOptions): Promise { + const { + users, + secretKey, + limits, + dateTime, + skipPasswordRequirement = true, + validationFailed = 0, + spinner, + } = options; + + const total = users.length; + const errorBreakdown = new Map(); + let processed = 0; + let successful = 0; + let failed = 0; + + const ctx: CreateContext = { + secretKey, + dateTime, + schedule: createApiScheduler(limits.concurrencyLimit, limits.rateLimit), + }; + + const progress = () => + spinner?.update( + `Importing users: [${processed}/${total}] (${successful} succeeded, ${failed} failed)`, + ); + + const recordFailure = (userId: string, message: string, code: string) => { + failed++; + processed++; + const normalized = normalizeErrorMessage(message); + errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); + importLogger({ userId, status: "error", error: message, code }, dateTime); + progress(); + }; + + const processUser = async (user: User): Promise => { + try { + const clerkUserId = await retryOn429(() => createUser(ctx, user, skipPasswordRequirement), { + onRetry: ({ message }) => + errorLogger( + { + userId: user.userId, + status: "429_retry", + errors: [{ code: "rate_limit_retry", message, longMessage: message }], + }, + dateTime, + ), + }); + successful++; + processed++; + importLogger({ userId: user.userId, status: "success", clerkUserId }, dateTime); + progress(); + } catch (error) { + if (error instanceof RateLimitExceededError) { + recordFailure(user.userId, error.message, "429"); + return; + } + + const apiError = error as BapiError; + const message = apiError.longMessage ?? apiError.message ?? "Unknown error"; + recordFailure(user.userId, message, String(apiError.status ?? "unknown")); + } + }; + + progress(); + await Promise.all(users.map((user) => processUser(user))); + + return { totalProcessed: total, successful, failed, validationFailed, errorBreakdown }; +} diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts new file mode 100644 index 000000000..07454ad98 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -0,0 +1,208 @@ +import { describe, expect, test } from "bun:test"; +import { createProgram } from "../../cli-program.ts"; +import { exportPlatformKeys } from "./export/registry.ts"; +import { transformerKeys } from "./transformers/registry.ts"; + +function findCommand(names: string[]) { + let current = createProgram().commands.find((cmd) => cmd.name() === names[0]); + for (const name of names.slice(1)) { + current = current?.commands.find((cmd) => cmd.name() === name); + } + return current; +} + +describe("registerMigrate", () => { + test("registers migrate as a top-level command group", () => { + const migrate = findCommand(["migrate"]); + expect(migrate).toBeDefined(); + expect(migrate?.description()).toContain("Migrate users"); + }); + + test("registers the run subcommand", () => { + expect(findCommand(["migrate", "run"])).toBeDefined(); + }); + + // Bare `clerk migrate` dispatches to `migrate run`, mirroring how bare + // `clerk deploy` dispatches to `deploy run`. + test("makes run the default subcommand, so bare `clerk migrate` starts the wizard", () => { + const run = findCommand(["migrate", "run"]); + expect(run as unknown as { _defaultCommandName?: unknown }).toBeDefined(); + const migrate = findCommand(["migrate"]) as unknown as { _defaultCommandName?: string }; + expect(migrate._defaultCommandName).toBe("run"); + }); + + test("keeps run visible in help, unlike deploy's hidden default", () => { + expect(findCommand(["migrate", "run"])?.parent?.commands.map((c) => c.name())).toContain("run"); + expect( + (findCommand(["migrate", "run"]) as unknown as { _hidden?: boolean })._hidden, + ).toBeFalsy(); + }); + + test.each([ + "--transformer", + "--file", + "--resume-after", + "--require-password", + "--skip-unsupported-providers", + "--firebase-signer-key", + "--firebase-salt-separator", + "--firebase-rounds", + "--firebase-mem-cost", + "--yes", + "--secret-key", + "--clerk-secret-key", + "--app", + "--instance", + ])("migrate run accepts %s", (flag) => { + const flags = findCommand(["migrate", "run"])?.options.map((option) => option.long); + expect(flags).toContain(flag); + }); + + test.each([[["transformers"]], [["transformers", "list"]]])("registers migrate %p", (names) => { + expect(findCommand(["migrate", ...names])).toBeDefined(); + }); + + test.each([ + [["export"]], + [["export", "clerk"]], + [["export", "auth0"]], + [["export", "supabase"]], + [["export", "authjs"]], + [["export", "betterauth"]], + [["export", "firebase"]], + ])("registers migrate %p", (names) => { + expect(findCommand(["migrate", ...names])).toBeDefined(); + }); + + // Bare `migrate export` runs the picker rather than defaulting to a + // platform, so nobody exports from the wrong place by pressing enter. + test("leaves export with no default subcommand", () => { + const group = findCommand(["migrate", "export"]) as unknown as { + _defaultCommandName?: string; + }; + expect(group._defaultCommandName).toBeFalsy(); + }); + + test("registers an export subcommand per registered platform", () => { + const registered = findCommand(["migrate", "export"])?.commands.map((c) => c.name()); + for (const key of exportPlatformKeys()) expect(registered).toContain(key); + }); + + test.each(["--output", "--secret-key", "--app", "--instance"])( + "export clerk accepts %s", + (flag) => { + expect(findCommand(["migrate", "export", "clerk"])?.options.map((o) => o.long)).toContain( + flag, + ); + }, + ); + + test.each(["--domain", "--client-id", "--client-secret", "--output"])( + "export auth0 accepts %s", + (flag) => { + expect(findCommand(["migrate", "export", "auth0"])?.options.map((o) => o.long)).toContain( + flag, + ); + }, + ); + + test.each(["supabase", "authjs", "betterauth"])("export %s accepts --db-url", (platform) => { + expect(findCommand(["migrate", "export", platform])?.options.map((o) => o.long)).toContain( + "--db-url", + ); + }); + + test("export firebase accepts --service-account", () => { + expect(findCommand(["migrate", "export", "firebase"])?.options.map((o) => o.long)).toContain( + "--service-account", + ); + }); + + test("documents the default output location in help", () => { + expect(findCommand(["migrate", "export", "clerk"])?.description()).toContain( + "./exports/clerk-export.json", + ); + expect(findCommand(["migrate", "export", "auth0"])?.description()).toContain( + "./exports/auth0-export.json", + ); + }); + + test("makes list the default transformers subcommand", () => { + const group = findCommand(["migrate", "transformers"]) as unknown as { + _defaultCommandName?: string; + }; + expect(group._defaultCommandName).toBe("list"); + }); + + test.each(["--json", "--transformer-file"])("transformers list accepts %s", (flag) => { + expect(findCommand(["migrate", "transformers", "list"])?.options.map((o) => o.long)).toContain( + flag, + ); + }); + + test("migrate run accepts --transformer-file", () => { + expect(findCommand(["migrate", "run"])?.options.map((o) => o.long)).toContain( + "--transformer-file", + ); + }); + + // Flat rather than under a noun group: it is the one command in this tree + // that destroys data in Clerk. + test("registers delete as a direct subcommand of migrate", () => { + expect(findCommand(["migrate", "delete"])).toBeDefined(); + expect(findCommand(["migrate", "delete"])?.description()).toContain("last migration"); + }); + + test.each(["--yes", "--secret-key", "--clerk-secret-key", "--app", "--instance"])( + "migrate delete accepts %s", + (flag) => { + expect(findCommand(["migrate", "delete"])?.options.map((o) => o.long)).toContain(flag); + }, + ); + + test.each([[["logs"]], [["logs", "list"]], [["logs", "clean"]], [["logs", "convert"]]])( + "registers migrate %p", + (names) => { + expect(findCommand(["migrate", ...names])).toBeDefined(); + }, + ); + + // Listing is read-only, so it is safe as the default for a bare + // `clerk migrate logs`. + test("makes list the default logs subcommand", () => { + const logs = findCommand(["migrate", "logs"]) as unknown as { _defaultCommandName?: string }; + expect(logs._defaultCommandName).toBe("list"); + }); + + test.each([ + [["logs", "list"], "--json"], + [["logs", "clean"], "--yes"], + [["logs", "convert"], "--all"], + ])("%s accepts %s", (names, flag) => { + expect(findCommand(["migrate", ...names])?.options.map((option) => option.long)).toContain( + flag, + ); + }); + + test("logs convert takes variadic file positionals", () => { + const args = findCommand(["migrate", "logs", "convert"])?.registeredArguments; + expect(args?.[0]?.variadic).toBe(true); + expect(args?.[0]?.required).toBe(false); + }); + + test("constrains --transformer to the registered transformers, for validation and completion", () => { + const option = findCommand(["migrate", "run"])?.options.find((o) => o.long === "--transformer"); + // Tracks the registry so adding a platform needs no edit here. + expect(option?.argChoices).toEqual(transformerKeys()); + }); + + test.each([ + ["-t", "--transformer"], + ["-f", "--file"], + ["-r", "--resume-after"], + ["-y", "--yes"], + ])("exposes %s as the short form of %s", (short, long) => { + const option = findCommand(["migrate", "run"])?.options.find((o) => o.long === long); + expect(option?.short).toBe(short); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts new file mode 100644 index 000000000..3d34ca55c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -0,0 +1,134 @@ +import { createOption } from "@commander-js/extra-typings"; +import type { Program } from "../../cli-program.ts"; +import { parseIntegerOption } from "../../lib/option-parsers.ts"; +import { deleteMigration } from "./delete.ts"; +import { registerMigrateExport } from "./export/index.ts"; +import { registerMigrateLogs } from "./logs/index.ts"; +import { run } from "./run.ts"; +import { list as transformersList } from "./transformers/list.ts"; +import { transformerKeys } from "./transformers/registry.ts"; + +const migrate = { run, delete: deleteMigration, transformersList }; + +export function registerMigrate(program: Program): void { + const migrateCommand = program + .command("migrate") + .description("Migrate users into Clerk from another auth provider") + .setExamples([ + { + command: "clerk migrate", + description: "Walk through a migration interactively", + }, + { + command: "clerk migrate run -y --transformer clerk --file users.json", + description: "Import users from a Clerk export", + }, + ]); + + // `isDefault` so bare `clerk migrate` runs the wizard, mirroring how bare + // `clerk deploy` dispatches to `deploy run`. Not hidden: unlike deploy's, + // this subcommand is documented and carries every flag. + migrateCommand + .command("run", { isDefault: true }) + .description("Import users from an exported JSON or CSV file") + .addOption( + createOption( + "-t, --transformer ", + "Source platform the file was exported from", + ).choices(transformerKeys()), + ) + .option( + "--transformer-file ", + "Path to a transformer you wrote, for a platform with no built-in", + ) + .option("-f, --file ", "Path to the exported user data (JSON or CSV)") + .option("-r, --resume-after ", "Skip every user up to and including this source ID") + .option("--require-password", "Import only users that have a password") + .option( + "--skip-unsupported-providers", + "Supabase: skip users whose only social provider is not enabled in Clerk", + ) + .option("--firebase-signer-key ", "Firebase base64 signer key") + .option("--firebase-salt-separator ", "Firebase base64 salt separator") + .option("--firebase-rounds ", "Firebase scrypt rounds", (value) => + parseIntegerOption(value, "--firebase-rounds", { min: 1 }), + ) + .option("--firebase-mem-cost ", "Firebase scrypt memory cost", (value) => + parseIntegerOption(value, "--firebase-mem-cost", { min: 1 }), + ) + .option("-y, --yes", "Skip the confirmation prompt") + .option("--secret-key ", "Backend API secret key to use") + .option("--clerk-secret-key ", "Deprecated alias for --secret-key") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate run -y --transformer clerk --file users.json", + description: "Import a Clerk Dashboard export", + }, + { + command: "clerk migrate run -y -t clerk -f users.csv --require-password", + description: "Import only the users that carry a password digest", + }, + { + command: "clerk migrate run -y -t clerk -f users.json -r user_2x9k", + description: "Resume a partial migration after the last imported user", + }, + { + command: "clerk migrate run -y -t supabase -f users.json --skip-unsupported-providers", + description: "Skip Supabase users whose only provider is not enabled in Clerk", + }, + ]) + .action((_opts, cmd) => + migrate.run(cmd.optsWithGlobals() as Parameters[0]), + ); + + // Flat, not under a noun group: this is the one command in the tree that + // destroys data in Clerk, and it is worth keeping short and prominent. + migrateCommand + .command("delete") + .description("Delete the users created by the last migration in this directory") + .option("-y, --yes", "Skip the confirmation prompt") + .option("--secret-key ", "Backend API secret key to use") + .option("--clerk-secret-key ", "Deprecated alias for --secret-key") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate delete", + description: "Undo the last migration after confirming", + }, + { command: "clerk migrate delete -y", description: "Undo without prompting" }, + ]) + .action((_opts, cmd) => + migrate.delete(cmd.optsWithGlobals() as Parameters[0]), + ); + + registerMigrateExport(migrateCommand); + + // A compiled binary has no source tree to grep, so the available mappings + // need a command rather than only appearing in the interactive picker. + const transformersCommand = migrateCommand + .command("transformers") + .description("Inspect the available source-platform transformers"); + + transformersCommand + .command("list", { isDefault: true }) + .description("List the built-in transformers, and any loaded from a file") + .option("--json", "Output as JSON") + .option("--transformer-file ", "Also list a transformer you wrote") + .setExamples([ + { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { + command: "clerk migrate transformers list --transformer-file ./my-transformer.ts", + description: "Include one you wrote", + }, + ]) + .action((_opts, cmd) => + migrate.transformersList( + cmd.optsWithGlobals() as Parameters[0], + ), + ); + + registerMigrateLogs(migrateCommand); +} diff --git a/packages/cli-core/src/commands/migrate/lib/analysis.test.ts b/packages/cli-core/src/commands/migrate/lib/analysis.test.ts new file mode 100644 index 000000000..322c2d753 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/analysis.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "bun:test"; +import { analyzeFields, hasValue } from "./analysis.ts"; + +describe("hasValue", () => { + test.each([ + ["a string", "x", true], + ["zero", 0, true], + ["false", false, true], + ["a populated array", ["a"], true], + ["an object", {}, true], + ["an empty string", "", false], + ["an empty array", [], false], + ["null", null, false], + ["undefined", undefined, false], + ])("treats %s as present: %p", (_label, value, expected) => { + expect(hasValue(value)).toBe(expected); + }); +}); + +describe("analyzeFields", () => { + test("returns zeroed counts for an empty file", () => { + const result = analyzeFields([]); + expect(result.totalUsers).toBe(0); + expect(result.identifiers.hasAnyIdentifier).toBe(0); + expect(result.fieldCounts).toEqual({}); + }); + + test("counts each identifier kind separately", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev" }, + { userId: "2", unverifiedEmailAddresses: ["b@x.dev"] }, + { userId: "3", phone: "+15555550100" }, + { userId: "4", unverifiedPhoneNumbers: ["+15555550101"] }, + { userId: "5", username: "carol" }, + ]); + + expect(result.identifiers).toMatchObject({ + verifiedEmails: 1, + unverifiedEmails: 1, + verifiedPhones: 1, + unverifiedPhones: 1, + username: 1, + hasAnyIdentifier: 5, + }); + }); + + test("counts emailAddresses towards verified emails", () => { + const result = analyzeFields([{ userId: "1", emailAddresses: ["a@x.dev"] }]); + expect(result.identifiers.verifiedEmails).toBe(1); + }); + + test("counts a user with several identifiers once", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev", phone: "+15555550100", username: "ada" }, + ]); + expect(result.identifiers.hasAnyIdentifier).toBe(1); + expect(result.identifiers.verifiedEmails).toBe(1); + expect(result.identifiers.verifiedPhones).toBe(1); + }); + + // These users cannot be imported under any instance configuration, which is + // what makes the count worth surfacing separately. + test("counts users carrying no identifier at all", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev" }, + { userId: "2", firstName: "Nobody" }, + { userId: "3" }, + ]); + expect(result.totalUsers).toBe(3); + expect(result.identifiers.hasAnyIdentifier).toBe(1); + }); + + test("counts the analyzed non-identifier fields", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev", firstName: "Ada", password: "d", totpSecret: "s" }, + { userId: "2", email: "b@x.dev", firstName: "Grace" }, + { userId: "3", email: "c@x.dev", lastName: "Hopper" }, + ]); + expect(result.fieldCounts).toEqual({ + firstName: 2, + lastName: 1, + password: 1, + totpSecret: 1, + }); + }); + + test("omits fields no user carries, rather than reporting them as zero", () => { + const result = analyzeFields([{ userId: "1", email: "a@x.dev" }]); + expect("password" in result.fieldCounts).toBe(false); + }); + + test("does not count an empty value as present", () => { + const result = analyzeFields([{ userId: "1", email: "a@x.dev", firstName: "", lastName: [] }]); + expect(result.fieldCounts.firstName).toBeUndefined(); + expect(result.fieldCounts.lastName).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/analysis.ts b/packages/cli-core/src/commands/migrate/lib/analysis.ts new file mode 100644 index 000000000..848ec1d76 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/analysis.ts @@ -0,0 +1,80 @@ +/** + * Counts what an import file actually contains, per field. + * + * Ported from the standalone migration-tool's `src/lib/analysis.ts`. Runs on + * transformed-but-unvalidated users so the report describes the whole file, + * including the rows that will be skipped. + */ + +import type { User } from "../types.ts"; + +/** Non-identifier fields the readiness report reports coverage for. */ +export const ANALYZED_FIELDS = [ + { key: "firstName", label: "First name" }, + { key: "lastName", label: "Last name" }, + { key: "password", label: "Password" }, + { key: "totpSecret", label: "TOTP secret" }, +] as const; + +export type IdentifierCounts = { + verifiedEmails: number; + unverifiedEmails: number; + verifiedPhones: number; + unverifiedPhones: number; + username: number; + /** Users with at least one identifier — the rest cannot be imported at all. */ + hasAnyIdentifier: number; +}; + +export type FieldAnalysis = { + identifiers: IdentifierCounts; + totalUsers: number; + fieldCounts: Record; +}; + +/** True for anything with real content — `0` and `false` count, `""` and `[]` do not. */ +export function hasValue(value: unknown): boolean { + if (value === undefined || value === null || value === "") return false; + if (Array.isArray(value)) return value.length > 0; + return true; +} + +export function analyzeFields(users: (User | Record)[]): FieldAnalysis { + const identifiers: IdentifierCounts = { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: 0, + }; + const fieldCounts: Record = {}; + + for (const entry of users) { + const user = entry as Record; + + for (const field of ANALYZED_FIELDS) { + if (hasValue(user[field.key])) { + fieldCounts[field.key] = (fieldCounts[field.key] ?? 0) + 1; + } + } + + const verifiedEmail = hasValue(user.email) || hasValue(user.emailAddresses); + const unverifiedEmail = hasValue(user.unverifiedEmailAddresses); + const verifiedPhone = hasValue(user.phone) || hasValue(user.phoneNumbers); + const unverifiedPhone = hasValue(user.unverifiedPhoneNumbers); + const username = hasValue(user.username); + + if (verifiedEmail) identifiers.verifiedEmails++; + if (unverifiedEmail) identifiers.unverifiedEmails++; + if (verifiedPhone) identifiers.verifiedPhones++; + if (unverifiedPhone) identifiers.unverifiedPhones++; + if (username) identifiers.username++; + + if (verifiedEmail || unverifiedEmail || verifiedPhone || unverifiedPhone || username) { + identifiers.hasAnyIdentifier++; + } + } + + return { identifiers, totalUsers: users.length, fieldCounts }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts new file mode 100644 index 000000000..a94479cd9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts @@ -0,0 +1,85 @@ +import { test, expect, describe, mock, beforeEach, afterAll } from "bun:test"; +import { stubFetch, useCaptureLog } from "../../../test/lib/stubs.ts"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { fetchInstanceSettings } from "./clerk-config.ts"; + +const USER_SETTINGS = { + attributes: { email_address: { enabled: true, required: true } }, +} as unknown as UserSettingsJSON; + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +describe("fetchInstanceSettings", () => { + const originalFetch = globalThis.fetch; + useCaptureLog(); + const mockFetch = mock(); + + beforeEach(() => { + mockFetch.mockReset(); + stubFetch(mockFetch); + }); + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + /** Routes the three hops: BAPI domains → FAPI dev browser → FAPI environment. */ + function route(domains: unknown): void { + mockFetch.mockImplementation((input: string | URL) => { + const url = String(input); + if (url.includes("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/v1/domains")) return Promise.resolve(json({ data: domains })); + if (url.includes("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/v1/dev_browser")) return Promise.resolve(json({ token: "jwt" })); + if (url.includes("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/v1/environment")) { + return Promise.resolve(json({ user_settings: USER_SETTINGS })); + } + throw new Error(`unexpected request: ${url}`); + }); + } + + test("reads settings off the primary domain's Frontend API", async () => { + route([{ is_satellite: false, frontend_api_url: "https://clerk.example.com" }]); + + expect(await fetchInstanceSettings("sk_test_abc")).toEqual(USER_SETTINGS); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.some((url) => url.includes("clerk.example.com/v1/dev_browser"))).toBe(true); + expect(urls.some((url) => url.includes("clerk.example.com/v1/environment"))).toBe(true); + }); + + test("prefers the primary domain over a satellite", async () => { + route([ + { is_satellite: true, frontend_api_url: "https://satellite.example.com" }, + { is_satellite: false, frontend_api_url: "https://clerk.example.com" }, + ]); + + await fetchInstanceSettings("sk_test_abc"); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.every((url) => !url.includes("satellite.example.com"))).toBe(true); + }); + + test("skips the dev browser bootstrap for a production key", async () => { + route([{ is_satellite: false, frontend_api_url: "https://clerk.example.com" }]); + + expect(await fetchInstanceSettings("sk_live_abc")).toEqual(USER_SETTINGS); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.some((url) => url.includes("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/v1/dev_browser"))).toBe(false); + }); + + // `null` means "unknown", so callers degrade rather than treating a failed + // lookup as "nothing is enabled". + test("returns null when no domain names a Frontend API URL", async () => { + route([{ is_satellite: false }]); + expect(await fetchInstanceSettings("sk_test_abc")).toBeNull(); + }); + + test("returns null when the domains lookup fails", async () => { + mockFetch.mockResolvedValue(new Response("nope", { status: 401 })); + expect(await fetchInstanceSettings("sk_test_abc")).toBeNull(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts new file mode 100644 index 000000000..7b6cd7dfa --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -0,0 +1,119 @@ +/** + * The destination instance's live user settings: which identifiers it accepts + * and requires, and which social providers it has enabled. + * + * Ported from the standalone migration-tool's `src/lib/clerk.ts`, rewritten + * onto the CLI's own primitives: the FAPI host comes from BAPI `/v1/domains` + * — a secret key is all `migrate run` is given — and the settings come from + * `lib/fapi.ts` rather than a bespoke fetch. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import { + bootstrapDevBrowser, + fetchUserSettings, + type UserSettingsJSON, +} from "../../../lib/fapi.ts"; +import { log } from "../../../lib/log.ts"; +import { detectInstanceType } from "./instance.ts"; + +/** + * Supabase provider keys whose Clerk strategy is not simply `oauth_`. + * + * Everything not listed here maps by prefix, which covers google, github, + * discord, spotify, twitch, notion, figma, gitlab, bitbucket and the rest. + */ +const CLERK_STRATEGY_ALIASES: Record = { + azure: "oauth_microsoft", + twitter: "oauth_x", + slack_oidc: "oauth_slack", + fly: "oauth_fly", +}; + +/** Supabase's provider key as Clerk's OAuth strategy name. */ +export function toClerkStrategy(provider: string): string { + return CLERK_STRATEGY_ALIASES[provider] ?? `oauth_${provider}`; +} + +/** Human label for a provider key, for report output. */ +export function providerLabel(provider: string): string { + const special: Record = { + github: "GitHub", + gitlab: "GitLab", + linkedin_oidc: "LinkedIn (OIDC)", + slack_oidc: "Slack (OIDC)", + twitter: "Twitter (X)", + azure: "Microsoft (Azure)", + workos: "WorkOS", + fly: "Fly.io", + }; + return special[provider] ?? provider.charAt(0).toUpperCase() + provider.slice(1); +} + +/** + * The Frontend API host of the instance a secret key addresses. + * + * `/v1/instance` carries no publishable key — for any instance, linked or not + * — so the primary domain's `frontend_api_url` is the only route from a secret + * key to the host its settings live behind. Every instance has at least one + * domain; satellites share the primary's Frontend API, so ordering only + * matters for tidiness. + */ +async function fetchFapiHost(secretKey: string): Promise { + const response = await bapiRequest({ method: "GET", path: "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/v1/domains", secretKey }); + const domains = (response.body as { data?: unknown })?.data; + if (!Array.isArray(domains)) return null; + + const primary = + domains.find((domain) => !(domain as { is_satellite?: boolean }).is_satellite) ?? domains[0]; + const frontendApiUrl = (primary as { frontend_api_url?: unknown })?.frontend_api_url; + if (typeof frontendApiUrl !== "string" || !frontendApiUrl) return null; + + return new URL(frontendApiUrl).host; +} + +/** + * Fetches the user settings for the instance a secret key addresses. + * + * @returns The settings, or `null` when they could not be read. Callers must + * treat `null` as "unknown" rather than as "nothing is enabled" — the + * readiness report degrades to a note, and provider skipping stands down. + */ +export async function fetchInstanceSettings(secretKey: string): Promise { + try { + const fapiHost = await fetchFapiHost(secretKey); + if (!fapiHost) { + log.debug("migrate: no domain on this instance named a Frontend API URL"); + return null; + } + + // Development FAPI rejects an environment request without a dev browser JWT. + const jwt = + detectInstanceType(secretKey) === "dev" ? await bootstrapDevBrowser(fapiHost) : undefined; + return await fetchUserSettings(fapiHost, jwt ? { jwt } : {}); + } catch (error) { + log.debug( + `migrate: could not read instance settings: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return null; + } +} + +/** The enabled social strategies (`oauth_google`, …) in a settings payload. */ +export function enabledSocialProviders(settings: UserSettingsJSON): string[] { + return Object.entries(settings.social ?? {}) + .filter(([, value]) => value?.enabled) + .map(([strategy]) => strategy); +} + +/** + * Convenience wrapper for callers that only need the enabled strategies. + * + * @returns `null` when the instance settings could not be read. + */ +export async function fetchEnabledSocialProviders(secretKey: string): Promise { + const settings = await fetchInstanceSettings(secretKey); + return settings ? enabledSocialProviders(settings) : null; +} diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts new file mode 100644 index 000000000..3fc969ee5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -0,0 +1,237 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { + createDbClient, + describeDbError, + detectDbType, + redactConnectionString, + sqlitePath, + withDbClient, +} from "./db.ts"; + +let workDir: string; +let dbPath: string; + +beforeAll(() => { + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-db-"))); + dbPath = path.join(workDir, "test.sqlite"); + + const db = new Database(dbPath, { create: true }); + db.run(`CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER)`); + db.run(`INSERT INTO "user" VALUES (?, ?, ?)`, ["u1", "a@x.dev", 1]); + db.run(`INSERT INTO "user" VALUES (?, ?, ?)`, ["u2", "b@x.dev", 0]); + db.close(); +}); + +afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("detectDbType", () => { + test.each([ + ["postgres://u:p@h/db", "postgres"], + ["postgresql://u:p@h/db", "postgres"], + ["POSTGRES://u:p@h/db", "postgres"], + ["mysql://u:p@h/db", "mysql"], + ["mysql2://u:p@h/db", "mysql"], + ["./db.sqlite", "sqlite"], + ["file:./db.sqlite", "sqlite"], + ["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/abs/path.db", "sqlite"], + [" postgres://u:p@h/db ", "postgres"], + ])("%s -> %s", (input, expected) => { + expect(detectDbType(input)).toBe(expected as never); + }); +}); + +describe("redactConnectionString", () => { + test.each([ + ["postgres://user:secret@host:5432/db", "postgres://***@host:5432/db"], + ["mysql://root:hunter2@127.0.0.1:3306/app", "mysql://***@127.0.0.1:3306/app"], + ["postgres://host/db", "postgres://host/db"], + ])("%s -> %s", (input, expected) => { + expect(redactConnectionString(input)).toBe(expected); + }); + + // An unencoded `@` in the password is the most common mistake, and it is + // exactly when the string ends up in an error message. Matching the first + // `@` would leave the rest of the password visible. + test("redacts a password containing an unencoded @", () => { + const redacted = redactConnectionString("postgres://user:pa@ss@host/db"); + expect(redacted).toBe("postgres://***@host/db"); + expect(redacted).not.toContain("ss"); + }); + + test("redacts a password containing a colon", () => { + expect(redactConnectionString("postgres://user:a:b:c@host/db")).toBe("postgres://***@host/db"); + }); + + test.each([["./db.sqlite"], ["/var/data/app.db"], ["file:./local.sqlite"]])( + "leaves the credential-free path %s alone", + (input) => { + expect(redactConnectionString(input)).toBe(input); + }, + ); +}); + +describe("sqlitePath", () => { + test.each([ + ["./db.sqlite", "./db.sqlite"], + ["file:./db.sqlite", "./db.sqlite"], + ["file:/abs/db.sqlite", "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/abs/db.sqlite"], + ["./db.sqlite?mode=ro", "./db.sqlite"], + [" ./db.sqlite ", "./db.sqlite"], + ])("%s -> %s", (input, expected) => { + expect(sqlitePath(input)).toBe(expected); + }); +}); + +describe("a sqlite client", () => { + test("connects and queries", async () => { + const client = await createDbClient(dbPath); + try { + const rows = await client.query<{ id: string }>(`SELECT id FROM "user" ORDER BY id`); + expect(rows.map((row) => row.id)).toEqual(["u1", "u2"]); + } finally { + await client.close(); + } + }); + + test("binds parameters", async () => { + const client = await createDbClient(dbPath); + try { + const rows = await client.query<{ email: string }>(`SELECT email FROM "user" WHERE id = ?`, [ + "u2", + ]); + expect(rows[0]?.email).toBe("b@x.dev"); + } finally { + await client.close(); + } + }); + + test("reports its dialect's placeholder and quoting", async () => { + const client = await createDbClient(dbPath); + try { + expect(client.dbType).toBe("sqlite"); + expect(client.placeholder(1)).toBe("?"); + expect(client.quote("user")).toBe('"user"'); + } finally { + await client.close(); + } + }); + + test("accepts a file: URL", async () => { + const client = await createDbClient(`file:${dbPath}`); + try { + expect(await client.query(`SELECT 1 AS n`)).toHaveLength(1); + } finally { + await client.close(); + } + }); + + // bun:sqlite opens lazily, so without an explicit probe a missing file would + // surface at the first real query, long after "connecting" finished. + test("fails at connect time when the file is missing, not mid-export", async () => { + await expect(createDbClient(path.join(workDir, "nope.sqlite"))).rejects.toThrow(CliError); + }); + + test("names the file in the failure", async () => { + await expect(createDbClient(path.join(workDir, "nope.sqlite"))).rejects.toThrow( + /Could not open the SQLite file/, + ); + }); +}); + +describe("withDbClient", () => { + test("returns the work's value", async () => { + expect(await withDbClient(dbPath, undefined, async () => "done")).toBe("done"); + }); + + test("closes the client even when the work throws", async () => { + // A leaked handle keeps the process alive after the export has written its + // file, which reads as a hang. + await expect( + withDbClient(dbPath, undefined, async () => { + throw new Error("boom"); + }), + ).rejects.toThrow(/boom/); + + // The file is still usable, so nothing is holding it open. + expect(await withDbClient(dbPath, undefined, async () => "reopened")).toBe("reopened"); + }); + + test("attaches a hint to a query failure, not just a connection failure", async () => { + await expect( + withDbClient(dbPath, undefined, (client) => client.query(`SELECT * FROM missing_table`)), + ).rejects.toThrow(/expected table was not found/); + }); + + test("passes a CliError through unchanged", async () => { + await expect( + withDbClient(dbPath, undefined, async () => { + throw new CliError("already explained"); + }), + ).rejects.toThrow(/already explained/); + }); +}); + +describe("describeDbError", () => { + const withCode = (code: string, message = "") => Object.assign(new Error(message), { code }); + + // Bun reports an unreachable host and a closed port identically, as + // "Connection closed" — precisely where a bare driver error helps least. + test.each([ + ["ERR_POSTGRES_CONNECTION_CLOSED", "Connection closed"], + ["ERR_MYSQL_CONNECTION_CLOSED", "Connection closed"], + ])("turns %s into host/port guidance", (code, message) => { + expect(describeDbError(withCode(code, message))).toMatch(/Check the host and port/); + }); + + test("gives Supabase the IPv4 add-on hint, which is the usual cause there", () => { + const hint = describeDbError( + withCode("ERR_POSTGRES_CONNECTION_CLOSED", "Connection closed"), + "supabase", + ); + expect(hint).toMatch(/pooler connection string/); + expect(hint).toMatch(/IPv4/); + }); + + test.each([ + ['password authentication failed for user "postgres"'], + ["Access denied for user 'root'@'localhost' (using password: YES)"], + ])("recognizes the rejected credentials in %p", (message) => { + expect(describeDbError(new Error(message))).toMatch(/rejected those credentials/); + }); + + test.each([ + ['relation "auth.users" does not exist'], + ["no such table: user"], + ["permission denied for table users"], + ])("recognizes the missing table in %p", (message) => { + expect(describeDbError(new Error(message))).toMatch(/table was not found|cannot read it/); + }); + + test("points Supabase at Auth being enabled and the postgres role", () => { + const hint = describeDbError(new Error('relation "auth.users" does not exist'), "supabase"); + expect(hint).toMatch(/Supabase Auth is enabled/); + expect(hint).toMatch(/postgres` role/); + }); + + test("recognizes an unopenable SQLite file", () => { + expect(describeDbError(new Error("unable to open database file"))).toMatch( + /Could not open the SQLite file/, + ); + }); + + test("still says something useful for an error it does not recognize", () => { + expect(describeDbError(new Error("something odd"))).toMatch(/Check the connection string/); + }); + + test("never echoes the error's own text, which could carry a connection string", () => { + const hint = describeDbError(new Error("failed for postgres://user:secret@host/db")); + expect(hint).not.toContain("secret"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts new file mode 100644 index 000000000..d048a3348 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -0,0 +1,227 @@ +/** + * One query interface over Postgres, MySQL and SQLite. + * + * Rewritten from the standalone migration-tool's `src/lib/db.ts`, which used + * `pg`, `mysql2` and `better-sqlite3`. None of those belong in a statically + * compiled binary — `better-sqlite3` is a native addon outright — so this runs + * on `Bun.sql` (Postgres and MySQL) and `bun:sqlite`, both built into the + * runtime. That swap is the entire reason the `engines.bun` floor exists. + * + * **Placeholders are not unified.** `Bun.sql` passes the query through to each + * server as written, so Postgres wants `$1` and MySQL wants `?` — verified + * against both. Rather than rewrite SQL strings (Postgres uses `?` as a JSONB + * operator, so a naive rewriter would corrupt real queries), callers ask the + * client for the placeholder and the identifier quoting they need. They already + * build per-dialect SQL for table casing, so this adds no new branching. + */ + +import { Database } from "bun:sqlite"; +import { SQL } from "bun"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; + +export type DbType = "postgres" | "mysql" | "sqlite"; + +export interface DbClient { + dbType: DbType; + query>(sql: string, params?: unknown[]): Promise; + /** The bind placeholder for the 1-indexed `position`. */ + placeholder(position: number): string; + /** Quotes an identifier for this dialect. */ + quote(identifier: string): string; + close(): Promise; +} + +/** + * Reads the database type from a connection string. + * + * Anything that is not a recognized URL scheme is treated as a SQLite path, + * matching how the standalone tool behaved and how users actually pass + * `./db.sqlite`. + */ +export function detectDbType(connectionString: string): DbType { + const lower = connectionString.trim().toLowerCase(); + if (lower.startsWith("postgresql://") || lower.startsWith("postgres://")) return "postgres"; + if (lower.startsWith("mysql://") || lower.startsWith("mysql2://")) return "mysql"; + return "sqlite"; +} + +/** + * Replaces any credentials in a connection string with `***`. + * + * Connection strings reach the CLI on the command line and end up in error + * messages and `--verbose` output. Bun's own errors do not echo them, and + * nothing here should either. + */ +export function redactConnectionString(connectionString: string): string { + // Greedy up to the LAST `@`: an unencoded `@` in the password is the most + // common connection-string mistake, and matching the first one would leave + // the rest of the password in the message. Everything before the final `@` + // is userinfo, so redacting all of it is always safe. + // Non-URL forms (SQLite paths) have no `://` and are left alone. + return connectionString.replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@"); +} + +/** Strips a `file:` prefix and any URL query, leaving a filesystem path. */ +export function sqlitePath(connectionString: string): string { + const trimmed = connectionString.trim(); + const withoutScheme = trimmed.startsWith("file:") ? trimmed.slice("file:".length) : trimmed; + return withoutScheme.split("?")[0] ?? withoutScheme; +} + +const QUOTING: Record string> = { + // Doubling the delimiter is the escape in every dialect here, so an + // identifier containing one cannot break out of the quotes. + postgres: (identifier) => `"${identifier.replace(/"/g, '""')}"`, + sqlite: (identifier) => `"${identifier.replace(/"/g, '""')}"`, + mysql: (identifier) => `\`${identifier.replace(/`/g, "``")}\``, +}; + +function bunSqlClient(connectionString: string, dbType: "postgres" | "mysql"): DbClient { + const sql = new SQL(connectionString); + + return { + dbType, + async query>(query: string, params: unknown[] = []) { + const rows = await sql.unsafe(query, params); + return (Array.isArray(rows) ? rows : []) as T[]; + }, + placeholder: dbType === "postgres" ? (position) => `$${position}` : () => "?", + quote: QUOTING[dbType], + async close() { + await sql.close(); + }, + }; +} + +function sqliteClient(connectionString: string): DbClient { + const database = new Database(sqlitePath(connectionString), { readonly: true }); + + return { + dbType: "sqlite", + query>(query: string, params: unknown[] = []) { + // bun:sqlite is synchronous; the Promise keeps one interface for callers. + return Promise.resolve(database.query(query).all(...(params as never[])) as T[]); + }, + placeholder: () => "?", + quote: QUOTING.sqlite, + close() { + database.close(); + return Promise.resolve(); + }, + }; +} + +/** + * Connects to the database a connection string names. + * + * @param platform - Tailors the failure hint; the same "Connection closed" + * means something different on Supabase than on a local SQLite file. + */ +export async function createDbClient( + connectionString: string, + platform?: DbPlatform, +): Promise { + const dbType = detectDbType(connectionString); + + try { + if (dbType === "sqlite") { + const client = sqliteClient(connectionString); + // bun:sqlite opens lazily, so a missing file would not surface until the + // first real query — long after the "connecting" spinner has stopped. + await client.query("SELECT 1"); + return client; + } + + const client = bunSqlClient(connectionString, dbType); + await client.query("SELECT 1"); + return client; + } catch (error) { + throw connectionError(error, connectionString, platform); + } +} + +export type DbPlatform = "supabase" | "betterauth" | "authjs"; + +/** + * Turns a driver error into something a user can act on. + * + * Rewritten rather than ported: the standalone tool matched on `pg`'s + * `ENOTFOUND`/`ETIMEDOUT`, which `Bun.sql` never emits. Bun reports both an + * unreachable host and a closed port as `ERR_*_CONNECTION_CLOSED` with the + * message "Connection closed" — exactly the case where a bare driver error + * helps least. + */ +export function describeDbError(error: unknown, platform?: DbPlatform): string { + const message = error instanceof Error ? error.message : String(error); + const code = (error as { code?: string })?.code ?? ""; + + if (code.includes("CONNECTION_CLOSED") || /connection closed|econnrefused/i.test(message)) { + if (platform === "supabase") { + return ( + "Could not reach the database. Check the host and port in the connection string.\n" + + "Supabase direct connections need the IPv4 add-on — use the pooler connection string\n" + + "(Dashboard → Connect → Session pooler), or enable IPv4 under Settings → Add-Ons." + ); + } + return "Could not reach the database. Check the host and port, and that the server accepts connections from here."; + } + + if (/password authentication failed|access denied/i.test(message)) { + return "The database rejected those credentials. Check the user and password in the connection string."; + } + + if (/does not exist|unknown database|no such table|permission denied/i.test(message)) { + if (platform === "supabase") { + return ( + "The auth.users table was not readable. It is created automatically when Supabase Auth is enabled.\n" + + "Check Authentication is enabled, and connect as the `postgres` role rather than an application role." + ); + } + return "The expected table was not found, or the user cannot read it. Check the database name and the user's SELECT permission."; + } + + if (/unable to open database|sqlitecantopen|no such file/i.test(message)) { + return "Could not open the SQLite file. Check the path, and that the file exists and is readable."; + } + + return "Check the connection string, that the server is running, and that it is reachable from here."; +} + +function connectionError( + error: unknown, + connectionString: string, + platform?: DbPlatform, +): CliError { + const message = error instanceof Error ? error.message : String(error); + return new CliError( + `Could not connect to ${redactConnectionString(connectionString)}: ${message}\n\n${describeDbError(error, platform)}`, + { code: ERROR_CODE.USAGE_ERROR }, + ); +} + +/** + * Runs `work` against a fresh client and always closes it. + * + * A leaked connection keeps the process alive after the export has written its + * file, which looks like a hang. + */ +export async function withDbClient( + connectionString: string, + platform: DbPlatform | undefined, + work: (client: DbClient) => Promise, +): Promise { + const client = await createDbClient(connectionString, platform); + try { + return await work(client); + } catch (error) { + // A query failure carries the same actionable hints as a connection one: + // a missing table is the most common thing that goes wrong here. + if (error instanceof CliError) throw error; + throw new CliError( + `${error instanceof Error ? error.message : String(error)}\n\n${describeDbError(error, platform)}`, + { code: ERROR_CODE.USAGE_ERROR }, + ); + } finally { + await client.close().catch(() => {}); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/instance.test.ts b/packages/cli-core/src/commands/migrate/lib/instance.test.ts new file mode 100644 index 000000000..6f6330f56 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/instance.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, test } from "bun:test"; +import { + DEV_USER_LIMIT, + detectInstanceType, + getDefaultConcurrencyLimit, + getDefaultRateLimit, + getRetryDelay, + resolveLimits, +} from "./instance.ts"; + +describe("detectInstanceType", () => { + test.each([ + ["sk_live_abc123", "prod"], + ["sk_test_abc123", "dev"], + ["sk_something_else", "dev"], + ["nonsense", "dev"], + ])("%s -> %s", (key, expected) => { + expect(detectInstanceType(key)).toBe(expected as "dev" | "prod"); + }); +}); + +describe("default limits", () => { + test.each([ + ["prod", 100], + ["dev", 10], + ])("%s instances get %i req/s", (instanceType, expected) => { + expect(getDefaultRateLimit(instanceType as "dev" | "prod")).toBe(expected); + }); + + test.each([ + [100, 9], + [10, 1], + [1, 1], + ])("a %i req/s limit yields %i concurrent calls", (rateLimit, expected) => { + expect(getDefaultConcurrencyLimit(rateLimit)).toBe(expected); + }); + + test("development instances are capped at 500 users", () => { + expect(DEV_USER_LIMIT).toBe(500); + }); +}); + +describe("resolveLimits", () => { + test("derives both limits from the key when nothing is overridden", () => { + expect(resolveLimits("sk_live_x", {})).toEqual({ + instanceType: "prod", + rateLimit: 100, + concurrencyLimit: 9, + }); + }); + + test("honours environment overrides", () => { + expect( + resolveLimits("sk_test_x", { + CLERK_MIGRATE_RATE_LIMIT: "50", + CLERK_MIGRATE_CONCURRENCY_LIMIT: "4", + }), + ).toEqual({ instanceType: "dev", rateLimit: 50, concurrencyLimit: 4 }); + }); + + test("derives concurrency from an overridden rate limit", () => { + expect(resolveLimits("sk_test_x", { CLERK_MIGRATE_RATE_LIMIT: "200" }).concurrencyLimit).toBe( + 19, + ); + }); + + test.each([["0"], ["-5"], ["fast"], [""]])( + "ignores the unusable override %p in favour of the default", + (value) => { + expect(resolveLimits("sk_test_x", { CLERK_MIGRATE_RATE_LIMIT: value }).rateLimit).toBe(10); + }, + ); +}); + +describe("getRetryDelay", () => { + test.each([ + [undefined, 10_000, 10_000, 10], + [15, 10_000, 15_000, 15], + [1, 10_000, 1000, 1], + ])("Retry-After %p -> %i ms", (retryAfter, fallback, delayMs, delaySeconds) => { + expect(getRetryDelay(retryAfter, fallback)).toEqual({ delayMs, delaySeconds }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/instance.ts b/packages/cli-core/src/commands/migrate/lib/instance.ts new file mode 100644 index 000000000..d094c11cf --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/instance.ts @@ -0,0 +1,93 @@ +/** + * Instance-type detection and the throughput limits that follow from it. + * + * Ported from the standalone migration-tool's `src/envs-constants.ts`, minus + * its dotenv/Zod env bootstrap: the secret key arrives from + * `resolveBapiSecretKey`, and only the two override knobs read the environment. + */ + +/** Development instances are capped at this many users by Clerk. */ +export const DEV_USER_LIMIT = 500; + +/** How many times a 429 is retried before the user is recorded as failed. */ +export const MAX_RETRIES = 5; + +/** Fallback backoff when a 429 response carries no `Retry-After`. */ +export const RETRY_DELAY_MS = 10_000; + +export type InstanceType = "dev" | "prod"; + +/** + * Derives the instance type from the secret key's prefix. + * + * @example detectInstanceType("sk_live_xxx") // "prod" + * @example detectInstanceType("sk_test_xxx") // "dev" + */ +export function detectInstanceType(secretKey: string): InstanceType { + return secretKey.split("_")[1] === "live" ? "prod" : "dev"; +} + +/** + * Clerk's documented `POST /v1/users` rate limits, as requests per second: + * 1000 per 10s for production, 100 per 10s for development. + */ +export function getDefaultRateLimit(instanceType: InstanceType): number { + return instanceType === "prod" ? 100 : 10; +} + +/** + * Concurrency that saturates ~95% of the rate limit, assuming ~100ms of API + * latency per call: N concurrent requests at 100ms each yield N * 10 req/s. + * + * Override with `CLERK_MIGRATE_CONCURRENCY_LIMIT` when actual latency differs. + */ +export function getDefaultConcurrencyLimit(rateLimit: number): number { + return Math.max(1, Math.floor(rateLimit * 0.095)); +} + +export type ResolvedLimits = { + instanceType: InstanceType; + rateLimit: number; + concurrencyLimit: number; +}; + +/** + * Resolves throughput limits for a run: defaults from the detected instance + * type, each overridable by an environment variable. + * + * Non-numeric or non-positive overrides are ignored in favour of the default + * rather than failing the run — an unusable limit would stall the import. + */ +export function resolveLimits( + secretKey: string, + env: Record = process.env, +): ResolvedLimits { + const instanceType = detectInstanceType(secretKey); + + const positive = (value: string | undefined): number | undefined => { + if (!value) return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; + }; + + const rateLimit = positive(env.CLERK_MIGRATE_RATE_LIMIT) ?? getDefaultRateLimit(instanceType); + const concurrencyLimit = + positive(env.CLERK_MIGRATE_CONCURRENCY_LIMIT) ?? getDefaultConcurrencyLimit(rateLimit); + + return { instanceType, rateLimit, concurrencyLimit }; +} + +/** + * Backoff for a 429, preferring the server's `Retry-After` over the default. + * + * @param retryAfterSeconds - `Retry-After` value from the response, if present. + * @param defaultDelayMs - Fallback delay in milliseconds. + */ +export function getRetryDelay( + retryAfterSeconds: number | undefined, + defaultDelayMs: number, +): { delayMs: number; delaySeconds: number } { + const delayMs = retryAfterSeconds ? retryAfterSeconds * 1000 : defaultDelayMs; + const delaySeconds = retryAfterSeconds || defaultDelayMs / 1000; + return { delayMs, delaySeconds }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.test.ts b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts new file mode 100644 index 000000000..1af83c1f0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts @@ -0,0 +1,200 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { classifyLogFile, findLogFile, formatSize, listLogFiles, readNdjson } from "./log-files.ts"; +import { getLogDir } from "./logger.ts"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logfiles-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); +}); + +/** Writes a log file with one NDJSON line per entry. */ +function writeLog(name: string, entries: unknown[]): string { + fs.mkdirSync(getLogDir(), { recursive: true }); + const filePath = path.join(getLogDir(), name); + fs.writeFileSync(filePath, entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n"); + return filePath; +} + +describe("classifyLogFile", () => { + test.each([ + ["migration-2026-01-01T12-00-00.log", "migration", "2026-01-01T12-00-00"], + ["user-deletion-2026-01-01T12-00-00.log", "deletion", "2026-01-01T12-00-00"], + ["export-2026-01-01T12-00-00.log", "export", "2026-01-01T12-00-00"], + ])("%s is a %s log from %s", (name, kind, timestamp) => { + expect(classifyLogFile(name)).toEqual({ kind: kind as never, timestamp }); + }); + + test.each([["random.log"], ["migration.log"], ["notes.txt"]])( + "%s is unrecognized rather than a parse failure", + (name) => { + expect(classifyLogFile(name)).toEqual({ kind: "unknown", timestamp: "" }); + }, + ); +}); + +describe("listLogFiles", () => { + test("returns nothing when the directory does not exist", () => { + expect(fs.existsSync(getLogDir())).toBe(false); + expect(listLogFiles()).toEqual([]); + }); + + test("returns nothing when the directory is empty", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + expect(listLogFiles()).toEqual([]); + }); + + test("reports kind, timestamp, size and entry count per file", () => { + writeLog("migration-2026-01-01T12-00-00.log", [{ userId: "u1" }, { userId: "u2" }]); + + const [file] = listLogFiles(); + expect(file).toMatchObject({ + name: "migration-2026-01-01T12-00-00.log", + kind: "migration", + timestamp: "2026-01-01T12-00-00", + entryCount: 2, + }); + expect(file?.sizeBytes).toBeGreaterThan(0); + }); + + test("ignores files that are not logs", () => { + writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + fs.writeFileSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"), "[]"); + fs.writeFileSync(path.join(getLogDir(), "notes.txt"), "hi"); + + expect(listLogFiles().map((file) => file.name)).toEqual(["migration-2026-01-01T12-00-00.log"]); + }); + + test("ignores subdirectories", () => { + fs.mkdirSync(path.join(getLogDir(), "nested.log"), { recursive: true }); + expect(listLogFiles()).toEqual([]); + }); + + test("returns the newest run first", () => { + writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + writeLog("migration-2026-03-01T12-00-00.log", [{ a: 1 }]); + writeLog("migration-2026-02-01T12-00-00.log", [{ a: 1 }]); + + expect(listLogFiles().map((file) => file.timestamp)).toEqual([ + "2026-03-01T12-00-00", + "2026-02-01T12-00-00", + "2026-01-01T12-00-00", + ]); + }); + + // Sorting on the filename would put every "user-deletion-" ahead of every + // "migration-", regardless of when the runs actually happened. + test("orders by timestamp across log kinds, not by the name's prefix", () => { + writeLog("user-deletion-2026-01-30T17-02-51.log", [{ a: 1 }]); + writeLog("migration-2026-02-01T09-14-22.log", [{ a: 1 }]); + + expect(listLogFiles().map((file) => file.kind)).toEqual(["migration", "deletion"]); + }); + + test("sorts unrecognized names last", () => { + writeLog("something-else.log", [{ a: 1 }]); + writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + + expect(listLogFiles().map((file) => file.name)).toEqual([ + "migration-2026-01-01T12-00-00.log", + "something-else.log", + ]); + }); + + test("does not count blank lines as entries", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync(path.join(getLogDir(), "migration-x.log"), '{"a":1}\n\n\n{"b":2}\n'); + expect(listLogFiles()[0]?.entryCount).toBe(2); + }); + + test("lists a log whose name does not match the convention", () => { + writeLog("something-else.log", [{ a: 1 }]); + expect(listLogFiles()[0]).toMatchObject({ kind: "unknown", timestamp: "", entryCount: 1 }); + }); +}); + +describe("findLogFile", () => { + beforeEach(() => { + writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + }); + + test("finds a log by name", () => { + expect(findLogFile("migration-2026-01-01T12-00-00.log")?.entryCount).toBe(1); + }); + + test("accepts a path and matches on the basename", () => { + expect(findLogFile("./logs/migration-2026-01-01T12-00-00.log")?.entryCount).toBe(1); + }); + + test("returns nothing for a name that is not there", () => { + expect(findLogFile("migration-nope.log")).toBeUndefined(); + }); +}); + +describe("readNdjson", () => { + test("parses one entry per line", () => { + const file = writeLog("migration-a.log", [{ userId: "u1" }, { userId: "u2" }]); + const { entries, errors } = readNdjson(file); + + expect(entries).toEqual([{ userId: "u1" }, { userId: "u2" }]); + expect(errors).toEqual([]); + }); + + test("skips blank lines without reporting them", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + const file = path.join(getLogDir(), "migration-b.log"); + fs.writeFileSync(file, '\n{"a":1}\n \n{"b":2}\n\n'); + + const { entries, errors } = readNdjson(file); + expect(entries).toHaveLength(2); + expect(errors).toEqual([]); + }); + + // A run killed mid-write leaves one truncated line; the complete entries + // before it are still worth having, so the read reports rather than aborts. + test("reports a malformed line by number and keeps the rest", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + const file = path.join(getLogDir(), "migration-c.log"); + fs.writeFileSync(file, '{"a":1}\n{"b":\n{"c":3}\n'); + + const { entries, errors } = readNdjson(file); + expect(entries).toEqual([{ a: 1 }, { c: 3 }]); + expect(errors).toHaveLength(1); + expect(errors[0]?.line).toBe(2); + }); + + test("numbers lines from one, counting blanks", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + const file = path.join(getLogDir(), "migration-d.log"); + fs.writeFileSync(file, '\n\n{"a":1}\nnot json\n'); + + expect(readNdjson(file).errors[0]?.line).toBe(4); + }); +}); + +describe("formatSize", () => { + test.each([ + [0, "0 B"], + [512, "512 B"], + [1024, "1.0 KB"], + [1536, "1.5 KB"], + [1024 * 1024, "1.0 MB"], + ])("%i bytes reads as %s", (bytes, expected) => { + expect(formatSize(bytes)).toBe(expected); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.ts b/packages/cli-core/src/commands/migrate/lib/log-files.ts new file mode 100644 index 000000000..1498ddd36 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/log-files.ts @@ -0,0 +1,141 @@ +/** + * Enumerating and reading the cwd-relative `./logs/` directory. + * + * The standalone migration-tool re-read the directory inside both of its log + * commands to build their pickers. `list`, `clean` and `convert` all share + * this instead, which is also what makes `logs list` nearly free. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { getLogDir } from "./logger.ts"; + +/** The run that produced a log file, read from its filename prefix. */ +export type LogKind = "migration" | "deletion" | "export" | "unknown"; + +const FILENAME_PATTERN = /^(migration|user-deletion|export)-(.+)\.log$/; + +const KIND_BY_PREFIX: Record = { + migration: "migration", + "user-deletion": "deletion", + export: "export", +}; + +export type LogFile = { + name: string; + path: string; + kind: LogKind; + /** Timestamp as recorded in the filename, or `""` for an unrecognized name. */ + timestamp: string; + sizeBytes: number; + /** Non-empty NDJSON lines, malformed ones included. */ + entryCount: number; +}; + +export function classifyLogFile(name: string): { kind: LogKind; timestamp: string } { + const match = FILENAME_PATTERN.exec(name); + if (!match) return { kind: "unknown", timestamp: "" }; + return { kind: KIND_BY_PREFIX[match[1] as string] ?? "unknown", timestamp: match[2] as string }; +} + +function countEntries(filePath: string): number { + try { + return fs + .readFileSync(filePath, "utf-8") + .split("\n") + .filter((line) => line.trim().length > 0).length; + } catch { + // An unreadable file still belongs in the listing; its count is unknown. + return 0; + } +} + +/** + * Every `.log` file in `./logs/`, newest first. + * + * @returns An empty array when the directory is absent — "no logs yet" and "no + * logs directory" are the same thing to every caller. + */ +export function listLogFiles(): LogFile[] { + const dir = getLogDir(); + if (!fs.existsSync(dir)) return []; + + const files: LogFile[] = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith(".log")) continue; + + const filePath = path.join(dir, name); + let stats: fs.Stats; + try { + stats = fs.statSync(filePath); + } catch { + continue; + } + if (!stats.isFile()) continue; + + files.push({ + name, + path: filePath, + ...classifyLogFile(name), + sizeBytes: stats.size, + entryCount: countEntries(filePath), + }); + } + + // Sort on the timestamp, not the filename: the kind prefix sorts first in a + // filename comparison, which would interleave a run from January ahead of one + // from March purely because "user-deletion" > "migration". Timestamps are + // ISO-ish and zero-padded, so lexical order is chronological. Names without + // one sort last, then alphabetically. + return files.sort( + (a, b) => b.timestamp.localeCompare(a.timestamp) || a.name.localeCompare(b.name), + ); +} + +/** Resolves a user-supplied name or path to a log file in `./logs/`. */ +export function findLogFile(nameOrPath: string): LogFile | undefined { + const wanted = path.basename(nameOrPath); + return listLogFiles().find((file) => file.name === wanted); +} + +export type NdjsonLineError = { + /** 1-indexed line number in the source file. */ + line: number; + message: string; +}; + +export type NdjsonReadResult = { + entries: unknown[]; + errors: NdjsonLineError[]; +}; + +/** + * Parses an NDJSON file line by line. + * + * Malformed lines are collected with their line numbers rather than aborting + * the read: a run killed mid-write leaves one truncated final line, and the + * hundreds of complete entries before it are still worth having. + */ +export function readNdjson(filePath: string): NdjsonReadResult { + const entries: unknown[] = []; + const errors: NdjsonLineError[] = []; + + const lines = fs.readFileSync(filePath, "utf-8").split("\n"); + for (const [index, line] of lines.entries()) { + if (line.trim().length === 0) continue; + try { + entries.push(JSON.parse(line)); + } catch (error) { + errors.push({ line: index + 1, message: (error as Error).message }); + } + } + + return { entries, errors }; +} + +/** Human-readable file size. */ +export function formatSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} diff --git a/packages/cli-core/src/commands/migrate/lib/logger.test.ts b/packages/cli-core/src/commands/migrate/lib/logger.test.ts new file mode 100644 index 000000000..db973f00a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/logger.test.ts @@ -0,0 +1,110 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + errorLogger, + getDateTimeStamp, + getLogDir, + getLogFilePath, + importLogger, + validationLogger, +} from "./logger.ts"; + +const DATE_TIME = "2026-01-01T12:00:00"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + // realpath so the comparison against process.cwd() survives macOS's + // /var -> /private/var symlink. + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logger-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); +}); + +function readEntries(): Record[] { + return fs + .readFileSync(getLogFilePath("migration", DATE_TIME), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); +} + +describe("log file paths", () => { + test("writes under the current working directory, not next to the binary", () => { + expect(getLogDir()).toBe(path.join(workDir, "logs")); + }); + + test("replaces the timestamp's colons so the name is valid on Windows", () => { + expect(path.basename(getLogFilePath("migration", DATE_TIME))).toBe( + "migration-2026-01-01T12-00-00.log", + ); + }); + + test("getDateTimeStamp drops milliseconds", () => { + expect(getDateTimeStamp()).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$/); + }); +}); + +describe("log writers", () => { + test("creates the logs directory on first write", () => { + expect(fs.existsSync(getLogDir())).toBe(false); + importLogger({ userId: "u1", status: "success", clerkUserId: "user_x" }, DATE_TIME); + expect(fs.existsSync(getLogDir())).toBe(true); + }); + + test("appends one NDJSON line per entry", () => { + importLogger({ userId: "u1", status: "success", clerkUserId: "user_x" }, DATE_TIME); + importLogger({ userId: "u2", status: "error", error: "boom", code: "422" }, DATE_TIME); + + const entries = readEntries(); + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ userId: "u1", status: "success", clerkUserId: "user_x" }); + expect(entries[1]).toEqual({ userId: "u2", status: "error", error: "boom", code: "422" }); + }); + + test("writes one line per error in a failed payload", () => { + errorLogger( + { + userId: "u1", + status: "422", + errors: [ + { code: "a", message: "short a", longMessage: "long a" }, + { code: "b", message: "short b" }, + ], + }, + DATE_TIME, + ); + + const entries = readEntries(); + expect(entries).toHaveLength(2); + expect(entries[0]).toMatchObject({ type: "User Creation Error", error: "long a" }); + // Falls back to `message` when the API omitted a long form. + expect(entries[1]).toMatchObject({ error: "short b" }); + }); + + test("records validation failures in the same run log", () => { + validationLogger( + { error: "missing identifier", path: ["email"], userId: "u3", row: 4 }, + DATE_TIME, + ); + expect(readEntries()[0]).toEqual({ + userId: "u3", + status: "fail", + error: "missing identifier", + path: ["email"], + row: 4, + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.ts b/packages/cli-core/src/commands/migrate/lib/logger.ts new file mode 100644 index 000000000..72cd98372 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/logger.ts @@ -0,0 +1,114 @@ +/** + * NDJSON migration logs. + * + * Ported from the standalone migration-tool's `src/logger.ts`, with one + * behavioural fix: logs are written relative to the current working directory + * rather than to `__dirname/../logs`. In a `bun build --compile` binary there + * is no source tree next to the executable, so the original path would land + * logs inside wherever the binary happens to live. + * + * Writes are synchronous appends so a run interrupted with Ctrl-C still leaves + * a complete record of everything already processed. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { log } from "../../../lib/log.ts"; +import type { + DeleteLogEntry, + ErrorLog, + ErrorPayload, + ExportLogEntry, + ImportLogEntry, + ValidationErrorPayload, +} from "../types.ts"; + +/** Absolute path of the cwd-relative `logs/` directory. */ +export function getLogDir(): string { + return path.join(process.cwd(), "logs"); +} + +/** Absolute path of the log file a run with this timestamp writes to. */ +export function getLogFilePath(logFile: string, dateTime: string): string { + // Colons are illegal in Windows filenames, and the timestamp is an ISO string. + return path.join(getLogDir(), `${logFile}-${dateTime}.log`.replace(/:/g, "-")); +} + +/** ISO timestamp without milliseconds — the log-file name discriminator. */ +export function getDateTimeStamp(): string { + return new Date().toISOString().split(".")[0] ?? ""; +} + +function appendToLogFile(fullPath: string, entry: unknown): void { + try { + fs.mkdirSync(path.dirname(fullPath), { recursive: true }); + fs.appendFileSync(fullPath, `${JSON.stringify(entry)}\n`); + } catch (error) { + // A broken log destination must not abort an in-flight migration; the run + // is still making real progress against the API. + log.warn(`Could not write migration log: ${(error as Error).message}`); + } +} + +/** Writes each error in a failed API call as its own NDJSON line. */ +export function errorLogger(payload: ErrorPayload, dateTime: string): void { + for (const err of payload.errors) { + const entry: ErrorLog = { + type: "User Creation Error", + userId: payload.userId, + status: payload.status, + error: err.longMessage ?? err.message, + }; + appendToLogFile(getLogFilePath("migration", dateTime), entry); + } +} + +/** Writes a user that failed schema validation before any API call. */ +export function validationLogger(payload: ValidationErrorPayload, dateTime: string): void { + appendToLogFile(getLogFilePath("migration", dateTime), { + userId: payload.userId, + status: "fail" as const, + error: payload.error, + path: payload.path, + row: payload.row, + }); +} + +/** Writes the outcome of one import attempt. */ +export function importLogger(entry: ImportLogEntry, dateTime: string): void { + appendToLogFile(getLogFilePath("migration", dateTime), entry); +} + +/** + * Writes the outcome of one deletion attempt. + * + * A separate `user-deletion-` file rather than another line in the migration + * log: undoing a migration is its own run, and mixing the two would make + * "what did this import do" unanswerable after an undo. + */ +export function deleteLogger(entry: DeleteLogEntry, dateTime: string): void { + appendToLogFile(getLogFilePath("user-deletion", dateTime), entry); +} + +/** + * Writes the outcome of exporting one user. + * + * Its own `export-` file for the same reason deletions get theirs: an export + * is a distinct run, and `migrate logs list` reports each kind separately. + */ +export function exportLogger(entry: ExportLogEntry, dateTime: string): void { + appendToLogFile(getLogFilePath("export", dateTime), entry); +} + +/** Writes each error in a failed deletion as its own NDJSON line. */ +export function deleteErrorLogger(payload: ErrorPayload, dateTime: string): void { + for (const err of payload.errors) { + const entry: ErrorLog = { + type: "User Deletion Error", + userId: payload.userId, + status: payload.status, + error: err.longMessage ?? err.message, + }; + appendToLogFile(getLogFilePath("user-deletion", dateTime), entry); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts new file mode 100644 index 000000000..cdac7d866 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts @@ -0,0 +1,322 @@ +import { describe, expect, test } from "bun:test"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import type { FieldAnalysis } from "./analysis.ts"; +import { buildReadinessReport, formatReadinessReport, type ReadinessItem } from "./readiness.ts"; + +/** Instance settings carrying only the attributes and providers a test names. */ +function settings(config: { + attributes?: Record; + social?: Record; +}): UserSettingsJSON { + return { + attributes: Object.fromEntries( + Object.entries(config.attributes ?? {}).map(([name, value]) => [ + name, + { enabled: value.enabled, required: value.required ?? false }, + ]), + ), + social: config.social ?? {}, + } as unknown as UserSettingsJSON; +} + +/** Field analysis with everything absent unless the test says otherwise. */ +function analysis(overrides: Partial & { totalUsers: number }): FieldAnalysis { + return { + identifiers: { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: overrides.totalUsers, + ...overrides.identifiers, + }, + fieldCounts: overrides.fieldCounts ?? {}, + totalUsers: overrides.totalUsers, + }; +} + +const item = (report: { items: ReadinessItem[] }, label: string) => + report.items.find((entry) => entry.label === label); + +describe("which rows appear", () => { + test("reports only the fields the file actually carries", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 3, identifiers: { verifiedEmails: 3 } as never }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }); + expect(report.items.map((entry) => entry.label)).toEqual(["Email"]); + }); + + test("counts verified and unverified identifiers together", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, unverifiedEmails: 2, hasAnyIdentifier: 5 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }); + expect(item(report, "Email")?.userCount).toBe(5); + }); + + test("groups rows into identifiers, auth and user model", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, username: 2, hasAnyIdentifier: 2 } as never, + fieldCounts: { password: 2, firstName: 2, lastName: 1 }, + }), + settings: settings({}), + }); + expect(report.items.map((entry) => [entry.label, entry.section])).toEqual([ + ["Email", "identifiers"], + ["Username", "identifiers"], + ["Password", "auth"], + ["First name", "model"], + ["Last name", "model"], + ]); + }); +}); + +describe("required in Clerk but missing from the file", () => { + // The expensive case: those users fail one at a time, mid-import, after + // earlier users have already been created. + test("flags an attribute Clerk requires that not every user has", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 10, username: 10 } as never, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }), + }); + + const email = item(report, "Email"); + expect(email?.blocking).toBe(true); + expect(email?.detail).toContain("3 users lack it"); + expect(report.blocking).toHaveLength(1); + }); + + test("does not flag a required attribute every user has", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + expect(report.blocking).toHaveLength(0); + }); + + test("does not flag an enabled-but-optional attribute that some users lack", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 10, hasAnyIdentifier: 10 } as never, + fieldCounts: { firstName: 2 }, + }), + settings: settings({ + attributes: { email_address: { enabled: true }, first_name: { enabled: true } }, + }), + }); + expect(report.blocking).toHaveLength(0); + }); + + test("uses the singular form for a single missing user", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 1, hasAnyIdentifier: 2, username: 2 } as never, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }), + }); + expect(item(report, "Email")?.detail).toContain("1 user lacks it"); + }); +}); + +describe("present in the file but disabled in Clerk", () => { + test("flags an attribute the instance has switched off", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, username: 3, hasAnyIdentifier: 3 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true }, username: { enabled: false } }, + }), + }); + + const username = item(report, "Username"); + expect(username?.blocking).toBe(true); + expect(username?.detail).toBe("not enabled in Clerk"); + }); + + test("flags a social provider users signed up with that Clerk lacks", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }), + providerCounts: { google: 3, discord: 1 }, + }); + + expect(item(report, "Google")?.blocking).toBe(false); + expect(item(report, "Discord")?.blocking).toBe(true); + expect(report.blocking.map((entry) => entry.label)).toEqual(["Discord"]); + }); + + test("maps a provider whose Clerk strategy name differs", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 1, identifiers: { verifiedEmails: 1 } as never }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_microsoft: { enabled: true } }, + }), + providerCounts: { azure: 1 }, + }); + expect(item(report, "Microsoft (Azure)")?.blocking).toBe(false); + }); + + test("ignores a provider no user actually signed up with", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 1, identifiers: { verifiedEmails: 1 } as never }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + providerCounts: { discord: 0 }, + }); + expect(item(report, "Discord")).toBeUndefined(); + }); +}); + +describe("when the instance settings cannot be read", () => { + const unreadable = () => + buildReadinessReport({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 3 } as never, + }), + settings: null, + }); + + test("marks the report as degraded rather than failing", () => { + expect(unreadable().settingsUnavailable).toBe(true); + }); + + // `null` means "not read", which must not be confused with `false` + // ("read, and it is off") — the latter blocks, the former cannot. + test("claims nothing about Clerk, so nothing blocks", () => { + const report = unreadable(); + expect(item(report, "Email")?.clerkEnabled).toBeNull(); + expect(report.blocking).toHaveLength(0); + }); + + test("still reports what the file contains", () => { + expect(unreadable().items.map((entry) => entry.label)).toEqual(["Email"]); + }); + + test("renders a note explaining the checks are coverage only", () => { + const output = formatReadinessReport(unreadable()).join("\n"); + expect(output).toContain("Could not read this instance's settings"); + expect(output).toContain("dashboard.clerk.com"); + }); +}); + +describe("file-level totals", () => { + test("counts users with no identifier at all", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 7 } as never, + }), + settings: settings({}), + }); + expect(report.withoutIdentifier).toBe(3); + }); + + test("carries the validation failure count through", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 2 }), + settings: settings({}), + validationFailed: 5, + }); + expect(report.validationFailed).toBe(5); + }); +}); + +describe("rendering", () => { + const blocked = () => + buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 8, username: 10 } as never, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }), + validationFailed: 2, + }); + + test("leads with the counts an operator needs before confirming", () => { + const output = formatReadinessReport(blocked()).join("\n"); + expect(output).toContain("10 users ready to import"); + expect(output).toContain("2 failed validation"); + expect(output).toContain("2 without any identifier"); + }); + + test("names the blocking rows and points at the dashboard", () => { + const output = formatReadinessReport(blocked()).join("\n"); + expect(output).toContain("1 setting needs attention"); + expect(output).toContain("3 users lack it"); + expect(output).toContain("dashboard.clerk.com"); + }); + + test("confirms a clean report when nothing blocks", () => { + const output = formatReadinessReport( + buildReadinessReport({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }), + ).join("\n"); + expect(output).toContain("Every field in this file is configured in Clerk"); + }); + + test("does not claim everything is configured when settings were unreadable", () => { + const output = formatReadinessReport( + buildReadinessReport({ analysis: analysis({ totalUsers: 1 }), settings: null }), + ).join("\n"); + expect(output).not.toContain("Every field in this file is configured"); + }); + + test("renders section headings only for sections that have rows", () => { + const output = formatReadinessReport( + buildReadinessReport({ + analysis: analysis({ + totalUsers: 1, + identifiers: { verifiedEmails: 1, hasAnyIdentifier: 1 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }), + ).join("\n"); + expect(output).toContain("Identifiers"); + expect(output).not.toContain("Social connections"); + expect(output).not.toContain("User model"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts new file mode 100644 index 000000000..353823415 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -0,0 +1,238 @@ +/** + * The Migration Readiness report: what the import file contains, cross- + * referenced against what the destination instance actually accepts. + * + * Ported from the standalone migration-tool's `displayCrossReference`. Split + * into a pure {@link buildReadinessReport} and a separate renderer so the + * cross-reference decisions are testable without parsing coloured output. + * + * The point of the report is to surface, *before* anything is written to + * Clerk, the two failure modes a migration only discovers halfway through: + * a field Clerk requires that some users lack, and a social provider users + * signed up with that Clerk has not enabled. + */ + +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { bold, dim, green, red, yellow } from "../../../lib/color.ts"; +// Pure attribute lookups, shared with the `users` create wizard. +import { isEnabled, isRequired, type AttributeName } from "../../users/interactive/attributes.ts"; +import type { FieldAnalysis } from "./analysis.ts"; +import { providerLabel, toClerkStrategy } from "./clerk-config.ts"; + +const DASHBOARD_URL = "https://dashboard.clerk.com/~/user-authentication"; + +export type ReadinessSection = "identifiers" | "auth" | "social" | "model"; + +/** + * One row of the report. + * + * @property clerkEnabled - `null` when the instance settings could not be read, + * which is different from `false` ("read, and it is off"). + * @property blocking - This row will cost users unless the operator acts. + */ +export type ReadinessItem = { + label: string; + section: ReadinessSection; + /** Users in the file that carry this field or provider. */ + userCount: number; + clerkEnabled: boolean | null; + clerkRequired: boolean | null; + blocking: boolean; + /** Why it blocks — omitted when it does not. */ + detail?: string; +}; + +export type ReadinessReport = { + totalUsers: number; + /** Users with no identifier at all; they cannot be imported under any settings. */ + withoutIdentifier: number; + validationFailed: number; + items: ReadinessItem[]; + /** Every item flagged `blocking`, in report order. */ + blocking: ReadinessItem[]; + /** True when the instance settings could not be read. */ + settingsUnavailable: boolean; +}; + +type BuildInput = { + analysis: FieldAnalysis; + /** `null` when no publishable key was available, or FAPI could not be read. */ + settings: UserSettingsJSON | null; + validationFailed?: number; + /** Source-platform provider key → user count. Supabase exports only. */ + providerCounts?: Record; +}; + +/** An identifier or user-model row, with its blocking verdict. */ +function buildAttributeItem( + label: string, + section: ReadinessSection, + attribute: AttributeName, + userCount: number, + settings: UserSettingsJSON | null, + totalUsers: number, +): ReadinessItem { + const enabled = settings ? isEnabled(settings, attribute) : null; + const required = settings ? isRequired(settings, attribute) : null; + const missing = totalUsers - userCount; + + // Required but not universal is the expensive case: those users fail one by + // one, mid-import, after earlier users have already been created. + if (required === true && missing > 0) { + return { + label, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: true, + detail: + missing === 1 + ? "required in Clerk, but 1 user lacks it" + : `required in Clerk, but ${missing} users lack it`, + }; + } + + // Present in the file but switched off in Clerk: the data is silently dropped. + if (enabled === false && userCount > 0) { + return { + label, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: true, + detail: "not enabled in Clerk", + }; + } + + return { + label, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: false, + }; +} + +/** + * Cross-references the file against the instance. + * + * A field absent from the file contributes no row — the report describes what + * is actually being imported, not every setting Clerk supports. + */ +export function buildReadinessReport(input: BuildInput): ReadinessReport { + const { analysis, settings, validationFailed = 0, providerCounts = {} } = input; + const total = analysis.totalUsers; + const items: ReadinessItem[] = []; + + const emailCount = analysis.identifiers.verifiedEmails + analysis.identifiers.unverifiedEmails; + const phoneCount = analysis.identifiers.verifiedPhones + analysis.identifiers.unverifiedPhones; + + const attributeRows: [string, ReadinessSection, AttributeName, number][] = [ + ["Email", "identifiers", "email_address", emailCount], + ["Phone", "identifiers", "phone_number", phoneCount], + ["Username", "identifiers", "username", analysis.identifiers.username], + ["Password", "auth", "password", analysis.fieldCounts.password ?? 0], + ["First name", "model", "first_name", analysis.fieldCounts.firstName ?? 0], + ["Last name", "model", "last_name", analysis.fieldCounts.lastName ?? 0], + ]; + + for (const [label, section, attribute, count] of attributeRows) { + if (count > 0) { + items.push(buildAttributeItem(label, section, attribute, count, settings, total)); + } + } + + for (const [provider, count] of Object.entries(providerCounts)) { + if (count === 0) continue; + const enabled = settings + ? (settings.social?.[toClerkStrategy(provider) as keyof typeof settings.social]?.enabled ?? + false) + : null; + items.push({ + label: providerLabel(provider), + section: "social", + userCount: count, + clerkEnabled: enabled, + clerkRequired: null, + blocking: enabled === false, + ...(enabled === false ? { detail: "not enabled in Clerk" } : {}), + }); + } + + return { + totalUsers: total, + withoutIdentifier: total - analysis.identifiers.hasAnyIdentifier, + validationFailed, + items, + blocking: items.filter((item) => item.blocking), + settingsUnavailable: settings === null, + }; +} + +const SECTION_ORDER: ReadinessSection[] = ["identifiers", "auth", "social", "model"]; +const SECTION_LABELS: Record = { + identifiers: "Identifiers", + auth: "Authentication", + social: "Social connections", + model: "User model", +}; + +function renderItem(item: ReadinessItem, total: number): string { + const coverage = item.userCount === total ? "all users" : `${item.userCount}/${total} users`; + + if (item.blocking) { + return ` ${yellow("⚠")} ${item.label} — ${yellow(item.detail ?? "needs attention")} — ${dim(coverage)}`; + } + if (item.clerkEnabled === true) { + return ` ${green("✓")} ${item.label} — ${dim(`enabled in Clerk — ${coverage}`)}`; + } + // Settings unavailable: state coverage without claiming anything about Clerk. + return ` ${yellow("○")} ${item.label} — ${dim(`${coverage} — check it is enabled in Clerk`)}`; +} + +/** Renders the report for a human, as lines. */ +export function formatReadinessReport(report: ReadinessReport): string[] { + const lines: string[] = [bold("Migration readiness")]; + + lines.push(` ${report.totalUsers} user${report.totalUsers === 1 ? "" : "s"} ready to import`); + if (report.validationFailed > 0) { + lines.push(` ${yellow(`${report.validationFailed} failed validation and will be skipped`)}`); + } + if (report.withoutIdentifier > 0) { + lines.push( + ` ${red(`${report.withoutIdentifier} without any identifier — cannot be imported`)}`, + ); + } + + if (report.settingsUnavailable) { + lines.push( + "", + ` ${yellow("○")} ${dim("Could not read this instance's settings, so the checks below are coverage only.")}`, + ` ${dim(` Verify your settings at ${DASHBOARD_URL}`)}`, + ); + } + + for (const section of SECTION_ORDER) { + const sectionItems = report.items.filter((item) => item.section === section); + if (sectionItems.length === 0) continue; + + lines.push("", bold(SECTION_LABELS[section])); + for (const item of sectionItems) lines.push(renderItem(item, report.totalUsers)); + } + + lines.push(""); + if (report.blocking.length > 0) { + const count = report.blocking.length; + lines.push( + yellow(`⚠ ${count} setting${count === 1 ? "" : "s"} need${count === 1 ? "s" : ""} attention`), + dim(` ${DASHBOARD_URL}`), + ); + } else if (!report.settingsUnavailable) { + lines.push(green("✓ Every field in this file is configured in Clerk")); + } + + return lines; +} diff --git a/packages/cli-core/src/commands/migrate/lib/retry.test.ts b/packages/cli-core/src/commands/migrate/lib/retry.test.ts new file mode 100644 index 000000000..ed7f10091 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/retry.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, test } from "bun:test"; +import { BapiError, CliError } from "../../../lib/errors.ts"; +import { RateLimitExceededError, readRetryAfter, retryOn429 } from "./retry.ts"; + +const rateLimited = (headers: Record = {}) => + new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow" }] }), + new Headers(headers), + ); + +const failed = (status: number) => + new BapiError( + status, + JSON.stringify({ errors: [{ code: "e", message: "nope" }] }), + new Headers(), + ); + +describe("readRetryAfter", () => { + test.each([ + ["12", 12], + ["0", undefined], + ["-1", undefined], + ["soon", undefined], + ])("Retry-After: %s -> %p", (header, expected) => { + expect(readRetryAfter(rateLimited({ "retry-after": header }))).toBe( + expected as number | undefined, + ); + }); + + test("falls back to the error body's retryAfter meta", () => { + const error = new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow", meta: { retryAfter: 7 } }] }), + new Headers(), + ); + expect(readRetryAfter(error)).toBe(7); + }); + + test("prefers the header over the body", () => { + const error = new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow", meta: { retryAfter: 7 } }] }), + new Headers({ "retry-after": "3" }), + ); + expect(readRetryAfter(error)).toBe(3); + }); + + test("returns undefined when neither carries a value", () => { + expect(readRetryAfter(rateLimited())).toBeUndefined(); + }); +}); + +describe("retryOn429", () => { + test("returns the value when the call succeeds first time", async () => { + expect(await retryOn429(async () => "ok")).toBe("ok"); + }); + + test("retries after a 429 and returns the eventual value", async () => { + let attempts = 0; + const result = await retryOn429( + async () => { + attempts++; + if (attempts === 1) throw rateLimited({ "retry-after": "1" }); + return "ok"; + }, + { defaultDelayMs: 5 }, + ); + + expect(result).toBe("ok"); + expect(attempts).toBe(2); + }); + + test("waits the interval the server asked for", async () => { + let attempts = 0; + const started = performance.now(); + + await retryOn429(async () => { + attempts++; + if (attempts === 1) throw rateLimited({ "retry-after": "1" }); + return "ok"; + }); + + expect(performance.now() - started).toBeGreaterThanOrEqual(900); + }); + + test("falls back to the default delay when no Retry-After is given", async () => { + let attempts = 0; + await retryOn429( + async () => { + attempts++; + if (attempts === 1) throw rateLimited(); + return "ok"; + }, + { defaultDelayMs: 5 }, + ); + expect(attempts).toBe(2); + }); + + test("reports each backoff to the caller so it can log against its own run", async () => { + const seen: { attempt: number; delaySeconds: number }[] = []; + let attempts = 0; + + await retryOn429( + async () => { + attempts++; + if (attempts <= 2) throw rateLimited(); + return "ok"; + }, + { + defaultDelayMs: 5, + onRetry: ({ attempt, delaySeconds }) => seen.push({ attempt, delaySeconds }), + }, + ); + + expect(seen.map((entry) => entry.attempt)).toEqual([1, 2]); + expect(seen[0]?.delaySeconds).toBe(0.005); + }); + + test("gives up after the ceiling, distinctly from an ordinary failure", async () => { + let attempts = 0; + + await expect( + retryOn429( + async () => { + attempts++; + throw rateLimited(); + }, + { maxRetries: 2, defaultDelayMs: 5 }, + ), + ).rejects.toThrow(RateLimitExceededError); + + // One initial attempt plus maxRetries retries. + expect(attempts).toBe(3); + }); + + // Only rate limiting is transient; retrying a 422 would just repeat it. + test.each([[400], [401], [404], [422], [500]])("lets a %i through untouched", async (status) => { + let attempts = 0; + + await expect( + retryOn429(async () => { + attempts++; + throw failed(status); + }), + ).rejects.toThrow(BapiError); + + expect(attempts).toBe(1); + }); + + test("lets a non-API error through untouched", async () => { + await expect(retryOn429(async () => Promise.reject(new CliError("boom")))).rejects.toThrow( + CliError, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/retry.ts b/packages/cli-core/src/commands/migrate/lib/retry.ts new file mode 100644 index 000000000..f9d1fc158 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/retry.ts @@ -0,0 +1,66 @@ +/** + * Rate-limit backoff, shared by `migrate run` and `migrate delete`. + * + * Both walk the whole user set through BAPI and hit the same limits, so they + * back off identically rather than approximately: extracting this is what + * makes "deletion retries the same as import" true by construction. + */ + +import { BapiError } from "../../../lib/errors.ts"; +import { MAX_RETRIES, RETRY_DELAY_MS, getRetryDelay } from "./instance.ts"; + +/** Seconds to wait per a 429's `Retry-After` header or error meta, if given. */ +export function readRetryAfter(error: BapiError): number | undefined { + const header = error.headers?.get("retry-after"); + if (header) { + const parsed = Number(header); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + const meta = error.meta?.retryAfter; + return typeof meta === "number" && meta > 0 ? meta : undefined; +} + +/** Raised once a 429 has been retried {@link MAX_RETRIES} times. */ +export class RateLimitExceededError extends Error { + constructor(public readonly attempts: number) { + super(`Rate limit exceeded after ${attempts} retries`); + this.name = "RateLimitExceededError"; + } +} + +export type RetryOptions = { + /** Called before each backoff, so the caller can log it against its own run. */ + onRetry?: (info: { attempt: number; delaySeconds: number; message: string }) => void; + maxRetries?: number; + /** Backoff when the response carries no `Retry-After`. */ + defaultDelayMs?: number; +}; + +/** + * Runs `fn`, backing off and retrying whenever BAPI answers 429. + * + * Anything other than a 429 propagates untouched — only rate limiting is + * transient. Exhausting the retries raises {@link RateLimitExceededError} so + * the caller can record it distinctly from an ordinary API failure. + */ +export async function retryOn429(fn: () => Promise, options: RetryOptions = {}): Promise { + const maxRetries = options.maxRetries ?? MAX_RETRIES; + const defaultDelayMs = options.defaultDelayMs ?? RETRY_DELAY_MS; + + for (let attempt = 0; ; attempt++) { + try { + return await fn(); + } catch (error) { + if (!(error instanceof BapiError) || error.status !== 429) throw error; + if (attempt >= maxRetries) throw new RateLimitExceededError(maxRetries); + + const { delayMs, delaySeconds } = getRetryDelay(readRetryAfter(error), defaultDelayMs); + options.onRetry?.({ + attempt: attempt + 1, + delaySeconds, + message: `Rate limit hit (429), retrying in ${delaySeconds}s (attempt ${attempt + 1}/${maxRetries})`, + }); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts b/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts new file mode 100644 index 000000000..68c44e50b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts @@ -0,0 +1,69 @@ +import { expect, test } from "bun:test"; +import { createApiScheduler } from "./scheduler.ts"; + +/** Resolves after `ms`, so a task can be held open while others queue behind it. */ +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test("never runs more tasks at once than the concurrency limit", async () => { + const schedule = createApiScheduler(3, 10_000); + let active = 0; + let peak = 0; + + await Promise.all( + Array.from({ length: 20 }, () => + schedule(async () => { + active++; + peak = Math.max(peak, active); + await wait(5); + active--; + }), + ), + ); + + expect(peak).toBe(3); + expect(active).toBe(0); +}); + +test("frees a slot when a task throws, instead of deadlocking the queue", async () => { + const schedule = createApiScheduler(1, 10_000); + + await expect(schedule(() => Promise.reject(new Error("boom")))).rejects.toThrow("boom"); + + // If release() had been skipped on the failure path, this would hang. + expect(await schedule(async () => "ok")).toBe("ok"); +}); + +test("paces calls to the rate limit", async () => { + // 100 req/s -> 10ms between starts; 5 calls span at least 4 intervals. + const schedule = createApiScheduler(5, 100); + const started = performance.now(); + + await Promise.all(Array.from({ length: 5 }, () => schedule(async () => {}))); + + expect(performance.now() - started).toBeGreaterThanOrEqual(35); +}); + +test("returns each task's own resolved value", async () => { + const schedule = createApiScheduler(2, 10_000); + const results = await Promise.all([1, 2, 3].map((n) => schedule(async () => n * 2))); + expect(results).toEqual([2, 4, 6]); +}); + +test("treats zero or negative limits as one", async () => { + const schedule = createApiScheduler(0, 10_000); + let active = 0; + let peak = 0; + + await Promise.all( + Array.from({ length: 4 }, () => + schedule(async () => { + active++; + peak = Math.max(peak, active); + await wait(2); + active--; + }), + ), + ); + + expect(peak).toBe(1); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/scheduler.ts b/packages/cli-core/src/commands/migrate/lib/scheduler.ts new file mode 100644 index 000000000..3f6704be9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/scheduler.ts @@ -0,0 +1,51 @@ +/** + * Concurrency gate plus rate pacing for BAPI calls. + * + * Replaces the standalone migration-tool's `p-limit` dependency: a bounded + * queue is a few lines, and the compiled binary carries one fewer package. + * + * Both limits apply to individual API calls rather than whole users, so a user + * with ten extra email addresses cannot burst past the instance's rate limit. + */ + +/** Runs `fn` once a slot is free and the pacing interval has elapsed. */ +export type ApiScheduler = (fn: () => Promise) => Promise; + +export function createApiScheduler(concurrencyLimit: number, rateLimit: number): ApiScheduler { + const maxConcurrent = Math.max(1, Math.floor(concurrencyLimit)); + const intervalMs = Math.ceil(1000 / Math.max(1, rateLimit)); + const waiting: (() => void)[] = []; + let active = 0; + let nextRequestAt = 0; + + function acquire(): Promise { + if (active < maxConcurrent) { + active++; + return Promise.resolve(); + } + return new Promise((resolve) => waiting.push(resolve)); + } + + function release(): void { + const next = waiting.shift(); + // Hand the slot straight to the next waiter; `active` is unchanged because + // the slot never actually frees up. + if (next) next(); + else active--; + } + + return async (fn) => { + await acquire(); + try { + const now = Date.now(); + const waitMs = Math.max(0, nextRequestAt - now); + nextRequestAt = Math.max(now, nextRequestAt) + intervalMs; + if (waitMs > 0) { + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + return await fn(); + } finally { + release(); + } + }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/settings.test.ts b/packages/cli-core/src/commands/migrate/lib/settings.test.ts new file mode 100644 index 000000000..579d508df --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/settings.test.ts @@ -0,0 +1,42 @@ +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { loadSettings, saveSettings } from "./settings.ts"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-")); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(path.join(workDir, ".settings"), { force: true }); +}); + +test("returns empty settings when the file is absent", () => { + expect(loadSettings()).toEqual({}); +}); + +test("round-trips the transformer key and file path", () => { + saveSettings({ key: "clerk", file: "users.json" }); + expect(loadSettings()).toEqual({ key: "clerk", file: "users.json" }); +}); + +test("writes to the current working directory", () => { + saveSettings({ key: "clerk" }); + expect(fs.existsSync(path.join(workDir, ".settings"))).toBe(true); +}); + +test("treats a corrupt settings file as empty rather than failing the run", () => { + fs.writeFileSync(path.join(workDir, ".settings"), "{not json"); + expect(loadSettings()).toEqual({}); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/settings.ts b/packages/cli-core/src/commands/migrate/lib/settings.ts new file mode 100644 index 000000000..fc9dfcd7e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/settings.ts @@ -0,0 +1,43 @@ +/** + * The cwd-relative `.settings` file: what this directory last migrated, and + * with which transformer. + * + * Ported from the standalone migration-tool's `src/lib/settings.ts`. Kept out + * of `~/.config/clerk/config.json` on purpose — that file is keyed by linked + * project identity, not by "which export file am I working through". + * + * Both halves fail silently: a missing, unreadable or unwritable `.settings` + * only costs the user a remembered default. + */ + +import fs from "node:fs"; +import path from "node:path"; +import type { Settings } from "../types.ts"; + +const SETTINGS_FILE = ".settings"; + +function settingsPath(): string { + return path.join(process.cwd(), SETTINGS_FILE); +} + +/** Reads saved settings, or `{}` when absent or corrupt. */ +export function loadSettings(): Settings { + try { + const file = settingsPath(); + if (fs.existsSync(file)) { + return JSON.parse(fs.readFileSync(file, "utf-8")) as Settings; + } + } catch { + // Corrupt or unreadable settings are indistinguishable from none. + } + return {}; +} + +/** Persists settings for the next run in this directory. */ +export function saveSettings(settings: Settings): void { + try { + fs.writeFileSync(settingsPath(), JSON.stringify(settings, null, 2)); + } catch { + // Read-only cwd; the run itself is unaffected. + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts b/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts new file mode 100644 index 000000000..00cde92c5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, test } from "bun:test"; +import { toClerkStrategy } from "./clerk-config.ts"; +import { + countProviders, + findDisabledProviders, + findUsersWithOnlyDisabledProviders, + getUserProviders, +} from "./supabase-providers.ts"; + +/** A Supabase row carrying the given providers, in the JSON export's shape. */ +const user = (id: string, providers: string[] | string | undefined) => ({ + id, + raw_app_meta_data: + providers === undefined ? undefined : JSON.stringify({ provider: "email", providers }), +}); + +describe("getUserProviders", () => { + test("reads providers from a JSON-string column, as a CSV export writes it", () => { + expect(getUserProviders(user("u1", ["email", "discord"]))).toEqual(["email", "discord"]); + }); + + test("reads providers from an object column, as a JSON export writes it", () => { + expect(getUserProviders({ id: "u1", raw_app_meta_data: { providers: ["google"] } })).toEqual([ + "google", + ]); + }); + + test("splits a delimited providers string", () => { + expect( + getUserProviders({ id: "u1", raw_app_meta_data: { providers: "email, discord" } }), + ).toEqual(["email", "discord"]); + }); + + test.each([ + ["missing column", { id: "u1" }], + ["unparseable column", { id: "u1", raw_app_meta_data: "{not json" }], + ["array column", { id: "u1", raw_app_meta_data: "[]" }], + ["no providers key", { id: "u1", raw_app_meta_data: '{"provider":"email"}' }], + ])("returns nothing for a %s", (_label, row) => { + expect(getUserProviders(row)).toEqual([]); + }); +}); + +describe("toClerkStrategy", () => { + test.each([ + ["google", "oauth_google"], + ["discord", "oauth_discord"], + ["github", "oauth_github"], + ["azure", "oauth_microsoft"], + ["twitter", "oauth_x"], + ["slack_oidc", "oauth_slack"], + ])("%s -> %s", (provider, strategy) => { + expect(toClerkStrategy(provider)).toBe(strategy); + }); +}); + +describe("countProviders", () => { + test("counts each provider across the export", () => { + expect( + countProviders([ + user("u1", ["email"]), + user("u2", ["email", "discord"]), + user("u3", ["discord"]), + ]), + ).toEqual({ email: 2, discord: 2 }); + }); +}); + +describe("findDisabledProviders", () => { + test("names the social providers Clerk does not have enabled", () => { + const rows = [user("u1", ["email", "google"]), user("u2", ["discord"])]; + expect(findDisabledProviders(rows, ["oauth_google"], toClerkStrategy)).toEqual(["discord"]); + }); + + test("never treats email or phone as disabled", () => { + const rows = [user("u1", ["email"]), user("u2", ["phone"]), user("u3", ["anonymous_users"])]; + expect(findDisabledProviders(rows, [], toClerkStrategy)).toEqual([]); + }); + + test("returns nothing when every provider is enabled", () => { + const rows = [user("u1", ["google"]), user("u2", ["github"])]; + expect(findDisabledProviders(rows, ["oauth_google", "oauth_github"], toClerkStrategy)).toEqual( + [], + ); + }); +}); + +describe("findUsersWithOnlyDisabledProviders", () => { + test("excludes a user whose sole provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", ["discord"])], ["discord"]); + expect([...result.excludedIds]).toEqual(["u1"]); + expect(result.byProvider).toEqual({ discord: 1 }); + }); + + test("keeps a user who can still sign in with email", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["email", "discord"])], + ["discord"], + ); + expect(result.excludedIds.size).toBe(0); + }); + + test("keeps a user who has another enabled social provider", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["google", "discord"])], + ["discord"], + ); + expect(result.excludedIds.size).toBe(0); + }); + + test("excludes a user whose every provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["discord", "twitch"])], + ["discord", "twitch"], + ); + expect([...result.excludedIds]).toEqual(["u1"]); + expect(result.byProvider).toEqual({ discord: 1, twitch: 1 }); + }); + + test("keeps a user with no provider data at all", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", undefined)], ["discord"]); + expect(result.excludedIds.size).toBe(0); + }); + + test("excludes nobody when no provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", ["discord"])], []); + expect(result.excludedIds.size).toBe(0); + }); + + test("reports a per-provider breakdown across many users", () => { + const result = findUsersWithOnlyDisabledProviders( + [ + user("u1", ["discord"]), + user("u2", ["discord"]), + user("u3", ["twitch"]), + user("u4", ["email", "discord"]), + ], + ["discord", "twitch"], + ); + expect([...result.excludedIds]).toEqual(["u1", "u2", "u3"]); + expect(result.byProvider).toEqual({ discord: 2, twitch: 1 }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts b/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts new file mode 100644 index 000000000..d123dd5e0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts @@ -0,0 +1,145 @@ +/** + * Cross-references the social providers in a Supabase export against what the + * destination Clerk instance has enabled. + * + * Ported from the standalone migration-tool's `src/lib/supabase.ts`, minus its + * hand-rolled CSV parser — the export is read through the same + * `readRawUsers` path every other Supabase read uses. + */ + +import { readRawUsers } from "./transform.ts"; + +/** + * Supabase lists these alongside social providers in `providers`, but they are + * built into Clerk and can never be "not enabled". + */ +export const NON_SOCIAL_PROVIDERS = new Set(["email", "phone", "anonymous_users"]); + +function parseMaybeJson(value: unknown): unknown { + if (typeof value !== "string") return value; + try { + return JSON.parse(value); + } catch { + return value; + } +} + +/** + * Reads a user's auth providers from `raw_app_meta_data`. + * + * The column arrives as a JSON string from a CSV export and as an object from + * a JSON one, and its `providers` value is itself sometimes a string. + */ +export function getUserProviders(user: Record): string[] { + const appMeta = parseMaybeJson(user.raw_app_meta_data); + if (!appMeta || typeof appMeta !== "object" || Array.isArray(appMeta)) return []; + + const providers = parseMaybeJson((appMeta as Record).providers); + if (Array.isArray(providers)) { + return providers.map((provider) => String(provider).trim()).filter(Boolean); + } + if (typeof providers === "string") { + return providers + .split(/[,|]/) + .map((provider) => provider.trim()) + .filter(Boolean); + } + return []; +} + +export type ProviderExclusions = { + /** Source IDs of users to skip. */ + excludedIds: Set; + /** How many excluded users each disabled provider accounts for. */ + byProvider: Record; +}; + +/** + * Finds the users whose *only* way in is a provider Clerk does not have + * enabled. + * + * A user keeps their place if any one of their providers still works — + * including email and phone. Excluding on "has at least one disabled provider" + * instead would drop users who could sign in perfectly well another way. + * + * @param disabled - Supabase provider keys not enabled in Clerk. + */ +export function findUsersWithOnlyDisabledProviders( + users: Record[], + disabled: string[], +): ProviderExclusions { + const empty: ProviderExclusions = { excludedIds: new Set(), byProvider: {} }; + if (disabled.length === 0) return empty; + + const disabledSet = new Set(disabled); + const excludedIds = new Set(); + const byProvider: Record = {}; + + for (const user of users) { + const providers = getUserProviders(user); + // No provider data means no basis to exclude — err towards importing. + if (providers.length === 0) continue; + + const hasUsableProvider = providers.some( + (provider) => NON_SOCIAL_PROVIDERS.has(provider) || !disabledSet.has(provider), + ); + if (hasUsableProvider) continue; + + excludedIds.add(String(user.id)); + for (const provider of providers.filter((p) => disabledSet.has(p))) { + byProvider[provider] = (byProvider[provider] ?? 0) + 1; + } + } + + return { excludedIds, byProvider }; +} + +/** Counts users per provider across the export, for reporting. */ +export function countProviders(users: Record[]): Record { + const counts: Record = {}; + for (const user of users) { + for (const provider of getUserProviders(user)) { + counts[provider] = (counts[provider] ?? 0) + 1; + } + } + return counts; +} + +/** + * The same counts, minus Supabase's pseudo-providers. + * + * Supabase lists `email` and `phone` in `providers` next to real connections, + * but Clerk has no `oauth_email` to enable — so anything cross-referencing + * against the instance's social settings must drop them, or every + * password-based user reads as "not enabled in Clerk". + */ +export function countSocialProviders(users: Record[]): Record { + return Object.fromEntries( + Object.entries(countProviders(users)).filter( + ([provider]) => !NON_SOCIAL_PROVIDERS.has(provider), + ), + ); +} + +/** + * Every social provider present in the export that Clerk does not have + * enabled. + * + * @param enabledStrategies - Clerk strategy names (`oauth_google`, …). + * @param toStrategy - Maps a Supabase provider key to its Clerk strategy. + */ +export function findDisabledProviders( + users: Record[], + enabledStrategies: string[], + toStrategy: (provider: string) => string, +): string[] { + const enabled = new Set(enabledStrategies); + return Object.keys(countSocialProviders(users)).filter( + (provider) => !enabled.has(toStrategy(provider)), + ); +} + +/** Reads a Supabase export and returns its raw rows for provider analysis. */ +export async function readSupabaseRows(file: string): Promise[]> { + return readRawUsers(file, "supabase"); +} diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts new file mode 100644 index 000000000..efb9f1e52 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -0,0 +1,218 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import clerkTransformer from "../transformers/clerk.ts"; +import { + consolidateClerkIdentifiers, + flattenObjectSelectively, + getFileType, + loadUsersFromFile, + normalizeUserData, + transformKeys, + transformUsers, + validatePreparedUsers, +} from "./transform.ts"; + +const DATE_TIME = "2026-01-01T00-00-00"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-transform-")); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("getFileType", () => { + test.each([ + ["users.json", "application/json"], + ["users.CSV", "text/csv"], + ["users.txt", undefined], + ["users", undefined], + ])("%s -> %p", (file, expected) => { + expect(getFileType(file)).toBe(expected as never); + }); +}); + +describe("flattenObjectSelectively", () => { + test("flattens only paths the transformer references", () => { + const result = flattenObjectSelectively( + { _id: { $oid: "123" }, meta: { keep: "nested" }, email: "a@example.com" }, + { "_id.$oid": "userId", email: "email" }, + ); + expect(result).toEqual({ + "_id.$oid": "123", + meta: { keep: "nested" }, + email: "a@example.com", + }); + }); + + test("leaves arrays intact", () => { + expect(flattenObjectSelectively({ tags: [{ a: 1 }] }, { "tags.a": "x" })).toEqual({ + tags: [{ a: 1 }], + }); + }); +}); + +describe("transformKeys", () => { + test("renames mapped fields and passes unmapped ones through", () => { + expect( + transformKeys( + { id: "u1", primary_email_address: "a@example.com", extra: "kept" }, + clerkTransformer, + ), + ).toEqual({ userId: "u1", email: "a@example.com", extra: "kept" }); + }); + + test.each([ + ["empty string", ""], + ["stringified empty object", '"{}"'], + ["null", null], + ])("drops fields whose value is %s", (_label, value) => { + expect(transformKeys({ id: "u1", first_name: value }, clerkTransformer)).toEqual({ + userId: "u1", + }); + }); +}); + +describe("normalizeUserData", () => { + test.each([ + ["comma-delimited emails", { email: "a@x.dev,b@x.dev" }, { email: ["a@x.dev", "b@x.dev"] }], + ["pipe-delimited emails", { email: "a@x.dev|b@x.dev" }, { email: ["a@x.dev", "b@x.dev"] }], + ["JSON array string", { email: '["a@x.dev"]' }, { email: ["a@x.dev"] }], + ["string boolean", { banned: "true" }, { banned: true }], + ["numeric boolean", { banned: 1 }, { banned: true }], + ["numeric string limit", { createOrganizationsLimit: "5" }, { createOrganizationsLimit: 5 }], + ["JSON metadata", { publicMetadata: '{"plan":"pro"}' }, { publicMetadata: { plan: "pro" } }], + ["date string", { createdAt: "2024-01-01" }, { createdAt: "2024-01-01T00:00:00.000Z" }], + ])("normalizes %s", (_label, input, expected) => { + expect(normalizeUserData(input)).toMatchObject(expected); + }); + + test("deletes fields that normalize to nothing", () => { + const result = normalizeUserData({ email: " ", publicMetadata: "", createdAt: "" }); + expect("email" in result).toBe(false); + expect("publicMetadata" in result).toBe(false); + expect("createdAt" in result).toBe(false); + }); + + test("leaves an unparseable date as-is for the schema to reject", () => { + expect(normalizeUserData({ createdAt: "yesterday" }).createdAt).toBe("yesterday"); + }); +}); + +describe("consolidateClerkIdentifiers", () => { + test("merges primary and verified emails, deduping", () => { + const user: Record = { + email: "a@x.dev", + emailAddresses: ["a@x.dev", "b@x.dev"], + unverifiedEmailAddresses: ["b@x.dev", "c@x.dev"], + }; + consolidateClerkIdentifiers(user); + expect(user.email).toEqual(["a@x.dev", "b@x.dev"]); + expect(user.emailAddresses).toBeUndefined(); + // b@x.dev is already verified, so it must not reappear as unverified. + expect(user.unverifiedEmailAddresses).toEqual(["c@x.dev"]); + }); + + test("drops the unverified list when every entry is already verified", () => { + const user: Record = { + phone: "+15555550100", + unverifiedPhoneNumbers: ["+15555550100"], + }; + consolidateClerkIdentifiers(user); + expect(user.phone).toEqual(["+15555550100"]); + expect("unverifiedPhoneNumbers" in user).toBe(false); + }); +}); + +describe("validatePreparedUsers", () => { + test("keeps valid users and counts the rest", () => { + const result = validatePreparedUsers( + [{ userId: "u1", email: "a@x.dev" }, { userId: "u2" }, { userId: "u3", username: "carol" }], + DATE_TIME, + ); + expect(result.users.map((user) => user.userId)).toEqual(["u1", "u3"]); + expect(result.validationFailed).toBe(1); + }); + + test("aborts the whole run on an unknown password hasher", () => { + expect(() => + validatePreparedUsers( + [{ userId: "u1", email: "a@x.dev", password: "d", passwordHasher: "rot13" }], + DATE_TIME, + ), + ).toThrow(CliError); + }); +}); + +describe("transformUsers", () => { + test("maps, consolidates and validates a Clerk export", () => { + const { transformedData, validationFailed } = transformUsers( + [ + { + id: "u1", + primary_email_address: "a@x.dev", + verified_email_addresses: ["a@x.dev", "b@x.dev"], + first_name: "Alice", + }, + ], + "clerk", + DATE_TIME, + ); + expect(validationFailed).toBe(0); + expect(transformedData[0]).toMatchObject({ + userId: "u1", + email: ["a@x.dev", "b@x.dev"], + firstName: "Alice", + }); + }); + + test("skips validation when asked, so analysis passes see every row", () => { + const { transformedData, validationFailed } = transformUsers( + [{ id: "u1" }], + "clerk", + DATE_TIME, + { + validate: false, + }, + ); + expect(transformedData).toHaveLength(1); + expect(validationFailed).toBe(0); + }); +}); + +describe("loadUsersFromFile", () => { + test("reads a JSON export", async () => { + fs.writeFileSync( + path.join(workDir, "users.json"), + JSON.stringify([{ id: "u1", primary_email_address: "a@x.dev" }]), + ); + const { users } = await loadUsersFromFile("users.json", "clerk", DATE_TIME); + expect(users).toHaveLength(1); + expect(users[0]?.userId).toBe("u1"); + }); + + test("reads a CSV export, including quoted commas", async () => { + fs.writeFileSync( + path.join(workDir, "users.csv"), + 'id,primary_email_address,verified_email_addresses\nu2,a@x.dev,"a@x.dev,b@x.dev"\n', + ); + const { users } = await loadUsersFromFile("users.csv", "clerk", DATE_TIME); + expect(users[0]?.userId).toBe("u2"); + expect(users[0]?.email).toEqual(["a@x.dev", "b@x.dev"]); + }); + + test("rejects a JSON file that is not an array of users", async () => { + fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); + await expect(loadUsersFromFile("wrapped.json", "clerk", DATE_TIME)).rejects.toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts new file mode 100644 index 000000000..9f1b4fe08 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -0,0 +1,469 @@ +/** + * The load → transform → validate pipeline. + * + * Ported from the standalone migration-tool's `src/migrate/functions.ts` and + * the transform helpers in its `src/lib/index.ts`. Two dependencies were + * dropped along the way: `mime-types` (an extension check covers the two + * formats we accept) and the repo-specific `/samples/` path special-case. + */ + +import fs from "node:fs"; +import path from "node:path"; +import csvParser from "csv-parser"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import { getTransformer } from "../transformers/registry.ts"; +import { + PASSWORD_HASHERS, + type TransformContext, + type TransformerRegistryEntry, + type User, +} from "../types.ts"; +import { userSchema } from "../validator.ts"; +import { validationLogger } from "./logger.ts"; + +export type FileType = "application/json" | "text/csv"; + +export type TransformOptions = { + /** Set `false` to keep invalid rows, for analysis passes that count fields. */ + validate?: boolean; + /** Per-run values `postTransform` may need. */ + context?: TransformContext; +}; + +/** Resolves an import path against the current working directory. */ +export function resolveImportFilePath(file: string): string { + return path.resolve(process.cwd(), file.trim()); +} + +export function fileExists(file: string): boolean { + return fs.existsSync(resolveImportFilePath(file)); +} + +/** + * Classifies an import file by extension. + * + * @returns The MIME type, or `undefined` for anything that is not JSON or CSV. + */ +export function getFileType(file: string): FileType | undefined { + const ext = path.extname(resolveImportFilePath(file)).toLowerCase(); + if (ext === ".json") return "application/json"; + if (ext === ".csv") return "text/csv"; + return undefined; +} + +// --- Field mapping --------------------------------------------------------- + +/** + * Flattens only the nested paths a transformer actually references. + * + * Lets a transformer map `"_id.$oid"` onto `userId` without flattening + * (and thereby mangling) metadata objects it does not mention. + */ +export function flattenObjectSelectively( + obj: Record, + transformer: Record, + prefix = "", +): Record { + const result: Record = {}; + + for (const [key, value] of Object.entries(obj)) { + const currentPath = prefix ? `${prefix}.${key}` : key; + const hasNestedMapping = Object.keys(transformer).some((mapped) => + mapped.startsWith(`${currentPath}.`), + ); + + if (hasNestedMapping && value && typeof value === "object" && !Array.isArray(value)) { + Object.assign( + result, + flattenObjectSelectively(value as Record, transformer, currentPath), + ); + } else { + result[currentPath] = value; + } + } + + return result; +} + +/** Renames source fields onto Clerk's import schema, dropping empty values. */ +export function transformKeys( + data: Record, + transformerConfig: { transformer: Record }, +): Record { + const transformed: Record = {}; + const { transformer } = transformerConfig; + const flat = flattenObjectSelectively(data, transformer); + + for (const [key, value] of Object.entries(flat)) { + if (value !== "" && value !== '"{}"' && value !== null) { + transformed[transformer[key] ?? key] = value; + } + } + + return transformed; +} + +// --- Value normalization --------------------------------------------------- + +function parseJsonValue(value: string): unknown { + const trimmed = value.trim(); + if (!trimmed) return value; + if (!["[", "{", '"'].includes(trimmed[0] ?? "")) return value; + try { + return JSON.parse(trimmed); + } catch { + return value; + } +} + +function parseDelimitedStrings(field: unknown): string[] { + if (Array.isArray(field)) return field as string[]; + if (typeof field === "string" && field) { + const parsed = parseJsonValue(field); + if (Array.isArray(parsed)) { + return parsed.map((value) => String(value).trim()).filter(Boolean); + } + return field + .split(/[,|]/) + .map((value) => value.trim()) + .filter(Boolean); + } + return []; +} + +function normalizeStringArrayField(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => String(item).trim()).filter(Boolean); + } + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const parsed = parseJsonValue(trimmed); + if (Array.isArray(parsed)) { + return parsed.map((item) => String(item).trim()).filter(Boolean); + } + if (typeof parsed === "string") { + const parsedString = parsed.trim(); + if (parsedString.includes(",") || parsedString.includes("|")) { + return parsedString + .split(/[,|]/) + .map((item) => item.trim()) + .filter(Boolean); + } + return parsedString; + } + return parsed; +} + +function normalizeBooleanField(value: unknown): unknown { + if (typeof value === "boolean") return value; + if (typeof value === "number") { + if (value === 1) return true; + if (value === 0) return false; + return value; + } + if (typeof value !== "string") return value; + + const normalized = value.trim().toLowerCase(); + if (["true", "1", "yes", "y"].includes(normalized)) return true; + if (["false", "0", "no", "n"].includes(normalized)) return false; + return value; +} + +function normalizeNumberField(value: unknown): unknown { + if (typeof value === "number") return value; + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? parsed : value; +} + +function normalizeMetadataField(value: unknown): unknown { + if (value === undefined || value === null || value === "") return undefined; + if (typeof value !== "string") return value; + + const parsed = parseJsonValue(value); + return typeof parsed === "string" ? value : parsed; +} + +function normalizeDateField(value: unknown): unknown { + if (value instanceof Date) return value.toISOString(); + if (typeof value === "number") { + const date = new Date(value); + return Number.isNaN(date.getTime()) ? value : date.toISOString(); + } + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const date = new Date(trimmed); + return Number.isNaN(date.getTime()) ? value : date.toISOString(); +} + +const ARRAY_FIELDS = [ + "email", + "emailAddresses", + "unverifiedEmailAddresses", + "phone", + "phoneNumbers", + "unverifiedPhoneNumbers", + "backupCodes", +] as const; + +const BOOLEAN_FIELDS = [ + "backupCodesEnabled", + "banned", + "bypassClientTrust", + "createOrganizationEnabled", + "deleteSelfEnabled", + "skipLegalChecks", + "skipPasswordChecks", +] as const; + +const METADATA_FIELDS = ["unsafeMetadata", "publicMetadata", "privateMetadata"] as const; + +const DATE_FIELDS = ["createdAt", "legalAcceptedAt"] as const; + +/** + * Coerces CSV's all-strings-everything into the shapes the schema expects. + * + * A field that normalizes to `undefined` is deleted rather than set, so an + * empty CSV column does not look like an explicitly-null value to Clerk. + */ +export function normalizeUserData(user: Record): Record { + const normalized = { ...user }; + + const setOrDelete = (field: string, value: unknown) => { + if (value === undefined) delete normalized[field]; + else normalized[field] = value; + }; + + for (const field of ARRAY_FIELDS) { + setOrDelete(field, normalizeStringArrayField(normalized[field])); + } + for (const field of BOOLEAN_FIELDS) { + normalized[field] = normalizeBooleanField(normalized[field]); + } + for (const field of METADATA_FIELDS) { + setOrDelete(field, normalizeMetadataField(normalized[field])); + } + for (const field of DATE_FIELDS) { + setOrDelete(field, normalizeDateField(normalized[field])); + } + setOrDelete( + "createOrganizationsLimit", + normalizeNumberField(normalized.createOrganizationsLimit), + ); + + return normalized; +} + +/** + * Merges a Clerk export's three email fields (and three phone fields) into the + * verified/unverified pair the schema models, deduping across all of them. + */ +export function consolidateClerkIdentifiers(user: Record): void { + const merge = (primaryKey: string, verifiedKey: string, unverifiedKey: string) => { + const primary = user[primaryKey] as string | undefined; + const verified = parseDelimitedStrings(user[verifiedKey]); + const unverified = parseDelimitedStrings(user[unverifiedKey]); + + const all: string[] = []; + if (primary) all.push(primary); + for (const value of verified) { + if (!all.includes(value)) all.push(value); + } + if (all.length > 0) user[primaryKey] = all; + delete user[verifiedKey]; + + const extraUnverified = unverified.filter((value) => !all.includes(value)); + if (extraUnverified.length > 0) user[unverifiedKey] = extraUnverified; + else delete user[unverifiedKey]; + }; + + merge("email", "emailAddresses", "unverifiedEmailAddresses"); + merge("phone", "phoneNumbers", "unverifiedPhoneNumbers"); +} + +// --- Validation ------------------------------------------------------------ + +/** + * Validates prepared users, logging each failure and dropping it from the run. + * + * An unrecognized `passwordHasher` is the one failure that aborts instead: + * importing those users would store credentials nobody can ever sign in with, + * and the fix is a one-word edit to the transformer. + */ +export function validatePreparedUsers( + users: Record[], + dateTime: string, +): { users: User[]; validationFailed: number } { + const validated: User[] = []; + let validationFailed = 0; + + for (let i = 0; i < users.length; i++) { + const user = users[i] as Record; + const result = userSchema.safeParse(user); + + if (result.success) { + validated.push(result.data); + continue; + } + + validationFailed++; + const firstIssue = result.error.issues[0]; + if (!firstIssue) continue; + + if (firstIssue.path.includes("passwordHasher") && user.passwordHasher) { + const invalidHasher = + typeof user.passwordHasher === "string" + ? user.passwordHasher + : JSON.stringify(user.passwordHasher); + throw new CliError( + `Invalid password hasher "${invalidHasher}" on user ${String(user.userId)} (row ${i + 1}).\n` + + `Expected one of: ${PASSWORD_HASHERS.join(", ")}`, + { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: "https://clerk.com/docs/guides/development/migrating/overview", + }, + ); + } + + validationLogger( + { + error: firstIssue.message, + path: firstIssue.path as (string | number)[], + userId: (user.userId as string) || `row-${i}`, + row: i, + }, + dateTime, + ); + } + + return { users: validated, validationFailed }; +} + +function addDefaultFields( + users: Record[], + transformer: TransformerRegistryEntry, +): Record[] { + if (!transformer.defaults) return users; + return users.map((user) => ({ ...user, ...transformer.defaults })); +} + +/** + * Maps, normalizes and (unless disabled) validates a batch of raw users. + * + * @param options.validate - Set `false` to get the mapped shape without + * dropping invalid rows, for analysis passes that count fields. + * @param options.context - Per-run values `postTransform` may need, e.g. + * Firebase's hash parameters. + */ +export function transformUsers( + users: Record[], + key: string, + dateTime: string, + options: TransformOptions = {}, +): { transformedData: User[]; validationFailed: number } { + const transformer = getTransformer(key); + const context = options.context ?? {}; + const transformed: Record[] = []; + + for (const user of users) { + const mapped = transformKeys(user, transformer); + + if (key === "clerk") { + consolidateClerkIdentifiers(mapped); + } + transformer.postTransform?.(mapped, context); + + transformed.push(normalizeUserData(mapped)); + } + + if (options.validate === false) { + return { transformedData: transformed as User[], validationFailed: 0 }; + } + + const result = validatePreparedUsers(transformed, dateTime); + return { transformedData: result.users, validationFailed: result.validationFailed }; +} + +// --- File loading ---------------------------------------------------------- + +async function readCsv(filePath: string): Promise[]> { + return new Promise((resolve, reject) => { + const users: Record[] = []; + fs.createReadStream(filePath) + .pipe(csvParser({ skipComments: true })) + .on("data", (row: Record) => users.push(row)) + .on("error", reject) + .on("end", () => resolve(users)); + }); +} + +async function readUsersFromFile( + file: string, + transformer: TransformerRegistryEntry, +): Promise[]> { + let filePath = resolveImportFilePath(file); + const type = getFileType(file); + let preExtracted: Record[] | undefined; + + if (transformer.preTransform) { + const result = await transformer.preTransform(filePath, type ?? ""); + filePath = result.filePath; + preExtracted = result.data; + } + + if (type === "text/csv") return readCsv(filePath); + if (preExtracted) return preExtracted; + + const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + if (!Array.isArray(parsed)) { + throw new CliError(`Expected ${file} to contain a JSON array of users, got ${typeof parsed}.`, { + code: ERROR_CODE.INVALID_JSON, + }); + } + return parsed as Record[]; +} + +/** + * Reads the export exactly as the transformer sees it, before any field + * mapping. + * + * Used by the Supabase provider cross-reference, which reads + * `raw_app_meta_data` — a column no transformer maps, so it is gone by the time + * users are transformed. + */ +export async function readRawUsers(file: string, key: string): Promise[]> { + return readUsersFromFile(file, getTransformer(key)); +} + +/** + * Reads a JSON or CSV export and returns the users ready to import. + * + * @param options - Passed through to {@link transformUsers}. + */ +export async function loadUsersFromFile( + file: string, + key: string, + dateTime: string, + options: TransformOptions = {}, +): Promise<{ users: User[]; validationFailed: number }> { + const transformer = getTransformer(key); + const raw = await readUsersFromFile(file, transformer); + const withDefaults = addDefaultFields(raw, transformer); + const { transformedData, validationFailed } = transformUsers( + withDefaults, + key, + dateTime, + options, + ); + return { users: transformedData, validationFailed }; +} diff --git a/packages/cli-core/src/commands/migrate/logs/clean.ts b/packages/cli-core/src/commands/migrate/logs/clean.ts new file mode 100644 index 000000000..32d60355d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/clean.ts @@ -0,0 +1,69 @@ +/** + * `clerk migrate logs clean` — delete the local log files. + * + * Ported from the standalone migration-tool's `src/clean-logs/index.ts`. + * + * Destructive, and it sits one word away from `clerk migrate delete`, which + * destroys something entirely different (users in a Clerk instance). So the + * confirmation is not optional: interactive runs prompt, and non-interactive + * ones must say `-y` rather than being allowed to assume. + */ + +import fs from "node:fs"; +import { throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { confirm } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { listLogFiles } from "../lib/log-files.ts"; +import { getLogDir } from "../lib/logger.ts"; + +export type LogsCleanOptions = { + yes?: boolean; +}; + +export async function clean(options: LogsCleanOptions = {}): Promise { + const files = listLogFiles(); + + if (files.length === 0) { + log.info(`No migration logs to clean in ${getLogDir()}.`); + return; + } + + const label = `${files.length} log file${files.length === 1 ? "" : "s"}`; + + if (!options.yes) { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate logs clean\` deletes ${label} from ${getLogDir()} and cannot prompt here. Pass -y to confirm.`, + undefined, + undefined, + [ + { + command: "clerk migrate logs clean -y", + description: "Delete every migration log without prompting", + }, + ], + ); + } + + const proceed = await confirm({ message: `Delete ${label}?`, default: false }); + if (!proceed) throwUserAbort(); + } + + let deleted = 0; + const failures: string[] = []; + + for (const file of files) { + try { + fs.unlinkSync(file.path); + deleted++; + } catch (error) { + failures.push(`${file.name}: ${(error as Error).message}`); + } + } + + for (const failure of failures) log.warn(`Could not delete ${failure}`); + + log.success(`Deleted ${deleted} log file${deleted === 1 ? "" : "s"}.`); + if (failures.length > 0) process.exitCode = 1; +} diff --git a/packages/cli-core/src/commands/migrate/logs/convert.ts b/packages/cli-core/src/commands/migrate/logs/convert.ts new file mode 100644 index 000000000..104a236fe --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/convert.ts @@ -0,0 +1,121 @@ +/** + * `clerk migrate logs convert` — NDJSON to a JSON array. + * + * Ported from the standalone migration-tool's `src/convert-logs/index.ts`, + * with two changes: files can be named as positionals or `--all` instead of + * only through a picker, and a malformed line is reported with its line number + * rather than aborting the whole file. + */ + +import fs from "node:fs"; +import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { multiselect } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { findLogFile, listLogFiles, readNdjson, type LogFile } from "../lib/log-files.ts"; +import { getLogDir } from "../lib/logger.ts"; + +export type LogsConvertOptions = { + all?: boolean; + files?: string[]; +}; + +/** The `.json` sibling a log converts into. */ +export function outputPathFor(file: LogFile): string { + return file.path.replace(/\.log$/, ".json"); +} + +/** + * Resolves which files to convert: explicit positionals, `--all`, or a + * multiselect when a human gave neither. + */ +async function resolveTargets(options: LogsConvertOptions): Promise { + const available = listLogFiles(); + + if (available.length === 0) { + log.info(`No migration logs to convert in ${getLogDir()}.`); + return []; + } + + if (options.files && options.files.length > 0) { + return options.files.map((name) => { + const found = findLogFile(name); + if (!found) { + throw new CliError(`No log file named ${name} in ${getLogDir()}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + }); + } + return found; + }); + } + + if (options.all) return available; + + if (isAgent() || !isHuman()) { + throwUsageError( + "`clerk migrate logs convert` needs a file to convert and cannot prompt here. Name one or more log files, or pass --all.", + undefined, + undefined, + [ + { command: "clerk migrate logs convert --all", description: "Convert every log file" }, + { + command: `clerk migrate logs convert ${available[0]?.name ?? "migration-....log"}`, + description: "Convert one log file", + }, + ], + ); + } + + const chosen = await multiselect({ + message: "Which log files should be converted to JSON?", + options: available.map((file) => ({ + value: file.name, + label: file.name, + hint: `${file.entryCount} entries`, + })), + }); + if (chosen.length === 0) throwUserAbort(); + + return available.filter((file) => chosen.includes(file.name)); +} + +export async function convert(options: LogsConvertOptions = {}): Promise { + const targets = await resolveTargets(options); + if (targets.length === 0) return; + + let converted = 0; + let malformed = 0; + + for (const file of targets) { + const output = outputPathFor(file); + + try { + const { entries, errors } = readNdjson(file.path); + + // Reported per line, so a truncated final line from an interrupted run + // is visible rather than silently missing from the output. + for (const error of errors) { + malformed++; + log.warn(`${file.name}:${error.line} is not valid JSON and was skipped — ${error.message}`); + } + + fs.writeFileSync(output, JSON.stringify(entries, null, 2)); + converted++; + const count = `${entries.length} ${entries.length === 1 ? "entry" : "entries"}`; + log.info(`${file.name} → ${output.split("/").pop()} ${dim(`(${count})`)}`); + } catch (error) { + log.warn(`Could not convert ${file.name}: ${(error as Error).message}`); + process.exitCode = 1; + } + } + + if (converted > 0) { + log.success( + `Converted ${converted} log file${converted === 1 ? "" : "s"}. Originals left in place.`, + ); + } + if (malformed > 0) { + log.warn(`${malformed} malformed line${malformed === 1 ? "" : "s"} skipped.`); + } +} diff --git a/packages/cli-core/src/commands/migrate/logs/index.ts b/packages/cli-core/src/commands/migrate/logs/index.ts new file mode 100644 index 000000000..50279c8ff --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/index.ts @@ -0,0 +1,72 @@ +import { createArgument } from "@commander-js/extra-typings"; +import type { Command } from "@commander-js/extra-typings"; +import { clean } from "./clean.ts"; +import { convert } from "./convert.ts"; +import { list } from "./list.ts"; + +const logs = { clean, convert, list }; + +/** + * Registers `logs list|clean|convert` under the `migrate` group. + * + * Noun-verb, matching every other group in the CLI (`config pull`, `users + * list`) rather than the standalone tool's `clean-logs`/`convert-logs`, which + * were npm script names. Grouping also disambiguates the two deletes in this + * tree: `migrate logs clean` removes local files, `migrate delete` removes + * users from a Clerk instance. + */ +export function registerMigrateLogs(migrateCommand: Command<[], Record>): void { + const logsCommand = migrateCommand + .command("logs") + .description("Inspect, convert and clean up local migration logs") + .setExamples([ + { command: "clerk migrate logs", description: "List the local migration logs" }, + { command: "clerk migrate logs clean -y", description: "Delete every migration log" }, + { + command: "clerk migrate logs convert --all", + description: "Convert every log to a JSON array", + }, + ]); + + // Listing is read-only, so it is safe as the default for a bare + // `clerk migrate logs`. + logsCommand + .command("list", { isDefault: true }) + .description("List the log files in ./logs/") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate logs list", description: "Show type, timestamp, size and entries" }, + { command: "clerk migrate logs list --json", description: "Machine-readable listing" }, + ]) + .action((_opts, cmd) => logs.list(cmd.optsWithGlobals() as Parameters[0])); + + logsCommand + .command("clean") + .description("Delete the log files in ./logs/") + .option("-y, --yes", "Skip the confirmation prompt") + .setExamples([ + { command: "clerk migrate logs clean", description: "Delete after confirming" }, + { command: "clerk migrate logs clean -y", description: "Delete without prompting" }, + ]) + .action((_opts, cmd) => logs.clean(cmd.optsWithGlobals() as Parameters[0])); + + logsCommand + .command("convert") + .description("Convert NDJSON logs to JSON arrays for analysis") + .addArgument(createArgument("[file...]", "Log files to convert. Omit to pick interactively.")) + .option("--all", "Convert every log file") + .setExamples([ + { command: "clerk migrate logs convert --all", description: "Convert every log file" }, + { + command: "clerk migrate logs convert migration-2026-01-01T12-00-00.log", + description: "Convert one log file", + }, + { command: "clerk migrate logs convert", description: "Pick files interactively" }, + ]) + .action((files, _opts, cmd) => + logs.convert({ + ...(cmd.optsWithGlobals() as Parameters[0]), + files, + }), + ); +} diff --git a/packages/cli-core/src/commands/migrate/logs/list.ts b/packages/cli-core/src/commands/migrate/logs/list.ts new file mode 100644 index 000000000..cf30e24e0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/list.ts @@ -0,0 +1,64 @@ +/** + * `clerk migrate logs list` — what is in `./logs/`. + * + * New in the CLI: the standalone tool enumerated the directory only to build + * its own pickers. Exposing it gives a human a "what did I just do" view and + * an agent a read-only way to inspect a migration without parsing NDJSON. + */ + +import { cyan, dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { formatSize, listLogFiles, type LogFile } from "../lib/log-files.ts"; +import { getLogDir } from "../lib/logger.ts"; + +export type LogsListOptions = { + json?: boolean; +}; + +function toJson(files: LogFile[]) { + return files.map((file) => ({ + name: file.name, + kind: file.kind, + timestamp: file.timestamp, + size_bytes: file.sizeBytes, + entry_count: file.entryCount, + path: file.path, + })); +} + +export function list(options: LogsListOptions = {}): void { + const files = listLogFiles(); + + if (options.json) { + log.data(JSON.stringify(toJson(files), null, 2)); + return; + } + + if (files.length === 0) { + log.info(`No migration logs in ${getLogDir()}.`); + return; + } + + const kindWidth = Math.max(...files.map((file) => file.kind.length), "TYPE".length) + 2; + const timeWidth = Math.max(...files.map((file) => file.timestamp.length), "TIMESTAMP".length) + 2; + const sizeWidth = Math.max(...files.map((file) => formatSize(file.sizeBytes).length), 4) + 2; + + log.info( + dim("TYPE".padEnd(kindWidth)) + + dim("TIMESTAMP".padEnd(timeWidth)) + + dim("SIZE".padEnd(sizeWidth)) + + dim("ENTRIES"), + ); + + for (const file of files) { + log.info( + cyan(file.kind.padEnd(kindWidth)) + + (file.timestamp || dim("—")).padEnd(timeWidth) + + dim(formatSize(file.sizeBytes).padEnd(sizeWidth)) + + String(file.entryCount), + ); + } + + log.info(""); + log.info(dim(`${files.length} log file${files.length === 1 ? "" : "s"} in ${getLogDir()}`)); +} diff --git a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts new file mode 100644 index 000000000..36bd10747 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts @@ -0,0 +1,177 @@ +/** + * The prompting half of `logs clean` and `logs convert`. + * + * Kept separate because `mock.module` registrations are process-lifetime, and + * `bun test --parallel` puts several files in each worker — so a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. Human mode itself needs no mock: `setMode` is the + * supported override. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +type ConfirmPrompt = { message: string; default?: boolean }; +type MultiselectPrompt = { + message: string; + options: { value: string; label: string; hint?: string }[]; +}; + +const mockConfirm = mock(async (_config: ConfirmPrompt) => true); +const mockMultiselect = mock(async (_config: MultiselectPrompt) => [] as string[]); + +mock.module("../../../lib/prompts.ts", () => ({ + confirm: (config: ConfirmPrompt) => mockConfirm(config), + multiselect: (config: MultiselectPrompt) => mockMultiselect(config), + text: async () => "", + password: async () => "", + editor: async () => "{}", +})); + +const { clean } = await import("./clean.ts"); +const { convert } = await import("./convert.ts"); +const { UserAbortError } = await import("../../../lib/errors.ts"); +const { getLogDir } = await import("../lib/logger.ts"); + +let originalMode: Mode; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; + +const MIGRATION = "migration-2026-01-01T12-00-00.log"; +const DELETION = "user-deletion-2026-02-01T12-00-00.log"; + +beforeAll(() => { + originalMode = getMode(); + setMode("human"); + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logs-int-"))); + process.chdir(workDir); +}); + +afterAll(() => { + setMode(originalMode); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + mockConfirm.mockReset(); + mockMultiselect.mockReset(); + mockConfirm.mockResolvedValue(true); + mockMultiselect.mockResolvedValue([]); + fs.rmSync(getLogDir(), { recursive: true, force: true }); + process.exitCode = 0; +}); + +function writeLog(name: string, entries: unknown[]): void { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync( + path.join(getLogDir(), name), + entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n", + ); +} + +describe("logs clean", () => { + test("prompts before deleting anything", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ a: 1 }]); + + await clean(); + + expect(mockConfirm).toHaveBeenCalledTimes(1); + expect(mockConfirm.mock.calls[0]?.[0]?.message).toContain("2 log files"); + expect(fs.readdirSync(getLogDir())).toEqual([]); + }); + + // Deleting on a stray enter would be the wrong default for a destructive + // command sitting next to `clerk migrate delete`. + test("defaults the prompt to no", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await clean(); + + expect(mockConfirm.mock.calls[0]?.[0]?.default).toBe(false); + }); + + test("declining leaves every file in place", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + mockConfirm.mockResolvedValue(false); + + await expect(clean()).rejects.toThrow(UserAbortError); + + expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + }); + + test("-y skips the prompt entirely", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await clean({ yes: true }); + + expect(mockConfirm).not.toHaveBeenCalled(); + expect(fs.readdirSync(getLogDir())).toEqual([]); + }); + + test("does not prompt when there is nothing to delete", async () => { + await clean(); + expect(mockConfirm).not.toHaveBeenCalled(); + }); +}); + +describe("logs convert", () => { + test("offers a multiselect when given neither files nor --all", async () => { + writeLog(MIGRATION, [{ a: 1 }, { b: 2 }]); + writeLog(DELETION, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([MIGRATION]); + + await convert(); + + const options = mockMultiselect.mock.calls[0]?.[0]?.options; + expect(options?.map((option) => option.value)).toEqual([DELETION, MIGRATION]); + expect(options?.[1]?.hint).toBe("2 entries"); + }); + + test("converts only what was selected", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([MIGRATION]); + + await convert(); + + expect(fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json"))).toEqual([ + "migration-2026-01-01T12-00-00.json", + ]); + }); + + test("selecting nothing aborts without writing", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([]); + + await expect(convert()).rejects.toThrow(UserAbortError); + + expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + }); + + test("does not prompt when --all was passed", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await convert({ all: true }); + + expect(mockMultiselect).not.toHaveBeenCalled(); + expect(captured.err).toContain("Converted 1 log file"); + }); + + test("does not prompt when files were named", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await convert({ files: [MIGRATION] }); + + expect(mockMultiselect).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/logs/logs.test.ts b/packages/cli-core/src/commands/migrate/logs/logs.test.ts new file mode 100644 index 000000000..5be2d232b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/logs.test.ts @@ -0,0 +1,219 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { clean } from "./clean.ts"; +import { convert } from "./convert.ts"; +import { list } from "./list.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logs-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); + process.exitCode = 0; +}); + +function writeLog(name: string, entries: unknown[]): void { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync( + path.join(getLogDir(), name), + entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n", + ); +} + +const MIGRATION = "migration-2026-01-01T12-00-00.log"; +const DELETION = "user-deletion-2026-02-01T12-00-00.log"; + +describe("logs list", () => { + test("says so plainly when there is no logs directory", () => { + list(); + expect(captured.err).toContain("No migration logs in"); + }); + + test("says so plainly when the directory is empty", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + list(); + expect(captured.err).toContain("No migration logs in"); + }); + + test("reports type, timestamp, size and entry count", () => { + writeLog(MIGRATION, [{ userId: "u1" }, { userId: "u2" }, { userId: "u3" }]); + + list(); + + expect(captured.err).toContain("TYPE"); + expect(captured.err).toContain("TIMESTAMP"); + expect(captured.err).toContain("SIZE"); + expect(captured.err).toContain("ENTRIES"); + expect(captured.err).toContain("migration"); + expect(captured.err).toContain("2026-01-01T12-00-00"); + expect(captured.err).toMatch(/\bB\b/); + expect(captured.err).toContain("3"); + }); + + test("lists every log kind", () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ a: 1 }]); + + list(); + + expect(captured.err).toContain("migration"); + expect(captured.err).toContain("deletion"); + expect(captured.err).toContain("2 log files"); + }); + + test("--json emits a machine-readable listing on stdout", () => { + writeLog(MIGRATION, [{ userId: "u1" }]); + + list({ json: true }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed).toHaveLength(1); + expect(parsed[0]).toMatchObject({ + name: MIGRATION, + kind: "migration", + timestamp: "2026-01-01T12-00-00", + entry_count: 1, + }); + }); + + test("--json emits an empty array rather than prose when there are no logs", () => { + list({ json: true }); + expect(JSON.parse(captured.out)).toEqual([]); + }); +}); + +describe("logs clean", () => { + test("says so plainly when there is nothing to clean", async () => { + await clean({ yes: true }); + expect(captured.err).toContain("No migration logs to clean"); + }); + + // Tests run non-TTY, which is the same signal an agent gives. + test("refuses without -y when it cannot prompt, and explains", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await expect(clean()).rejects.toThrow(/cannot prompt here.*Pass -y/s); + expect(fs.existsSync(path.join(getLogDir(), MIGRATION))).toBe(true); + }); + + test("names how many files are at stake when it refuses", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ a: 1 }]); + + await expect(clean()).rejects.toThrow(/2 log files/); + }); + + test("-y deletes the log files and reports the count", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ a: 1 }]); + + await clean({ yes: true }); + + expect(fs.readdirSync(getLogDir())).toEqual([]); + expect(captured.err).toContain("Deleted 2 log files"); + }); + + test("leaves converted JSON output alone", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + fs.writeFileSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"), "[]"); + + await clean({ yes: true }); + + expect(fs.readdirSync(getLogDir())).toEqual(["migration-2026-01-01T12-00-00.json"]); + }); +}); + +describe("logs convert", () => { + test("says so plainly when there is nothing to convert", async () => { + await convert({ all: true }); + expect(captured.err).toContain("No migration logs to convert"); + }); + + test("writes a JSON array alongside the original, leaving it intact", async () => { + writeLog(MIGRATION, [{ userId: "u1" }, { userId: "u2" }]); + + await convert({ files: [MIGRATION] }); + + const output = path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"); + expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([ + { userId: "u1" }, + { userId: "u2" }, + ]); + expect(fs.existsSync(path.join(getLogDir(), MIGRATION))).toBe(true); + expect(captured.err).toContain("Originals left in place"); + }); + + test("--all converts every log file", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ b: 2 }]); + + await convert({ all: true }); + + const written = fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json")); + expect(written.sort()).toEqual([ + "migration-2026-01-01T12-00-00.json", + "user-deletion-2026-02-01T12-00-00.json", + ]); + }); + + test("accepts a path and resolves it against ./logs/", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await convert({ files: [`./logs/${MIGRATION}`] }); + + expect(fs.existsSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"))).toBe(true); + }); + + test("fails clearly on a file that is not there", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await expect(convert({ files: ["migration-nope.log"] })).rejects.toThrow(CliError); + }); + + // Silently dropping the line would leave a JSON array that looks complete. + test("reports a malformed line by number and converts the rest", async () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync(path.join(getLogDir(), MIGRATION), '{"a":1}\n{"b":\n{"c":3}\n'); + + await convert({ files: [MIGRATION] }); + + expect(captured.err).toContain(`${MIGRATION}:2`); + expect(captured.err).toContain("1 malformed line skipped"); + + const output = path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"); + expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([{ a: 1 }, { c: 3 }]); + }); + + test("refuses without a target when it cannot prompt, naming the alternatives", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await expect(convert()).rejects.toThrow(/cannot prompt here/); + expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + }); + + test("reports the entry count per converted file", async () => { + writeLog(MIGRATION, [{ a: 1 }, { b: 2 }, { c: 3 }]); + + await convert({ all: true }); + + expect(captured.err).toContain("3 entries"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts new file mode 100644 index 000000000..ef5a05f85 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -0,0 +1,124 @@ +/** + * Keeps README.md and the command tree honest about each other. + * + * This README documents six export platforms, six transformers and three log + * subcommands across ~600 lines. Checking it by eye at review time does not + * scale, and a doc that names a flag the binary rejects is worse than no doc: + * the reader trusts it and gets a usage error. + * + * Both directions are checked — every example must resolve, and every flag must + * be written down — so neither renaming a flag nor adding one passes silently. + */ + +import { describe, expect, test } from "bun:test"; +import type { Command } from "commander"; +import { createProgram } from "../../cli-program.ts"; + +const README = await Bun.file(new URL("./README.md", import.meta.url)).text(); + +/** Fenced blocks only, so prose that merely mentions a flag is not parsed. */ +function fencedBlocks(markdown: string): string[] { + return [...markdown.matchAll(/^```[a-z]*\n([\s\S]*?)^```/gm)].map((match) => match[1] ?? ""); +} + +/** + * Every `clerk migrate …` invocation the README puts in front of a reader, from + * fenced blocks and inline backticks alike — both get copied. + */ +function documentedCommands(markdown: string): string[] { + const found = new Set(); + + for (const block of fencedBlocks(markdown)) { + // Line continuations first: the Firebase example spans three lines. + for (const line of block.replace(/\\\n\s*/g, " ").split("\n")) { + const start = line.indexOf("clerk migrate"); + // A command never contains a backtick or a `#`; the sample error output + // that quotes `clerk migrate` mid-sentence does. + if (start !== -1) found.add(line.slice(start).split(/[`#]/)[0]!.trim()); + } + } + + for (const match of markdown.matchAll(/`(clerk migrate[^`]*)`/g)) { + found.add(match[1]!.trim()); + } + + return [...found]; +} + +/** Walks as deep as the tree allows; the first flag or positional stops it. */ +function resolve(tokens: string[]): { command: Command; rest: string[] } { + let command = createProgram() as Command; + let index = 0; + for (; index < tokens.length; index++) { + const child = command.commands.find( + (candidate) => + candidate.name() === tokens[index] || candidate.aliases().includes(tokens[index]!), + ); + if (!child) break; + command = child; + } + return { command, rest: tokens.slice(index) }; +} + +function flagsOf(command: Command): string[] { + return command.options.flatMap( + (option) => [option.short, option.long].filter(Boolean) as string[], + ); +} + +/** Every command under `migrate`, so no subcommand escapes the flag sweep. */ +function migrateTree(): { path: string; command: Command }[] { + const collected: { path: string; command: Command }[] = []; + const visit = (command: Command, path: string) => { + collected.push({ path, command }); + for (const child of command.commands) { + if (child.name() !== "help") visit(child, `${path} ${child.name()}`); + } + }; + visit(resolve(["migrate"]).command, "migrate"); + return collected; +} + +const EXAMPLES = documentedCommands(README); + +/** One case per (example, flag) pair, so a failure names the exact flag. */ +const FLAG_USES: [string, string][] = EXAMPLES.flatMap((example) => + example + .split(/\s+/) + .filter((token) => token.startsWith("-")) + .map((token) => [example, token.split("=")[0]!] as [string, string]), +); + +describe("migrate README", () => { + // Guards the extractor: a regex that silently matched nothing would make + // every check below pass vacuously. + test("finds the documented examples", () => { + expect(EXAMPLES.length).toBeGreaterThan(20); + expect(FLAG_USES.length).toBeGreaterThan(20); + }); + + test.each(EXAMPLES)("`%s` resolves to a real command", (example) => { + const tokens = example.split(/\s+/).slice(1); + const { command, rest } = resolve(tokens); + const firstFlag = rest.findIndex((token) => token.startsWith("-")); + const positionals = firstFlag === -1 ? rest : rest.slice(0, firstFlag); + // Leftover words before any flag are positionals — only some commands take + // them, and a subcommand that does not exist lands here too. + if (positionals.length > 0) expect(command.registeredArguments.length).toBeGreaterThan(0); + // A parent means at least `migrate` resolved. Checking the name instead + // would be wrong: `migrate export clerk` is itself named `clerk`. + expect(command.parent).not.toBeNull(); + }); + + test.each(FLAG_USES)("`%s` uses %s, which the command accepts", (example, flag) => { + const { command } = resolve(example.split(/\s+/).slice(1)); + expect(flagsOf(command)).toContain(flag); + }); + + test.each(migrateTree())("$path documents every flag it accepts", ({ command }) => { + const undocumented = flagsOf(command).filter( + (flag) => flag.startsWith("--") && flag !== "--help" && !README.includes(flag), + ); + expect(undocumented).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts new file mode 100644 index 000000000..ef518506c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -0,0 +1,301 @@ +/** + * The human-mode half of `migrate run`: the wizard fills in missing flags, the + * readiness report renders, and declining the confirmation writes nothing. + * + * Kept in its own file because `mock.module` registrations are process-lifetime, + * and `bun test --parallel` puts several files in each worker — so a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. Human mode itself needs no mock: `setMode` is the + * supported override. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { getMode, setMode, type Mode } from "../../mode.ts"; +import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +const mockSelect = mock(async () => "clerk" as unknown); +const mockText = mock(async () => "export.json" as unknown); +let confirmAnswer = true; +let originalMode: Mode; + +mock.module("../../lib/listage.ts", () => ({ + ...listageStubs, + select: (...args: unknown[]) => mockSelect(...(args as [])), +})); + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../lib/prompts.ts", () => ({ + confirm: async () => confirmAnswer, + multiselect: async () => [], + text: (...args: unknown[]) => mockText(...(args as [])), + password: async () => "", + editor: async () => "{}", +})); + +const { run } = await import("./run.ts"); +const { deleteMigration } = await import("./delete.ts"); +const { UserAbortError } = await import("../../lib/errors.ts"); +const { loadSettings, saveSettings } = await import("./lib/settings.ts"); + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { method: string; url: string; body: unknown }[]; + +const EXPORT = [ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", primary_email_address: "b@x.dev" }, +]; + +const baseOptions = { transformer: "clerk", file: "export.json", secretKey: "sk_test_x" }; + +beforeAll(() => { + originalMode = getMode(); + setMode("human"); + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); + process.chdir(workDir); +}); + +afterAll(() => { + setMode(originalMode); + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + confirmAnswer = true; + mockSelect.mockReset(); + mockText.mockReset(); + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("export.json"); + fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); + stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); +}); + +afterEach(() => { + process.exitCode = 0; +}); + +/** Stubs BAPI plus the FAPI environment lookup the readiness report needs. */ +function stubInstanceSettings(settings: { attributes?: object; social?: object } | null) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + if (url.endsWith("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/v1/domains")) { + if (!settings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.includes("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/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.includes("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/v1/environment")) return Response.json({ user_settings: settings }); + return Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; +} + +const created = () => requests.filter((r) => r.url.endsWith("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/v1/users")); + +describe("the wizard fills in missing flags", () => { + test("bare `clerk migrate` prompts for the transformer and file, then imports", async () => { + await run({ secretKey: "sk_test_x" }); + + expect(mockSelect).toHaveBeenCalledTimes(1); + expect(mockText).toHaveBeenCalledTimes(1); + expect(created()).toHaveLength(2); + }); + + test("asks only for what the flags did not supply", async () => { + await run({ ...baseOptions, transformer: "clerk" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(mockText).not.toHaveBeenCalled(); + }); + + test("prompts for the file when only the transformer was passed", async () => { + await run({ transformer: "clerk", secretKey: "sk_test_x" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(mockText).toHaveBeenCalledTimes(1); + }); + + test("records the wizard's answers for the next run", async () => { + await run({ secretKey: "sk_test_x" }); + + expect(loadSettings()).toMatchObject({ key: "clerk", file: "export.json" }); + }); +}); + +describe("the readiness report", () => { + test("renders before the confirmation", async () => { + await run(baseOptions); + expect(captured.err).toContain("Migration readiness"); + expect(captured.err).toContain("2 users ready to import"); + }); + + // The whole point of the report: seeing what will go wrong, then backing out + // before a single user exists in the destination instance. + test("declining afterwards writes nothing to Clerk", async () => { + confirmAnswer = false; + + await expect(run(baseOptions)).rejects.toThrow(UserAbortError); + + expect(captured.err).toContain("Migration readiness"); + expect(created()).toHaveLength(0); + }); + + test("accepting proceeds with the import", async () => { + confirmAnswer = true; + + await run(baseOptions); + + expect(created()).toHaveLength(2); + }); + + test("flags a field Clerk requires that not every user has", async () => { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", username: "bob" }, + ]), + ); + + await run(baseOptions); + + expect(captured.err).toContain("1 user lacks it"); + expect(captured.err).toContain("1 setting needs attention"); + }); + + test("degrades to a note when the instance settings cannot be read", async () => { + stubInstanceSettings(null); + + await run(baseOptions); + + expect(captured.err).toContain("Could not read this instance's settings"); + expect(created()).toHaveLength(2); + }); + + test("is skipped for a -y run, which pays for no extra round-trips", async () => { + await run({ ...baseOptions, yes: true }); + + expect(requests.some((r) => r.url.endsWith("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/v1/domains"))).toBe(false); + expect(captured.err).not.toContain("Migration readiness"); + expect(created()).toHaveLength(2); + }); +}); + +describe("guards that still apply interactively", () => { + test("the dev-instance 500-user cap", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify( + Array.from({ length: 501 }, (_, i) => ({ + id: `u${i}`, + primary_email_address: `u${i}@x.dev`, + })), + ), + ); + + await expect(run(baseOptions)).rejects.toThrow(/development instance/); + expect(created()).toHaveLength(0); + }); + + test("an unrecognized password hasher aborts before any request", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + await expect(run(baseOptions)).rejects.toThrow(/Invalid password hasher/); + expect(created()).toHaveLength(0); + }); +}); + +describe("migrate delete confirmation", () => { + /** Answers the external-id lookup, then the deletes. */ + function stubDeleteTargets(present: Record) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ method: init?.method ?? "GET", url, body: null }); + + if (url.includes("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/v1/users?")) { + const asked = new URL(url).searchParams.getAll("external_id"); + return Response.json( + asked + .filter((externalId) => externalId in present) + .map((externalId) => ({ id: present[externalId], external_id: externalId })), + ); + } + return Response.json({ deleted: true }); + }) as unknown as typeof fetch; + } + + const deleted = () => requests.filter((r) => r.method === "DELETE"); + + beforeEach(() => { + saveSettings({ key: "clerk", file: "export.json" }); + stubDeleteTargets({ legacy_a: "user_1", legacy_b: "user_2" }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "legacy_a", primary_email_address: "a@x.dev" }, + { id: "legacy_b", primary_email_address: "b@x.dev" }, + ]), + ); + }); + + test("reports the count and confirms before deleting", async () => { + confirmAnswer = true; + + await deleteMigration({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("About to delete 2 users"); + expect(deleted()).toHaveLength(2); + }); + + // The undo for a bad undo does not exist, so declining must cost nothing. + test("declining deletes nobody", async () => { + confirmAnswer = false; + + await expect(deleteMigration({ secretKey: "sk_test_x" })).rejects.toThrow(UserAbortError); + + expect(deleted()).toHaveLength(0); + }); + + test("-y skips the prompt", async () => { + confirmAnswer = false; + + await deleteMigration({ yes: true, secretKey: "sk_test_x" }); + + expect(deleted()).toHaveLength(2); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts new file mode 100644 index 000000000..e5408e422 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -0,0 +1,749 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../lib/errors.ts"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { getLogDir } from "./lib/logger.ts"; +import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; +import { loadSettings } from "./lib/settings.ts"; +import { applyResumeAfter, resolveFirebaseHashConfig, run, validateRunOptions } from "./run.ts"; +import type { FirebaseHashConfig, User } from "./types.ts"; + +let workDir: string; +let originalCwd: string; + +const users = (...ids: string[]): User[] => ids.map((userId) => ({ userId }) as User); + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-"))); + process.chdir(workDir); + fs.writeFileSync(path.join(workDir, "users.json"), "[]"); + fs.writeFileSync(path.join(workDir, "users.txt"), ""); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("validateRunOptions", () => { + test("accepts a transformer and an existing JSON file", () => { + expect(validateRunOptions({ transformer: "clerk", file: "users.json" })).toEqual({ + transformer: "clerk", + file: "users.json", + }); + }); + + test.each([ + ["no transformer", { file: "users.json" }, /--transformer/], + ["an unknown transformer", { transformer: "okta", file: "users.json" }, /Unknown transformer/], + ["no file", { transformer: "clerk" }, /--file/], + ["a missing file", { transformer: "clerk", file: "nope.json" }, /File not found/], + [ + "an unsupported extension", + { transformer: "clerk", file: "users.txt" }, + /Unsupported file type/, + ], + ])("rejects %s", (_label, options, message) => { + expect(() => validateRunOptions(options)).toThrow(message); + }); + + test("names the valid transformers when one is missing", () => { + expect(() => validateRunOptions({ file: "users.json" })).toThrow(/clerk/); + }); +}); + +describe("resolveFirebaseHashConfig", () => { + const ALL = { + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }; + + test("builds the config when all four flags are present", () => { + expect(resolveFirebaseHashConfig(ALL)).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + // A digest built from a partial set is well-formed but verifies against + // nothing, so every migrated user would silently fail to sign in. + test.each([ + ["firebaseSignerKey", "--firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost"], + ] as const)("rejects a set missing %s, naming the flag", (omit, flag) => { + const partial = { ...ALL }; + delete (partial as Record)[omit]; + expect(() => resolveFirebaseHashConfig(partial)).toThrow(new RegExp(flag)); + }); + + test("names every missing flag at once", () => { + expect(() => resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" })).toThrow( + /--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/, + ); + }); + + test("falls back to saved settings when no flag is passed", () => { + const saved: FirebaseHashConfig = { + base64_signer_key: "S", + base64_salt_separator: "B", + rounds: 8, + mem_cost: 14, + }; + expect(resolveFirebaseHashConfig({}, saved)).toEqual(saved); + }); + + test("prefers flags over saved settings", () => { + const saved: FirebaseHashConfig = { + base64_signer_key: "OLD", + base64_salt_separator: "B", + rounds: 1, + mem_cost: 1, + }; + expect(resolveFirebaseHashConfig(ALL, saved)?.base64_signer_key).toBe("SIGNER"); + }); + + test("returns nothing when neither flags nor settings supply a config", () => { + expect(resolveFirebaseHashConfig({})).toBeUndefined(); + }); +}); + +describe("applyResumeAfter", () => { + test("returns everything when no ID is given", () => { + expect(applyResumeAfter(users("a", "b"), undefined)).toHaveLength(2); + }); + + test("skips up to and including the named user", () => { + expect(applyResumeAfter(users("a", "b", "c"), "b").map((u) => u.userId)).toEqual(["c"]); + }); + + test("returns nothing when the named user is last", () => { + expect(applyResumeAfter(users("a", "b"), "b")).toEqual([]); + }); + + test("throws rather than silently re-importing everyone", () => { + expect(() => applyResumeAfter(users("a"), "zz")).toThrow(CliError); + }); +}); + +describe("run", () => { + const captured = useCaptureLog(); + let originalFetch: typeof globalThis.fetch; + let requests: { method: string; url: string; body: unknown }[]; + + const export2 = [ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d1", + password_hasher: "bcrypt", + }, + { id: "u2", primary_email_address: "b@x.dev" }, + ]; + + beforeAll(() => { + originalFetch = globalThis.fetch; + }); + + beforeEach(() => { + requests = []; + delete process.env.CLERK_MIGRATE_RATE_LIMIT; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ + method: init?.method ?? "GET", + url: input.toString(), + body: init?.body ? JSON.parse(init.body as string) : null, + }); + return new Response(JSON.stringify({ id: "user_created" }), { status: 200 }); + }) as typeof fetch; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + process.exitCode = 0; + }); + + const baseOptions = { + transformer: "clerk", + file: "export.json", + yes: true, + secretKey: "sk_test_x", + }; + + test("imports every user in the file end to end", async () => { + await run(baseOptions); + + const created = requests.filter((r) => r.url.endsWith("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/v1/users")); + expect(created).toHaveLength(2); + expect(created[0]?.method).toBe("POST"); + expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "u1", + "u2", + ]); + expect(captured.err).toContain("Imported:"); + }); + + test("writes a timestamped NDJSON log for the run", async () => { + await run(baseOptions); + + const logs = fs.readdirSync(getLogDir()); + expect(logs).toHaveLength(1); + expect(logs[0]).toMatch(/^migration-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + + const entries = fs + .readFileSync(path.join(getLogDir(), logs[0] as string), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(entries.filter((e) => e.status === "success")).toHaveLength(2); + }); + + test("records the run's key and file in .settings", async () => { + await run(baseOptions); + expect(loadSettings()).toEqual({ key: "clerk", file: "export.json" }); + }); + + test("--require-password imports only the users that have one", async () => { + await run({ ...baseOptions, requirePassword: true }); + + const created = requests.filter((r) => r.url.endsWith("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/v1/users")); + expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u1"]); + expect(captured.err).toContain("skipping 1 user(s) without a password"); + }); + + test("--resume-after skips everyone up to and including that ID", async () => { + await run({ ...baseOptions, resumeAfter: "u1" }); + + const created = requests.filter((r) => r.url.endsWith("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/v1/users")); + expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u2"]); + }); + + test("logs validation failures and imports the rest", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify([...export2, { id: "u3" }])); + + await run(baseOptions); + + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + expect(captured.err).toContain("1 user(s) failed validation"); + }); + + test("warns that --clerk-secret-key is deprecated but still honours it", async () => { + await run({ ...baseOptions, secretKey: undefined, clerkSecretKey: "sk_test_x" }); + + expect(captured.err).toContain("--clerk-secret-key is deprecated"); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + }); + + test("refuses to exceed the development-instance user limit", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify( + Array.from({ length: 501 }, (_, i) => ({ + id: `u${i}`, + primary_email_address: `u${i}@x.dev`, + })), + ), + ); + + await expect(run(baseOptions)).rejects.toThrow(/development instance/); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(0); + }); + + test("aborts before any API call when the hasher is unrecognized", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + await expect(run(baseOptions)).rejects.toThrow(/Invalid password hasher/); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(0); + }); + + test("exits non-zero when some users failed", async () => { + globalThis.fetch = (async () => + new Response(JSON.stringify({ errors: [{ code: "e", message: "taken" }] }), { + status: 422, + })) as unknown as typeof fetch; + + await run(baseOptions); + expect(process.exitCode).toBe(1); + }); + + // Tests run non-TTY, so `isHuman()` is false and the wizard path is never + // reached — the same guard an agent hits. + describe("without --transformer or --file", () => { + test.each([ + [{}, /--transformer and --file /], + [{ transformer: "clerk" }, /--file /], + [{ file: "export.json" }, /--transformer /], + ])("names the missing flags rather than prompting (%p)", async (partial, expected) => { + await expect(run({ ...partial, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + expected, + ); + expect(requests).toHaveLength(0); + }); + + test("explains that it cannot prompt", async () => { + await expect(run({ yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /cannot prompt in agent mode/, + ); + }); + }); + + describe("readiness report", () => { + /** Stubs BAPI plus the FAPI environment lookup the report depends on. */ + function stubInstanceSettings( + settings: { attributes?: object; social?: object } | null, + onUsers?: () => Response, + ) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + if (url.endsWith("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/v1/domains")) { + if (!settings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.includes("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/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.includes("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/v1/environment")) return Response.json({ user_settings: settings }); + return onUsers ? onUsers() : Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; + } + + const created = () => requests.filter((r) => r.url.endsWith("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/v1/users")); + + // `-y` means nobody is watching, so the two extra round-trips buy nothing. + test("is skipped for a -y run", async () => { + stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); + + await run(baseOptions); + + expect(requests.some((r) => r.url.endsWith("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/v1/domains"))).toBe(false); + expect(captured.err).not.toContain("Migration readiness"); + expect(created()).toHaveLength(2); + }); + + test("renders before any user is created, and flags a required-but-missing field", async () => { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }); + // One user has no email, so a required email address will cost them. + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", username: "bob" }, + ]), + ); + + await run({ ...baseOptions, yes: false }); + + expect(captured.err).toContain("Migration readiness"); + expect(captured.err).toContain("1 user lacks it"); + + // The report was printed before the first POST /v1/users. + const reportIndex = requests.findIndex((r) => r.url.includes("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/v1/environment")); + const firstCreate = requests.findIndex((r) => r.url.endsWith("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/v1/users")); + expect(reportIndex).toBeGreaterThanOrEqual(0); + expect(reportIndex).toBeLessThan(firstCreate); + }); + + test("degrades to a note when the instance settings cannot be read", async () => { + stubInstanceSettings(null); + + await run({ ...baseOptions, yes: false }); + + expect(captured.err).toContain("Could not read this instance's settings"); + expect(created()).toHaveLength(2); + }); + + test("cross-references supabase providers against the instance", async () => { + stubInstanceSettings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["discord"]}', + }, + ]), + ); + + await run({ ...baseOptions, transformer: "supabase", yes: false }); + + expect(captured.err).toContain("Social connections"); + expect(captured.err).toContain("Discord"); + expect(captured.err).toContain("not enabled in Clerk"); + }); + + // Supabase lists `email` and `phone` in `providers` alongside real social + // connections, and Clerk has no `oauth_email` to enable — so counting them + // as social flagged every password user as a blocking problem. + test("leaves supabase's email and phone pseudo-providers out of the social section", async () => { + stubInstanceSettings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email","discord"]}', + }, + ]), + ); + + await run({ ...baseOptions, transformer: "supabase", yes: false }); + + const social = captured.err.slice(captured.err.indexOf("Social connections")); + expect(social).toContain("Discord"); + expect(social).not.toContain("Email"); + expect(social).not.toContain("Phone"); + }); + }); + + describe("--transformer-file", () => { + const CUSTOM = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email", given: "firstName", pw: "password" }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { if (!user.firstName) delete user.firstName; }, + };`; + + let customFile: string; + let customCounter = 0; + + beforeEach(() => { + // A fresh filename each time: dynamic import() caches by URL, so reusing + // one would silently return a previous test's module. + customFile = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, customFile), CUSTOM); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { account_ref: "mp_1", contact_email: "a@x.dev", given: "Ada", pw: "$2b$10$hash" }, + { account_ref: "mp_2", contact_email: "b@x.dev", given: "", pw: "$2b$10$hash" }, + ]), + ); + }); + + afterEach(() => { + __resetCustomTransformersForTesting(); + }); + + const created = () => requests.filter((r) => r.url.endsWith("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/v1/users")); + + test("imports through a user-authored transformer", async () => { + await run({ + file: "export.json", + transformerFile: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_1", + "mp_2", + ]); + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("transformer from"); + }); + + test("applies the custom transformer's defaults and postTransform", async () => { + await run({ + file: "export.json", + transformerFile: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ first_name: "Ada", password_hasher: "bcrypt" }); + // postTransform dropped the empty given name rather than sending "". + expect("first_name" in (bodies[1] ?? {})).toBe(false); + }); + + // No sensible precedence between "the one you wrote" and "the one we ship". + test("conflicts with --transformer rather than picking one", async () => { + await expect( + run({ + transformer: "clerk", + file: "export.json", + transformerFile: customFile, + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/both name a transformer. Pass one or the other/); + expect(created()).toHaveLength(0); + }); + + test("fails before any request when the file is not there", async () => { + await expect( + run({ + file: "export.json", + transformerFile: "./nope.ts", + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/No transformer file at/); + expect(requests).toHaveLength(0); + }); + + test("fails before any request when the file is malformed", async () => { + const bad = `./bad-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, bad), + `export default { key: "x", label: "X", transformer: {} };`, + ); + + await expect( + run({ file: "export.json", transformerFile: bad, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/no source field maps to `userId`/); + expect(requests).toHaveLength(0); + }); + + test("still requires --file", async () => { + await expect( + run({ transformerFile: customFile, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/--file/); + }); + }); + + describe("per-platform imports", () => { + /** One realistic record per platform, in that platform's export shape. */ + const PLATFORMS: [string, unknown, string][] = [ + [ + "auth0", + [ + { + user_id: "auth0|1", + email: "a@x.dev", + email_verified: true, + given_name: "Ada", + family_name: "L", + }, + ], + "auth0|1", + ], + ["authjs", [{ id: "aj1", email: "a@x.dev", email_verified: "2024-01-01T00:00:00Z" }], "aj1"], + [ + "betterauth", + [{ user_id: "ba1", email: "a@x.dev", email_verified: true, password_hash: "$2a$10$h" }], + "ba1", + ], + [ + "supabase", + [ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-06-29 20:25:06+00", + encrypted_password: "$2b$10$h", + }, + ], + "sb1", + ], + ]; + + test.each(PLATFORMS)( + "%s transforms, validates and imports its export", + async (key, records, externalId) => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(records)); + + await run({ ...baseOptions, transformer: key }); + + const created = requests.filter((r) => r.url.endsWith("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/v1/users")); + expect(created).toHaveLength(1); + expect((created[0]?.body as { external_id: string } | undefined)?.external_id).toBe( + externalId, + ); + }, + ); + + test("firebase imports its wrapped export and builds the scrypt digest", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify({ + users: [ + { + localId: "fb1", + email: "a@x.dev", + emailVerified: true, + passwordHash: "SGFzaA==", + salt: "U2FsdA==", + }, + ], + }), + ); + + await run({ + ...baseOptions, + transformer: "firebase", + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }); + + const body = requests.find((r) => r.url.endsWith("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/v1/users"))?.body as Record< + string, + unknown + >; + expect(body).toMatchObject({ + external_id: "fb1", + password_digest: "SGFzaA==$U2FsdA==$SIGNER$Bw==$8$14", + password_hasher: "scrypt_firebase", + }); + }); + + test("a partial firebase flag set fails before anything is read", async () => { + await expect( + run({ ...baseOptions, transformer: "firebase", firebaseSignerKey: "SIGNER" }), + ).rejects.toThrow(/--firebase-salt-separator/); + expect(requests).toHaveLength(0); + }); + + test("an unknown transformer fails listing the valid keys", async () => { + await expect(run({ ...baseOptions, transformer: "okta" })).rejects.toThrow( + /Unknown transformer "okta".*clerk.*supabase/s, + ); + }); + }); + + describe("--skip-unsupported-providers", () => { + const supabaseExport = [ + { + id: "sb_email", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email"]}', + }, + { + id: "sb_discord", + email: "b@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["discord"]}', + }, + { + id: "sb_both", + email: "c@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email","discord"]}', + }, + ]; + + /** Stubs BAPI plus the FAPI environment lookup the check depends on. */ + function stubInstance(enabledSocial: Record | null) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + if (url.endsWith("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/v1/domains")) { + if (!enabledSocial) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.includes("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/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.includes("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/v1/environment")) { + return Response.json({ user_settings: { social: enabledSocial } }); + } + return Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; + } + + const created = () => + requests + .filter((r) => r.url.endsWith("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/v1/users")) + .map((r) => (r.body as { external_id: string }).external_id); + + beforeEach(() => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(supabaseExport)); + }); + + test("skips only the user whose sole provider is disabled", async () => { + stubInstance({ oauth_google: { enabled: true }, oauth_discord: { enabled: false } }); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(created()).toEqual(["sb_email", "sb_both"]); + expect(captured.err).toContain("skipping 1 user(s)"); + expect(captured.err).toContain("discord: 1"); + }); + + test("imports everyone when the provider is enabled", async () => { + stubInstance({ oauth_discord: { enabled: true } }); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(created()).toHaveLength(3); + }); + + // A failed lookup must not be read as "nothing is enabled" — that would + // silently drop every social user. + test("imports everyone when the instance config cannot be read", async () => { + stubInstance(null); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(created()).toHaveLength(3); + expect(captured.err).toContain("Could not read the instance's enabled providers"); + }); + + test("is a no-op with a warning on a non-supabase transformer", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); + + await run({ ...baseOptions, skipUnsupportedProviders: true }); + + expect(created()).toHaveLength(2); + expect(captured.err).toContain("only applies to supabase"); + }); + + test("records the flag in .settings", async () => { + stubInstance({ oauth_discord: { enabled: true } }); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(loadSettings().skipUnsupportedProviders).toBe(true); + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts new file mode 100644 index 000000000..0d518fb3c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -0,0 +1,519 @@ +/** + * `clerk migrate run` — non-interactive user import. + * + * Ported from the standalone migration-tool's `src/migrate/cli.ts` + * (`runNonInteractive`), with auth moved onto the CLI's standard secret-key + * resolution chain and every failure raised as a `CliError` instead of + * `console.error` + `process.exit`. + * + * The interactive wizard that a bare `clerk migrate` will launch is a separate + * command; this path is the one an agent or a script drives. + */ + +import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; +import { bold, dim, green, red, yellow } from "../../lib/color.ts"; +import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; +import { log } from "../../lib/log.ts"; +import { confirm } from "../../lib/prompts.ts"; +import { withGutter, withSpinner } from "../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../mode.ts"; +import { importUsers } from "./import-users.ts"; +import { analyzeFields } from "./lib/analysis.ts"; +import { + enabledSocialProviders, + fetchInstanceSettings, + toClerkStrategy, +} from "./lib/clerk-config.ts"; +import { buildReadinessReport, formatReadinessReport } from "./lib/readiness.ts"; +import { DEV_USER_LIMIT, resolveLimits } from "./lib/instance.ts"; +import { getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; +import { loadSettings, saveSettings } from "./lib/settings.ts"; +import { + countSocialProviders, + findDisabledProviders, + findUsersWithOnlyDisabledProviders, + readSupabaseRows, +} from "./lib/supabase-providers.ts"; +import { fileExists, getFileType, loadUsersFromFile } from "./lib/transform.ts"; +import { loadCustomTransformer } from "./transformers/load-custom.ts"; +import { registerCustomTransformer, transformerKeys } from "./transformers/registry.ts"; +import type { FirebaseHashConfig, ImportSummary, User } from "./types.ts"; +import { runWizard, throwAgentFlagsRequired } from "./wizard.ts"; + +export type MigrateRunOptions = { + transformer?: string; + file?: string; + resumeAfter?: string; + requirePassword?: boolean; + yes?: boolean; + secretKey?: string; + /** Deprecated alias for `--secret-key`, kept for existing prompts and docs. */ + clerkSecretKey?: string; + app?: string; + instance?: string; + /** Path to a user-authored transformer, for a platform with no built-in. */ + transformerFile?: string; + /** Supabase: drop users whose only social provider is disabled in Clerk. */ + skipUnsupportedProviders?: boolean; + firebaseSignerKey?: string; + firebaseSaltSeparator?: string; + firebaseRounds?: number; + firebaseMemCost?: number; +}; + +const FIREBASE_FLAGS = [ + ["firebaseSignerKey", "--firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost"], +] as const; + +/** + * Resolves Firebase's four hash parameters from flags, falling back to + * `.settings` when none were passed. + * + * The four are required as a set: a digest built from a partial set is + * well-formed but verifies against nothing, so every migrated user would fail + * to sign in with no error at import time. + * + * @returns The config, or `undefined` when none was supplied — which is fine + * for an export that carries no password hashes. + */ +export function resolveFirebaseHashConfig( + options: MigrateRunOptions, + saved?: FirebaseHashConfig, +): FirebaseHashConfig | undefined { + const provided = FIREBASE_FLAGS.filter(([key]) => options[key] !== undefined); + + if (provided.length === 0) return saved; + + if (provided.length < FIREBASE_FLAGS.length) { + const missing = FIREBASE_FLAGS.filter(([key]) => options[key] === undefined).map( + ([, flag]) => flag, + ); + throwUsageError( + `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + + "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + "https://clerk.com/docs/guides/development/migrating/firebase", + ); + } + + return { + base64_signer_key: options.firebaseSignerKey as string, + base64_salt_separator: options.firebaseSaltSeparator as string, + rounds: options.firebaseRounds as number, + mem_cost: options.firebaseMemCost as number, + }; +} + +/** + * Validates the flags a run needs before anything is read or sent. + * + * @returns The transformer key and file path, both guaranteed present. + */ +export function validateRunOptions(options: MigrateRunOptions): { + transformer: string; + file: string; +} { + const valid = transformerKeys(); + + // A custom transformer has already been loaded and registered by the time + // this runs, so its key is resolvable even though it is not in `valid`. + if (options.transformerFile) { + if (!options.file) { + throwUsageError( + "Missing required option --file (path to a JSON or CSV export).", + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: + "clerk migrate run -y --transformer-file ./my-transformer.ts --file users.json", + description: "Import with a custom transformer", + }, + ], + ); + } + if (!fileExists(options.file)) { + throw new CliError(`File not found: ${options.file}`, { code: ERROR_CODE.FILE_NOT_FOUND }); + } + if (!getFileType(options.file)) { + throwUsageError(`Unsupported file type for ${options.file}. Provide a .json or .csv file.`); + } + return { transformer: options.transformer as string, file: options.file }; + } + + if (!options.transformer) { + throwUsageError( + `Missing required option --transformer. Valid values: ${valid.join(", ")}.`, + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: "clerk migrate run -y --transformer clerk --file users.json", + description: "Import a Clerk export", + }, + ], + ); + } + if (!valid.includes(options.transformer)) { + throwUsageError( + `Unknown transformer "${options.transformer}". Valid values: ${valid.join(", ")}.`, + ); + } + if (!options.file) { + throwUsageError( + "Missing required option --file (path to a JSON or CSV export).", + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: "clerk migrate run -y --transformer clerk --file users.json", + description: "Import a Clerk export", + }, + ], + ); + } + if (!fileExists(options.file)) { + throw new CliError(`File not found: ${options.file}`, { code: ERROR_CODE.FILE_NOT_FOUND }); + } + if (!getFileType(options.file)) { + throwUsageError(`Unsupported file type for ${options.file}. Provide a .json or .csv file.`); + } + + return { transformer: options.transformer, file: options.file }; +} + +/** + * Drops every user up to and including `resumeAfter`. + * + * @throws CliError when the ID is not in the file — silently importing the + * whole set would duplicate everything the previous run already created. + */ +export function applyResumeAfter(users: User[], resumeAfter: string | undefined): User[] { + if (!resumeAfter) return users; + + const index = users.findIndex((user) => user.userId === resumeAfter); + if (index === -1) { + throw new CliError(`Could not find user ID "${resumeAfter}" in the import file.`, { + code: ERROR_CODE.USAGE_ERROR, + }); + } + return users.slice(index + 1); +} + +function formatSummary(summary: ImportSummary, logFile: string): string { + const inFile = summary.totalProcessed + summary.validationFailed; + const lines = [ + `${bold("Total users in file:")} ${inFile}`, + `${green("Imported:")} ${summary.successful}`, + `${red("Failed:")} ${summary.failed}`, + ]; + + if (summary.validationFailed > 0) { + lines.push(`${yellow("Failed validation:")} ${summary.validationFailed}`); + } + if (summary.errorBreakdown.size > 0) { + lines.push("", bold("Error breakdown:")); + for (const [error, count] of summary.errorBreakdown) { + lines.push(` ${count} user${count === 1 ? "" : "s"}: ${error}`); + } + } + lines.push("", dim(`Log: ${logFile}`)); + + return lines.join("\n"); +} + +/** + * Drops users whose only way into Clerk is a social provider the destination + * instance has not enabled. + * + * Only meaningful for Supabase exports — it is the one platform whose export + * records per-user providers. If the instance's configuration cannot be read, + * nobody is dropped: a failed lookup must not be mistaken for "no providers + * are enabled". + */ +async function skipDisabledProviderUsers( + users: User[], + file: string, + transformer: string, + secretKey: string, +): Promise { + if (transformer !== "supabase") { + log.warn(`--skip-unsupported-providers only applies to supabase exports; ignoring.`); + return users; + } + + const settings = await withSpinner("Checking enabled providers", () => + fetchInstanceSettings(secretKey), + ); + const enabled = settings ? enabledSocialProviders(settings) : null; + if (!enabled) { + log.warn( + "Could not read the instance's enabled providers; importing every user. Re-run with --verbose for details.", + ); + return users; + } + + const rows = await readSupabaseRows(file); + const disabled = findDisabledProviders(rows, enabled, toClerkStrategy); + if (disabled.length === 0) { + log.info("Every provider in this export is enabled in Clerk; no users skipped."); + return users; + } + + const { excludedIds, byProvider } = findUsersWithOnlyDisabledProviders(rows, disabled); + if (excludedIds.size === 0) { + log.info( + `${disabled.join(", ")} not enabled in Clerk, but every user has another way to sign in; none skipped.`, + ); + return users; + } + + const breakdown = Object.entries(byProvider) + .map(([provider, count]) => `${provider}: ${count}`) + .join(", "); + log.warn( + `--skip-unsupported-providers: skipping ${excludedIds.size} user(s) whose only provider is not enabled in Clerk (${breakdown}).`, + ); + + return users.filter((user) => !excludedIds.has(user.userId)); +} + +/** + * Prints the Migration Readiness report: what the file contains, cross- + * referenced against what the destination instance accepts. + * + * Rendered immediately before the confirmation prompt, so declining that + * prompt aborts with nothing written to Clerk. + * + * Skipped only for `-y`, which says "don't ask, don't lecture" and should not + * pay for two extra network round-trips. Agent mode still gets it: an agent + * driving a migration can act on "this field is required and 40 users lack it" + * exactly as a human would. + */ +async function showReadinessReport(input: { + users: User[]; + file: string; + transformer: string; + secretKey: string; + validationFailed: number; + skipReport: boolean; +}): Promise { + if (input.skipReport) return; + + const settings = await withSpinner("Checking instance settings", () => + fetchInstanceSettings(input.secretKey), + ); + + // Only Supabase exports record per-user providers, so only they can be + // cross-referenced against the instance's social connections. + let providerCounts: Record | undefined; + if (input.transformer === "supabase") { + try { + providerCounts = countSocialProviders(await readSupabaseRows(input.file)); + } catch (error) { + log.debug(`migrate: could not read providers for the readiness report: ${String(error)}`); + } + } + + const report = buildReadinessReport({ + analysis: analyzeFields(input.users), + settings, + validationFailed: input.validationFailed, + providerCounts, + }); + + log.blank(); + for (const line of formatReadinessReport(report)) log.info(line); + log.blank(); +} + +/** + * Fills in a missing `--transformer`/`--file` interactively, or explains what + * to pass. + * + * Agent mode is the CLI's existing non-interactive signal, so an agent that + * runs bare `clerk migrate` gets a usage error naming the flags rather than a + * prompt it cannot answer. + */ +async function resolveMissingOptions(options: MigrateRunOptions): Promise { + const missing = { transformer: !options.transformer, file: !options.file }; + if (!missing.transformer && !missing.file) return options; + + if (isAgent() || !isHuman()) { + throwAgentFlagsRequired(missing); + } + + // A partial Firebase flag set is a usage error whether or not the wizard is + // filling in the rest, so it is checked before any prompt. + const firebaseHashConfig = resolveFirebaseHashConfig(options); + const answers = await runWizard({ + transformer: options.transformer, + file: options.file, + firebaseHashConfig, + }); + + return { + ...options, + transformer: answers.transformer, + file: answers.file, + ...(answers.firebaseHashConfig + ? { + firebaseSignerKey: answers.firebaseHashConfig.base64_signer_key, + firebaseSaltSeparator: answers.firebaseHashConfig.base64_salt_separator, + firebaseRounds: answers.firebaseHashConfig.rounds, + firebaseMemCost: answers.firebaseHashConfig.mem_cost, + } + : {}), + }; +} + +/** + * Loads and registers a `--transformer-file`, so the rest of the run treats it + * exactly like a built-in. + * + * @returns The options with `transformer` set to the loaded entry's key. + */ +async function applyCustomTransformer(options: MigrateRunOptions): Promise { + if (!options.transformerFile) return options; + + // Both name a transformer, and there is no sensible precedence between "the + // one you wrote" and "the one we ship" — say so rather than picking. + if (options.transformer) { + throwUsageError( + "--transformer and --transformer-file both name a transformer. Pass one or the other.", + undefined, + undefined, + [ + { + command: "clerk migrate run -y --transformer-file ./my-transformer.ts --file users.json", + description: "Use a transformer you wrote", + }, + { + command: "clerk migrate run -y --transformer clerk --file users.json", + description: "Use a built-in transformer", + }, + ], + ); + } + + const custom = await loadCustomTransformer(options.transformerFile); + registerCustomTransformer(custom); + log.info(`Loaded the \`${custom.key}\` transformer from ${options.transformerFile}.`); + + return { ...options, transformer: custom.key }; +} + +export async function run(rawOptions: MigrateRunOptions): Promise { + if (rawOptions.clerkSecretKey) { + log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); + } + + rawOptions = await applyCustomTransformer(rawOptions); + const options = await resolveMissingOptions(rawOptions); + const secretKeyOption = options.secretKey ?? options.clerkSecretKey; + + const { transformer, file } = validateRunOptions(options); + const saved = loadSettings(); + const firebaseHashConfig = resolveFirebaseHashConfig(options, saved.firebaseHashConfig); + + await withGutter("Migrating users to Clerk", async () => { + const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const limits = resolveLimits(secretKey); + const dateTime = getDateTimeStamp(); + const logFile = getLogFilePath("migration", dateTime); + + const { users: loaded, validationFailed } = await withSpinner( + `Loading users from ${file}`, + () => loadUsersFromFile(file, transformer, dateTime, { context: { firebaseHashConfig } }), + "Users loaded", + ); + + let users = applyResumeAfter(loaded, options.resumeAfter); + if (options.resumeAfter) { + log.info(`Resuming after ${options.resumeAfter} (${loaded.length - users.length} skipped).`); + } + + if (options.skipUnsupportedProviders) { + users = await skipDisabledProviderUsers(users, file, transformer, secretKey); + } + + if (options.requirePassword) { + const withPassword = users.filter((user) => Boolean(user.password)); + const dropped = users.length - withPassword.length; + if (dropped > 0) { + log.info(`--require-password: skipping ${dropped} user(s) without a password.`); + } + users = withPassword; + } + + if (validationFailed > 0) { + log.warn( + `${validationFailed} user(s) failed validation and will be skipped. See ${logFile}.`, + ); + } + + if (users.length === 0) { + log.warn("No users left to import."); + return; + } + + if (limits.instanceType === "dev" && users.length > DEV_USER_LIMIT) { + throw new CliError( + `Cannot import ${users.length} users into a development instance — the limit is ${DEV_USER_LIMIT}.\n` + + "Target a production instance, or reduce the import file.", + { code: ERROR_CODE.USAGE_ERROR }, + ); + } + + log.info( + `Importing ${users.length} user(s) via the ${transformer} transformer into ` + + `${target ?? "the resolved instance"} (${limits.instanceType}).`, + ); + + await showReadinessReport({ + users, + file, + transformer, + secretKey, + validationFailed, + skipReport: Boolean(options.yes), + }); + + if (!options.yes && isHuman() && !isAgent()) { + const proceed = await confirm({ + message: `Import ${users.length} user(s)?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + saveSettings({ + key: transformer, + file, + ...(options.skipUnsupportedProviders ? { skipUnsupportedProviders: true } : {}), + ...(firebaseHashConfig ? { firebaseHashConfig } : {}), + }); + + const summary = await withSpinner( + `Importing users: [0/${users.length}]`, + (spinner) => + importUsers({ + users, + secretKey, + limits, + dateTime, + skipPasswordRequirement: !options.requirePassword, + validationFailed, + spinner, + }), + "Import complete", + ); + + log.raw(formatSummary(summary, logFile)); + + if (summary.failed > 0) process.exitCode = 1; + }); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/auth0.ts b/packages/cli-core/src/commands/migrate/transformers/auth0.ts new file mode 100644 index 000000000..c1595d139 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/auth0.ts @@ -0,0 +1,43 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification } from "./shared.ts"; + +/** + * Auth0 → Clerk transformer. + * + * Works with Auth0's Export Users API. `user_id` is a `provider|id` string + * (`auth0|abc123`, `github|12345`) and is carried through as the Clerk user's + * `external_id`. + * + * Auth0 does not include password hashes in a standard export — they have to + * be requested from Auth0 support. When present they are bcrypt (`$2a$`/`$2b$`, + * 10 rounds), which is why `passwordHasher` defaults to `bcrypt`. + */ +const auth0Transformer = { + key: "auth0", + label: "Auth0", + description: + "Works with Auth0's Export Users API. Password hashes require a support request to Auth0.", + transformer: { + user_id: "userId", + email: "email", + email_verified: "emailVerified", + username: "username", + given_name: "firstName", + family_name: "lastName", + phone_number: "phone", + phone_verified: "phoneVerified", + passwordHash: "password", + user_metadata: "publicMetadata", + app_metadata: "privateMetadata", + created_at: "createdAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "boolean"); + routeByVerification(user, "phone", "phoneVerified", "boolean"); + }, + defaults: { + passwordHasher: "bcrypt" as const, + }, +} satisfies TransformerRegistryEntry; + +export default auth0Transformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/authjs.ts b/packages/cli-core/src/commands/migrate/transformers/authjs.ts new file mode 100644 index 000000000..3391b1804 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/authjs.ts @@ -0,0 +1,38 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, splitName } from "./shared.ts"; + +/** + * Auth.js (formerly NextAuth) → Clerk transformer. + * + * Auth.js has no export tool and no fixed user table, so this assumes the + * common shape: `SELECT id, name, email, email_verified, created_at FROM users`. + * A different schema means editing the mapping below or supplying a custom + * transformer file. + * + * `email_verified` is a nullable timestamp rather than a boolean — any value + * means verified. + * + * No password default: Auth.js's core is passwordless (OAuth and email links), + * so users arrive without a digest and are imported with + * `skip_password_requirement`. + */ +const authjsTransformer = { + key: "authjs", + label: "Auth.js (NextAuth)", + description: + "Assumes an export of `SELECT id, name, email, email_verified, created_at FROM users`. `name` is split into firstName and lastName.", + transformer: { + id: "userId", + email: "email", + email_verified: "emailVerified", + name: "name", + created_at: "createdAt", + updated_at: "updatedAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "timestamp"); + splitName(user); + }, +} satisfies TransformerRegistryEntry; + +export default authjsTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/betterauth.ts b/packages/cli-core/src/commands/migrate/transformers/betterauth.ts new file mode 100644 index 000000000..f32d2f31c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/betterauth.ts @@ -0,0 +1,46 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, splitName } from "./shared.ts"; + +/** + * Better Auth → Clerk transformer. + * + * Works with `clerk migrate export betterauth`, which joins the user table + * with the credential account row to pick up the bcrypt `password_hash`. + * + * Better Auth plugins add columns Clerk has no equivalent for + * (`display_username`, `role`, `ban_reason`, `two_factor_enabled`). They need + * no handling: the schema strips anything it does not declare. `banned` is the + * exception, because that one *is* a Clerk field. + */ +const betterAuthTransformer = { + key: "betterauth", + label: "Better Auth", + description: + "Works with the Better Auth export. Supports bcrypt passwords and the admin plugin's banned flag.", + transformer: { + user_id: "userId", + email: "email", + email_verified: "emailVerified", + name: "name", + password_hash: "password", + username: "username", + phone_number: "phone", + phone_number_verified: "phoneVerified", + created_at: "createdAt", + updated_at: "updatedAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "boolean"); + routeByVerification(user, "phone", "phoneVerified", "boolean"); + splitName(user); + + // Only carry `banned` when it is actually true — Better Auth writes false + // for every user that was never banned, and sending that to Clerk is noise. + if (user.banned !== true) delete user.banned; + }, + defaults: { + passwordHasher: "bcrypt" as const, + }, +} satisfies TransformerRegistryEntry; + +export default betterAuthTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/clerk.ts b/packages/cli-core/src/commands/migrate/transformers/clerk.ts new file mode 100644 index 000000000..8fb094839 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/clerk.ts @@ -0,0 +1,45 @@ +import type { TransformerRegistryEntry } from "../types.ts"; + +/** + * Clerk → Clerk transformer, for moving users between Clerk instances + * (typically development → production). + * + * Maps the Dashboard's user export format onto the import schema. + */ +const clerkTransformer = { + key: "clerk", + label: "Clerk", + description: + "Migrate between Clerk instances (e.g. development to production, or to another Clerk application). Export your users from the Clerk Dashboard first.", + transformer: { + id: "userId", + primary_email_address: "email", + verified_email_addresses: "emailAddresses", + unverified_email_addresses: "unverifiedEmailAddresses", + first_name: "firstName", + last_name: "lastName", + password_digest: "password", + password_hasher: "passwordHasher", + primary_phone_number: "phone", + verified_phone_numbers: "phoneNumbers", + unverified_phone_numbers: "unverifiedPhoneNumbers", + username: "username", + totp_secret: "totpSecret", + backup_codes_enabled: "backupCodesEnabled", + backup_codes: "backupCodes", + public_metadata: "publicMetadata", + unsafe_metadata: "unsafeMetadata", + private_metadata: "privateMetadata", + // Account state a Dashboard export carries and `POST /v1/users` accepts. + // Unmapped, these survive the export and are then silently stripped at + // validation — losing original signup dates on a dev → prod migration. + created_at: "createdAt", + legal_accepted_at: "legalAcceptedAt", + banned: "banned", + create_organization_enabled: "createOrganizationEnabled", + create_organizations_limit: "createOrganizationsLimit", + delete_self_enabled: "deleteSelfEnabled", + }, +} satisfies TransformerRegistryEntry; + +export default clerkTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/firebase.ts b/packages/cli-core/src/commands/migrate/transformers/firebase.ts new file mode 100644 index 000000000..73966c1b2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/firebase.ts @@ -0,0 +1,121 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import type { PreTransformResult, TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, splitName, toIsoDate } from "./shared.ts"; + +/** + * Column order of `firebase auth:export --format=csv`, which writes no header + * row. Without these the CSV parser would treat the first user as the header. + */ +const FIREBASE_CSV_HEADERS = + "localId,email,emailVerified,passwordHash,passwordSalt,displayName,photoUrl," + + "googleId,googleEmail,googleDisplayName,googlePhotoUrl," + + "facebookId,facebookEmail,facebookDisplayName,facebookPhotoUrl," + + "twitterId,twitterEmail,twitterDisplayName,twitterPhotoUrl," + + "githubId,githubEmail,githubDisplayName,githubPhotoUrl," + + "createdAt,lastSignedInAt,phoneNumber,disabled,customAttributes,providerUserInfo"; + +/** + * Firebase → Clerk transformer. + * + * Handles both shapes `firebase auth:export` produces: a headerless CSV, and + * JSON wrapped in `{ users: [...] }`. + * + * Firebase's scrypt is a modified variant, so Clerk needs the project's four + * hash parameters alongside each digest. They arrive on the run's + * {@link TransformContext} from `--firebase-*` flags or saved `.settings`. + * + * See https://clerk.com/docs/guides/development/migrating/firebase + */ +const firebaseTransformer = { + key: "firebase", + label: "Firebase", + description: + "Works with `firebase auth:export` (CSV or JSON). Requires the project's four password hash parameters to migrate passwords.", + + preTransform: (filePath: string, fileType: string): PreTransformResult => { + if (fileType === "text/csv") { + // Written to the OS temp dir rather than the user's cwd: this is a + // parsing artifact, not a migration output like ./logs. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-firebase-")); + const withHeaders = path.join(tmpDir, path.basename(filePath)); + fs.writeFileSync( + withHeaders, + `${FIREBASE_CSV_HEADERS}\n${fs.readFileSync(filePath, "utf-8")}`, + ); + return { filePath: withHeaders }; + } + + if (fileType === "application/json") { + const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + if (Array.isArray(parsed)) return { filePath, data: parsed as Record[] }; + + const users = (parsed as { users?: unknown })?.users; + if (Array.isArray(users)) return { filePath, data: users as Record[] }; + + throw new CliError( + "Invalid Firebase JSON export: expected `{ users: [...] }` or an array of users.", + { code: ERROR_CODE.INVALID_JSON }, + ); + } + + return { filePath }; + }, + + transformer: { + localId: "userId", + email: "email", + emailVerified: "emailVerified", + passwordHash: "passwordHash", + passwordSalt: "salt", + phoneNumber: "phone", + displayName: "name", + }, + + postTransform: (user, context) => { + const passwordHash = user.passwordHash; + const salt = user.salt; + + if (passwordHash && salt) { + const config = context.firebaseHashConfig; + if (!config) { + throw new CliError( + "This export contains Firebase password hashes, which need the project's hash parameters to import.\n" + + "Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters, then pass:\n" + + " --firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost", + { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: "https://clerk.com/docs/guides/development/migrating/firebase", + }, + ); + } + + // Clerk's scrypt_firebase hasher expects every parameter inline: + // hash$salt$signerKey$saltSeparator$rounds$memCost + user.password = [ + passwordHash, + salt, + config.base64_signer_key, + config.base64_salt_separator, + config.rounds, + config.mem_cost, + ].join("$"); + + delete user.passwordHash; + delete user.salt; + } + + routeByVerification(user, "email", "emailVerified", "boolean"); + // Firebase exports timestamps as Unix milliseconds, often as strings. + user.createdAt = toIsoDate(user.createdAt, true); + splitName(user); + }, + + defaults: { + passwordHasher: "scrypt_firebase" as const, + }, +} satisfies TransformerRegistryEntry; + +export default firebaseTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/list.test.ts b/packages/cli-core/src/commands/migrate/transformers/list.test.ts new file mode 100644 index 000000000..9f432e1d9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/list.test.ts @@ -0,0 +1,116 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { list } from "./list.ts"; +import { transformers } from "./registry.ts"; + +const captured = useCaptureLog(); + +// eslint-disable-next-line no-control-regex +const stripAnsi = (value: string) => value.replace(/\[[0-9;]*m/g, ""); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-tlist-"))); + process.chdir(workDir); + fs.writeFileSync( + path.join(workDir, "custom.ts"), + `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId" }, + };`, + ); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("human output", () => { + test.each([...transformers])("lists the $key transformer with its label", async (transformer) => { + await list(); + expect(captured.err).toContain(transformer.key); + expect(captured.err).toContain(transformer.label); + }); + + // `log.info` auto-highlights backticked spans, so the rendered description + // carries colour codes the source string does not. + test.each([...transformers])("includes the $key description", async (transformer) => { + await list(); + expect(stripAnsi(captured.err)).toContain(transformer.description); + }); + + test("counts the built-ins", async () => { + await list(); + expect(captured.err).toContain(`${transformers.length} built-in transformers`); + }); + + // A compiled binary has no source tree to grep, so the way to extend it has + // to be discoverable from the list itself. + test("says how to add one when none is loaded", async () => { + await list(); + expect(captured.err).toContain("--transformer-file"); + }); + + test("appends a custom transformer and names its source", async () => { + await list({ transformerFile: "./custom.ts" }); + + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("custom — ./custom.ts"); + expect(captured.err).toContain("plus 1 loaded from --transformer-file"); + }); + + test("drops the how-to hint once one is loaded", async () => { + await list({ transformerFile: "./custom.ts" }); + expect(captured.err).not.toContain("Migrating from something else?"); + }); +}); + +describe("--json", () => { + test("emits every built-in on stdout", async () => { + await list({ json: true }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed).toHaveLength(transformers.length); + expect(parsed.map((entry) => entry.key)).toEqual(transformers.map((entry) => entry.key)); + }); + + test("reports key, label, description and the userId source field", async () => { + await list({ json: true }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed[0]).toMatchObject({ + key: "clerk", + label: "Clerk", + built_in: true, + maps_to_user_id: "id", + }); + }); + + test("marks a custom transformer as not built in", async () => { + await list({ json: true, transformerFile: "./custom.ts" }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed.at(-1)).toMatchObject({ + key: "myplatform", + built_in: false, + source: "./custom.ts", + maps_to_user_id: "account_ref", + }); + }); +}); + +describe("a bad --transformer-file", () => { + test("fails rather than listing only the built-ins", async () => { + await expect(list({ transformerFile: "./nope.ts" })).rejects.toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/list.ts b/packages/cli-core/src/commands/migrate/transformers/list.ts new file mode 100644 index 000000000..349a53a8f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/list.ts @@ -0,0 +1,67 @@ +/** + * `clerk migrate transformers list` — which source platforms are available. + * + * New in the CLI. The standalone tool's interactive picker was the only place + * these were listed, which was fine when the user had the source tree to grep. + * A compiled binary's users have neither, so the list is a command. + */ + +import { bold, cyan, dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import type { TransformerRegistryEntry } from "../types.ts"; +import { loadCustomTransformer } from "./load-custom.ts"; +import { transformers } from "./registry.ts"; + +export type TransformersListOptions = { + json?: boolean; + transformerFile?: string; +}; + +type Listed = TransformerRegistryEntry & { builtIn: boolean; source?: string }; + +function toJson(entries: Listed[]) { + return entries.map((entry) => ({ + key: entry.key, + label: entry.label, + description: entry.description, + built_in: entry.builtIn, + ...(entry.source ? { source: entry.source } : {}), + maps_to_user_id: + Object.entries(entry.transformer).find(([, target]) => target === "userId")?.[0] ?? null, + })); +} + +export async function list(options: TransformersListOptions = {}): Promise { + const entries: Listed[] = transformers.map((entry) => ({ ...entry, builtIn: true })); + + if (options.transformerFile) { + const custom = await loadCustomTransformer(options.transformerFile); + entries.push({ ...custom, builtIn: false, source: options.transformerFile }); + } + + if (options.json) { + log.data(JSON.stringify(toJson(entries), null, 2)); + return; + } + + for (const entry of entries) { + const suffix = entry.builtIn ? "" : ` ${dim(`(custom — ${entry.source})`)}`; + log.info(`${cyan(bold(entry.key))} ${entry.label}${suffix}`); + log.info(` ${dim(entry.description)}`); + log.info(""); + } + + const custom = entries.length - transformers.length; + log.info( + dim( + `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + + (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), + ), + ); + + if (custom === 0) { + log.info( + dim("Migrating from something else? Write a transformer and pass --transformer-file."), + ); + } +} diff --git a/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts b/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts new file mode 100644 index 000000000..5dff6ad0b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts @@ -0,0 +1,222 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { loadCustomTransformer, validateTransformer } from "./load-custom.ts"; +import { __resetCustomTransformersForTesting } from "./registry.ts"; + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-custom-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +afterEach(() => { + __resetCustomTransformersForTesting(); +}); + +/** + * Writes a transformer file with a unique name. + * + * Names must not repeat: a dynamic `import()` caches by URL, so reusing one + * would silently return the previous test's module. + */ +function writeTransformer(source: string, ext = "ts"): string { + const name = `custom-${counter++}.${ext}`; + fs.writeFileSync(path.join(workDir, name), source); + return `./${name}`; +} + +const VALID = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email" }, +};`; + +describe("loadCustomTransformer", () => { + test("loads a user-authored TypeScript transformer", async () => { + const entry = await loadCustomTransformer(writeTransformer(VALID)); + + expect(entry).toMatchObject({ + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId", contact_email: "email" }, + }); + }); + + test("loads plain JavaScript too", async () => { + const entry = await loadCustomTransformer(writeTransformer(VALID, "js")); + expect(entry.key).toBe("myplatform"); + }); + + // The file is the user's own code; the CLI must transpile whatever they wrote. + test("transpiles TypeScript syntax the runtime has to strip", async () => { + const entry = await loadCustomTransformer( + writeTransformer(` + interface Entry { key: string; label: string; transformer: Record } + const mapping = { my_id: "userId" } as const; + const custom: Entry = { key: "tsplatform", label: "TS", transformer: { ...mapping } }; + export default custom satisfies Entry; + `), + ); + expect(entry.key).toBe("tsplatform"); + }); + + test("carries the optional hooks through", async () => { + const entry = await loadCustomTransformer( + writeTransformer(`export default { + key: "hooked", label: "Hooked", + transformer: { id: "userId" }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { user.firstName = "set"; }, + };`), + ); + + expect(entry.defaults).toEqual({ passwordHasher: "bcrypt" }); + const user: Record = {}; + entry.postTransform?.(user, {}); + expect(user.firstName).toBe("set"); + }); + + test("supplies a description when the author omitted one", async () => { + const entry = await loadCustomTransformer( + writeTransformer( + `export default { key: "bare", label: "Bare", transformer: { id: "userId" } };`, + ), + ); + expect(entry.description).toBe("Custom transformer"); + }); + + test("reports a path that is not there", async () => { + await expect(loadCustomTransformer("./nope.ts")).rejects.toThrow(/No transformer file at/); + }); + + test("reports a directory given instead of a file", async () => { + fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); + await expect(loadCustomTransformer("./adir")).rejects.toThrow(/is a directory/); + }); + + test("reports a file that does not parse, quoting the syntax error", async () => { + await expect( + loadCustomTransformer(writeTransformer("export default { key: ,,, }")), + ).rejects.toThrow(/Could not load/); + }); + + test("reports a file that throws while loading", async () => { + await expect( + loadCustomTransformer(writeTransformer(`throw new Error("boom"); export default {};`)), + ).rejects.toThrow(/Could not load .*boom/s); + }); + + test("points at a named export when the default is missing", async () => { + const file = writeTransformer( + `export const myPlatform = { key: "x", label: "X", transformer: { a: "userId" } };`, + ); + + await expect(loadCustomTransformer(file)).rejects.toThrow( + /has no default export.*`myPlatform`.*did you mean `export default`/s, + ); + }); + + test("reports a missing default with no named exports to suggest", async () => { + await expect(loadCustomTransformer(writeTransformer("const unused = 1;"))).rejects.toThrow( + /has no default export\.$/m, + ); + }); +}); + +describe("validateTransformer", () => { + const valid = { + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId" }, + }; + + test("accepts a minimal valid entry", () => { + expect(validateTransformer(valid, "f.ts").key).toBe("myplatform"); + }); + + test.each([ + ["a null default export", null, /is null, not an object/], + ["a number default export", 42, /is number, not an object/], + ["a string default export", "nope", /is string, not an object/], + ])("rejects %s", (_label, value, expected) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(expected); + }); + + test.each([ + ["key", { ...valid, key: undefined }], + ["key", { ...valid, key: "" }], + ["key", { ...valid, key: " " }], + ["key", { ...valid, key: 7 }], + ["label", { ...valid, label: undefined }], + ["label", { ...valid, label: "" }], + ])("rejects a bad %s naming the field", (field, value) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test("rejects a non-string description", () => { + expect(() => validateTransformer({ ...valid, description: 7 }, "f.ts")).toThrow( + /`description` must be a string/, + ); + }); + + test.each([ + ["missing", { ...valid, transformer: undefined }], + ["null", { ...valid, transformer: null }], + ["an array", { ...valid, transformer: [] }], + ["a string", { ...valid, transformer: "id" }], + ])("rejects a transformer mapping that is %s", (_label, value) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(/`transformer`|`transformer\./); + }); + + test("names the offending entry when a mapping target is not a field name", () => { + expect(() => + validateTransformer({ ...valid, transformer: { account_ref: "userId", bad: 7 } }, "f.ts"), + ).toThrow(/`transformer.bad` must map to a Clerk field name, got number/); + }); + + // Without it the import runs to completion and creates every user with no + // external_id — which is what makes a migration reversible. + test("rejects a mapping with no userId target", () => { + expect(() => validateTransformer({ ...valid, transformer: { a: "email" } }, "f.ts")).toThrow( + /no source field maps to `userId`/, + ); + }); + + test.each([ + ["defaults", { ...valid, defaults: "nope" }], + ["preTransform", { ...valid, preTransform: "nope" }], + ["postTransform", { ...valid, postTransform: 7 }], + ])("rejects a %s of the wrong type", (field, value) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test.each([["clerk"], ["auth0"], ["supabase"]])( + "rejects %s, which would shadow a built-in", + (key) => { + expect(() => validateTransformer({ ...valid, key }, "f.ts")).toThrow( + /already a built-in transformer/, + ); + }, + ); + + test("names the file in every message, so the author knows which one", () => { + expect(() => validateTransformer({}, "./their-file.ts")).toThrow(/\.\/their-file\.ts/); + }); + + test("raises CliError, so the global handler formats it", () => { + expect(() => validateTransformer({}, "f.ts")).toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/load-custom.ts b/packages/cli-core/src/commands/migrate/transformers/load-custom.ts new file mode 100644 index 000000000..1ac82a719 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/load-custom.ts @@ -0,0 +1,164 @@ +/** + * Loading a user-authored transformer at runtime. + * + * In the standalone migration-tool, supporting a new platform meant adding a + * file to `src/transformers/` and one line to the registry — the user had the + * source tree. A compiled binary has neither a source tree to edit nor a way + * for an end user to rebuild it, so `--transformer-file` restores that + * extensibility by importing a file from the user's own project instead. + * + * **Verified before this was built on:** a `bun build --compile` executable can + * `import()` an arbitrary external `.ts` file at runtime, including TypeScript + * that needs transpiling. Bun's transpiler is part of the runtime, not only the + * bundler. Confirmed with a throwaway compiled binary on darwin-arm64, + * linux-arm64 (glibc), linux-arm64-musl and linux-x64. + * + * The file is user-supplied code the CLI executes, so its shape is validated + * up front and rejected with a specific message rather than crashing deep in + * the transform pipeline on a missing field. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import type { TransformerRegistryEntry } from "../types.ts"; +import { transformers } from "./registry.ts"; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; + +function invalid(problem: string, file: string): never { + throw new CliError(`${file} is not a valid transformer: ${problem}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); +} + +/** + * Checks a loaded value against the registry entry shape. + * + * Every failure names the specific field and what was wrong with it — the + * author is writing this file by hand against a shape they cannot see. + * + * @param file - Path as the user typed it, for the error message. + */ +export function validateTransformer(value: unknown, file: string): TransformerRegistryEntry { + if (value === null || typeof value !== "object") { + invalid(`the default export is ${value === null ? "null" : typeof value}, not an object`, file); + } + + const entry = value as Record; + + for (const field of ["key", "label"] as const) { + if (typeof entry[field] !== "string" || entry[field].trim().length === 0) { + invalid(`\`${field}\` must be a non-empty string`, file); + } + } + + if (entry.description !== undefined && typeof entry.description !== "string") { + invalid("`description` must be a string when present", file); + } + + // Arrays are objects, and an author who wrote `transformer: []` should hear + // that rather than the downstream "no field maps to userId". + if ( + entry.transformer === null || + typeof entry.transformer !== "object" || + Array.isArray(entry.transformer) + ) { + invalid("`transformer` must be an object mapping source fields to Clerk fields", file); + } + + const mapping = entry.transformer as Record; + for (const [source, target] of Object.entries(mapping)) { + if (typeof target !== "string" || target.length === 0) { + invalid( + `\`transformer.${source}\` must map to a Clerk field name, got ${typeof target}`, + file, + ); + } + } + + // Without this the import runs to completion and creates every user with no + // external_id, which is what makes a migration re-runnable and reversible. + if (!Object.values(mapping).includes("userId")) { + invalid( + "no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id", + file, + ); + } + + if ( + entry.defaults !== undefined && + (entry.defaults === null || typeof entry.defaults !== "object" || Array.isArray(entry.defaults)) + ) { + invalid("`defaults` must be an object when present", file); + } + + for (const hook of ["preTransform", "postTransform"] as const) { + if (entry[hook] !== undefined && typeof entry[hook] !== "function") { + invalid(`\`${hook}\` must be a function when present`, file); + } + } + + if (transformers.some((builtIn) => builtIn.key === entry.key)) { + invalid( + `\`key\` is "${String(entry.key)}", which is already a built-in transformer. Choose another key`, + file, + ); + } + + return { + ...(entry as unknown as TransformerRegistryEntry), + description: (entry.description as string | undefined) ?? "Custom transformer", + }; +} + +/** + * Imports and validates a user-authored transformer. + * + * @throws CliError when the path is missing, the module fails to load, or the + * exported value does not match the registry entry shape. + */ +export async function loadCustomTransformer(file: string): Promise { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No transformer file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + if (fs.statSync(resolved).isDirectory()) { + throw new CliError(`${resolved} is a directory, not a transformer file.`, { + code: ERROR_CODE.USAGE_ERROR, + }); + } + + let module: Record; + try { + // A file URL rather than a bare path: an absolute POSIX path happens to + // work, but a Windows path (`C:\...`) is not a valid import specifier. + module = (await import(Bun.pathToFileURL(resolved).href)) as Record; + } catch (error) { + throw new CliError( + `Could not load ${file}: ${(error as Error).message}\n` + + "The file must be valid JavaScript or TypeScript that this CLI can import.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + if (module.default === undefined) { + // Point at what they probably meant rather than just restating the rule. + const named = Object.keys(module).filter((key) => key !== "default"); + const hint = + named.length > 0 + ? ` Found named export${named.length === 1 ? "" : "s"} ${named.map((n) => `\`${n}\``).join(", ")} — did you mean \`export default\`?` + : ""; + throw new CliError(`${file} has no default export.${hint}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + } + + return validateTransformer(module.default, file); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/registry.ts b/packages/cli-core/src/commands/migrate/transformers/registry.ts new file mode 100644 index 000000000..6279577a0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/registry.ts @@ -0,0 +1,74 @@ +/** + * Transformer registry. + * + * `migrate run` reads this array to resolve `--transformer` and to list the + * valid choices in help output and tab-completion. + * + * To add a platform: create `transformers/.ts` exporting a + * `TransformerRegistryEntry`, then add it to the array below. + */ + +import type { TransformerRegistryEntry } from "../types.ts"; +import auth0Transformer from "./auth0.ts"; +import authjsTransformer from "./authjs.ts"; +import betterAuthTransformer from "./betterauth.ts"; +import clerkTransformer from "./clerk.ts"; +import firebaseTransformer from "./firebase.ts"; +import supabaseTransformer from "./supabase.ts"; + +export const transformers: TransformerRegistryEntry[] = [ + clerkTransformer, + auth0Transformer, + authjsTransformer, + betterAuthTransformer, + firebaseTransformer, + supabaseTransformer, +]; + +/** + * Transformers loaded from a user's `--transformer-file` for this invocation. + * + * Kept beside the built-ins rather than pushed into them, so the shipped list + * is never mutated and `--transformer`'s choices stay exactly the built-in + * keys. One CLI invocation loads at most one, so this holding a single entry is + * the normal case; the array shape just avoids a special case in the lookups. + */ +const customTransformers: TransformerRegistryEntry[] = []; + +export function registerCustomTransformer(entry: TransformerRegistryEntry): void { + customTransformers.push(entry); +} + +/** Test-only: drops anything a previous test registered. */ +export function __resetCustomTransformersForTesting(): void { + customTransformers.length = 0; +} + +/** Built-ins plus whatever `--transformer-file` loaded. */ +export function allTransformers(): TransformerRegistryEntry[] { + return [...transformers, ...customTransformers]; +} + +/** + * The built-in keys, for `--transformer`'s choices and tab-completion. + * + * Deliberately excludes custom transformers: they are selected by path via + * `--transformer-file`, and Commander resolves these choices once at + * registration time, before any file could have been loaded. + */ +export function transformerKeys(): string[] { + return transformers.map((entry) => entry.key); +} + +/** + * Looks up a transformer by key, custom ones included. + * + * @throws Error when no transformer is registered under that key. + */ +export function getTransformer(key: string): TransformerRegistryEntry { + const transformer = allTransformers().find((entry) => entry.key === key); + if (!transformer) { + throw new Error(`Transformer not found for key: ${key}`); + } + return transformer; +} diff --git a/packages/cli-core/src/commands/migrate/transformers/shared.ts b/packages/cli-core/src/commands/migrate/transformers/shared.ts new file mode 100644 index 000000000..c4040230a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/shared.ts @@ -0,0 +1,93 @@ +/** + * Helpers shared by more than one transformer. + * + * Every source platform records verification as a sibling field of the + * identifier, and several ship a single `name` string where Clerk wants a + * first/last pair — so both live here rather than being copied five times. + */ + +/** + * How a platform records that an identifier is verified. + * + * - `boolean` — a true/false flag (Auth0, Better Auth, Firebase). A CSV export + * turns these into the *strings* `"true"`/`"false"`, so `"false"` must not + * be mistaken for a truthy value. + * - `timestamp` — a nullable confirmation time (Auth.js `email_verified`, + * Supabase `email_confirmed_at`). Any real value means verified. + */ +export type VerificationStyle = "boolean" | "timestamp"; + +/** CSV exports write SQL NULL as one of these rather than an empty cell. */ +const NULLISH_STRINGS = new Set(["", "null", "nil", "undefined", "\\n"]); + +export function isVerified(value: unknown, style: VerificationStyle): boolean { + if (value === null || value === undefined) return false; + + if (style === "boolean") { + return value === true || value === 1 || value === "true" || value === "1"; + } + + if (value instanceof Date) return !Number.isNaN(value.getTime()); + if (typeof value === "number") return true; + return typeof value === "string" && !NULLISH_STRINGS.has(value.trim().toLowerCase()); +} + +/** + * Routes an identifier to its verified or unverified field, then drops the + * platform's verification marker. + * + * An unverified identifier must not go on `POST /v1/users`'s primary field: + * Clerk creates those verified, which would silently promote an address the + * source platform never confirmed. + */ +export function routeByVerification( + user: Record, + field: "email" | "phone", + verifiedField: string, + style: VerificationStyle, +): void { + const value = user[field]; + if (value && !isVerified(user[verifiedField], style)) { + user[field === "email" ? "unverifiedEmailAddresses" : "unverifiedPhoneNumbers"] = value; + delete user[field]; + } + delete user[verifiedField]; +} + +/** + * Splits a single display name into `firstName` and `lastName`. + * + * Only splits when there are at least two words — a one-word name would + * otherwise produce a first name with no last name, which several instance + * configurations reject. + */ +export function splitName(user: Record, field = "name"): void { + const name = user[field]; + if (!name || typeof name !== "string") return; + + const parts = name.trim().split(/\s+/); + if (parts.length > 1) { + user.firstName = parts[0]; + user.lastName = parts.slice(1).join(" "); + } + delete user[field]; +} + +/** + * Converts a source timestamp to ISO 8601, leaving it untouched when it does + * not parse so the schema reports it as a validation failure with the original + * value visible in the log. + * + * @param epochMillis - Treat a bare number (or numeric string) as Unix + * milliseconds, which is how Firebase exports timestamps. + */ +export function toIsoDate(value: unknown, epochMillis = false): unknown { + if (value === undefined || value === null || value === "") return value; + + const parsed = + epochMillis && (typeof value === "number" || /^\d+$/.test(String(value))) + ? new Date(Number(value)) + : new Date(String(value)); + + return Number.isNaN(parsed.getTime()) ? value : parsed.toISOString(); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/supabase.ts b/packages/cli-core/src/commands/migrate/transformers/supabase.ts new file mode 100644 index 000000000..1efde4b6a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/supabase.ts @@ -0,0 +1,70 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, toIsoDate } from "./shared.ts"; + +/** + * Supabase Auth → Clerk transformer. + * + * Works with a `auth.users` export, per + * https://supabase.com/docs/guides/auth/managing-user-data#exporting-users + * + * Supabase records verification as a nullable confirmation timestamp + * (`email_confirmed_at`) rather than a boolean, and stores timestamps in + * PostgreSQL's format (`2024-06-29 20:25:06.126079+00`). + */ + +/** Discord writes display names as `name#0`; the suffix reads as a URL to Clerk. */ +const DISCORD_DISCRIMINATOR = /#\d+$/; + +function stripDiscriminator(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + return value.replace(DISCORD_DISCRIMINATOR, "").trim() || undefined; +} + +const supabaseTransformer = { + key: "supabase", + label: "Supabase", + description: + "Works with a Supabase `auth.users` export. Use --skip-unsupported-providers to drop users whose only social provider is not enabled in Clerk.", + transformer: { + id: "userId", + email: "email", + email_confirmed_at: "emailConfirmedAt", + first_name: "firstName", + last_name: "lastName", + encrypted_password: "password", + phone: "phone", + phone_confirmed_at: "phoneConfirmedAt", + raw_user_meta_data: "publicMetadata", + created_at: "createdAt", + }, + postTransform: (user) => { + user.createdAt = toIsoDate(user.createdAt); + routeByVerification(user, "email", "emailConfirmedAt", "timestamp"); + routeByVerification(user, "phone", "phoneConfirmedAt", "timestamp"); + + // A basic SQL export has no first_name/last_name columns; the name lives in + // user metadata instead, under whichever key the provider happened to use. + if (!user.firstName && user.publicMetadata && typeof user.publicMetadata === "object") { + const meta = user.publicMetadata as Record; + const displayName = stripDiscriminator(meta.display_name ?? meta.first_name ?? meta.name); + if (displayName) { + const parts = displayName.split(/\s+/); + user.firstName = parts[0]; + if (parts.length > 1 && !user.lastName) user.lastName = parts.slice(1).join(" "); + } + } + + for (const field of ["firstName", "lastName"] as const) { + if (typeof user[field] === "string") { + const cleaned = stripDiscriminator(user[field]); + if (cleaned) user[field] = cleaned; + else delete user[field]; + } + } + }, + defaults: { + passwordHasher: "bcrypt" as const, + }, +} satisfies TransformerRegistryEntry; + +export default supabaseTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts b/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts new file mode 100644 index 000000000..ae492f01d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts @@ -0,0 +1,405 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { loadUsersFromFile, transformUsers } from "../lib/transform.ts"; +import type { FirebaseHashConfig } from "../types.ts"; +import { getTransformer, transformerKeys, transformers } from "./registry.ts"; +import { isVerified } from "./shared.ts"; + +const DATE_TIME = "2026-01-01T00:00:00"; + +const FIREBASE_HASH: FirebaseHashConfig = { + base64_signer_key: "SIGNERKEY==", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, +}; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-transformers-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +/** Writes `records` to a uniquely-named file and loads it through `key`. */ +async function load(key: string, records: unknown, ext = "json", context = {}) { + const file = `${key}-${Math.abs(JSON.stringify(records).length)}-${ext}.${ext}`; + fs.writeFileSync( + path.join(workDir, file), + typeof records === "string" ? records : JSON.stringify(records), + ); + return loadUsersFromFile(file, key, DATE_TIME, { context }); +} + +const one = (key: string, record: Record, context = {}) => + transformUsers([record], key, DATE_TIME, { validate: false, context }).transformedData[0] as + | Record + | undefined; + +describe("registry", () => { + test("registers all six platforms", () => { + expect(transformerKeys()).toEqual([ + "clerk", + "auth0", + "authjs", + "betterauth", + "firebase", + "supabase", + ]); + }); + + test.each([...transformers])("$key maps a source field to userId", (transformer) => { + expect(Object.values(transformer.transformer)).toContain("userId"); + }); + + test.each([...transformers])("$key carries a label and description", (transformer) => { + expect(transformer.label.length).toBeGreaterThan(0); + expect(transformer.description.length).toBeGreaterThan(0); + }); + + test("throws for an unregistered key", () => { + expect(() => getTransformer("okta")).toThrow(/Transformer not found/); + }); +}); + +describe("isVerified", () => { + // A CSV export stringifies everything, so the boolean style must read + // "false" as false. Treating it as truthy would mark unconfirmed addresses + // verified on import — the exact thing the routing exists to prevent. + test.each([ + [true, true], + ["true", true], + [1, true], + ["1", true], + [false, false], + ["false", false], + [0, false], + ["0", false], + ["", false], + [null, false], + [undefined, false], + ])("boolean style: %p -> %p", (value, expected) => { + expect(isVerified(value, "boolean")).toBe(expected); + }); + + // The timestamp style is presence-based: any real confirmation time counts, + // and SQL NULL arrives from a CSV export as one of several spellings. + test.each([ + ["2024-06-29 20:25:06+00", true], + ["2024-01-15T10:30:00.000Z", true], + ["", false], + [" ", false], + ["null", false], + ["NULL", false], + ["\\N", false], + [null, false], + [undefined, false], + ])("timestamp style: %p -> %p", (value, expected) => { + expect(isVerified(value, "timestamp")).toBe(expected); + }); +}); + +describe("auth0", () => { + const base = { user_id: "auth0|abc", email: "a@x.dev", passwordHash: "$2b$10$hash" }; + + test("maps identity, name and metadata onto the Clerk schema", async () => { + const { users } = await load("auth0", [ + { ...base, email_verified: true, given_name: "Ada", family_name: "Lovelace" }, + ]); + expect(users[0]).toMatchObject({ + userId: "auth0|abc", + email: "a@x.dev", + firstName: "Ada", + lastName: "Lovelace", + password: "$2b$10$hash", + passwordHasher: "bcrypt", + }); + }); + + test.each([ + [true, "email", undefined], + [false, undefined, "a@x.dev"], + [undefined, undefined, "a@x.dev"], + ])("email_verified=%p routes the address correctly", (verified, kept, unverified) => { + const user = one("auth0", { ...base, email_verified: verified }); + expect(user?.email).toBe(kept ? "a@x.dev" : undefined); + expect(user?.unverifiedEmailAddresses).toBe(unverified); + }); + + test("routes an unverified phone away from the primary field", () => { + const user = one("auth0", { ...base, phone_number: "+15555550100", phone_verified: false }); + expect(user?.phone).toBeUndefined(); + expect(user?.unverifiedPhoneNumbers).toBe("+15555550100"); + }); + + test("drops the platform's verification markers", () => { + const user = one("auth0", { ...base, email_verified: true, phone_verified: true }); + expect("emailVerified" in (user ?? {})).toBe(false); + expect("phoneVerified" in (user ?? {})).toBe(false); + }); + + test("keeps user_metadata public and app_metadata private", async () => { + const { users } = await load("auth0", [ + { + ...base, + email_verified: true, + user_metadata: { theme: "dark" }, + app_metadata: { plan: "pro" }, + }, + ]); + expect(users[0]?.publicMetadata).toEqual({ theme: "dark" }); + expect(users[0]?.privateMetadata).toEqual({ plan: "pro" }); + }); +}); + +describe("authjs", () => { + const base = { id: "cuid1", email: "a@x.dev" }; + + test("treats a confirmation timestamp as verified", () => { + const user = one("authjs", { ...base, email_verified: "2024-01-15T10:30:00.000Z" }); + expect(user?.email).toBe("a@x.dev"); + expect(user?.unverifiedEmailAddresses).toBeUndefined(); + }); + + test.each([[null], [""], [undefined]])("treats email_verified=%p as unverified", (value) => { + const user = one("authjs", { ...base, email_verified: value }); + expect(user?.unverifiedEmailAddresses).toBe("a@x.dev"); + }); + + test.each([ + ["Jane Doe", "Jane", "Doe"], + ["Mary Jane Watson", "Mary", "Jane Watson"], + [" Ada Lovelace ", "Ada", "Lovelace"], + ])("splits %p into %p / %p", (name, firstName, lastName) => { + const user = one("authjs", { ...base, name }); + expect(user?.firstName).toBe(firstName); + expect(user?.lastName).toBe(lastName); + }); + + test("leaves a single-word name unsplit rather than inventing a last name", () => { + const user = one("authjs", { ...base, name: "Prince" }); + expect(user?.firstName).toBeUndefined(); + expect(user?.lastName).toBeUndefined(); + expect("name" in (user ?? {})).toBe(false); + }); + + test("imports without a password, since Auth.js core is passwordless", async () => { + const { users } = await load("authjs", [{ ...base, email_verified: "2024-01-01" }]); + expect(users[0]?.password).toBeUndefined(); + expect(users[0]?.passwordHasher).toBeUndefined(); + }); +}); + +describe("betterauth", () => { + const base = { user_id: "ba1", email: "a@x.dev", email_verified: true }; + + test("maps the credential hash and defaults the hasher to bcrypt", async () => { + const { users } = await load("betterauth", [{ ...base, password_hash: "$2a$10$hash" }]); + expect(users[0]).toMatchObject({ password: "$2a$10$hash", passwordHasher: "bcrypt" }); + }); + + test("routes an unverified phone", () => { + const user = one("betterauth", { + ...base, + phone_number: "+15555550100", + phone_number_verified: false, + }); + expect(user?.unverifiedPhoneNumbers).toBe("+15555550100"); + }); + + test.each([ + [true, true], + [false, undefined], + [undefined, undefined], + ])("banned=%p is carried through as %p", (banned, expected) => { + expect(one("betterauth", { ...base, banned })?.banned).toBe(expected as boolean | undefined); + }); + + test("drops plugin-only columns during validation", async () => { + const { users } = await load("betterauth", [ + { ...base, role: "admin", display_username: "ADA", two_factor_enabled: true }, + ]); + const user = users[0] as Record; + expect("role" in user).toBe(false); + expect("display_username" in user).toBe(false); + expect("two_factor_enabled" in user).toBe(false); + }); +}); + +describe("firebase", () => { + const base = { localId: "fb1", email: "a@x.dev", emailVerified: true }; + const withHash = { ...base, passwordHash: "SGFzaA==", salt: "U2FsdA==" }; + + test("builds the scrypt digest Clerk expects, parameters inline", async () => { + const { users } = await load("firebase", { users: [withHash] }, "json", { + firebaseHashConfig: FIREBASE_HASH, + }); + expect(users[0]?.password).toBe("SGFzaA==$U2FsdA==$SIGNERKEY==$Bw==$8$14"); + expect(users[0]?.passwordHasher).toBe("scrypt_firebase"); + }); + + test("refuses to import hashes without the project's hash parameters", async () => { + await expect(load("firebase", { users: [withHash] })).rejects.toThrow( + /Firebase password hashes/, + ); + }); + + test("imports a passwordless export with no hash parameters at all", async () => { + const { users } = await load("firebase", { users: [base] }); + expect(users).toHaveLength(1); + expect(users[0]?.password).toBeUndefined(); + }); + + test("unwraps the { users: [...] } export shape", async () => { + const { users } = await load("firebase", { users: [base, { ...base, localId: "fb2" }] }); + expect(users.map((u) => u.userId)).toEqual(["fb1", "fb2"]); + }); + + test("accepts a bare array too", async () => { + const { users } = await load("firebase", [base]); + expect(users).toHaveLength(1); + }); + + test("rejects a JSON export that is neither", async () => { + await expect(load("firebase", { records: [] })).rejects.toThrow(CliError); + }); + + test("prepends headers to a headerless CSV export", async () => { + const csv = "fb9,a@x.dev,true,,,Ada Lovelace,,,,,,,,,,,,,,,,,,1704067200000,,,,,\n"; + const { users } = await load("firebase", csv, "csv"); + expect(users[0]).toMatchObject({ userId: "fb9", email: "a@x.dev", firstName: "Ada" }); + }); + + test.each([ + ["1704067200000", "2024-01-01T00:00:00.000Z"], + [1704067200000, "2024-01-01T00:00:00.000Z"], + ])("converts the Unix-millisecond createdAt %p", (createdAt, expected) => { + expect(one("firebase", { ...base, createdAt })?.createdAt).toBe(expected); + }); + + test.each([ + [true, true], + ["true", true], + [false, false], + ["false", false], + ])("emailVerified=%p keeps the address primary: %p", (emailVerified, verified) => { + const user = one("firebase", { ...base, emailVerified }); + expect(user?.email !== undefined).toBe(verified); + }); +}); + +describe("supabase", () => { + const base = { id: "sb1", email: "a@x.dev", email_confirmed_at: "2024-06-29 20:25:06.126079+00" }; + + test("maps the bcrypt password and converts the PostgreSQL timestamp", async () => { + const { users } = await load("supabase", [ + { ...base, encrypted_password: "$2b$10$hash", created_at: "2024-06-29 20:25:06.126079+00" }, + ]); + expect(users[0]).toMatchObject({ + password: "$2b$10$hash", + passwordHasher: "bcrypt", + createdAt: "2024-06-29T20:25:06.126Z", + }); + }); + + test.each([ + ["2024-06-29 20:25:06+00", true], + [null, false], + ["", false], + ])("email_confirmed_at=%p means verified: %p", (confirmedAt, verified) => { + const user = one("supabase", { ...base, email_confirmed_at: confirmedAt }); + expect(user?.email !== undefined).toBe(verified); + }); + + test("falls back to user metadata for a missing first name", () => { + const user = one("supabase", { + ...base, + raw_user_meta_data: { display_name: "Ada Lovelace" }, + }); + expect(user?.firstName).toBe("Ada"); + expect(user?.lastName).toBe("Lovelace"); + }); + + test("prefers explicit name columns over metadata", () => { + const user = one("supabase", { + ...base, + first_name: "Grace", + raw_user_meta_data: { display_name: "Ada Lovelace" }, + }); + expect(user?.firstName).toBe("Grace"); + }); + + test.each([ + ["ada#0", "ada"], + ["ada#1234", "ada"], + ])("strips the Discord discriminator from %p", (displayName, expected) => { + const user = one("supabase", { ...base, raw_user_meta_data: { display_name: displayName } }); + expect(user?.firstName).toBe(expected); + }); + + test("drops a name that was nothing but a discriminator", () => { + const user = one("supabase", { ...base, first_name: "#0" }); + expect(user?.firstName).toBeUndefined(); + }); +}); + +describe("invalid records", () => { + const INVALID: [string, Record][] = [ + ["auth0", { user_id: "a1" }], + ["authjs", { id: "a2" }], + ["betterauth", { user_id: "a3" }], + ["firebase", { localId: "a4" }], + ["supabase", { id: "a5" }], + ]; + + test.each(INVALID)( + "%s logs a user with no identifier instead of crashing", + async (key, record) => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); + + const { users, validationFailed } = await load(key, [ + record, + { ...record, ...identifierFor(key) }, + ]); + + expect(validationFailed).toBe(1); + expect(users).toHaveLength(1); + + const logged = fs + .readdirSync(getLogDir()) + .flatMap((name) => + fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n"), + ) + .map((line) => JSON.parse(line) as Record); + expect(logged.some((entry) => entry.status === "fail")).toBe(true); + }, + ); + + test.each(INVALID)("%s logs a malformed email rather than sending it", async (key, record) => { + const { users, validationFailed } = await load(key, [ + { ...record, ...identifierFor(key, "not-an-email") }, + ]); + expect(validationFailed).toBe(1); + expect(users).toHaveLength(0); + }); +}); + +/** The per-platform source field that becomes a Clerk identifier. */ +function identifierFor(key: string, email = "ok@x.dev"): Record { + if (key === "auth0") return { email, email_verified: true }; + if (key === "authjs") return { email, email_verified: "2024-01-01" }; + if (key === "betterauth") return { email, email_verified: true }; + if (key === "firebase") return { email, emailVerified: true }; + return { email, email_confirmed_at: "2024-01-01 00:00:00+00" }; +} diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts new file mode 100644 index 000000000..40ba3a5ec --- /dev/null +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -0,0 +1,193 @@ +/** + * Shared types for `clerk migrate`. + * + * Ported from the standalone migration-tool's `src/types.ts`. The Clerk API + * error shape is declared locally rather than imported from `@clerk/types`, + * because this command family talks to BAPI through `lib/bapi.ts` instead of + * `@clerk/backend`. + */ + +import type * as z from "zod"; +import type { userSchema } from "./validator.ts"; + +/** + * Password hashing algorithms Clerk can verify on import. + * + * When migrating users with existing passwords, the source platform's hasher + * must be named so Clerk can validate the digest instead of rejecting it. + */ +export const PASSWORD_HASHERS = [ + "argon2i", + "argon2id", + "awscognito", + "bcrypt", + "bcrypt_peppered", + "bcrypt_sha256_django", + "hmac_sha256_utf16_b64", + "md5", + "md5_salted", + "pbkdf2_sha1", + "pbkdf2_sha256", + "pbkdf2_sha256_django", + "pbkdf2_sha512", + "pbkdf2_sha512_hex", + "scrypt_firebase", + "scrypt_werkzeug", + "sha256", + "sha256_salted", + "md5_phpass", + "ldap_ssha", + "sha512_symfony", +] as const; + +/** A user that has passed schema validation and is ready to import. */ +export type User = z.infer; + +/** Union of all registered transformer keys (e.g. `"clerk"`). */ +export type TransformerKey = string; + +/** + * One error entry as returned in a Clerk API error response body. + * + * Local mirror of `@clerk/types`' `ClerkAPIError` covering only the fields the + * migration logs read. + */ +export type ClerkApiError = { + code: string; + message: string; + longMessage?: string; +}; + +/** A failed user-creation attempt, as handed to the error logger. */ +export type ErrorPayload = { + userId: string; + status: string; + errors: ClerkApiError[]; +}; + +/** A user that failed schema validation before any API call was made. */ +export type ValidationErrorPayload = { + error: string; + path: (string | number)[]; + userId: string; + row: number; +}; + +/** A formatted error line as written to the NDJSON log. */ +export type ErrorLog = { + type: string; + userId: string; + status: string; + error: string | undefined; +}; + +/** One import attempt as written to the NDJSON log. */ +export type ImportLogEntry = { + userId: string; + status: "success" | "error"; + clerkUserId?: string; + error?: string; + code?: string; +}; + +/** One exported user as written to the NDJSON log. */ +export type ExportLogEntry = { + /** The source platform's ID for this user. */ + userId: string; + status: "success" | "error"; + error?: string; +}; + +/** One deletion attempt as written to the NDJSON log. */ +export type DeleteLogEntry = { + /** The source platform's ID — the Clerk user's `external_id`. */ + userId: string; + clerkUserId?: string; + status: "success" | "error"; + error?: string; + code?: string; +}; + +/** Totals for a completed import run. */ +export type ImportSummary = { + totalProcessed: number; + successful: number; + failed: number; + validationFailed: number; + errorBreakdown: Map; +}; + +/** + * Per-directory migration state, persisted to a cwd-relative `.settings` file. + * + * Deliberately not routed through `~/.config/clerk/config.json`: that file is + * keyed by linked-project identity, which is a different concept from "which + * file did I last migrate with". + */ +export type Settings = { + key?: string; + file?: string; + skipUnsupportedProviders?: boolean; + firebaseHashConfig?: FirebaseHashConfig; +}; + +/** + * Firebase's scrypt parameters, needed to rebuild a password hash Clerk can + * verify. + * + * Found in the Firebase console under Authentication → Users → (⋮) → Password + * hash parameters. All four are required together; a partial set produces a + * digest that silently fails every sign-in. + */ +export type FirebaseHashConfig = { + base64_signer_key: string; + base64_salt_separator: string; + rounds: number; + mem_cost: number; +}; + +/** + * Per-run values a transformer may need but cannot read from the user record. + * + * Passed to `postTransform` rather than held in module state so two runs in one + * process — or two test files — cannot see each other's configuration. + */ +export type TransformContext = { + firebaseHashConfig?: FirebaseHashConfig; +}; + +/** + * Result of a transformer's `preTransform` hook. + * + * @property filePath - Path to read from; may differ from the input (e.g. a + * temp file with generated CSV headers). + * @property data - Users already extracted from a wrapper object, when the + * source format nests them. + */ +export type PreTransformResult = { + filePath: string; + data?: Record[]; +}; + +/** + * A platform transformer: how to get from one source export shape to Clerk's + * import shape. + * + * @property transformer - Source field path → Clerk field name. + * @property defaults - Values merged into every user from this platform. + * @property preTransform - Runs before field mapping. + * @property postTransform - Mutates a user after field mapping, given the + * run's {@link TransformContext}. + */ +export type TransformerRegistryEntry = { + key: string; + label: string; + description: string; + transformer: Record; + defaults?: Record; + preTransform?: ( + filePath: string, + fileType: string, + ) => PreTransformResult | Promise; + postTransform?: (user: Record, context: TransformContext) => void; +}; diff --git a/packages/cli-core/src/commands/migrate/validator.test.ts b/packages/cli-core/src/commands/migrate/validator.test.ts new file mode 100644 index 000000000..d398a4dc9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/validator.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, test } from "bun:test"; +import { PASSWORD_HASHERS } from "./types.ts"; +import { userSchema } from "./validator.ts"; + +const base = { userId: "user_1", email: "a@example.com" }; + +describe("userSchema identifiers", () => { + const IDENTIFIER_CASES = [ + ["email", { email: "a@example.com" }, true], + ["emailAddresses array", { emailAddresses: ["a@example.com"] }, true], + ["unverified email", { unverifiedEmailAddresses: ["a@example.com"] }, true], + ["phone", { phone: "+15555550100" }, true], + ["unverified phone", { unverifiedPhoneNumbers: ["+15555550100"] }, true], + ["username", { username: "alice" }, true], + ["nothing", {}, false], + ["empty email array", { email: [] }, false], + ["empty username", { username: "" }, false], + ] as const; + + test.each([...IDENTIFIER_CASES])( + "accepts a user identified by %s: %p -> %p", + (_label, fields, ok) => { + expect(userSchema.safeParse({ userId: "user_1", ...fields }).success).toBe(ok); + }, + ); + + test("reports the identifier failure against the email path", () => { + const result = userSchema.safeParse({ userId: "user_1" }); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues[0]?.path).toEqual(["email"]); + }); +}); + +describe("userSchema passwords", () => { + test("rejects a password without a hasher", () => { + const result = userSchema.safeParse({ ...base, password: "digest" }); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues[0]?.path).toEqual(["passwordHasher"]); + }); + + test("accepts a password with a valid hasher", () => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: "bcrypt" }).success, + ).toBe(true); + }); + + test("rejects an unknown hasher", () => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: "rot13" }).success, + ).toBe(false); + }); + + test.each([...PASSWORD_HASHERS])("accepts the %s hasher", (hasher) => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: hasher }).success, + ).toBe(true); + }); +}); + +describe("userSchema field types", () => { + const FIELD_CASES = [ + ["valid email", { email: "a@example.com" }, true], + ["malformed email", { email: "not-an-email" }, false], + ["email array with one bad entry", { email: ["a@example.com", "nope"] }, false], + ["userId missing", { userId: undefined }, false], + ["valid createdAt", { createdAt: "2024-01-01T00:00:00Z" }, true], + ["unparseable createdAt", { createdAt: "yesterday" }, false], + ["integer org limit", { createOrganizationsLimit: 3 }, true], + ["fractional org limit", { createOrganizationsLimit: 1.5 }, false], + ["metadata object", { publicMetadata: { plan: "pro" } }, true], + ["metadata string", { publicMetadata: "pro" }, false], + ["backupCodes array", { backupCodes: ["a", "b"] }, true], + ] as const; + + test.each([...FIELD_CASES])("%s -> %p", (_label, fields, ok) => { + expect(userSchema.safeParse({ ...base, ...fields }).success).toBe(ok); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/validator.ts b/packages/cli-core/src/commands/migrate/validator.ts new file mode 100644 index 000000000..2ebc31166 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/validator.ts @@ -0,0 +1,99 @@ +/** + * Zod schema every user is validated against before it reaches BAPI. + * + * Ported from the standalone migration-tool's `src/migrate/validator.ts`. + * + * ============================================================================ + * ONLY EDIT THIS IF YOU ARE ADDING A NEW FIELD. + * Adding support for a new source platform means adding a transformer, not + * touching the schema. + * ============================================================================ + */ + +import * as z from "zod"; +import { PASSWORD_HASHERS } from "./types.ts"; + +const metadataSchema = z.record(z.string(), z.unknown()); + +const dateStringSchema = z.string().refine((value) => !Number.isNaN(new Date(value).getTime()), { + message: "Expected a valid date string", +}); + +/** Zod enum of the password hashers Clerk accepts on import. */ +export const passwordHasherEnum = z.enum(PASSWORD_HASHERS); + +/** + * Validates user data before sending it to Clerk. + * + * Everything is optional except: + * - `userId`, required for tracking, logging and `--resume-after` + * - `passwordHasher`, required whenever `password` is present + * - at least one identifier (email, phone or username) + * + * Identifier fields accept either a single value or an array. + */ +export const userSchema = z + .object({ + userId: z.string(), + // Email fields + email: z.union([z.email(), z.array(z.email())]).optional(), + emailAddresses: z.union([z.email(), z.array(z.email())]).optional(), + unverifiedEmailAddresses: z.union([z.email(), z.array(z.email())]).optional(), + // Phone fields + phone: z.union([z.string(), z.array(z.string())]).optional(), + phoneNumbers: z.union([z.string(), z.array(z.string())]).optional(), + unverifiedPhoneNumbers: z.union([z.string(), z.array(z.string())]).optional(), + // User info + username: z.string().optional(), + firstName: z.string().optional(), + lastName: z.string().optional(), + // Password + password: z.string().optional(), + passwordHasher: passwordHasherEnum.optional(), + // 2FA + totpSecret: z.string().optional(), + backupCodesEnabled: z.boolean().optional(), + backupCodes: z.array(z.string()).optional(), + // Metadata + unsafeMetadata: metadataSchema.optional(), + publicMetadata: metadataSchema.optional(), + privateMetadata: metadataSchema.optional(), + // Additional Clerk API fields + banned: z.boolean().optional(), + bypassClientTrust: z.boolean().optional(), + createOrganizationEnabled: z.boolean().optional(), + createOrganizationsLimit: z.number().int().optional(), + createdAt: dateStringSchema.optional(), + deleteSelfEnabled: z.boolean().optional(), + legalAcceptedAt: dateStringSchema.optional(), + skipLegalChecks: z.boolean().optional(), + skipPasswordChecks: z.boolean().optional(), + }) + .refine((data) => !data.password || data.passwordHasher, { + message: "passwordHasher is required when password is provided", + path: ["passwordHasher"], + }) + .refine( + (data) => { + const hasValue = (field: unknown): boolean => { + if (!field) return false; + if (typeof field === "string") return field.length > 0; + if (Array.isArray(field)) return field.length > 0; + return false; + }; + return ( + hasValue(data.email) || + hasValue(data.emailAddresses) || + hasValue(data.unverifiedEmailAddresses) || + hasValue(data.phone) || + hasValue(data.phoneNumbers) || + hasValue(data.unverifiedPhoneNumbers) || + hasValue(data.username) + ); + }, + { + message: + "User must have at least one identifier (email, phone, unverified email, unverified phone, or username)", + path: ["email"], + }, + ); diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts new file mode 100644 index 000000000..f3633665d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -0,0 +1,267 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +useCaptureLog(); + +type Prompt = { message: string; default?: string; validate?: (v?: string) => string | undefined }; +type SelectPrompt = Prompt & { choices: { name: string; value: string }[] }; + +// Registered at file top, before the wizard (or anything it imports) loads. +// This file is the only consumer of the mocked prompt modules. +const mockSelect = mock(async (_config: SelectPrompt) => undefined as unknown); +const mockText = mock(async (_config: Prompt) => "" as unknown); + +mock.module("../../lib/listage.ts", () => ({ + ...listageStubs, + select: (config: SelectPrompt) => mockSelect(config), +})); + +mock.module("../../lib/prompts.ts", () => ({ + confirm: async () => true, + text: (config: Prompt) => mockText(config), + password: async () => "", + editor: async () => "{}", +})); + +const { runWizard, throwAgentFlagsRequired } = await import("./wizard.ts"); +const { saveSettings } = await import("./lib/settings.ts"); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-"))); + process.chdir(workDir); + fs.writeFileSync(path.join(workDir, "users.json"), "[]"); + fs.writeFileSync(path.join(workDir, "other.csv"), ""); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + mockSelect.mockReset(); + mockText.mockReset(); + fs.rmSync(path.join(workDir, ".settings"), { force: true }); +}); + +/** The config object the wizard passed to its Nth `text`/`select` prompt. */ +const textCall = (index: number): Prompt | undefined => mockText.mock.calls[index]?.[0]; +const selectCall = (index: number): SelectPrompt | undefined => mockSelect.mock.calls[index]?.[0]; + +describe("transformer picker", () => { + test("is built from the registry, so every platform appears", async () => { + mockSelect.mockResolvedValue("auth0"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.choices.map((choice) => choice.value)).toEqual([ + "clerk", + "auth0", + "authjs", + "betterauth", + "firebase", + "supabase", + ]); + }); + + test("labels each choice with the transformer's display name", async () => { + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.choices.map((choice) => choice.name)).toContain("Better Auth"); + }); + + test("is skipped when --transformer was already passed", async () => { + mockText.mockResolvedValue("users.json"); + + const result = await runWizard({ transformer: "clerk" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(result.transformer).toBe("clerk"); + }); +}); + +describe("defaults from the previous run", () => { + test("pre-selects the last transformer and pre-fills the last file", async () => { + saveSettings({ key: "supabase", file: "other.csv" }); + mockSelect.mockResolvedValue("supabase"); + mockText.mockResolvedValue("other.csv"); + + await runWizard({}); + + expect(selectCall(0)?.default).toBe("supabase"); + expect(textCall(0)?.default).toBe("other.csv"); + }); + + test("offers no default when nothing has been saved", async () => { + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.default).toBeUndefined(); + expect(textCall(0)?.default).toBeUndefined(); + }); + + // A saved key from a build that has since dropped that transformer would + // otherwise pre-select a value the picker cannot offer. + test("ignores a saved transformer that is no longer registered", async () => { + saveSettings({ key: "okta" }); + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.default).toBeUndefined(); + }); +}); + +describe("file prompt validation", () => { + const validate = async () => { + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + await runWizard({}); + return textCall(0)?.validate; + }; + + test.each([ + ["users.json", undefined], + ["other.csv", undefined], + ])("accepts %s", async (file, expected) => { + expect((await validate())?.(file)).toBe(expected as undefined); + }); + + test("rejects an empty answer", async () => { + expect((await validate())?.("")).toMatch(/required/); + }); + + test("rejects a file that does not exist", async () => { + expect((await validate())?.("missing.json")).toMatch(/File not found/); + }); + + test("rejects an unsupported extension", async () => { + fs.writeFileSync(path.join(workDir, "notes.txt"), ""); + expect((await validate())?.("notes.txt")).toMatch(/\.json or \.csv/); + }); +}); + +describe("firebase hash parameters", () => { + test("are asked for when the firebase transformer is picked", async () => { + mockSelect.mockResolvedValue("firebase"); + mockText + .mockResolvedValueOnce("users.json") + .mockResolvedValueOnce("SIGNER") + .mockResolvedValueOnce("Bw==") + .mockResolvedValueOnce("8") + .mockResolvedValueOnce("14"); + + const result = await runWizard({}); + + expect(result.firebaseHashConfig).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + // Pressing enter through the signer key is how a user says "this export has + // no passwords" — the remaining three would be meaningless without it. + test("stop being asked when the signer key is left blank", async () => { + mockSelect.mockResolvedValue("firebase"); + mockText.mockResolvedValueOnce("users.json").mockResolvedValueOnce(" "); + + const result = await runWizard({}); + + expect(result.firebaseHashConfig).toBeUndefined(); + expect(mockText).toHaveBeenCalledTimes(2); + }); + + test("are pre-filled from the previous run", async () => { + saveSettings({ + firebaseHashConfig: { + base64_signer_key: "SAVED", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }, + }); + mockSelect.mockResolvedValue("firebase"); + mockText + .mockResolvedValueOnce("users.json") + .mockResolvedValueOnce("SAVED") + .mockResolvedValueOnce("Bw==") + .mockResolvedValueOnce("8") + .mockResolvedValueOnce("14"); + + await runWizard({}); + + expect(textCall(1)?.default).toBe("SAVED"); + expect(textCall(3)?.default).toBe("8"); + }); + + test("are not asked for on a non-firebase transformer", async () => { + mockSelect.mockResolvedValue("auth0"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(mockText).toHaveBeenCalledTimes(1); + }); + + test("are not asked for when the flags already supplied them", async () => { + mockSelect.mockResolvedValue("firebase"); + mockText.mockResolvedValue("users.json"); + + const config = { + base64_signer_key: "FLAG", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }; + const result = await runWizard({ firebaseHashConfig: config }); + + expect(mockText).toHaveBeenCalledTimes(1); + expect(result.firebaseHashConfig).toEqual(config); + }); + + test.each([["0"], ["-1"], ["1.5"], ["many"]])("rejects %p as a rounds value", async (value) => { + mockSelect.mockResolvedValue("firebase"); + mockText + .mockResolvedValueOnce("users.json") + .mockResolvedValueOnce("SIGNER") + .mockResolvedValueOnce("Bw==") + .mockResolvedValueOnce("8") + .mockResolvedValueOnce("14"); + + await runWizard({}); + + expect(textCall(3)?.validate?.(value)).toMatch(/positive whole number/); + }); +}); + +describe("throwAgentFlagsRequired", () => { + test.each([ + [{ transformer: true, file: true }, /--transformer and --file /], + [{ transformer: true, file: false }, /--transformer \./], + [{ transformer: false, file: true }, /--file \./], + ])("names only the flags that are missing (%p)", (missing, expected) => { + expect(() => throwAgentFlagsRequired(missing)).toThrow(expected); + }); + + test("says why it cannot prompt", () => { + expect(() => throwAgentFlagsRequired({ transformer: true, file: true })).toThrow( + /cannot prompt in agent mode/, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts new file mode 100644 index 000000000..3082a1b63 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -0,0 +1,167 @@ +/** + * The interactive path behind a bare `clerk migrate`. + * + * Ported from the standalone migration-tool's `src/migrate/cli.ts` interactive + * flow. Every answer is pre-filled from the previous run's `.settings`, so a + * repeat migration is mostly pressing enter. + * + * Agent mode never reaches here — `run` raises a usage error naming the flags + * instead, because an agent cannot answer a prompt. + */ + +import { throwUsageError } from "../../lib/errors.ts"; +import { select } from "../../lib/listage.ts"; +import { log } from "../../lib/log.ts"; +import { text } from "../../lib/prompts.ts"; +import { loadSettings } from "./lib/settings.ts"; +import { fileExists, getFileType } from "./lib/transform.ts"; +import { transformers } from "./transformers/registry.ts"; +import type { FirebaseHashConfig } from "./types.ts"; + +export type WizardResult = { + transformer: string; + file: string; + firebaseHashConfig?: FirebaseHashConfig; +}; + +/** Trims a description down to a single readable hint line. */ +function hint(description: string): string { + const firstSentence = description.split(". ")[0] ?? description; + return firstSentence.length > 96 ? `${firstSentence.slice(0, 93)}...` : firstSentence; +} + +async function pickTransformer(defaultKey: string | undefined): Promise { + // Built from the registry, so a new platform appears here with no second + // place to update. + return select({ + message: "Which platform are you migrating from?", + choices: transformers.map((entry) => ({ + name: entry.label, + value: entry.key, + description: hint(entry.description), + })), + default: defaultKey && transformers.some((t) => t.key === defaultKey) ? defaultKey : undefined, + }); +} + +async function askFile(defaultFile: string | undefined): Promise { + return text({ + message: "Path to the exported user file (JSON or CSV)", + default: defaultFile, + validate: (value) => { + const file = value?.trim(); + if (!file) return "A file path is required"; + if (!fileExists(file)) return `File not found: ${file}`; + if (!getFileType(file)) return "Provide a .json or .csv file"; + return undefined; + }, + }); +} + +/** + * Collects Firebase's four hash parameters. + * + * Asked as a set because a partial set produces a digest that verifies against + * nothing. Pressing enter through all four leaves the config unset, which is + * correct for an export with no password hashes. + */ +async function askFirebaseHashConfig( + saved: FirebaseHashConfig | undefined, +): Promise { + log.info( + "Firebase password hashes need the project's hash parameters. Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + ); + log.info(dimIfSaved(saved)); + + const signerKey = ( + await text({ + message: "base64 signer key (leave blank if this export has no passwords)", + default: saved?.base64_signer_key, + }) + ).trim(); + if (!signerKey) return undefined; + + const saltSeparator = ( + await text({ + message: "base64 salt separator", + default: saved?.base64_salt_separator, + validate: (value) => (value?.trim() ? undefined : "Required alongside the signer key"), + }) + ).trim(); + + const rounds = await askNumber("rounds", saved?.rounds); + const memCost = await askNumber("mem cost", saved?.mem_cost); + + return { + base64_signer_key: signerKey, + base64_salt_separator: saltSeparator, + rounds, + mem_cost: memCost, + }; +} + +function dimIfSaved(saved: FirebaseHashConfig | undefined): string { + return saved + ? "Saved parameters found — press enter to reuse them." + : "Leave the signer key blank if this export carries no passwords."; +} + +async function askNumber(label: string, defaultValue: number | undefined): Promise { + const answer = await text({ + message: label, + default: defaultValue === undefined ? undefined : String(defaultValue), + validate: (value) => { + const parsed = Number(value?.trim()); + return Number.isInteger(parsed) && parsed > 0 ? undefined : "Enter a positive whole number"; + }, + }); + return Number(answer.trim()); +} + +/** + * Fills in whichever of transformer and file were not passed as flags. + * + * @param provided - Flags the caller already supplied; those are not asked for. + */ +export async function runWizard(provided: { + transformer?: string; + file?: string; + firebaseHashConfig?: FirebaseHashConfig; +}): Promise { + const saved = loadSettings(); + + const transformer = provided.transformer ?? (await pickTransformer(saved.key)); + const file = provided.file ?? (await askFile(saved.file)); + + let firebaseHashConfig = provided.firebaseHashConfig; + if (transformer === "firebase" && !firebaseHashConfig) { + firebaseHashConfig = await askFirebaseHashConfig(saved.firebaseHashConfig); + } + + return { transformer, file, ...(firebaseHashConfig ? { firebaseHashConfig } : {}) }; +} + +/** + * The error an agent gets instead of a prompt. + * + * Names exactly the flags that are missing, so the caller can retry without + * guessing which of the two it forgot. + */ +export function throwAgentFlagsRequired(missing: { transformer: boolean; file: boolean }): never { + const flags = [ + missing.transformer ? "--transformer " : undefined, + missing.file ? "--file " : undefined, + ].filter(Boolean); + + throwUsageError( + `\`clerk migrate\` is interactive and cannot prompt in agent mode. Pass ${flags.join(" and ")}.`, + undefined, + undefined, + [ + { + command: `clerk migrate run -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, + description: "Run non-interactively", + }, + ], + ); +} From f32b73f598f510fa362ece6fdb757f274da19346 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 12:47:58 -0400 Subject: [PATCH 002/141] test(migrate): support multiselect prompt stubs --- packages/cli-core/src/test/integration/lib/harness.ts | 8 +++++++- packages/cli-core/src/test/lib/stubs.ts | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/test/integration/lib/harness.ts b/packages/cli-core/src/test/integration/lib/harness.ts index 6d80b75cf..2ad58c8a7 100644 --- a/packages/cli-core/src/test/integration/lib/harness.ts +++ b/packages/cli-core/src/test/integration/lib/harness.ts @@ -110,7 +110,7 @@ mock.module( // ── Prompt queue (drives lib/prompts.ts and lib/listage.ts mocks) ──────────── -type PromptType = "select" | "search" | "input" | "confirm" | "password" | "editor"; +type PromptType = "select" | "search" | "input" | "confirm" | "password" | "editor" | "multiselect"; const promptQueues: Record = { select: [], @@ -119,6 +119,7 @@ const promptQueues: Record = { confirm: [], password: [], editor: [], + multiselect: [], }; function dequeuePrompt(name: PromptType) { @@ -159,6 +160,7 @@ export const mockPrompts = { input: (...responses: string[]) => promptQueues.input.push(...responses), password: (...responses: string[]) => promptQueues.password.push(...responses), editor: (...responses: string[]) => promptQueues.editor.push(...responses), + multiselect: (...responses: unknown[][]) => promptQueues.multiselect.push(...responses), }; function resetPromptQueues() { @@ -198,8 +200,12 @@ mock.module("../../../lib/listage.ts", () => ({ }, })); +// Every export of the real module must appear here: a missing one is a module +// link error at import time, not a failed prompt, so it takes down every test +// in the file the moment any command imports it. mock.module("../../../lib/prompts.ts", () => ({ confirm: dequeuePrompt("confirm"), + multiselect: dequeuePrompt("multiselect"), text: dequeuePrompt("input"), password: dequeuePrompt("password"), editor: dequeuePrompt("editor"), diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index 70598ff99..efc4e5301 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -213,9 +213,14 @@ export const gitStubs = { * Stubs for `lib/prompts.ts` — the @clack/prompts-backed wrapper. Default * responses return benign values so tests can mock the module without * configuring each prompt explicitly. + * + * Must cover every export of the real module: an omission is a module link + * error at import time, which takes down the whole test file rather than + * failing one prompt. */ export const libPromptsStubs = { confirm: async () => true, + multiselect: async () => [], text: async () => "", password: async () => "", editor: async () => "{}", From 9a56d9fb4e7d251508428bfce2c7cf56f2d3a86d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 12:52:05 -0400 Subject: [PATCH 003/141] docs(migrate): mention migration command --- CLAUDE.md | 4 +++- README.md | 1 + scripts/check-bun-version.ts | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index bd4118cba..49f96e04a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,7 @@ Default to using Bun instead of Node.js. - `Bun.serve()` supports WebSockets, HTTPS, and routes. Don't use `express`. - `bun:sqlite` for SQLite. Don't use `better-sqlite3`. - `Bun.redis` for Redis. Don't use `ioredis`. -- `Bun.sql` for Postgres. Don't use `pg` or `postgres.js`. +- `Bun.sql` for Postgres and MySQL. Don't use `pg`, `postgres.js`, or `mysql2`. - `WebSocket` is built-in. Don't use `ws`. - Prefer `Bun.file` over `node:fs`'s readFile/writeFile - Bun.$`ls` instead of execa. @@ -56,6 +56,8 @@ When running multiple test files directly with `bun test`, always pass `--isolat These flags require Bun >= 1.3.13 — older versions silently ignore them and lose isolation. `bun run test` and `bun run test:e2e` run `scripts/check-bun-version.ts` first, which fails fast when the installed Bun is older than the `engines.bun` floor in package.json. +The same floor also covers `Bun.sql`'s MySQL adapter used by the DB-backed export commands: MySQL support landed in Bun 1.2.21, but binary columns (password hashes) only decoded correctly from 1.3.6. See the header of `scripts/check-bun-version.ts`. + ## Versioning The `CLI_VERSION` global is injected at compile time via `bun build --compile --define "CLI_VERSION=..."`. Local `build:compile` omits it, so the binary reports `0.0.0-dev`. The CI release workflow injects the real version. diff --git a/README.md b/README.md index ab88d248d..0454faada 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Commands: update [options] Update the Clerk CLI to the latest version deploy Deploy a Clerk application to production webhooks Stream webhook events to a local handler and verify their signatures + migrate Migrate users into Clerk from another auth provider help [command] Display help for command bird Play Clerk Bird, a Flappy Bird game in your terminal ``` diff --git a/scripts/check-bun-version.ts b/scripts/check-bun-version.ts index a05b2d4b7..74f576892 100644 --- a/scripts/check-bun-version.ts +++ b/scripts/check-bun-version.ts @@ -9,6 +9,13 @@ * producing hundreds of order-dependent failures. Bun does not enforce * `engines.bun` at install time, so this preflight fails loudly instead. * + * A second, lower constraint rides along: the DB-backed `clerk migrate export` + * commands read MySQL through `Bun.sql` rather than `mysql2`. Verified against + * MySQL 8.4 -- the adapter landed in Bun 1.2.21, but VARBINARY/BLOB columns + * came back as lossily decoded strings until 1.3.6, which would silently + * corrupt exported password hashes. The 1.3.13 floor above already covers it; + * do not drop below 1.3.6 if the `--parallel` requirement ever goes away. + * * Usage: * bun run scripts/check-bun-version.ts */ From 080eb66cda435731c2178b3033523e48ad199907 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 12:52:53 -0400 Subject: [PATCH 004/141] docs(changeset): add migrate changeset --- .changeset/migrate-cli.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/migrate-cli.md diff --git a/.changeset/migrate-cli.md b/.changeset/migrate-cli.md new file mode 100644 index 000000000..42864a475 --- /dev/null +++ b/.changeset/migrate-cli.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Add `clerk migrate` for importing users, exporting from supported auth providers, reviewing migration logs, undoing a migration, and extending imports with custom transformers. From d94307e2dd0853886c4347b92a524c08c58a188f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 15:52:29 -0400 Subject: [PATCH 005/141] refactor(migrate): keep migration state in the CLI config, not a cwd `.settings` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clerk migrate run` wrote a `.settings` file into the working directory to remember what it last imported. The CLI cannot gitignore that on the user's behalf, so it lands inside the repository being migrated — and it carried the Firebase signer key, a secret, as plaintext JSON. That state now lives in the `migrations` section of the CLI's own config file, keyed by project through `getProjectKey()` (linked profile, then git remote, then directory). This is the shape `clerk webhooks listen` already uses for its relay token, so `migrations` sits beside `relay` with the same accessor pair. The Firebase hash parameters are dropped from persistence rather than moved: remembering a secret writes it to disk wherever the file lives. They now fall back to `CLERK_FIREBASE_SIGNER_KEY`, `CLERK_FIREBASE_SALT_SEPARATOR`, `CLERK_FIREBASE_ROUNDS` and `CLERK_FIREBASE_MEM_COST`, so a repeat run still need not re-type four flags, and `.env.local` is already gitignored. No migration path for existing `.settings` files: `clerk migrate` is unreleased, so nothing in the wild has one. --- .../cli-core/src/commands/migrate/README.md | 36 ++++++-- .../src/commands/migrate/delete.test.ts | 40 +++++---- .../cli-core/src/commands/migrate/delete.ts | 21 ++--- .../src/commands/migrate/lib/settings.test.ts | 60 ++++++++++--- .../src/commands/migrate/lib/settings.ts | 56 ++++++------ .../commands/migrate/run-interactive.test.ts | 14 ++- .../cli-core/src/commands/migrate/run.test.ts | 87 ++++++++++++++----- packages/cli-core/src/commands/migrate/run.ts | 53 +++++++---- .../commands/migrate/transformers/firebase.ts | 3 +- .../cli-core/src/commands/migrate/types.ts | 14 --- .../src/commands/migrate/wizard.test.ts | 30 +++---- .../cli-core/src/commands/migrate/wizard.ts | 40 ++++----- packages/cli-core/src/lib/config.test.ts | 39 +++++++++ packages/cli-core/src/lib/config.ts | 43 ++++++++- packages/cli-core/src/test/lib/stubs.ts | 3 + 15 files changed, 362 insertions(+), 177 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index f18e38223..17eecf60b 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -35,9 +35,10 @@ clerk migrate ``` It picks the transformer from a list built off the registry, asks for the file, -collects Firebase's hash parameters when they are needed, and pre-fills every -answer from the last run's `.settings` so a repeat migration is mostly pressing -enter. Anything already passed as a flag is not asked for. +collects Firebase's hash parameters when they are needed, and pre-fills the +platform and file from the last run so a repeat migration is mostly pressing +enter. Anything already passed as a flag is not asked for. Firebase's hash +parameters are never pre-filled — see [below](#--firebase--firebase). Then it prints the [Migration Readiness report](#migration-readiness-report) and waits for confirmation. Declining writes nothing to Clerk. @@ -312,8 +313,8 @@ destroys data **in Clerk**, and is worth keeping short and prominent. (Contrast #### What it will and will not touch -`.settings` is the only record of what a run created, so that is what -identifies the migration being undone. Without it the command fails and +The saved migration record is the only account of what a run created, so that +is what identifies the migration being undone. Without it the command fails and explains — deleting nothing silently would look like a successful undo. Users are found with `GET /v1/users?external_id=…`, 100 IDs per request. Only a @@ -525,7 +526,21 @@ clerk migrate run -y -t firebase -f users.json \ All four are **required as a set** — supplying some but not all is a usage error naming what is missing. A partial set produces a well-formed digest that verifies against nothing, so users would import successfully and then be unable -to sign in. They are saved to `.settings` and reused on the next run. +to sign in. + +They are **never saved**: the signer key is a Firebase secret, and remembering +it would mean writing it to disk in plaintext. To avoid re-passing all four on +every run, set them in the environment (`.env.local` is already gitignored): + +| Variable | Flag | +| ------------------------------- | --------------------------- | +| `CLERK_FIREBASE_SIGNER_KEY` | `--firebase-signer-key` | +| `CLERK_FIREBASE_SALT_SEPARATOR` | `--firebase-salt-separator` | +| `CLERK_FIREBASE_ROUNDS` | `--firebase-rounds` | +| `CLERK_FIREBASE_MEM_COST` | `--firebase-mem-cost` | + +Flags win over the environment, and the two can be mixed as long as all four +end up supplied. An export with no password hashes needs no parameters at all. @@ -666,10 +681,13 @@ rather than "which project is linked here". | `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | | `./logs/export-.log` | NDJSON: one line per exported user | | `./exports/-export.json` | The export itself, unless `--output` says otherwise | -| `./.settings` | The transformer key and file path of the last run | -`.settings` is what `migrate delete` reads to know which migration to undo, so -it is load-bearing rather than a convenience. +The transformer and file of the last run are **not** written here. They go to +the `migrations` section of the CLI's own config file (`clerk config --help` +names its location), keyed by project the same way a linked profile is. That is +what `migrate delete` reads to know which migration to undo, so it is +load-bearing rather than a convenience — and it has no business being written +into the repository being migrated. Log writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. Use the last diff --git a/packages/cli-core/src/commands/migrate/delete.test.ts b/packages/cli-core/src/commands/migrate/delete.test.ts index 5336e2312..ef110ed00 100644 --- a/packages/cli-core/src/commands/migrate/delete.test.ts +++ b/packages/cli-core/src/commands/migrate/delete.test.ts @@ -2,6 +2,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } fr import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { _setConfigDir } from "../../lib/config.ts"; import { CliError } from "../../lib/errors.ts"; import { useCaptureLog } from "../../test/lib/stubs.ts"; import { @@ -22,6 +23,7 @@ const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurr const DATE_TIME = "2026-01-01T00:00:00"; let workDir: string; +let configDir: string; let originalCwd: string; let originalFetch: typeof globalThis.fetch; let requests: { method: string; url: string }[]; @@ -35,19 +37,23 @@ beforeAll(() => { originalCwd = process.cwd(); originalFetch = globalThis.fetch; workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-delete-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-delete-config-")); + _setConfigDir(configDir); process.chdir(workDir); }); afterAll(() => { globalThis.fetch = originalFetch; + _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); }); beforeEach(() => { requests = []; fs.rmSync(getLogDir(), { recursive: true, force: true }); - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); }); @@ -91,27 +97,27 @@ const logEntries = () => const deleteCalls = () => requests.filter((r) => r.method === "DELETE").map((r) => r.url); describe("resolveMigrationToUndo", () => { - test("reads the file and transformer from .settings", () => { - saveSettings({ key: "clerk", file: "export.json" }); - expect(resolveMigrationToUndo()).toEqual({ file: "export.json", key: "clerk" }); + test("reads the file and transformer from the saved migration", async () => { + await saveSettings({ transformer: "clerk", file: "export.json" }); + expect(await resolveMigrationToUndo()).toEqual({ file: "export.json", key: "clerk" }); }); // Deleting nothing silently would look like a successful undo. - test("explains when there is no .settings at all", () => { - expect(() => resolveMigrationToUndo()).toThrow(/no `.settings` from a previous/); + test("explains when there is no saved migration at all", async () => { + await expect(resolveMigrationToUndo()).rejects.toThrow(/no record of a previous/); }); test.each([ - ["no file", { key: "clerk" }], + ["no file", { transformer: "clerk" }], ["no transformer", { file: "export.json" }], - ])("explains when .settings has %s", (_label, settings) => { - saveSettings(settings); - expect(() => resolveMigrationToUndo()).toThrow(CliError); + ])("explains when the saved migration has %s", async (_label, settings) => { + await saveSettings(settings); + await expect(resolveMigrationToUndo()).rejects.toThrow(CliError); }); - test("explains when the migration file has since been removed", () => { - saveSettings({ key: "clerk", file: "gone.json" }); - expect(() => resolveMigrationToUndo()).toThrow(/no longer there/); + test("explains when the migration file has since been removed", async () => { + await saveSettings({ transformer: "clerk", file: "gone.json" }); + await expect(resolveMigrationToUndo()).rejects.toThrow(/no longer there/); }); }); @@ -344,8 +350,8 @@ describe("deleteMigratedUsers", () => { describe("deleteMigration", () => { const baseOptions = { yes: true, secretKey: "sk_test_x" }; - beforeEach(() => { - saveSettings({ key: "clerk", file: "export.json" }); + beforeEach(async () => { + await saveSettings({ transformer: "clerk", file: "export.json" }); }); test("deletes the users the last run created", async () => { @@ -398,8 +404,8 @@ describe("deleteMigration", () => { expect(deleteCalls()).toHaveLength(0); }); - test("fails before any API call when there is no .settings", async () => { - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + test("fails before any API call when there is no saved migration", async () => { + fs.rmSync(path.join(configDir, "config.json"), { force: true }); stubBapi({ legacy_a: "user_1" }); await expect(deleteMigration(baseOptions)).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 5cc78dbef..8b14abe55 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -64,28 +64,29 @@ export type MigratedUser = { /** * Resolves which migration is being undone. * - * `.settings` is the only record of that — this command has no independent way - * to know what a previous run created, which is why it is coupled to `run`. + * The saved migration record is the only account of that — this command has no + * independent way to know what a previous run created, which is why it is + * coupled to `run`. */ -export function resolveMigrationToUndo(): { file: string; key: string } { - const settings = loadSettings(); +export async function resolveMigrationToUndo(): Promise<{ file: string; key: string }> { + const settings = await loadSettings(); - if (!settings.file || !settings.key) { + if (!settings.file || !settings.transformer) { throw new CliError( - "No migration to undo: this directory has no `.settings` from a previous `clerk migrate run`.\n" + - "Run `clerk migrate delete` from the directory you migrated from.", + "No migration to undo: this project has no record of a previous `clerk migrate run`.\n" + + "Run `clerk migrate delete` from the project you migrated from.", { code: ERROR_CODE.FILE_NOT_FOUND }, ); } if (!fileExists(settings.file)) { throw new CliError( - `The migration file ${settings.file} named in .settings is no longer there, so the users it created cannot be identified.`, + `The migration file ${settings.file} is no longer there, so the users it created cannot be identified.`, { code: ERROR_CODE.FILE_NOT_FOUND }, ); } - return { file: settings.file, key: settings.key }; + return { file: settings.file, key: settings.transformer }; } /** @@ -261,7 +262,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); diff --git a/packages/cli-core/src/commands/migrate/lib/settings.test.ts b/packages/cli-core/src/commands/migrate/lib/settings.test.ts index 579d508df..7ad3cf9ed 100644 --- a/packages/cli-core/src/commands/migrate/lib/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/settings.test.ts @@ -1,15 +1,19 @@ -import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { afterAll, afterEach, beforeAll, beforeEach, expect, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { _setConfigDir, getMigrationEntry, getProjectKey } from "../../../lib/config.ts"; import { loadSettings, saveSettings } from "./settings.ts"; let workDir: string; +let configDir: string; let originalCwd: string; beforeAll(() => { originalCwd = process.cwd(); - workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-")); + // Realpath'd because the project key is derived from `process.cwd()`, which + // resolves the /var → /private/var symlink macOS puts in front of tmpdir. + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-"))); process.chdir(workDir); }); @@ -19,24 +23,52 @@ afterAll(() => { }); beforeEach(() => { - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-config-")); + _setConfigDir(configDir); }); -test("returns empty settings when the file is absent", () => { - expect(loadSettings()).toEqual({}); +afterEach(() => { + _setConfigDir(undefined); + fs.rmSync(configDir, { recursive: true, force: true }); }); -test("round-trips the transformer key and file path", () => { - saveSettings({ key: "clerk", file: "users.json" }); - expect(loadSettings()).toEqual({ key: "clerk", file: "users.json" }); +test("returns empty settings when nothing was saved", async () => { + expect(await loadSettings()).toEqual({}); }); -test("writes to the current working directory", () => { - saveSettings({ key: "clerk" }); - expect(fs.existsSync(path.join(workDir, ".settings"))).toBe(true); +test("round-trips the transformer key and file path", async () => { + await saveSettings({ transformer: "clerk", file: "users.json" }); + expect(await loadSettings()).toEqual({ transformer: "clerk", file: "users.json" }); }); -test("treats a corrupt settings file as empty rather than failing the run", () => { - fs.writeFileSync(path.join(workDir, ".settings"), "{not json"); - expect(loadSettings()).toEqual({}); +test("writes to the CLI config file, not the working directory", async () => { + await saveSettings({ transformer: "clerk" }); + + expect(fs.existsSync(path.join(workDir, ".settings"))).toBe(false); + const config = JSON.parse(fs.readFileSync(path.join(configDir, "config.json"), "utf-8")); + expect(config.migrations).toEqual({ [await getProjectKey(workDir)]: { transformer: "clerk" } }); +}); + +test("keys the record by project, so another directory does not see it", async () => { + await saveSettings({ transformer: "clerk", file: "users.json" }); + + const elsewhere = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-other-"))); + try { + expect(await getMigrationEntry(await getProjectKey(elsewhere))).toBeUndefined(); + } finally { + fs.rmSync(elsewhere, { recursive: true, force: true }); + } +}); + +test("treats a corrupt config file as empty rather than failing the run", async () => { + fs.writeFileSync(path.join(configDir, "config.json"), "{not json"); + expect(await loadSettings()).toEqual({}); +}); + +test("leaves the run standing when the config cannot be written", async () => { + fs.rmSync(configDir, { recursive: true, force: true }); + fs.writeFileSync(configDir, "not a directory"); + + await saveSettings({ transformer: "clerk" }); + expect(await loadSettings()).toEqual({}); }); diff --git a/packages/cli-core/src/commands/migrate/lib/settings.ts b/packages/cli-core/src/commands/migrate/lib/settings.ts index fc9dfcd7e..958bf1cd2 100644 --- a/packages/cli-core/src/commands/migrate/lib/settings.ts +++ b/packages/cli-core/src/commands/migrate/lib/settings.ts @@ -1,43 +1,39 @@ /** - * The cwd-relative `.settings` file: what this directory last migrated, and - * with which transformer. + * What this project last migrated, and with which transformer. * - * Ported from the standalone migration-tool's `src/lib/settings.ts`. Kept out - * of `~/.config/clerk/config.json` on purpose — that file is keyed by linked - * project identity, not by "which export file am I working through". + * Kept in the CLI's own config file under `migrations`, keyed by project — the + * same shape `clerk webhooks listen` files its relay token under. An earlier + * version wrote a `.settings` file into the user's cwd instead, which the CLI + * cannot gitignore on the user's behalf and which put migration state inside + * the repository being migrated. * - * Both halves fail silently: a missing, unreadable or unwritable `.settings` - * only costs the user a remembered default. + * Both halves fail silently: an unreadable or unwritable config only costs the + * user a remembered default, so it must not take the run down with it. */ -import fs from "node:fs"; -import path from "node:path"; -import type { Settings } from "../types.ts"; +import { + getMigrationEntry, + getProjectKey, + setMigrationEntry, + type MigrationEntry, +} from "../../../lib/config.ts"; +import { log } from "../../../lib/log.ts"; -const SETTINGS_FILE = ".settings"; - -function settingsPath(): string { - return path.join(process.cwd(), SETTINGS_FILE); -} - -/** Reads saved settings, or `{}` when absent or corrupt. */ -export function loadSettings(): Settings { +/** Reads saved settings, or `{}` when absent or unreadable. */ +export async function loadSettings(): Promise { try { - const file = settingsPath(); - if (fs.existsSync(file)) { - return JSON.parse(fs.readFileSync(file, "utf-8")) as Settings; - } - } catch { - // Corrupt or unreadable settings are indistinguishable from none. + return (await getMigrationEntry(await getProjectKey(process.cwd()))) ?? {}; + } catch (error) { + log.debug(`config: could not read migration settings — ${error}`); + return {}; } - return {}; } -/** Persists settings for the next run in this directory. */ -export function saveSettings(settings: Settings): void { +/** Persists settings for the next run in this project. */ +export async function saveSettings(settings: MigrationEntry): Promise { try { - fs.writeFileSync(settingsPath(), JSON.stringify(settings, null, 2)); - } catch { - // Read-only cwd; the run itself is unaffected. + await setMigrationEntry(await getProjectKey(process.cwd()), settings); + } catch (error) { + log.debug(`config: could not save migration settings — ${error}`); } } diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index ef518506c..c5355da20 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -40,10 +40,12 @@ const { run } = await import("./run.ts"); const { deleteMigration } = await import("./delete.ts"); const { UserAbortError } = await import("../../lib/errors.ts"); const { loadSettings, saveSettings } = await import("./lib/settings.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); const captured = useCaptureLog(); let workDir: string; +let configDir: string; let originalCwd: string; let originalFetch: typeof globalThis.fetch; let requests: { method: string; url: string; body: unknown }[]; @@ -61,14 +63,18 @@ beforeAll(() => { originalCwd = process.cwd(); originalFetch = globalThis.fetch; workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-config-")); + _setConfigDir(configDir); process.chdir(workDir); }); afterAll(() => { setMode(originalMode); globalThis.fetch = originalFetch; + _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); }); beforeEach(() => { @@ -79,7 +85,7 @@ beforeEach(() => { mockSelect.mockResolvedValue("clerk"); mockText.mockResolvedValue("export.json"); fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); }); @@ -137,7 +143,7 @@ describe("the wizard fills in missing flags", () => { test("records the wizard's answers for the next run", async () => { await run({ secretKey: "sk_test_x" }); - expect(loadSettings()).toMatchObject({ key: "clerk", file: "export.json" }); + expect(await loadSettings()).toMatchObject({ transformer: "clerk", file: "export.json" }); }); }); @@ -261,8 +267,8 @@ describe("migrate delete confirmation", () => { const deleted = () => requests.filter((r) => r.method === "DELETE"); - beforeEach(() => { - saveSettings({ key: "clerk", file: "export.json" }); + beforeEach(async () => { + await saveSettings({ transformer: "clerk", file: "export.json" }); stubDeleteTargets({ legacy_a: "user_1", legacy_b: "user_2" }); fs.writeFileSync( path.join(workDir, "export.json"), diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index e5408e422..ff9ffbbae 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2,15 +2,17 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } fr import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { _setConfigDir } from "../../lib/config.ts"; import { CliError } from "../../lib/errors.ts"; import { useCaptureLog } from "../../test/lib/stubs.ts"; import { getLogDir } from "./lib/logger.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; import { loadSettings } from "./lib/settings.ts"; import { applyResumeAfter, resolveFirebaseHashConfig, run, validateRunOptions } from "./run.ts"; -import type { FirebaseHashConfig, User } from "./types.ts"; +import type { User } from "./types.ts"; let workDir: string; +let configDir: string; let originalCwd: string; const users = (...ids: string[]): User[] => ids.map((userId) => ({ userId }) as User); @@ -18,14 +20,18 @@ const users = (...ids: string[]): User[] => ids.map((userId) => ({ userId }) as beforeAll(() => { originalCwd = process.cwd(); workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-config-")); + _setConfigDir(configDir); process.chdir(workDir); fs.writeFileSync(path.join(workDir, "users.json"), "[]"); fs.writeFileSync(path.join(workDir, "users.txt"), ""); }); afterAll(() => { + _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); }); describe("validateRunOptions", () => { @@ -91,27 +97,62 @@ describe("resolveFirebaseHashConfig", () => { ); }); - test("falls back to saved settings when no flag is passed", () => { - const saved: FirebaseHashConfig = { - base64_signer_key: "S", - base64_salt_separator: "B", - rounds: 8, - mem_cost: 14, + describe("environment fallback", () => { + const ENV = { + CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER", + CLERK_FIREBASE_SALT_SEPARATOR: "Bw==", + CLERK_FIREBASE_ROUNDS: "8", + CLERK_FIREBASE_MEM_COST: "14", }; - expect(resolveFirebaseHashConfig({}, saved)).toEqual(saved); - }); - test("prefers flags over saved settings", () => { - const saved: FirebaseHashConfig = { - base64_signer_key: "OLD", - base64_salt_separator: "B", - rounds: 1, - mem_cost: 1, - }; - expect(resolveFirebaseHashConfig(ALL, saved)?.base64_signer_key).toBe("SIGNER"); + afterEach(() => { + for (const name of Object.keys(ENV)) delete process.env[name]; + }); + + const setEnv = (vars: Partial) => Object.assign(process.env, vars); + + test("builds the config when no flag is passed", () => { + setEnv(ENV); + expect(resolveFirebaseHashConfig({})).toEqual({ + base64_signer_key: "ENV_SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + test("prefers a flag over the environment", () => { + setEnv(ENV); + expect(resolveFirebaseHashConfig(ALL)?.base64_signer_key).toBe("SIGNER"); + }); + + // Half from the environment and half from flags is still a complete set. + test("fills only the gaps the flags left", () => { + setEnv({ CLERK_FIREBASE_ROUNDS: "8", CLERK_FIREBASE_MEM_COST: "14" }); + expect( + resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER", firebaseSaltSeparator: "Bw==" }), + ).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + test("still demands the full set when the environment supplies only part", () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); + expect(() => resolveFirebaseHashConfig({})).toThrow(/--firebase-salt-separator/); + }); + + // An empty var is how a shell spells "unset", and treating it as set would + // demand the other three for a config nobody asked for. + test("ignores an empty variable", () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "" }); + expect(resolveFirebaseHashConfig({})).toBeUndefined(); + }); }); - test("returns nothing when neither flags nor settings supply a config", () => { + test("returns nothing when neither flags nor the environment supply a config", () => { expect(resolveFirebaseHashConfig({})).toBeUndefined(); }); }); @@ -157,7 +198,7 @@ describe("run", () => { requests = []; delete process.env.CLERK_MIGRATE_RATE_LIMIT; fs.rmSync(getLogDir(), { recursive: true, force: true }); - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { requests.push({ @@ -209,9 +250,9 @@ describe("run", () => { expect(entries.filter((e) => e.status === "success")).toHaveLength(2); }); - test("records the run's key and file in .settings", async () => { + test("records the run's transformer and file for the next run", async () => { await run(baseOptions); - expect(loadSettings()).toEqual({ key: "clerk", file: "export.json" }); + expect(await loadSettings()).toEqual({ transformer: "clerk", file: "export.json" }); }); test("--require-password imports only the users that have one", async () => { @@ -738,12 +779,12 @@ describe("run", () => { expect(captured.err).toContain("only applies to supabase"); }); - test("records the flag in .settings", async () => { + test("records the flag for the next run", async () => { stubInstance({ oauth_discord: { enabled: true } }); await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); - expect(loadSettings().skipUnsupportedProviders).toBe(true); + expect((await loadSettings()).skipUnsupportedProviders).toBe(true); }); }); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 0d518fb3c..cd1e1934f 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -27,7 +27,7 @@ import { import { buildReadinessReport, formatReadinessReport } from "./lib/readiness.ts"; import { DEV_USER_LIMIT, resolveLimits } from "./lib/instance.ts"; import { getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; -import { loadSettings, saveSettings } from "./lib/settings.ts"; +import { saveSettings } from "./lib/settings.ts"; import { countSocialProviders, findDisabledProviders, @@ -62,15 +62,38 @@ export type MigrateRunOptions = { }; const FIREBASE_FLAGS = [ - ["firebaseSignerKey", "--firebase-signer-key"], - ["firebaseSaltSeparator", "--firebase-salt-separator"], - ["firebaseRounds", "--firebase-rounds"], - ["firebaseMemCost", "--firebase-mem-cost"], + ["firebaseSignerKey", "--firebase-signer-key", "CLERK_FIREBASE_SIGNER_KEY"], + ["firebaseSaltSeparator", "--firebase-salt-separator", "CLERK_FIREBASE_SALT_SEPARATOR"], + ["firebaseRounds", "--firebase-rounds", "CLERK_FIREBASE_ROUNDS"], + ["firebaseMemCost", "--firebase-mem-cost", "CLERK_FIREBASE_MEM_COST"], ] as const; +const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); + /** - * Resolves Firebase's four hash parameters from flags, falling back to - * `.settings` when none were passed. + * Overlays the `CLERK_FIREBASE_*` environment variables onto whichever flags + * were not passed. + * + * The signer key is a Firebase secret, so it is read rather than stored: the + * CLI never persists these, and `.env.local` is already gitignored and already + * where the CLI keeps a project's local secrets. + */ +function withFirebaseEnv(options: MigrateRunOptions): MigrateRunOptions { + const merged = { ...options }; + for (const [key, , envVar] of FIREBASE_FLAGS) { + if (merged[key] !== undefined) continue; + const value = process.env[envVar]; + if (value === undefined || value.trim() === "") continue; + // A non-numeric round count is left to fail the flag's own validation + // rather than silently becoming NaN. + (merged as Record)[key] = FIREBASE_NUMERIC.has(key) ? Number(value) : value; + } + return merged; +} + +/** + * Resolves Firebase's four hash parameters from flags, falling back to the + * `CLERK_FIREBASE_*` environment variables. * * The four are required as a set: a digest built from a partial set is * well-formed but verifies against nothing, so every migrated user would fail @@ -80,12 +103,12 @@ const FIREBASE_FLAGS = [ * for an export that carries no password hashes. */ export function resolveFirebaseHashConfig( - options: MigrateRunOptions, - saved?: FirebaseHashConfig, + rawOptions: MigrateRunOptions, ): FirebaseHashConfig | undefined { + const options = withFirebaseEnv(rawOptions); const provided = FIREBASE_FLAGS.filter(([key]) => options[key] !== undefined); - if (provided.length === 0) return saved; + if (provided.length === 0) return undefined; if (provided.length < FIREBASE_FLAGS.length) { const missing = FIREBASE_FLAGS.filter(([key]) => options[key] === undefined).map( @@ -415,8 +438,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const secretKeyOption = options.secretKey ?? options.clerkSecretKey; const { transformer, file } = validateRunOptions(options); - const saved = loadSettings(); - const firebaseHashConfig = resolveFirebaseHashConfig(options, saved.firebaseHashConfig); + const firebaseHashConfig = resolveFirebaseHashConfig(options); await withGutter("Migrating users to Clerk", async () => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); @@ -490,11 +512,12 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (!proceed) throwUserAbort(); } - saveSettings({ - key: transformer, + // The Firebase hash parameters are deliberately not among these: the signer + // key is a secret, and remembering it would write it to disk in plaintext. + await saveSettings({ + transformer, file, ...(options.skipUnsupportedProviders ? { skipUnsupportedProviders: true } : {}), - ...(firebaseHashConfig ? { firebaseHashConfig } : {}), }); const summary = await withSpinner( diff --git a/packages/cli-core/src/commands/migrate/transformers/firebase.ts b/packages/cli-core/src/commands/migrate/transformers/firebase.ts index 73966c1b2..9c9d09c6e 100644 --- a/packages/cli-core/src/commands/migrate/transformers/firebase.ts +++ b/packages/cli-core/src/commands/migrate/transformers/firebase.ts @@ -25,7 +25,8 @@ const FIREBASE_CSV_HEADERS = * * Firebase's scrypt is a modified variant, so Clerk needs the project's four * hash parameters alongside each digest. They arrive on the run's - * {@link TransformContext} from `--firebase-*` flags or saved `.settings`. + * {@link TransformContext} from `--firebase-*` flags or the matching + * `CLERK_FIREBASE_*` environment variables; they are never persisted. * * See https://clerk.com/docs/guides/development/migrating/firebase */ diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts index 40ba3a5ec..904267037 100644 --- a/packages/cli-core/src/commands/migrate/types.ts +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -117,20 +117,6 @@ export type ImportSummary = { errorBreakdown: Map; }; -/** - * Per-directory migration state, persisted to a cwd-relative `.settings` file. - * - * Deliberately not routed through `~/.config/clerk/config.json`: that file is - * keyed by linked-project identity, which is a different concept from "which - * file did I last migrate with". - */ -export type Settings = { - key?: string; - file?: string; - skipUnsupportedProviders?: boolean; - firebaseHashConfig?: FirebaseHashConfig; -}; - /** * Firebase's scrypt parameters, needed to rebuild a password hash Clerk can * verify. diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts index f3633665d..e913a0064 100644 --- a/packages/cli-core/src/commands/migrate/wizard.test.ts +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -28,27 +28,33 @@ mock.module("../../lib/prompts.ts", () => ({ const { runWizard, throwAgentFlagsRequired } = await import("./wizard.ts"); const { saveSettings } = await import("./lib/settings.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); let workDir: string; +let configDir: string; let originalCwd: string; beforeAll(() => { originalCwd = process.cwd(); workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-config-")); + _setConfigDir(configDir); process.chdir(workDir); fs.writeFileSync(path.join(workDir, "users.json"), "[]"); fs.writeFileSync(path.join(workDir, "other.csv"), ""); }); afterAll(() => { + _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); }); beforeEach(() => { mockSelect.mockReset(); mockText.mockReset(); - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); }); /** The config object the wizard passed to its Nth `text`/`select` prompt. */ @@ -93,7 +99,7 @@ describe("transformer picker", () => { describe("defaults from the previous run", () => { test("pre-selects the last transformer and pre-fills the last file", async () => { - saveSettings({ key: "supabase", file: "other.csv" }); + await saveSettings({ transformer: "supabase", file: "other.csv" }); mockSelect.mockResolvedValue("supabase"); mockText.mockResolvedValue("other.csv"); @@ -116,7 +122,7 @@ describe("defaults from the previous run", () => { // A saved key from a build that has since dropped that transformer would // otherwise pre-select a value the picker cannot offer. test("ignores a saved transformer that is no longer registered", async () => { - saveSettings({ key: "okta" }); + await saveSettings({ transformer: "okta" }); mockSelect.mockResolvedValue("clerk"); mockText.mockResolvedValue("users.json"); @@ -187,27 +193,21 @@ describe("firebase hash parameters", () => { expect(mockText).toHaveBeenCalledTimes(2); }); - test("are pre-filled from the previous run", async () => { - saveSettings({ - firebaseHashConfig: { - base64_signer_key: "SAVED", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }, - }); + // The signer key is a Firebase secret, so it is never written to disk and so + // there is nothing to offer back. A repeat run passes it as a flag or env var. + test("are never pre-filled, because they are not saved", async () => { mockSelect.mockResolvedValue("firebase"); mockText .mockResolvedValueOnce("users.json") - .mockResolvedValueOnce("SAVED") + .mockResolvedValueOnce("SIGNER") .mockResolvedValueOnce("Bw==") .mockResolvedValueOnce("8") .mockResolvedValueOnce("14"); await runWizard({}); - expect(textCall(1)?.default).toBe("SAVED"); - expect(textCall(3)?.default).toBe("8"); + expect(textCall(1)?.default).toBeUndefined(); + expect(textCall(3)?.default).toBeUndefined(); }); test("are not asked for on a non-firebase transformer", async () => { diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index 3082a1b63..d5b556edb 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -2,8 +2,9 @@ * The interactive path behind a bare `clerk migrate`. * * Ported from the standalone migration-tool's `src/migrate/cli.ts` interactive - * flow. Every answer is pre-filled from the previous run's `.settings`, so a - * repeat migration is mostly pressing enter. + * flow. The platform and file are pre-filled from the previous run, so a repeat + * migration is mostly pressing enter. Firebase's hash parameters are not: the + * signer key is a secret, and the CLI does not keep those. * * Agent mode never reaches here — `run` raises a usage error naming the flags * instead, because an agent cannot answer a prompt. @@ -65,18 +66,17 @@ async function askFile(defaultFile: string | undefined): Promise { * nothing. Pressing enter through all four leaves the config unset, which is * correct for an export with no password hashes. */ -async function askFirebaseHashConfig( - saved: FirebaseHashConfig | undefined, -): Promise { +async function askFirebaseHashConfig(): Promise { log.info( "Firebase password hashes need the project's hash parameters. Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", ); - log.info(dimIfSaved(saved)); + log.info( + "Set CLERK_FIREBASE_SIGNER_KEY, CLERK_FIREBASE_SALT_SEPARATOR, CLERK_FIREBASE_ROUNDS and CLERK_FIREBASE_MEM_COST to skip these prompts on the next run.", + ); const signerKey = ( await text({ message: "base64 signer key (leave blank if this export has no passwords)", - default: saved?.base64_signer_key, }) ).trim(); if (!signerKey) return undefined; @@ -84,32 +84,21 @@ async function askFirebaseHashConfig( const saltSeparator = ( await text({ message: "base64 salt separator", - default: saved?.base64_salt_separator, validate: (value) => (value?.trim() ? undefined : "Required alongside the signer key"), }) ).trim(); - const rounds = await askNumber("rounds", saved?.rounds); - const memCost = await askNumber("mem cost", saved?.mem_cost); - return { base64_signer_key: signerKey, base64_salt_separator: saltSeparator, - rounds, - mem_cost: memCost, + rounds: await askNumber("rounds"), + mem_cost: await askNumber("mem cost"), }; } -function dimIfSaved(saved: FirebaseHashConfig | undefined): string { - return saved - ? "Saved parameters found — press enter to reuse them." - : "Leave the signer key blank if this export carries no passwords."; -} - -async function askNumber(label: string, defaultValue: number | undefined): Promise { +async function askNumber(label: string): Promise { const answer = await text({ message: label, - default: defaultValue === undefined ? undefined : String(defaultValue), validate: (value) => { const parsed = Number(value?.trim()); return Number.isInteger(parsed) && parsed > 0 ? undefined : "Enter a positive whole number"; @@ -128,14 +117,17 @@ export async function runWizard(provided: { file?: string; firebaseHashConfig?: FirebaseHashConfig; }): Promise { - const saved = loadSettings(); + const saved = await loadSettings(); - const transformer = provided.transformer ?? (await pickTransformer(saved.key)); + const transformer = provided.transformer ?? (await pickTransformer(saved.transformer)); const file = provided.file ?? (await askFile(saved.file)); let firebaseHashConfig = provided.firebaseHashConfig; if (transformer === "firebase" && !firebaseHashConfig) { - firebaseHashConfig = await askFirebaseHashConfig(saved.firebaseHashConfig); + // Never prefilled: the signer key is a secret the CLI does not keep. A + // repeat run supplies it through `--firebase-*` or `CLERK_FIREBASE_*`, + // which short-circuits this prompt entirely. + firebaseHashConfig = await askFirebaseHashConfig(); } return { transformer, file, ...(firebaseHashConfig ? { firebaseHashConfig } : {}) }; diff --git a/packages/cli-core/src/lib/config.test.ts b/packages/cli-core/src/lib/config.test.ts index c8deae64a..d69ed52fe 100644 --- a/packages/cli-core/src/lib/config.test.ts +++ b/packages/cli-core/src/lib/config.test.ts @@ -11,6 +11,9 @@ const { clearAuth, getProfile, setProfile, + getMigrationEntry, + setMigrationEntry, + getProjectKey, listProfiles, resolveProfile, resolveInstanceId, @@ -83,6 +86,42 @@ describe("config", () => { expect(await getAuth()).toBeUndefined(); }); + test("setMigrationEntry and getMigrationEntry", async () => { + expect(await getMigrationEntry("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/projects/my-app")).toBeUndefined(); + await setMigrationEntry("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/projects/my-app", { transformer: "clerk", file: "users.json" }); + expect(await getMigrationEntry("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/projects/my-app")).toEqual({ + transformer: "clerk", + file: "users.json", + }); + expect(await getMigrationEntry("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/projects/other")).toBeUndefined(); + }); + + // readConfig rebuilds the document field by field, so a key it does not know + // about is dropped on the next write rather than merely ignored. + // readConfig rebuilds the document field by field, so a section it does not + // know about is dropped on the next write rather than merely ignored. + test("migrations survive a write to another section", async () => { + await setMigrationEntry("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/projects/my-app", { transformer: "clerk" }); + await setProfile("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/projects/my-app", { + workspaceId: "org_abc", + appId: "app_def", + instances: { development: "ins_ghi" }, + }); + + expect(await getMigrationEntry("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/projects/my-app")).toEqual({ transformer: "clerk" }); + }); + + test("getProjectKey prefers the linked profile's key over the directory", async () => { + expect(await getProjectKey("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/projects/unlinked")).toBe("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/projects/unlinked"); + + await setProfile("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/projects/linked", { + workspaceId: "org_abc", + appId: "app_def", + instances: { development: "ins_ghi" }, + }); + expect(await getProjectKey("/projects/linked/src")).toBe("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/projects/linked"); + }); + test("setProfile and getProfile", async () => { const profile = { workspaceId: "org_abc", diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index c0bccb29f..8139d9ca0 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -50,11 +50,19 @@ interface RelayEntry { token: string; } +/** What `clerk migrate run` last imported for a project, and how. */ +interface MigrationEntry { + transformer?: string; + file?: string; + skipUnsupportedProviders?: boolean; +} + interface ClerkConfig { environment?: string; auth?: Record; profiles: Record; relay?: Record; + migrations?: Record; } function defaultConfig(): ClerkConfig { @@ -86,6 +94,13 @@ function migrateRawConfig(raw: Record): ClerkConfig { config.relay = relay; } + // Not validated per entry the way `relay` is: every field is optional, so + // there is no key whose absence marks an entry as junk. A malformed one costs + // a remembered default, not a failed run. + if (raw.migrations && typeof raw.migrations === "object" && !Array.isArray(raw.migrations)) { + config.migrations = raw.migrations as Record; + } + if (raw.auth && typeof raw.auth === "object") { const auth = raw.auth as Record; if (typeof auth.userId === "string") { @@ -207,6 +222,18 @@ export async function setRelayEntry(key: string, entry: RelayEntry): Promise { + const config = await readConfig(); + return config.migrations?.[key]; +} + +export async function setMigrationEntry(key: string, entry: MigrationEntry): Promise { + const config = await readConfig(); + if (!config.migrations) config.migrations = {}; + config.migrations[key] = entry; + await writeConfig(config); +} + type ResolvedVia = "remote" | "git-common-dir" | "directory"; export async function resolveProfile(cwd: string): Promise< @@ -258,6 +285,20 @@ export async function resolveProfile(cwd: string): Promise< return undefined; } +/** + * The key a per-project record (e.g. `migrations`) is filed under. + * + * Prefers the linked profile's own key so the record sits beside the profile it + * belongs to, and so it survives `clerk link` being re-run from a subdirectory. + * Falls back to the git remote and then the directory, because a project that + * has never been linked still deserves to be remembered. + */ +export async function getProjectKey(cwd: string): Promise { + const resolved = await resolveProfile(cwd); + if (resolved) return resolved.path; + return (await getGitNormalizedRemote(cwd)) ?? cwd; +} + const INSTANCE_ALIASES: Record = { dev: "development", development: "development", @@ -397,4 +438,4 @@ export async function resolveAppContext( }; } -export type { Auth, Profile, ClerkConfig, AppContextOptions }; +export type { Auth, Profile, ClerkConfig, MigrationEntry, AppContextOptions }; diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index efc4e5301..89edb89d1 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -146,6 +146,9 @@ export const configStubs = { listProfiles: noop, getRelayEntry: noop, setRelayEntry: noop, + getMigrationEntry: noop, + setMigrationEntry: noop, + getProjectKey: async () => "", resolveProfile: noop, resolveProfileOrAutolink: noop, resolveInstanceId: () => ({ id: "", label: "" }), From 2f22633d887473ca56a239ee2964410638050f86 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 16:40:04 -0400 Subject: [PATCH 006/141] feat(migrate): add `clerk migrate settings`, and resolve migration env vars the way the secret key does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related gaps. **The env vars did not work from a file.** `CLERK_FIREBASE_*`, `AUTH0_*` and the `*_DB_URL` vars were read straight off `process.env`, which the shipped binary never populates from a `.env` file — it is compiled with `--no-compile-autoload-dotenv`. Only an exported shell variable reached them. The secret key avoided this by parsing the project's env files itself. That lookup is now shared: `findKeyInProject` moves out of `keyless-target.ts` into `lib/dotenv.ts` as `findEnvValue`, and every migration value goes through it. Each resolution reports its source, so `--verbose` names the file a value came from. Left on `process.env`: `CLERK_MIGRATE_RATE_LIMIT`, `CLERK_MIGRATE_CONCURRENCY_LIMIT` and `FIREBASE_AUTH_EMULATOR_HOST` — runtime knobs rather than project config, and `resolveLimits` is sync on a hot path. **There was no way to see or change what a run would pick up.** `clerk migrate settings` lists every setting with its value and its source, `set` changes one, `clear` forgets them. Values are split by what they are, not by which command wrote them: project state to the CLI config, credentials to `.env.clerk-migrate`. That file is the migration's own rather than the app's `.env.local`, because a Firebase signer key is of no use to the application being migrated and does not belong in the file its developers read daily. It is added to `.gitignore` on creation — reusing `ensureGitignoreEntry`, promoted from `keyless.ts` to `lib/git.ts` for the second caller — and deleted when `clear` removes its last value. Credentials are redacted everywhere they are shown, `--json` included. --- .../cli-core/src/commands/migrate/README.md | 70 ++++++- .../src/commands/migrate/export/auth0.test.ts | 18 +- .../src/commands/migrate/export/auth0.ts | 11 +- .../migrate/export/db-exports.test.ts | 34 +++- .../src/commands/migrate/export/db-options.ts | 5 +- .../cli-core/src/commands/migrate/index.ts | 2 + .../src/commands/migrate/lib/env-file.test.ts | 121 ++++++++++++ .../src/commands/migrate/lib/env-file.ts | 123 +++++++++++++ .../cli-core/src/commands/migrate/run.test.ts | 39 ++-- packages/cli-core/src/commands/migrate/run.ts | 33 ++-- .../src/commands/migrate/settings/clear.ts | 58 ++++++ .../src/commands/migrate/settings/index.ts | 75 ++++++++ .../src/commands/migrate/settings/list.ts | 87 +++++++++ .../src/commands/migrate/settings/registry.ts | 111 +++++++++++ .../src/commands/migrate/settings/set.ts | 48 +++++ .../migrate/settings/settings.test.ts | 172 ++++++++++++++++++ packages/cli-core/src/lib/dotenv.ts | 68 +++++++ packages/cli-core/src/lib/git.ts | 21 ++- packages/cli-core/src/lib/keyless-target.ts | 47 +---- packages/cli-core/src/lib/keyless.ts | 13 +- .../src/test/integration/lib/harness.ts | 1 + 21 files changed, 1038 insertions(+), 119 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/env-file.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/env-file.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/clear.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/index.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/list.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/registry.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/set.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/settings.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 17eecf60b..1e479a8f8 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -441,6 +441,54 @@ clerk migrate transformers list --transformer-file ./my-transformer.ts `--json` gives an agent the same data, including which source field each transformer maps to `userId`. +### `clerk migrate settings` + +What a run in this directory would pick up, and where each value comes from. + +```sh +clerk migrate settings # list +clerk migrate settings set transformer firebase +clerk migrate settings set firebase-signer-key abc123… +clerk migrate settings clear -y +``` + +``` +SETTING VALUE SOURCE +transformer firebase clerk config +file ./users.json clerk config +firebase-signer-key aVer…3456 .env.clerk-migrate +firebase-rounds — not set +``` + +The source column is the point. A migration reads from flags, the environment, +two of the app's env files and the CLI's config, so when a run picks up a stale +value the question is never "what is it" but "which of those won". + +| Command | Description | +| ----------------------------- | ----------------------------------------------------- | +| `settings` / `settings list` | Show every setting, its value and its source | +| `settings list --json` | The same, machine-readable | +| `settings set ` | Change one setting | +| `settings clear [-y]` | Forget this project's settings and delete its secrets | + +#### Where each setting is kept + +Two stores, split by what the value **is** rather than by which command wrote it: + +| Store | Holds | Why | +| -------------------- | --------------------------------------------------- | ---------------------------------------------------------------- | +| CLI config | `transformer`, `file`, `skip-unsupported-providers` | Project state, not secret, useless outside the CLI | +| `.env.clerk-migrate` | `firebase-*` | Credentials: gitignored on write, and hand-editable for rotation | + +`.env.clerk-migrate` is the migration's own file rather than the app's +`.env.local`, because a Firebase signer key is of no use to the application +being migrated and does not belong in the file its developers read daily. The +CLI adds it to `.gitignore` the first time it writes it, and deletes it when +`settings clear` removes the last value. + +Credentials are redacted wherever they are displayed, including under `--json`, +so the output is safe to paste into an issue. + ### Custom transformers (`--transformer-file`) Migrating from a platform with no built-in, without recompiling the CLI: @@ -528,9 +576,10 @@ naming what is missing. A partial set produces a well-formed digest that verifies against nothing, so users would import successfully and then be unable to sign in. -They are **never saved**: the signer key is a Firebase secret, and remembering -it would mean writing it to disk in plaintext. To avoid re-passing all four on -every run, set them in the environment (`.env.local` is already gitignored): +They never go into the CLI's config: the signer key is a Firebase secret, and +that file is not a secret store. To avoid re-passing all four on every run, set +them once with [`clerk migrate settings`](#clerk-migrate-settings), or export +them yourself: | Variable | Flag | | ------------------------------- | --------------------------- | @@ -539,8 +588,9 @@ every run, set them in the environment (`.env.local` is already gitignored): | `CLERK_FIREBASE_ROUNDS` | `--firebase-rounds` | | `CLERK_FIREBASE_MEM_COST` | `--firebase-mem-cost` | -Flags win over the environment, and the two can be mixed as long as all four -end up supplied. +Resolution order is flag, then exported variable, then `.env.clerk-migrate`, +then the app's `.env.local`/`.env`. The sources can be mixed as long as all four +end up supplied. Run with `--verbose` to see which one each came from. An export with no password hashes needs no parameters at all. @@ -681,13 +731,13 @@ rather than "which project is linked here". | `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | | `./logs/export-.log` | NDJSON: one line per exported user | | `./exports/-export.json` | The export itself, unless `--output` says otherwise | +| `./.env.clerk-migrate` | Migration credentials, written by `settings set` and gitignored | The transformer and file of the last run are **not** written here. They go to -the `migrations` section of the CLI's own config file (`clerk config --help` -names its location), keyed by project the same way a linked profile is. That is -what `migrate delete` reads to know which migration to undo, so it is -load-bearing rather than a convenience — and it has no business being written -into the repository being migrated. +the `migrations` section of the CLI's own config file, keyed by project the +same way a linked profile is. That is what `migrate delete` reads to know which +migration to undo, so it is load-bearing rather than a convenience — and it has +no business being written into the repository being migrated. Log writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. Use the last diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index b48b89819..58ce5add2 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -15,6 +15,9 @@ import { resolveAuth0Credentials, } from "./auth0.ts"; +/** A cwd with no `.env` files, so these tests exercise only the injected env. */ +const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); + const captured = useCaptureLog(); const CREDENTIALS = { domain: "t.auth0.com", clientId: "cid", clientSecret: "csec" }; @@ -88,22 +91,25 @@ describe("resolveAuth0Credentials", () => { test("prefers flags", async () => { const resolved = await resolveAuth0Credentials( { domain: "flag.auth0.com", clientId: "f", clientSecret: "s" }, + NO_ENV_FILES, { AUTH0_DOMAIN: "env.auth0.com" }, ); expect(resolved.domain).toBe("flag.auth0.com"); }); test("falls back to the environment", async () => { - const resolved = await resolveAuth0Credentials( - {}, - { AUTH0_DOMAIN: "env.auth0.com", AUTH0_CLIENT_ID: "e", AUTH0_CLIENT_SECRET: "s" }, - ); + const resolved = await resolveAuth0Credentials({}, NO_ENV_FILES, { + AUTH0_DOMAIN: "env.auth0.com", + AUTH0_CLIENT_ID: "e", + AUTH0_CLIENT_SECRET: "s", + }); expect(resolved).toEqual({ domain: "env.auth0.com", clientId: "e", clientSecret: "s" }); }); test("normalizes a domain that came with a scheme", async () => { const resolved = await resolveAuth0Credentials( { domain: "https://t.auth0.com/", clientId: "c", clientSecret: "s" }, + NO_ENV_FILES, {}, ); expect(resolved.domain).toBe("t.auth0.com"); @@ -111,14 +117,14 @@ describe("resolveAuth0Credentials", () => { // Tests run non-TTY, the same signal an agent gives. test("names every missing credential at once rather than one at a time", async () => { - await expect(resolveAuth0Credentials({}, {})).rejects.toThrow( + await expect(resolveAuth0Credentials({}, NO_ENV_FILES, {})).rejects.toThrow( /--domain \(or AUTH0_DOMAIN\), --client-id \(or AUTH0_CLIENT_ID\), --client-secret \(or AUTH0_CLIENT_SECRET\)/, ); }); test("names only what is actually missing", async () => { await expect( - resolveAuth0Credentials({ domain: "t.auth0.com", clientId: "c" }, {}), + resolveAuth0Credentials({ domain: "t.auth0.com", clientId: "c" }, NO_ENV_FILES, {}), ).rejects.toThrow(/Missing: --client-secret \(or AUTH0_CLIENT_SECRET\)\./); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 0e94933b5..3d90d661c 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -21,6 +21,7 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt, text } from "../../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; @@ -64,12 +65,16 @@ export function normalizeAuth0Domain(domain: string): string { */ export async function resolveAuth0Credentials( options: ExportAuth0Options, + cwd: string = process.cwd(), env: Record = process.env, ): Promise { + const fromEnv = async (name: string): Promise => + (await findMigrateEnvValue([name], cwd, env))?.value; + const resolved = { - domain: options.domain ?? env.AUTH0_DOMAIN, - clientId: options.clientId ?? env.AUTH0_CLIENT_ID, - clientSecret: options.clientSecret ?? env.AUTH0_CLIENT_SECRET, + domain: options.domain ?? (await fromEnv("AUTH0_DOMAIN")), + clientId: options.clientId ?? (await fromEnv("AUTH0_CLIENT_ID")), + clientSecret: options.clientSecret ?? (await fromEnv("AUTH0_CLIENT_SECRET")), }; const missing = ( diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index e7601df39..15116d255 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -27,6 +27,9 @@ import { import { buildSupabaseExport } from "./supabase.ts"; import { looksLikeConnectionString, resolveDbUrl } from "./db-options.ts"; +/** A cwd with no `.env` files, so these tests exercise only the injected env. */ +const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); + const captured = useCaptureLog(); let workDir: string; @@ -105,32 +108,45 @@ describe("resolveDbUrl", () => { const config = { platform: "authjs" as const, envVar: "AUTHJS_DB_URL", prompt: "url" }; test("prefers the flag", async () => { - const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, { + const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, NO_ENV_FILES, { AUTHJS_DB_URL: "mysql://u:p@h/db", }); expect(url).toBe("postgres://u:p@h/db"); }); test("falls back to the environment variable", async () => { - expect(await resolveDbUrl({}, config, { AUTHJS_DB_URL: "mysql://u:p@h/db" })).toBe( - "mysql://u:p@h/db", - ); + expect( + await resolveDbUrl({}, config, NO_ENV_FILES, { AUTHJS_DB_URL: "mysql://u:p@h/db" }), + ).toBe("mysql://u:p@h/db"); }); test("rejects a flag that is not a connection string, naming the encoding trap", async () => { - await expect(resolveDbUrl({ dbUrl: "not a url" }, config, {})).rejects.toThrow(/URL-encode it/); + await expect(resolveDbUrl({ dbUrl: "not a url" }, config, NO_ENV_FILES, {})).rejects.toThrow( + /URL-encode it/, + ); }); test("warns and moves on when the environment variable is unusable", async () => { // Tests run non-TTY, so it then hits the agent-mode branch. - await expect(resolveDbUrl({}, config, { AUTHJS_DB_URL: "garbage" })).rejects.toThrow( - /cannot prompt here/, - ); + await expect( + resolveDbUrl({}, config, NO_ENV_FILES, { AUTHJS_DB_URL: "garbage" }), + ).rejects.toThrow(/cannot prompt here/); expect(captured.err).toContain("AUTHJS_DB_URL is not a valid connection string"); }); + // The env var reaching process.env is the runtime's job; this is the fallback + // for when it did not, and is the rung the secret key has always had. + test("falls back to a .env file when the variable is not in the environment", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-dburl-env-")); + fs.writeFileSync(path.join(dir, ".env.local"), "AUTHJS_DB_URL=postgres://u:p@h/db\n"); + + expect(await resolveDbUrl({}, config, dir, {})).toBe("postgres://u:p@h/db"); + }); + test("names both the flag and the variable when it cannot prompt", async () => { - await expect(resolveDbUrl({}, config, {})).rejects.toThrow(/--db-url.*AUTHJS_DB_URL/s); + await expect(resolveDbUrl({}, config, NO_ENV_FILES, {})).rejects.toThrow( + /--db-url.*AUTHJS_DB_URL/s, + ); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 880e177da..999e6d5be 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -11,6 +11,7 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { detectDbType, redactConnectionString, type DbPlatform } from "../lib/db.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; export type DbExportOptions = { dbUrl?: string; @@ -57,6 +58,7 @@ export function looksLikeConnectionString(value: string): boolean { export async function resolveDbUrl( options: DbExportOptions, config: ResolveConfig, + cwd: string = process.cwd(), env: Record = process.env, ): Promise { const fromFlag = options.dbUrl?.trim(); @@ -70,7 +72,8 @@ export async function resolveDbUrl( return fromFlag; } - const fromEnv = env[config.envVar]?.trim(); + const located = await findMigrateEnvValue([config.envVar], cwd, env); + const fromEnv = located?.value.trim(); if (fromEnv) { if (looksLikeConnectionString(fromEnv)) return fromEnv; // Falling through silently would make the prompt look unexplained. diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index 3d34ca55c..a1a56e1c7 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -4,6 +4,7 @@ import { parseIntegerOption } from "../../lib/option-parsers.ts"; import { deleteMigration } from "./delete.ts"; import { registerMigrateExport } from "./export/index.ts"; import { registerMigrateLogs } from "./logs/index.ts"; +import { registerMigrateSettings } from "./settings/index.ts"; import { run } from "./run.ts"; import { list as transformersList } from "./transformers/list.ts"; import { transformerKeys } from "./transformers/registry.ts"; @@ -131,4 +132,5 @@ export function registerMigrate(program: Program): void { ); registerMigrateLogs(migrateCommand); + registerMigrateSettings(migrateCommand); } diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.test.ts b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts new file mode 100644 index 000000000..7c190a9d7 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts @@ -0,0 +1,121 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + clearMigrateEnvValues, + findMigrateEnvValue, + MIGRATE_ENV_FILE, + writeMigrateEnvValues, +} from "./env-file.ts"; + +let workDir: string; + +const envFile = () => path.join(workDir, MIGRATE_ENV_FILE); +const read = (file: string) => fs.readFileSync(path.join(workDir, file), "utf-8"); + +beforeEach(() => { + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-envfile-"))); +}); + +afterEach(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("writeMigrateEnvValues", () => { + test("creates the file and gitignores it", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).toBe("CLERK_FIREBASE_ROUNDS=8\n"); + expect(read(".gitignore")).toContain(MIGRATE_ENV_FILE); + }); + + test("appends to an existing .gitignore without duplicating the entry", async () => { + fs.writeFileSync(path.join(workDir, ".gitignore"), "node_modules\n"); + + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_MEM_COST: "14" }, workDir); + + expect(read(".gitignore")).toBe(`node_modules\n${MIGRATE_ENV_FILE}\n`); + }); + + // The header `mergeEnvVars` adds is right for an app's shared .env and wrong + // here — one `settings set` per key would stack one header per call. + test("adds no section header, however many times it is called", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_MEM_COST: "14" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_SIGNER_KEY: "k" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).not.toContain("#"); + }); + + test("updates a key in place rather than appending a second copy", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "10" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).toBe("CLERK_FIREBASE_ROUNDS=10\n"); + }); + + // The file is meant to be hand-editable, so a write must not flatten it. + test("preserves hand-written comments and unrelated keys", async () => { + fs.writeFileSync(envFile(), "# my note\nOTHER=keep\n"); + + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).toBe("# my note\nOTHER=keep\nCLERK_FIREBASE_ROUNDS=8\n"); + }); +}); + +describe("findMigrateEnvValue", () => { + test("reads a value out of the file", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_SIGNER_KEY: "from-file" }, workDir); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_SIGNER_KEY"], workDir, {}); + expect(located).toEqual({ value: "from-file", source: MIGRATE_ENV_FILE }); + }); + + test("beats the app's own .env.local", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=1\n"); + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, {}); + expect(located?.value).toBe("8"); + }); + + // An exported variable is the one thing an operator can change per-invocation. + test("loses to an exported environment variable", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "99", + }); + expect(located).toEqual({ value: "99", source: "CLERK_FIREBASE_ROUNDS env var" }); + }); + + test("returns nothing when the setting is absent everywhere", async () => { + expect(await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, {})).toBeUndefined(); + }); +}); + +describe("clearMigrateEnvValues", () => { + test("removes only the named settings", async () => { + fs.writeFileSync(envFile(), "OTHER=keep\nCLERK_FIREBASE_ROUNDS=8\n"); + + expect(await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir)).toEqual([ + "CLERK_FIREBASE_ROUNDS", + ]); + expect(read(MIGRATE_ENV_FILE)).toBe("OTHER=keep\n"); + }); + + // Left behind, it reads as "there is config here" when there is not. + test("deletes the file when nothing but comments would remain", async () => { + fs.writeFileSync(envFile(), "# a note\nCLERK_FIREBASE_ROUNDS=8\n"); + + await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir); + expect(fs.existsSync(envFile())).toBe(false); + }); + + test("reports nothing dropped when there is no file", async () => { + expect(await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir)).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.ts b/packages/cli-core/src/commands/migrate/lib/env-file.ts new file mode 100644 index 000000000..4d7a61f0a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/env-file.ts @@ -0,0 +1,123 @@ +/** + * `.env.clerk-migrate` — the migration's own env file. + * + * Migration credentials are a Firebase signer key, an Auth0 client secret, a + * database URL: things the app being migrated has no use for. Writing them into + * the app's `.env.local` mixes two unrelated sets of config in the file a + * developer reads every day, so they get their own. + * + * Read ahead of `.env`/`.env.local`, so a value set here wins over a stale one + * left in the app's file. An exported shell variable still beats both — that is + * {@link findEnvValue}'s contract for every value the CLI resolves. + * + * Always added to `.gitignore` on write. The CLI creating a credential-bearing + * file in someone's repository without that is how one ends up committed. + */ + +import { unlink } from "node:fs/promises"; +import { join } from "node:path"; +import { + findEnvValue, + parseEnvFile, + serializeEnvFile, + type EnvLine, + type LocatedEnvValue, +} from "../../../lib/dotenv.ts"; +import { ensureGitignoreEntry } from "../../../lib/git.ts"; +import { log } from "../../../lib/log.ts"; + +export const MIGRATE_ENV_FILE = ".env.clerk-migrate"; + +/** Lowest priority first: the migration's own file overrides the app's. */ +const MIGRATE_ENV_FILES = [".env", ".env.local", MIGRATE_ENV_FILE] as const; + +/** Resolves a migration setting: environment first, then the project's env files. */ +export async function findMigrateEnvValue( + names: string[], + cwd: string = process.cwd(), + env: Record = process.env, +): Promise { + const located = await findEnvValue(cwd, names, { env, files: MIGRATE_ENV_FILES }); + if (located) log.debug(`migrate: ${names[0]} from ${located.source}`); + return located; +} + +/** + * Merges `values` into the parsed file: existing keys update in place, new ones + * append. + * + * Deliberately not `mergeEnvVars` from `lib/dotenv.ts`. That one prepends a + * `# Clerk` section header when the file holds none of the keys being written, + * which is right for `env pull` dropping Clerk keys into an app's shared `.env` + * — and wrong here twice over: every key in this file is already Clerk's, and + * writing one setting at a time means the check fires again on every call, + * stacking a fresh header per `settings set`. + */ +function mergeMigrateEnv(lines: EnvLine[], values: Record): EnvLine[] { + const remaining = { ...values }; + + const merged = lines.map((line): EnvLine => { + if (line.type !== "entry" || !(line.key in remaining)) return line; + const value = remaining[line.key]!; + delete remaining[line.key]; + return { type: "entry", key: line.key, value, raw: `${line.key}=${value}` }; + }); + + for (const [key, value] of Object.entries(remaining)) { + merged.push({ type: "entry", key, value, raw: `${key}=${value}` }); + } + return merged; +} + +/** + * Writes settings into `.env.clerk-migrate`, creating and gitignoring it first. + * + * Existing comments, blank lines and key order survive — the file is meant to + * be hand-edited, so rewriting it wholesale would discard the user's notes. + */ +export async function writeMigrateEnvValues( + values: Record, + cwd: string = process.cwd(), +): Promise { + const target = join(cwd, MIGRATE_ENV_FILE); + const existing = await Bun.file(target) + .text() + .catch(() => ""); + + await Bun.write(target, serializeEnvFile(mergeMigrateEnv(parseEnvFile(existing), values))); + await ensureGitignoreEntry(cwd, MIGRATE_ENV_FILE); + + return MIGRATE_ENV_FILE; +} + +/** Removes the named settings from `.env.clerk-migrate`, leaving the rest. */ +export async function clearMigrateEnvValues( + names: string[], + cwd: string = process.cwd(), +): Promise { + const target = join(cwd, MIGRATE_ENV_FILE); + const existing = await Bun.file(target) + .text() + .catch(() => ""); + if (!existing) return []; + + const dropped: string[] = []; + const kept = parseEnvFile(existing).filter((line) => { + if (line.type !== "entry" || !names.includes(line.key)) return true; + dropped.push(line.key); + return false; + }); + + if (dropped.length === 0) return dropped; + + // A file holding nothing but the comments that described the settings it no + // longer has is worse than no file: it reads as "there is config here". + if (kept.some((line) => line.type === "entry")) { + await Bun.write(target, serializeEnvFile(kept)); + } else { + await unlink(target).catch(() => {}); + log.debug(`migrate: removed empty ${MIGRATE_ENV_FILE}`); + } + + return dropped; +} diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index ff9ffbbae..330d70645 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -69,8 +69,8 @@ describe("resolveFirebaseHashConfig", () => { firebaseMemCost: 14, }; - test("builds the config when all four flags are present", () => { - expect(resolveFirebaseHashConfig(ALL)).toEqual({ + test("builds the config when all four flags are present", async () => { + expect(await resolveFirebaseHashConfig(ALL)).toEqual({ base64_signer_key: "SIGNER", base64_salt_separator: "Bw==", rounds: 8, @@ -85,14 +85,14 @@ describe("resolveFirebaseHashConfig", () => { ["firebaseSaltSeparator", "--firebase-salt-separator"], ["firebaseRounds", "--firebase-rounds"], ["firebaseMemCost", "--firebase-mem-cost"], - ] as const)("rejects a set missing %s, naming the flag", (omit, flag) => { + ] as const)("rejects a set missing %s, naming the flag", async (omit, flag) => { const partial = { ...ALL }; delete (partial as Record)[omit]; - expect(() => resolveFirebaseHashConfig(partial)).toThrow(new RegExp(flag)); + await expect(resolveFirebaseHashConfig(partial)).rejects.toThrow(new RegExp(flag)); }); - test("names every missing flag at once", () => { - expect(() => resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" })).toThrow( + test("names every missing flag at once", async () => { + await expect(resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" })).rejects.toThrow( /--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/, ); }); @@ -111,9 +111,9 @@ describe("resolveFirebaseHashConfig", () => { const setEnv = (vars: Partial) => Object.assign(process.env, vars); - test("builds the config when no flag is passed", () => { + test("builds the config when no flag is passed", async () => { setEnv(ENV); - expect(resolveFirebaseHashConfig({})).toEqual({ + expect(await resolveFirebaseHashConfig({})).toEqual({ base64_signer_key: "ENV_SIGNER", base64_salt_separator: "Bw==", rounds: 8, @@ -121,16 +121,19 @@ describe("resolveFirebaseHashConfig", () => { }); }); - test("prefers a flag over the environment", () => { + test("prefers a flag over the environment", async () => { setEnv(ENV); - expect(resolveFirebaseHashConfig(ALL)?.base64_signer_key).toBe("SIGNER"); + expect((await resolveFirebaseHashConfig(ALL))?.base64_signer_key).toBe("SIGNER"); }); // Half from the environment and half from flags is still a complete set. - test("fills only the gaps the flags left", () => { + test("fills only the gaps the flags left", async () => { setEnv({ CLERK_FIREBASE_ROUNDS: "8", CLERK_FIREBASE_MEM_COST: "14" }); expect( - resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER", firebaseSaltSeparator: "Bw==" }), + await resolveFirebaseHashConfig({ + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + }), ).toEqual({ base64_signer_key: "SIGNER", base64_salt_separator: "Bw==", @@ -139,21 +142,21 @@ describe("resolveFirebaseHashConfig", () => { }); }); - test("still demands the full set when the environment supplies only part", () => { + test("still demands the full set when the environment supplies only part", async () => { setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); - expect(() => resolveFirebaseHashConfig({})).toThrow(/--firebase-salt-separator/); + await expect(resolveFirebaseHashConfig({})).rejects.toThrow(/--firebase-salt-separator/); }); // An empty var is how a shell spells "unset", and treating it as set would // demand the other three for a config nobody asked for. - test("ignores an empty variable", () => { + test("ignores an empty variable", async () => { setEnv({ CLERK_FIREBASE_SIGNER_KEY: "" }); - expect(resolveFirebaseHashConfig({})).toBeUndefined(); + expect(await resolveFirebaseHashConfig({})).toBeUndefined(); }); }); - test("returns nothing when neither flags nor the environment supply a config", () => { - expect(resolveFirebaseHashConfig({})).toBeUndefined(); + test("returns nothing when neither flags nor the environment supply a config", async () => { + expect(await resolveFirebaseHashConfig({})).toBeUndefined(); }); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index cd1e1934f..9b6e9bdf0 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -19,6 +19,7 @@ import { withGutter, withSpinner } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; import { importUsers } from "./import-users.ts"; import { analyzeFields } from "./lib/analysis.ts"; +import { findMigrateEnvValue } from "./lib/env-file.ts"; import { enabledSocialProviders, fetchInstanceSettings, @@ -71,29 +72,31 @@ const FIREBASE_FLAGS = [ const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); /** - * Overlays the `CLERK_FIREBASE_*` environment variables onto whichever flags - * were not passed. + * Overlays the `CLERK_FIREBASE_*` values onto whichever flags were not passed. * - * The signer key is a Firebase secret, so it is read rather than stored: the - * CLI never persists these, and `.env.local` is already gitignored and already - * where the CLI keeps a project's local secrets. + * Resolved through {@link findMigrateEnvValue}: the environment first, then + * `.env.clerk-migrate`, then the app's own `.env` files. The signer key is a + * Firebase secret, so it is never written to the CLI's config — + * `.env.clerk-migrate` is gitignored on creation. */ -function withFirebaseEnv(options: MigrateRunOptions): MigrateRunOptions { +async function withFirebaseEnv(options: MigrateRunOptions): Promise { const merged = { ...options }; for (const [key, , envVar] of FIREBASE_FLAGS) { if (merged[key] !== undefined) continue; - const value = process.env[envVar]; - if (value === undefined || value.trim() === "") continue; + const located = await findMigrateEnvValue([envVar]); + if (!located || located.value.trim() === "") continue; // A non-numeric round count is left to fail the flag's own validation // rather than silently becoming NaN. - (merged as Record)[key] = FIREBASE_NUMERIC.has(key) ? Number(value) : value; + (merged as Record)[key] = FIREBASE_NUMERIC.has(key) + ? Number(located.value) + : located.value; } return merged; } /** * Resolves Firebase's four hash parameters from flags, falling back to the - * `CLERK_FIREBASE_*` environment variables. + * `CLERK_FIREBASE_*` environment variables and the project's `.env` files. * * The four are required as a set: a digest built from a partial set is * well-formed but verifies against nothing, so every migrated user would fail @@ -102,10 +105,10 @@ function withFirebaseEnv(options: MigrateRunOptions): MigrateRunOptions { * @returns The config, or `undefined` when none was supplied — which is fine * for an export that carries no password hashes. */ -export function resolveFirebaseHashConfig( +export async function resolveFirebaseHashConfig( rawOptions: MigrateRunOptions, -): FirebaseHashConfig | undefined { - const options = withFirebaseEnv(rawOptions); +): Promise { + const options = await withFirebaseEnv(rawOptions); const provided = FIREBASE_FLAGS.filter(([key]) => options[key] !== undefined); if (provided.length === 0) return undefined; @@ -370,7 +373,7 @@ async function resolveMissingOptions(options: MigrateRunOptions): Promise { const secretKeyOption = options.secretKey ?? options.clerkSecretKey; const { transformer, file } = validateRunOptions(options); - const firebaseHashConfig = resolveFirebaseHashConfig(options); + const firebaseHashConfig = await resolveFirebaseHashConfig(options); await withGutter("Migrating users to Clerk", async () => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); diff --git a/packages/cli-core/src/commands/migrate/settings/clear.ts b/packages/cli-core/src/commands/migrate/settings/clear.ts new file mode 100644 index 000000000..933d1e540 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/clear.ts @@ -0,0 +1,58 @@ +/** + * `clerk migrate settings clear` — forget this project's migration settings. + * + * Clears both stores by default. The credentials half is the reason this + * command exists: after a migration finishes, a Firebase signer key sitting in + * the repo has no further use, and "delete the file yourself" is a step people + * skip. + * + * `migrate delete` reads the saved transformer and file to know what to undo, + * so clearing is confirmed unless `-y` — an operator who clears and then wants + * to undo has no record left to undo from. + */ + +import { throwUserAbort } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { confirm } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { clearMigrateEnvValues, MIGRATE_ENV_FILE } from "../lib/env-file.ts"; +import { loadSettings, saveSettings } from "../lib/settings.ts"; +import { SETTINGS } from "./registry.ts"; + +export type SettingsClearOptions = { + yes?: boolean; +}; + +const ENV_VARS = SETTINGS.filter((s) => s.store === "env").map((s) => s.envVar as string); + +export async function clear(options: SettingsClearOptions = {}): Promise { + const saved = await loadSettings(); + const hadConfig = Object.keys(saved).length > 0; + + if (!options.yes && isHuman() && !isAgent()) { + if (hadConfig && saved.file) { + log.warn( + `\`clerk migrate delete\` uses the saved file (${saved.file}) to identify the users the last run created. ` + + "Clearing it leaves nothing to undo from.", + ); + } + const proceed = await confirm({ + message: "Clear this project's migration settings?", + default: false, + }); + if (!proceed) throwUserAbort(); + } + + if (hadConfig) await saveSettings({}); + const dropped = await clearMigrateEnvValues(ENV_VARS); + + if (!hadConfig && dropped.length === 0) { + log.info("No migration settings to clear for this project."); + return; + } + + if (hadConfig) log.success("Cleared the saved transformer and file."); + if (dropped.length > 0) { + log.success(`Removed ${dropped.length} credential(s) from ${MIGRATE_ENV_FILE}.`); + } +} diff --git a/packages/cli-core/src/commands/migrate/settings/index.ts b/packages/cli-core/src/commands/migrate/settings/index.ts new file mode 100644 index 000000000..7ce5b6f44 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/index.ts @@ -0,0 +1,75 @@ +import { createArgument } from "@commander-js/extra-typings"; +import type { Command } from "@commander-js/extra-typings"; +import { clear } from "./clear.ts"; +import { list } from "./list.ts"; +import { SETTING_NAMES } from "./registry.ts"; +import { set } from "./set.ts"; + +const settings = { clear, list, set }; + +/** + * Registers `settings list|set|clear` under the `migrate` group. + * + * Noun-verb like every other group in the tree, and listing is the default + * because it is the read-only one — a bare `clerk migrate settings` should show, + * never change. + */ +export function registerMigrateSettings( + migrateCommand: Command<[], Record>, +): void { + const settingsCommand = migrateCommand + .command("settings") + .description("Inspect and change this project's saved migration settings") + .setExamples([ + { + command: "clerk migrate settings", + description: "Show every setting and where it resolves from", + }, + { + command: "clerk migrate settings set firebase-signer-key abc123", + description: "Save a credential to the gitignored .env.clerk-migrate", + }, + { command: "clerk migrate settings clear -y", description: "Forget this project's settings" }, + ]); + + settingsCommand + .command("list", { isDefault: true }) + .description("Show each setting, its value and which source supplied it") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate settings list", description: "Credentials shown redacted" }, + { command: "clerk migrate settings list --json", description: "Machine-readable listing" }, + ]) + .action((_opts, cmd) => + settings.list(cmd.optsWithGlobals() as Parameters[0]), + ); + + settingsCommand + .command("set") + .description("Set one setting for this project") + .addArgument(createArgument("", "Setting to change").choices(SETTING_NAMES)) + .addArgument(createArgument("", "New value")) + .setExamples([ + { + command: "clerk migrate settings set transformer firebase", + description: "Remember the source platform", + }, + { + command: "clerk migrate settings set firebase-signer-key abc123", + description: "Write a credential to .env.clerk-migrate", + }, + ]) + .action((name, value) => settings.set(name, value)); + + settingsCommand + .command("clear") + .description("Forget the saved settings and remove the saved credentials") + .option("-y, --yes", "Skip the confirmation prompt") + .setExamples([ + { command: "clerk migrate settings clear", description: "Clear after confirming" }, + { command: "clerk migrate settings clear -y", description: "Clear without prompting" }, + ]) + .action((_opts, cmd) => + settings.clear(cmd.optsWithGlobals() as Parameters[0]), + ); +} diff --git a/packages/cli-core/src/commands/migrate/settings/list.ts b/packages/cli-core/src/commands/migrate/settings/list.ts new file mode 100644 index 000000000..9fb1683d0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/list.ts @@ -0,0 +1,87 @@ +/** + * `clerk migrate settings` — what a run in this project would pick up, and + * where each value is coming from. + * + * The source column is the point. A migration reads from flags, the + * environment, two of the app's env files and the CLI's config; when a run uses + * a stale value, the question is never "what is it" but "which of those is + * winning". Credentials are redacted, so this is safe to paste into an issue. + */ + +import { cyan, dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; +import { loadSettings } from "../lib/settings.ts"; +import { displayValue, SETTINGS, type SettingDef } from "./registry.ts"; + +export type SettingsListOptions = { + json?: boolean; +}; + +interface ResolvedSetting { + setting: SettingDef; + value?: string; + source?: string; +} + +async function resolveAll(): Promise { + const saved = await loadSettings(); + + return Promise.all( + SETTINGS.map(async (setting): Promise => { + if (setting.store === "config") { + const value = saved[setting.configKey as keyof typeof saved]; + return value === undefined + ? { setting } + : { setting, value: String(value), source: "clerk config" }; + } + + const located = await findMigrateEnvValue([setting.envVar as string]); + return located ? { setting, value: located.value, source: located.source } : { setting }; + }), + ); +} + +function toJson(resolved: ResolvedSetting[]) { + return resolved.map(({ setting, value, source }) => ({ + name: setting.name, + store: setting.store, + // Redacted here too: `--json` is what gets piped into a log or a ticket. + value: value === undefined ? null : displayValue(setting, value), + set: value !== undefined, + secret: Boolean(setting.secret), + source: source ?? null, + })); +} + +export async function list(options: SettingsListOptions = {}): Promise { + const resolved = await resolveAll(); + + if (options.json) { + log.data(JSON.stringify(toJson(resolved), null, 2)); + return; + } + + const nameWidth = Math.max(...SETTINGS.map((s) => s.name.length), "SETTING".length) + 2; + const valueWidth = + Math.max( + ...resolved.map(({ setting, value }) => + value === undefined ? 1 : displayValue(setting, value).length, + ), + "VALUE".length, + ) + 2; + + log.info(dim("SETTING".padEnd(nameWidth)) + dim("VALUE".padEnd(valueWidth)) + dim("SOURCE")); + + for (const { setting, value, source } of resolved) { + const shown = value === undefined ? dim("—") : displayValue(setting, value); + log.info( + cyan(setting.name.padEnd(nameWidth)) + + shown.padEnd(valueWidth + (value === undefined ? dim("—").length - 1 : 0)) + + dim(source ?? "not set"), + ); + } + + log.blank(); + log.info(dim("Credentials are shown redacted. `clerk migrate settings set `.")); +} diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts new file mode 100644 index 000000000..8063607a2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -0,0 +1,111 @@ +/** + * What `clerk migrate settings` can show and change. + * + * Two stores, split by what the value is rather than by which command wrote it: + * + * - **config** — what this project last migrated. Not secret, not per-machine + * secret material, and useless to anyone but the CLI, so it lives in the + * CLI's own config file keyed by project. + * - **env** — credentials. They go to `.env.clerk-migrate`, which is + * gitignored on write and hand-editable, because a credential belongs + * somewhere the user can rotate it without the CLI's help. + * + * A setting is listed here exactly once; `list`, `set` and `clear` all read + * this table rather than each keeping their own idea of what exists. + */ + +export type SettingStore = "config" | "env"; + +export interface SettingDef { + /** What the user types: `clerk migrate settings set `. */ + name: string; + store: SettingStore; + description: string; + /** For `env` settings, the variable read at run time. */ + envVar?: string; + /** For `config` settings, the key on the saved migration entry. */ + configKey?: "transformer" | "file" | "skipUnsupportedProviders"; + /** Redact when displaying — the value is a credential. */ + secret?: boolean; + /** Reject a value the run would only fail on later. */ + validate?: (value: string) => string | undefined; +} + +const positiveInteger = (value: string): string | undefined => { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? undefined : "Expected a positive whole number"; +}; + +const boolean = (value: string): string | undefined => + ["true", "false"].includes(value) ? undefined : "Expected true or false"; + +export const SETTINGS: SettingDef[] = [ + { + name: "transformer", + store: "config", + configKey: "transformer", + description: "Source platform the last run imported from", + }, + { + name: "file", + store: "config", + configKey: "file", + description: "Export file the last run imported", + }, + { + name: "skip-unsupported-providers", + store: "config", + configKey: "skipUnsupportedProviders", + description: "Supabase: skip users whose only social provider is disabled in Clerk", + validate: boolean, + }, + { + name: "firebase-signer-key", + store: "env", + envVar: "CLERK_FIREBASE_SIGNER_KEY", + description: "Firebase base64 signer key", + secret: true, + }, + { + name: "firebase-salt-separator", + store: "env", + envVar: "CLERK_FIREBASE_SALT_SEPARATOR", + description: "Firebase base64 salt separator", + }, + { + name: "firebase-rounds", + store: "env", + envVar: "CLERK_FIREBASE_ROUNDS", + description: "Firebase scrypt rounds", + validate: positiveInteger, + }, + { + name: "firebase-mem-cost", + store: "env", + envVar: "CLERK_FIREBASE_MEM_COST", + description: "Firebase scrypt memory cost", + validate: positiveInteger, + }, +]; + +export const SETTING_NAMES = SETTINGS.map((setting) => setting.name); + +export function findSetting(name: string): SettingDef | undefined { + return SETTINGS.find((setting) => setting.name === name); +} + +/** + * Shows enough of a credential to recognise it, never enough to use it. + * + * Anything short enough that head-and-tail would leak most of it is masked + * whole: a 10-character key shown as `abcd…wxyz` has given away 8 of them. + */ +export function redact(value: string): string { + if (value.length < 16) return "•".repeat(8); + return `${value.slice(0, 4)}…${value.slice(-4)}`; +} + +/** The display value for a setting: redacted when it is a credential. */ +export function displayValue(setting: SettingDef, value: string): string { + return setting.secret ? redact(value) : value; +} diff --git a/packages/cli-core/src/commands/migrate/settings/set.ts b/packages/cli-core/src/commands/migrate/settings/set.ts new file mode 100644 index 000000000..1b0e1d997 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/set.ts @@ -0,0 +1,48 @@ +/** + * `clerk migrate settings set ` — change one setting. + * + * Which store it lands in is a property of the setting, not a flag: a + * credential always goes to `.env.clerk-migrate`, project state always goes to + * the CLI config. Letting the caller choose would mean a signer key could be + * put somewhere that is not gitignored. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { writeMigrateEnvValues } from "../lib/env-file.ts"; +import { loadSettings, saveSettings } from "../lib/settings.ts"; +import { displayValue, findSetting, SETTING_NAMES } from "./registry.ts"; + +export async function set(name: string, value: string): Promise { + const setting = findSetting(name); + if (!setting) { + throwUsageError( + `Unknown setting "${name}". Valid names: ${SETTING_NAMES.join(", ")}.`, + undefined, + undefined, + [ + { + command: "clerk migrate settings", + description: "List the settings and their current values", + }, + ], + ); + } + + const invalid = setting.validate?.(value); + if (invalid) throwUsageError(`Invalid value for ${name}: ${invalid}.`); + + if (setting.store === "env") { + const file = await writeMigrateEnvValues({ [setting.envVar as string]: value }); + log.success(`Set \`${name}\` in ${file} (gitignored).`); + return; + } + + const saved = await loadSettings(); + await saveSettings({ + ...saved, + [setting.configKey as string]: + setting.configKey === "skipUnsupportedProviders" ? value === "true" : value, + }); + log.success(`Set \`${name}\` to ${displayValue(setting, value)} for this project.`); +} diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts new file mode 100644 index 000000000..9bee68233 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -0,0 +1,172 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { MIGRATE_ENV_FILE } from "../lib/env-file.ts"; +import { loadSettings, saveSettings } from "../lib/settings.ts"; +import { clear } from "./clear.ts"; +import { list } from "./list.ts"; +import { redact } from "./registry.ts"; +import { set } from "./set.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let configDir: string; +let originalCwd: string; + +const envFileContent = () => fs.readFileSync(path.join(workDir, MIGRATE_ENV_FILE), "utf-8"); + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-cmd-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-cfg-")); + _setConfigDir(configDir); + process.chdir(workDir); +}); + +afterAll(() => { + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.rmSync(path.join(workDir, MIGRATE_ENV_FILE), { force: true }); + fs.rmSync(path.join(workDir, ".gitignore"), { force: true }); +}); + +afterEach(() => { + process.exitCode = 0; +}); + +describe("redact", () => { + test("shows head and tail of a long value", () => { + expect(redact("aVeryLongSignerKeyValue123456")).toBe("aVer…3456"); + }); + + // Head-and-tail on a short value gives away most of it. + test.each([["short"], ["0123456789"], ["123456789012345"]])("masks %p whole", (value) => { + expect(redact(value)).toBe("••••••••"); + }); +}); + +describe("set", () => { + test("writes a credential to the gitignored env file, not the CLI config", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + + expect(envFileContent()).toContain("CLERK_FIREBASE_SIGNER_KEY=aVeryLongSignerKeyValue123456"); + expect(await loadSettings()).toEqual({}); + expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toContain(MIGRATE_ENV_FILE); + }); + + test("writes project state to the CLI config, not the env file", async () => { + await set("transformer", "firebase"); + + expect(await loadSettings()).toEqual({ transformer: "firebase" }); + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + }); + + test("keeps the settings it is not changing", async () => { + await saveSettings({ transformer: "clerk", file: "users.json" }); + await set("file", "other.json"); + + expect(await loadSettings()).toEqual({ transformer: "clerk", file: "other.json" }); + }); + + test("stores a boolean setting as a boolean", async () => { + await set("skip-unsupported-providers", "true"); + expect(await loadSettings()).toEqual({ skipUnsupportedProviders: true }); + }); + + test.each([ + ["firebase-rounds", "zero", /positive whole number/], + ["skip-unsupported-providers", "yes", /true or false/], + ])("rejects an invalid value for %s", async (name, value, message) => { + await expect(set(name, value)).rejects.toThrow(message); + }); + + test("names the valid settings when given an unknown one", async () => { + await expect(set("nope", "x")).rejects.toThrow(/firebase-signer-key/); + }); + + // A run would fail on it later; failing at write time keeps the bad value out + // of the file entirely. + test("writes nothing when the value is rejected", async () => { + await expect(set("firebase-rounds", "-1")).rejects.toThrow(); + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + }); +}); + +describe("list", () => { + test("names the source each value resolved from", async () => { + await set("transformer", "firebase"); + await set("firebase-salt-separator", "Bw=="); + captured.clear(); + + await list(); + + expect(captured.err).toContain("clerk config"); + expect(captured.err).toContain(MIGRATE_ENV_FILE); + }); + + test("redacts a credential but not the rest", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + await set("transformer", "firebase"); + captured.clear(); + + await list(); + + expect(captured.err).toContain("aVer…3456"); + expect(captured.err).not.toContain("aVeryLongSignerKeyValue123456"); + expect(captured.err).toContain("firebase"); + }); + + // --json is what gets piped into a ticket or a CI log. + test("redacts in JSON output too", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + captured.clear(); + + await list({ json: true }); + + expect(captured.out).not.toContain("aVeryLongSignerKeyValue123456"); + expect(JSON.parse(captured.out)).toContainEqual( + expect.objectContaining({ name: "firebase-signer-key", value: "aVer…3456", secret: true }), + ); + }); + + test("marks everything as unset in a fresh project", async () => { + await list({ json: true }); + expect(JSON.parse(captured.out).every((entry: { set: boolean }) => !entry.set)).toBe(true); + }); +}); + +describe("clear", () => { + test("empties both stores", async () => { + await set("transformer", "firebase"); + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + + await clear({ yes: true }); + + expect(await loadSettings()).toEqual({}); + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + }); + + test("says so rather than claiming to have cleared nothing", async () => { + await clear({ yes: true }); + expect(captured.err).toContain("No migration settings to clear"); + }); + + test("leaves settings the migration does not own", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "OTHER=keep\n"); + await set("firebase-rounds", "8"); + + await clear({ yes: true }); + + expect(envFileContent()).toBe("OTHER=keep\n"); + }); +}); diff --git a/packages/cli-core/src/lib/dotenv.ts b/packages/cli-core/src/lib/dotenv.ts index 2b5d0cb20..16bc28187 100644 --- a/packages/cli-core/src/lib/dotenv.ts +++ b/packages/cli-core/src/lib/dotenv.ts @@ -28,6 +28,74 @@ export async function findExistingEnvFile(cwd: string, fallback: string): Promis return fallback; } +/** + * The env files read back when resolving a value, as opposed to written to. + * + * Deliberately shorter than {@link ENV_FILE_CANDIDATES}: the runtime has + * already loaded every `.env*` variant it recognises into `process.env`, which + * {@link findEnvValue} checks first. This list only has to cover the case where + * the CLI's own process did not load the file — a different cwd at startup, or + * a runtime with no dotenv support. + */ +const ENV_FILES = [".env", ".env.local"]; + +export interface FindEnvValueOptions { + /** Injectable in tests; defaults to the real environment. */ + env?: Record; + /** Lowest priority first — a later file overrides an earlier one. */ + files?: readonly string[]; +} + +export interface LocatedEnvValue { + value: string; + /** Where it came from, for `--verbose` (`CLERK_SECRET_KEY env var`, `.env.local`). */ + source: string; +} + +/** + * Looks for a value under any of `names`, in the order the app itself would + * resolve one: the environment first, then env files with a later file + * overriding an earlier one. + * + * This is the CLI's one way to read a project-level setting. Reading + * `process.env` directly instead skips the file fallback and reports no source, + * so a command that does it cannot explain where its input came from. + */ +export async function findEnvValue( + cwd: string, + names: string[], + options: FindEnvValueOptions = {}, +): Promise { + const { env = process.env, files = ENV_FILES } = options; + + for (const name of new Set(names)) { + const value = env[name]; + if (value) return { value, source: `${name} env var` }; + } + + // Priority is by name, not by position: the framework-specific name beats + // the generic fallback even when the generic one appears later in the same + // file. Within one name, a later file still overrides an earlier one. + const foundByName = new Map(); + for (const envFile of files) { + const file = Bun.file(join(cwd, envFile)); + if (!(await file.exists())) continue; + + for (const line of parseEnvFile(await file.text())) { + if (line.type !== "entry" || !line.value) continue; + if (names.includes(line.key)) { + foundByName.set(line.key, { value: line.value, source: envFile }); + } + } + } + + for (const name of names) { + const located = foundByName.get(name); + if (located) return located; + } + return undefined; +} + export type EnvLine = | { type: "comment"; raw: string } | { type: "blank" } diff --git a/packages/cli-core/src/lib/git.ts b/packages/cli-core/src/lib/git.ts index da4f4e955..697a57de8 100644 --- a/packages/cli-core/src/lib/git.ts +++ b/packages/cli-core/src/lib/git.ts @@ -1,4 +1,4 @@ -import { resolve } from "node:path"; +import { join, resolve } from "node:path"; import { log } from "./log.ts"; const $ = Bun.$; @@ -100,3 +100,22 @@ export function normalizeGitRemoteUrl(raw: string): string { return url.toLowerCase(); } + +/** + * Adds `entry` to the project's `.gitignore` unless it is already listed. + * + * The CLI writes files into a user's repository that must not be committed — + * the keyless breadcrumb, and the migration settings file. Creating one without + * this is how a live credential ends up in a tracked file. + */ +export async function ensureGitignoreEntry(cwd: string, entry: string): Promise { + const gitignorePath = join(cwd, ".gitignore"); + const content = await Bun.file(gitignorePath) + .text() + .catch(() => ""); + const lines = content.split("\n").map((l) => l.trim()); + if (lines.includes(entry)) return; + const separator = content && !content.endsWith("\n") ? "\n" : ""; + await Bun.write(gitignorePath, `${content}${separator}${entry}\n`); + log.debug(`git: added ${entry} to .gitignore`); +} diff --git a/packages/cli-core/src/lib/keyless-target.ts b/packages/cli-core/src/lib/keyless-target.ts index c1f7f5b08..82fa439d3 100644 --- a/packages/cli-core/src/lib/keyless-target.ts +++ b/packages/cli-core/src/lib/keyless-target.ts @@ -11,7 +11,7 @@ import { join } from "node:path"; import { bapiRequest } from "./bapi.ts"; import { resolveAppContext, resolveProfile } from "./config.ts"; import { getStoredSession, hasAccountCredentials, type OAuthSession } from "./credential-store.ts"; -import { parseEnvFile } from "./dotenv.ts"; +import { findEnvValue } from "./dotenv.ts"; import { CliError, ERROR_CODE, throwUsageError } from "./errors.ts"; import { decodePublishableKey } from "./fapi.ts"; import { detectPublishableKeyName, detectSecretKeyName } from "./framework.ts"; @@ -38,8 +38,6 @@ export type InstanceTarget = | { kind: "account"; ctx: AccountContext; label: string } | { kind: "keyless"; keyless: KeylessTarget; label: string }; -const ENV_FILES = [".env", ".env.local"]; - /** * Where the Clerk SDKs park the keys for a keyless app they created themselves * (running `next dev` with no keys configured). Shape: @@ -71,45 +69,6 @@ export async function readSdkKeylessApp( } } -interface LocatedKey { - value: string; - source: string; -} - -/** - * Looks for a key under any of `names`, in the order the app itself would - * resolve one: the environment first, then env files with a later file - * overriding an earlier one. - */ -async function findKeyInProject(cwd: string, names: string[]): Promise { - for (const name of new Set(names)) { - const value = process.env[name]; - if (value) return { value, source: `${name} env var` }; - } - - // Priority is by name, not by position: the framework-specific name beats - // the generic fallback even when the generic one appears later in the same - // file. Within one name, a later file still overrides an earlier one. - const foundByName = new Map(); - for (const envFile of ENV_FILES) { - const file = Bun.file(join(cwd, envFile)); - if (!(await file.exists())) continue; - - for (const line of parseEnvFile(await file.text())) { - if (line.type !== "entry" || !line.value) continue; - if (names.includes(line.key)) { - foundByName.set(line.key, { value: line.value, source: envFile }); - } - } - } - - for (const name of names) { - const located = foundByName.get(name); - if (located) return located; - } - return undefined; -} - /** * The instance secret key a keyless project keeps locally. Falls back to the * keys an SDK created for itself, which it only does when nothing else supplies @@ -117,7 +76,7 @@ async function findKeyInProject(cwd: string, names: string[]): Promise { const names = [await detectSecretKeyName(cwd), "CLERK_SECRET_KEY"]; - const located = await findKeyInProject(cwd, names); + const located = await findEnvValue(cwd, names); const found = located ? { secretKey: located.value, source: located.source } @@ -136,7 +95,7 @@ async function sdkKeylessTarget(cwd: string): Promise /** The publishable key a keyless project holds locally, when one can be found. */ export async function findLocalPublishableKey(cwd: string): Promise { const names = [await detectPublishableKeyName(cwd), "CLERK_PUBLISHABLE_KEY"]; - const located = await findKeyInProject(cwd, names); + const located = await findEnvValue(cwd, names); return located?.value ?? (await readSdkKeylessApp(cwd))?.publishableKey; } diff --git a/packages/cli-core/src/lib/keyless.ts b/packages/cli-core/src/lib/keyless.ts index 46db0a1a7..1e2e3ca96 100644 --- a/packages/cli-core/src/lib/keyless.ts +++ b/packages/cli-core/src/lib/keyless.ts @@ -5,6 +5,7 @@ import { detectPublishableKeyName, detectSecretKeyName, detectEnvFile } from "./ import { parseEnvFile, mergeEnvVars, serializeEnvFile } from "./dotenv.ts"; import { BapiError } from "./errors.ts"; import { loggedFetch } from "./fetch.ts"; +import { ensureGitignoreEntry } from "./git.ts"; import { log } from "./log.ts"; const BREADCRUMB_DIR = ".clerk"; @@ -113,18 +114,6 @@ function breadcrumbPath(cwd: string): string { return join(cwd, BREADCRUMB_DIR, BREADCRUMB_FILE); } -async function ensureGitignoreEntry(cwd: string, entry: string): Promise { - const gitignorePath = join(cwd, ".gitignore"); - const content = await Bun.file(gitignorePath) - .text() - .catch(() => ""); - const lines = content.split("\n").map((l) => l.trim()); - if (lines.includes(entry)) return; - const separator = content && !content.endsWith("\n") ? "\n" : ""; - await Bun.write(gitignorePath, `${content}${separator}${entry}\n`); - log.debug(`Added ${entry} to .gitignore`); -} - export async function writeKeylessBreadcrumb(cwd: string, claimToken: string): Promise { await ensureGitignoreEntry(cwd, BREADCRUMB_DIR + "/"); await mkdir(join(cwd, BREADCRUMB_DIR), { recursive: true }); diff --git a/packages/cli-core/src/test/integration/lib/harness.ts b/packages/cli-core/src/test/integration/lib/harness.ts index 2ad58c8a7..6853e9ca3 100644 --- a/packages/cli-core/src/test/integration/lib/harness.ts +++ b/packages/cli-core/src/test/integration/lib/harness.ts @@ -91,6 +91,7 @@ mock.module( getGitRepoIdentifier: async () => mockState.gitRepoIdentifier, getGitNormalizedRemote: async () => mockState.gitNormalizedRemote, normalizeGitRemoteUrl: (url: string) => url, + ensureGitignoreEntry: async () => {}, }) satisfies typeof import("../../../lib/git.ts"), ); From 194d8f6048a7f351afd98b0fb01816812bae9472 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 16:53:11 -0400 Subject: [PATCH 007/141] feat(migrate): explain each setting in `migrate settings`, and fix the column alignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The list showed seven kebab-case identifiers and nothing else, so `file` and `skip-unsupported-providers` read as jargon rather than as anything a user could act on. A description column now carries the prose the registry already held. The names stay kebab-case on purpose: each one is identical to the `migrate run` flag it backs, so `firebase-signer-key` and `--firebase-signer-key` are one knob reached two ways rather than two spellings to learn. Sentence case belongs in the description, which is where it now is. Also fixes the alignment. `column()` pads to the visible width before colouring; the previous code coloured first and then hand-compensated for the escape bytes with `dim("—").length - 1`, which only held for unset rows — any row with a value pulled `SOURCE` and everything after it out of line. --- .../cli-core/src/commands/migrate/README.md | 14 ++++-- .../src/commands/migrate/settings/list.ts | 50 +++++++++++++------ .../src/commands/migrate/settings/registry.ts | 16 ++++-- .../migrate/settings/settings.test.ts | 37 ++++++++++++++ 4 files changed, 94 insertions(+), 23 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 1e479a8f8..d8d9aeb63 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -453,13 +453,17 @@ clerk migrate settings clear -y ``` ``` -SETTING VALUE SOURCE -transformer firebase clerk config -file ./users.json clerk config -firebase-signer-key aVer…3456 .env.clerk-migrate -firebase-rounds — not set +SETTING VALUE SOURCE DESCRIPTION +transformer firebase clerk config Source platform the export came from +file users.json clerk config Export file to import users from +firebase-signer-key aVer…3456 .env.clerk-migrate Firebase base64 signer key +firebase-rounds — not set Firebase scrypt rounds ``` +Setting names are kebab-case and identical to the `migrate run` flag each one +backs, so `firebase-signer-key` here is `--firebase-signer-key` there rather +than a second spelling to learn. The description column carries the prose. + The source column is the point. A migration reads from flags, the environment, two of the app's env files and the CLI's config, so when a run picks up a stale value the question is never "what is it" but "which of those won". diff --git a/packages/cli-core/src/commands/migrate/settings/list.ts b/packages/cli-core/src/commands/migrate/settings/list.ts index 9fb1683d0..f5e9e43ec 100644 --- a/packages/cli-core/src/commands/migrate/settings/list.ts +++ b/packages/cli-core/src/commands/migrate/settings/list.ts @@ -46,6 +46,7 @@ function toJson(resolved: ResolvedSetting[]) { return resolved.map(({ setting, value, source }) => ({ name: setting.name, store: setting.store, + description: setting.description, // Redacted here too: `--json` is what gets piped into a log or a ticket. value: value === undefined ? null : displayValue(setting, value), set: value !== undefined, @@ -54,6 +55,16 @@ function toJson(resolved: ResolvedSetting[]) { })); } +/** + * Pads to a visible width, then colours. + * + * Colouring first and padding after would count the ANSI escape bytes towards + * the width and pull every later column left by however many they took. + */ +function column(text: string, width: number, paint: (value: string) => string): string { + return paint(text) + " ".repeat(Math.max(0, width - text.length)); +} + export async function list(options: SettingsListOptions = {}): Promise { const resolved = await resolveAll(); @@ -62,23 +73,34 @@ export async function list(options: SettingsListOptions = {}): Promise { return; } - const nameWidth = Math.max(...SETTINGS.map((s) => s.name.length), "SETTING".length) + 2; - const valueWidth = - Math.max( - ...resolved.map(({ setting, value }) => - value === undefined ? 1 : displayValue(setting, value).length, - ), - "VALUE".length, - ) + 2; + const cells = resolved.map(({ setting, value, source }) => ({ + setting, + name: setting.name, + value: value === undefined ? "—" : displayValue(setting, value), + unset: value === undefined, + source: source ?? "not set", + })); + + const width = (header: string, pick: (cell: (typeof cells)[number]) => string) => + Math.max(header.length, ...cells.map((cell) => pick(cell).length)) + 2; - log.info(dim("SETTING".padEnd(nameWidth)) + dim("VALUE".padEnd(valueWidth)) + dim("SOURCE")); + const nameWidth = width("SETTING", (c) => c.name); + const valueWidth = width("VALUE", (c) => c.value); + const sourceWidth = width("SOURCE", (c) => c.source); + + log.info( + column("SETTING", nameWidth, dim) + + column("VALUE", valueWidth, dim) + + column("SOURCE", sourceWidth, dim) + + dim("DESCRIPTION"), + ); - for (const { setting, value, source } of resolved) { - const shown = value === undefined ? dim("—") : displayValue(setting, value); + for (const cell of cells) { log.info( - cyan(setting.name.padEnd(nameWidth)) + - shown.padEnd(valueWidth + (value === undefined ? dim("—").length - 1 : 0)) + - dim(source ?? "not set"), + column(cell.name, nameWidth, cyan) + + column(cell.value, valueWidth, cell.unset ? dim : (value) => value) + + column(cell.source, sourceWidth, dim) + + dim(cell.setting.description), ); } diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index 8063607a2..9d358a5f5 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -17,7 +17,15 @@ export type SettingStore = "config" | "env"; export interface SettingDef { - /** What the user types: `clerk migrate settings set `. */ + /** + * What the user types: `clerk migrate settings set `. + * + * Kebab-case, and identical to the `migrate run` flag it backs. A setting and + * its flag are the same knob reached two ways, so `firebase-signer-key` here + * and `--firebase-signer-key` there must not drift into two spellings the + * user has to learn separately. Sentence-case prose belongs in + * `description`, which is what the list renders alongside it. + */ name: string; store: SettingStore; description: string; @@ -44,19 +52,19 @@ export const SETTINGS: SettingDef[] = [ name: "transformer", store: "config", configKey: "transformer", - description: "Source platform the last run imported from", + description: "Source platform the export came from", }, { name: "file", store: "config", configKey: "file", - description: "Export file the last run imported", + description: "Export file to import users from", }, { name: "skip-unsupported-providers", store: "config", configKey: "skipUnsupportedProviders", - description: "Supabase: skip users whose only social provider is disabled in Clerk", + description: "Supabase: skip users with no provider enabled in Clerk", validate: boolean, }, { diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index 9bee68233..b1a606789 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -139,6 +139,43 @@ describe("list", () => { ); }); + // The names are kebab-case because they mirror the `migrate run` flags; the + // description column is what makes the list readable. + test("explains each setting in prose", async () => { + await list(); + + expect(captured.err).toContain("Source platform the export came from"); + expect(captured.err).toContain("Export file to import users from"); + }); + + test("carries the description into JSON too", async () => { + await list({ json: true }); + + expect(JSON.parse(captured.out)).toContainEqual( + expect.objectContaining({ name: "file", description: "Export file to import users from" }), + ); + }); + + // Colouring before padding counts the ANSI bytes towards the column width, + // which pulls later columns left on exactly the rows that have a value. + test("starts the description at one column, set or not", async () => { + await set("transformer", "supabase"); + captured.clear(); + + await list(); + + const plain = captured.err.replaceAll(/\u001B\[\d+m/g, ""); + const columnOf = (description: string) => + plain + .split("\n") + .find((row) => row.includes(description)) + ?.indexOf(description); + + expect(columnOf("Source platform the export came from")).toBe( + columnOf("Export file to import users from") as number, + ); + }); + test("marks everything as unset in a fresh project", async () => { await list({ json: true }); expect(JSON.parse(captured.out).every((entry: { set: boolean }) => !entry.set)).toBe(true); From 909003bf91ae5dfaabbb6081d77e2cc62848ef24 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 16:59:29 -0400 Subject: [PATCH 008/141] feat(migrate): move the Supabase scope of `skip-unsupported-providers` to a trailing parenthetical Matches how the README heading already scopes it (`--skip-unsupported-providers (Supabase)`), and leaves the description reading as one sentence rather than a label plus a colon. --- packages/cli-core/src/commands/migrate/settings/registry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index 9d358a5f5..e034ec60d 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -64,7 +64,7 @@ export const SETTINGS: SettingDef[] = [ name: "skip-unsupported-providers", store: "config", configKey: "skipUnsupportedProviders", - description: "Supabase: skip users with no provider enabled in Clerk", + description: "Skip users with no provider enabled in Clerk (Supabase)", validate: boolean, }, { From 922890ad1bbe9fdad7f4e626d2c86f2708424c9c Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:11:42 -0400 Subject: [PATCH 009/141] feat(migrate): wrap the logs and transformers subcommands in the gutter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `migrate logs list|clean|convert` and `migrate transformers list` printed flush-left with no intro/outro frame, while `migrate run`, `migrate delete` and every `migrate export` already wrapped — as do the pre-existing commands they mirror (`apps list`, `mcp list`, `unlink`, `config pull`). Follows `commands/mcp/list.ts`: the `--json` early return stays outside the gutter so machine-readable output is unchanged, and only the human path wraps. `withGutter` no-ops outside human mode, so agent output is untouched, and it turns a cancelled prompt into `└ Paused` rather than `└ Failed` — which `logs clean` and `logs convert` both needed. --- .../src/commands/migrate/logs/clean.ts | 73 +++++++++--------- .../src/commands/migrate/logs/convert.ts | 75 ++++++++++--------- .../src/commands/migrate/logs/list.ts | 50 +++++++------ .../migrate/logs/logs-interactive.test.ts | 24 ++++++ .../src/commands/migrate/logs/logs.test.ts | 66 +++++++++++++--- .../migrate/transformers/list.test.ts | 30 ++++++++ .../src/commands/migrate/transformers/list.ts | 37 ++++----- 7 files changed, 234 insertions(+), 121 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/logs/clean.ts b/packages/cli-core/src/commands/migrate/logs/clean.ts index 32d60355d..3c609f5ad 100644 --- a/packages/cli-core/src/commands/migrate/logs/clean.ts +++ b/packages/cli-core/src/commands/migrate/logs/clean.ts @@ -13,6 +13,7 @@ import fs from "node:fs"; import { throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { confirm } from "../../../lib/prompts.ts"; +import { withGutter } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { listLogFiles } from "../lib/log-files.ts"; import { getLogDir } from "../lib/logger.ts"; @@ -22,48 +23,50 @@ export type LogsCleanOptions = { }; export async function clean(options: LogsCleanOptions = {}): Promise { - const files = listLogFiles(); + await withGutter("Cleaning migration logs", async () => { + const files = listLogFiles(); - if (files.length === 0) { - log.info(`No migration logs to clean in ${getLogDir()}.`); - return; - } + if (files.length === 0) { + log.info(`No migration logs to clean in ${getLogDir()}.`); + return; + } - const label = `${files.length} log file${files.length === 1 ? "" : "s"}`; + const label = `${files.length} log file${files.length === 1 ? "" : "s"}`; - if (!options.yes) { - if (isAgent() || !isHuman()) { - throwUsageError( - `\`clerk migrate logs clean\` deletes ${label} from ${getLogDir()} and cannot prompt here. Pass -y to confirm.`, - undefined, - undefined, - [ - { - command: "clerk migrate logs clean -y", - description: "Delete every migration log without prompting", - }, - ], - ); - } + if (!options.yes) { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate logs clean\` deletes ${label} from ${getLogDir()} and cannot prompt here. Pass -y to confirm.`, + undefined, + undefined, + [ + { + command: "clerk migrate logs clean -y", + description: "Delete every migration log without prompting", + }, + ], + ); + } - const proceed = await confirm({ message: `Delete ${label}?`, default: false }); - if (!proceed) throwUserAbort(); - } + const proceed = await confirm({ message: `Delete ${label}?`, default: false }); + if (!proceed) throwUserAbort(); + } - let deleted = 0; - const failures: string[] = []; + let deleted = 0; + const failures: string[] = []; - for (const file of files) { - try { - fs.unlinkSync(file.path); - deleted++; - } catch (error) { - failures.push(`${file.name}: ${(error as Error).message}`); + for (const file of files) { + try { + fs.unlinkSync(file.path); + deleted++; + } catch (error) { + failures.push(`${file.name}: ${(error as Error).message}`); + } } - } - for (const failure of failures) log.warn(`Could not delete ${failure}`); + for (const failure of failures) log.warn(`Could not delete ${failure}`); - log.success(`Deleted ${deleted} log file${deleted === 1 ? "" : "s"}.`); - if (failures.length > 0) process.exitCode = 1; + log.success(`Deleted ${deleted} log file${deleted === 1 ? "" : "s"}.`); + if (failures.length > 0) process.exitCode = 1; + }); } diff --git a/packages/cli-core/src/commands/migrate/logs/convert.ts b/packages/cli-core/src/commands/migrate/logs/convert.ts index 104a236fe..33adf2bae 100644 --- a/packages/cli-core/src/commands/migrate/logs/convert.ts +++ b/packages/cli-core/src/commands/migrate/logs/convert.ts @@ -12,6 +12,7 @@ import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../../ import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { multiselect } from "../../../lib/prompts.ts"; +import { withGutter } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { findLogFile, listLogFiles, readNdjson, type LogFile } from "../lib/log-files.ts"; import { getLogDir } from "../lib/logger.ts"; @@ -81,41 +82,47 @@ async function resolveTargets(options: LogsConvertOptions): Promise { } export async function convert(options: LogsConvertOptions = {}): Promise { - const targets = await resolveTargets(options); - if (targets.length === 0) return; - - let converted = 0; - let malformed = 0; - - for (const file of targets) { - const output = outputPathFor(file); - - try { - const { entries, errors } = readNdjson(file.path); - - // Reported per line, so a truncated final line from an interrupted run - // is visible rather than silently missing from the output. - for (const error of errors) { - malformed++; - log.warn(`${file.name}:${error.line} is not valid JSON and was skipped — ${error.message}`); + // The multiselect lives inside the gutter so cancelling it closes with + // `└ Paused` rather than leaving a half-drawn frame. + await withGutter("Converting migration logs", async () => { + const targets = await resolveTargets(options); + if (targets.length === 0) return; + + let converted = 0; + let malformed = 0; + + for (const file of targets) { + const output = outputPathFor(file); + + try { + const { entries, errors } = readNdjson(file.path); + + // Reported per line, so a truncated final line from an interrupted run + // is visible rather than silently missing from the output. + for (const error of errors) { + malformed++; + log.warn( + `${file.name}:${error.line} is not valid JSON and was skipped — ${error.message}`, + ); + } + + fs.writeFileSync(output, JSON.stringify(entries, null, 2)); + converted++; + const count = `${entries.length} ${entries.length === 1 ? "entry" : "entries"}`; + log.info(`${file.name} → ${output.split("/").pop()} ${dim(`(${count})`)}`); + } catch (error) { + log.warn(`Could not convert ${file.name}: ${(error as Error).message}`); + process.exitCode = 1; } - - fs.writeFileSync(output, JSON.stringify(entries, null, 2)); - converted++; - const count = `${entries.length} ${entries.length === 1 ? "entry" : "entries"}`; - log.info(`${file.name} → ${output.split("/").pop()} ${dim(`(${count})`)}`); - } catch (error) { - log.warn(`Could not convert ${file.name}: ${(error as Error).message}`); - process.exitCode = 1; } - } - if (converted > 0) { - log.success( - `Converted ${converted} log file${converted === 1 ? "" : "s"}. Originals left in place.`, - ); - } - if (malformed > 0) { - log.warn(`${malformed} malformed line${malformed === 1 ? "" : "s"} skipped.`); - } + if (converted > 0) { + log.success( + `Converted ${converted} log file${converted === 1 ? "" : "s"}. Originals left in place.`, + ); + } + if (malformed > 0) { + log.warn(`${malformed} malformed line${malformed === 1 ? "" : "s"} skipped.`); + } + }); } diff --git a/packages/cli-core/src/commands/migrate/logs/list.ts b/packages/cli-core/src/commands/migrate/logs/list.ts index cf30e24e0..1ef7f9405 100644 --- a/packages/cli-core/src/commands/migrate/logs/list.ts +++ b/packages/cli-core/src/commands/migrate/logs/list.ts @@ -8,6 +8,7 @@ import { cyan, dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; +import { withGutter } from "../../../lib/spinner.ts"; import { formatSize, listLogFiles, type LogFile } from "../lib/log-files.ts"; import { getLogDir } from "../lib/logger.ts"; @@ -26,7 +27,7 @@ function toJson(files: LogFile[]) { })); } -export function list(options: LogsListOptions = {}): void { +export async function list(options: LogsListOptions = {}): Promise { const files = listLogFiles(); if (options.json) { @@ -34,31 +35,34 @@ export function list(options: LogsListOptions = {}): void { return; } - if (files.length === 0) { - log.info(`No migration logs in ${getLogDir()}.`); - return; - } - - const kindWidth = Math.max(...files.map((file) => file.kind.length), "TYPE".length) + 2; - const timeWidth = Math.max(...files.map((file) => file.timestamp.length), "TIMESTAMP".length) + 2; - const sizeWidth = Math.max(...files.map((file) => formatSize(file.sizeBytes).length), 4) + 2; + await withGutter("Listing migration logs", async () => { + if (files.length === 0) { + log.info(`No migration logs in ${getLogDir()}.`); + return; + } - log.info( - dim("TYPE".padEnd(kindWidth)) + - dim("TIMESTAMP".padEnd(timeWidth)) + - dim("SIZE".padEnd(sizeWidth)) + - dim("ENTRIES"), - ); + const kindWidth = Math.max(...files.map((file) => file.kind.length), "TYPE".length) + 2; + const timeWidth = + Math.max(...files.map((file) => file.timestamp.length), "TIMESTAMP".length) + 2; + const sizeWidth = Math.max(...files.map((file) => formatSize(file.sizeBytes).length), 4) + 2; - for (const file of files) { log.info( - cyan(file.kind.padEnd(kindWidth)) + - (file.timestamp || dim("—")).padEnd(timeWidth) + - dim(formatSize(file.sizeBytes).padEnd(sizeWidth)) + - String(file.entryCount), + dim("TYPE".padEnd(kindWidth)) + + dim("TIMESTAMP".padEnd(timeWidth)) + + dim("SIZE".padEnd(sizeWidth)) + + dim("ENTRIES"), ); - } - log.info(""); - log.info(dim(`${files.length} log file${files.length === 1 ? "" : "s"} in ${getLogDir()}`)); + for (const file of files) { + log.info( + cyan(file.kind.padEnd(kindWidth)) + + (file.timestamp || dim("—")).padEnd(timeWidth) + + dim(formatSize(file.sizeBytes).padEnd(sizeWidth)) + + String(file.entryCount), + ); + } + + log.info(""); + log.info(dim(`${files.length} log file${files.length === 1 ? "" : "s"} in ${getLogDir()}`)); + }); } diff --git a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts index 36bd10747..4a7cf9eb1 100644 --- a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts @@ -175,3 +175,27 @@ describe("logs convert", () => { expect(mockMultiselect).not.toHaveBeenCalled(); }); }); + +// withGutter turns a UserAbortError into `└ Paused`; a real failure would close +// with `└ Failed`. Declining a prompt is not a failure, so the two must not swap. +describe("cancelling inside the gutter", () => { + test("declining the logs clean confirm closes with Paused, not Failed", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + mockConfirm.mockResolvedValue(false); + + await expect(clean()).rejects.toThrow(UserAbortError); + + expect(captured.err).toContain("Paused"); + expect(captured.err).not.toContain("Failed"); + }); + + test("selecting nothing in the logs convert multiselect closes with Paused", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([]); + + await expect(convert()).rejects.toThrow(UserAbortError); + + expect(captured.err).toContain("Paused"); + expect(captured.err).not.toContain("Failed"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/logs/logs.test.ts b/packages/cli-core/src/commands/migrate/logs/logs.test.ts index 5be2d232b..d0db18fb6 100644 --- a/packages/cli-core/src/commands/migrate/logs/logs.test.ts +++ b/packages/cli-core/src/commands/migrate/logs/logs.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { clean } from "./clean.ts"; @@ -42,21 +43,21 @@ const MIGRATION = "migration-2026-01-01T12-00-00.log"; const DELETION = "user-deletion-2026-02-01T12-00-00.log"; describe("logs list", () => { - test("says so plainly when there is no logs directory", () => { - list(); + test("says so plainly when there is no logs directory", async () => { + await list(); expect(captured.err).toContain("No migration logs in"); }); - test("says so plainly when the directory is empty", () => { + test("says so plainly when the directory is empty", async () => { fs.mkdirSync(getLogDir(), { recursive: true }); - list(); + await list(); expect(captured.err).toContain("No migration logs in"); }); - test("reports type, timestamp, size and entry count", () => { + test("reports type, timestamp, size and entry count", async () => { writeLog(MIGRATION, [{ userId: "u1" }, { userId: "u2" }, { userId: "u3" }]); - list(); + await list(); expect(captured.err).toContain("TYPE"); expect(captured.err).toContain("TIMESTAMP"); @@ -68,21 +69,21 @@ describe("logs list", () => { expect(captured.err).toContain("3"); }); - test("lists every log kind", () => { + test("lists every log kind", async () => { writeLog(MIGRATION, [{ a: 1 }]); writeLog(DELETION, [{ a: 1 }]); - list(); + await list(); expect(captured.err).toContain("migration"); expect(captured.err).toContain("deletion"); expect(captured.err).toContain("2 log files"); }); - test("--json emits a machine-readable listing on stdout", () => { + test("--json emits a machine-readable listing on stdout", async () => { writeLog(MIGRATION, [{ userId: "u1" }]); - list({ json: true }); + await list({ json: true }); const parsed = JSON.parse(captured.out) as Record[]; expect(parsed).toHaveLength(1); @@ -94,8 +95,8 @@ describe("logs list", () => { }); }); - test("--json emits an empty array rather than prose when there are no logs", () => { - list({ json: true }); + test("--json emits an empty array rather than prose when there are no logs", async () => { + await list({ json: true }); expect(JSON.parse(captured.out)).toEqual([]); }); }); @@ -217,3 +218,44 @@ describe("logs convert", () => { expect(captured.err).toContain("3 entries"); }); }); + +describe("human-mode frame", () => { + let originalMode: Mode; + + beforeAll(() => { + originalMode = getMode(); + setMode("human"); + }); + + afterAll(() => { + setMode(originalMode); + }); + + test("logs list wraps its output in an intro/outro gutter", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await list(); + + expect(captured.err).toContain("\u250c"); + expect(captured.err).toContain("Listing migration logs"); + expect(captured.err).toContain("\u2514"); + expect(captured.err).toContain("Done"); + }); + + test("--json stays outside the gutter, on stdout only", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await list({ json: true }); + + expect(JSON.parse(captured.out)).toHaveLength(1); + expect(captured.err).not.toContain("\u250c"); + }); + + test("a failure inside logs convert closes with Failed and still throws", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await expect(convert({ files: ["nope.log"] })).rejects.toThrow(CliError); + + expect(captured.err).toContain("Failed"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/list.test.ts b/packages/cli-core/src/commands/migrate/transformers/list.test.ts index 9f432e1d9..26179ee9f 100644 --- a/packages/cli-core/src/commands/migrate/transformers/list.test.ts +++ b/packages/cli-core/src/commands/migrate/transformers/list.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { list } from "./list.ts"; import { transformers } from "./registry.ts"; @@ -114,3 +115,32 @@ describe("a bad --transformer-file", () => { await expect(list({ transformerFile: "./nope.ts" })).rejects.toThrow(CliError); }); }); + +describe("human-mode frame", () => { + let originalMode: Mode; + + beforeAll(() => { + originalMode = getMode(); + setMode("human"); + }); + + afterAll(() => { + setMode(originalMode); + }); + + test("wraps its output in an intro/outro gutter", async () => { + await list(); + + expect(captured.err).toContain("┌"); + expect(captured.err).toContain("Listing transformers"); + expect(captured.err).toContain("└"); + expect(captured.err).toContain("Done"); + }); + + test("--json stays outside the gutter, on stdout only", async () => { + await list({ json: true }); + + expect(() => JSON.parse(captured.out)).not.toThrow(); + expect(captured.err).not.toContain("┌"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/list.ts b/packages/cli-core/src/commands/migrate/transformers/list.ts index 349a53a8f..37da1e0a5 100644 --- a/packages/cli-core/src/commands/migrate/transformers/list.ts +++ b/packages/cli-core/src/commands/migrate/transformers/list.ts @@ -8,6 +8,7 @@ import { bold, cyan, dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; +import { withGutter } from "../../../lib/spinner.ts"; import type { TransformerRegistryEntry } from "../types.ts"; import { loadCustomTransformer } from "./load-custom.ts"; import { transformers } from "./registry.ts"; @@ -44,24 +45,26 @@ export async function list(options: TransformersListOptions = {}): Promise return; } - for (const entry of entries) { - const suffix = entry.builtIn ? "" : ` ${dim(`(custom — ${entry.source})`)}`; - log.info(`${cyan(bold(entry.key))} ${entry.label}${suffix}`); - log.info(` ${dim(entry.description)}`); - log.info(""); - } - - const custom = entries.length - transformers.length; - log.info( - dim( - `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + - (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), - ), - ); + await withGutter("Listing transformers", async () => { + for (const entry of entries) { + const suffix = entry.builtIn ? "" : ` ${dim(`(custom — ${entry.source})`)}`; + log.info(`${cyan(bold(entry.key))} ${entry.label}${suffix}`); + log.info(` ${dim(entry.description)}`); + log.info(""); + } - if (custom === 0) { + const custom = entries.length - transformers.length; log.info( - dim("Migrating from something else? Write a transformer and pass --transformer-file."), + dim( + `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + + (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), + ), ); - } + + if (custom === 0) { + log.info( + dim("Migrating from something else? Write a transformer and pass --transformer-file."), + ); + } + }); } From 05b65365b2fab5707621294d380e3b978985d603 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:13:51 -0400 Subject: [PATCH 010/141] feat(migrate): follow the CLI's spinner, next-steps, pluralization and icon conventions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five formatting mismatches between `migrate` and every command around it. **Summaries escaped the gutter.** `run` and `delete` printed their final summary through `log.raw`, the un-prefixed channel meant for machine-readable output, so it landed flush-left and broke the `┌ … │ … └` frame. `log.info` runs each line through `applyPrefix`, so only the channel changes. **Spinners lost the `...` convention.** All 39 pre-existing `withSpinner` call sites end their message in `...` and none passes a done-message — the vanishing ellipsis *is* the completion signal. Migrate had inverted both halves. Adds the ellipsis to every message and `spinner.update()` string, and drops the four done-message arguments so the stop text derives from the message like everywhere else. **No Next steps blocks.** Every comparable command closes with one. Adds `MIGRATE_DONE`, `MIGRATE_DELETE` and `MIGRATE_EXPORT`, wired through the gutter's `setNextSteps`. `reportExport` now returns the steps rather than hand-rolling a `dim("Next: …")` line, which keeps the six export modules' call shape intact. An export of zero users returns none — there is nothing to import — so `setNextSteps` ignores an empty list instead of rendering a header with no bullets under it. **`user(s)` pluralization.** The CLI's form is `${n} thing${n === 1 ? "" : "s"}`. Migrate used `user(s)`/`row(s)` in ten places while using the correct form in others. **`●`/`○` status icons**, which appear nowhere else in the codebase, become the established `✓`/`✗`/`!` vocabulary from `doctor` and `init`. --- .../cli-core/src/commands/migrate/README.md | 15 ++--- .../src/commands/migrate/delete.test.ts | 4 +- .../cli-core/src/commands/migrate/delete.ts | 29 +++++----- .../src/commands/migrate/export/auth0.test.ts | 11 +++- .../src/commands/migrate/export/auth0.ts | 28 +++++----- .../src/commands/migrate/export/authjs.ts | 22 ++++---- .../src/commands/migrate/export/betterauth.ts | 20 ++++--- .../src/commands/migrate/export/clerk.test.ts | 38 ++++++++++++- .../src/commands/migrate/export/clerk.ts | 26 ++++----- .../migrate/export/db-exports.test.ts | 2 +- .../commands/migrate/export/firebase.test.ts | 11 +++- .../src/commands/migrate/export/firebase.ts | 30 +++++----- .../src/commands/migrate/export/shared.ts | 26 +++++---- .../src/commands/migrate/export/supabase.ts | 20 ++++--- .../src/commands/migrate/import-users.ts | 2 +- .../src/commands/migrate/lib/readiness.ts | 4 +- .../cli-core/src/commands/migrate/run.test.ts | 6 +- packages/cli-core/src/commands/migrate/run.ts | 56 ++++++++++--------- .../src/commands/migrate/settings/clear.ts | 4 +- packages/cli-core/src/lib/next-steps.ts | 10 ++++ packages/cli-core/src/lib/spinner.ts | 4 +- 21 files changed, 227 insertions(+), 141 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index d8d9aeb63..f74ab3606 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -165,13 +165,14 @@ import it, not after: ``` Field coverage - ● 3/3 have an email address - ○ 0/3 have a phone number - ○ 1/3 have a username - ○ 2/3 have a password (not exportable — see below) - -Exported 3 user(s) to /project/exports/clerk-export.json -Next: clerk migrate run --transformer clerk --file exports/clerk-export.json + ✓ 3/3 have an email address + ✗ 0/3 have a phone number + ! 1/3 have a username + ! 2/3 have a password (not exportable — see below) + +Exported 3 users to /project/exports/clerk-export.json +└ Next steps + → Run `clerk migrate run --transformer clerk --file exports/clerk-export.json` to import them ``` Every export also writes `logs/export-.log`, so `migrate logs list` diff --git a/packages/cli-core/src/commands/migrate/delete.test.ts b/packages/cli-core/src/commands/migrate/delete.test.ts index ef110ed00..01711ec97 100644 --- a/packages/cli-core/src/commands/migrate/delete.test.ts +++ b/packages/cli-core/src/commands/migrate/delete.test.ts @@ -382,7 +382,7 @@ describe("deleteMigration", () => { await deleteMigration(baseOptions); expect(deleteCalls()).toEqual([expect.stringContaining("/v1/users/user_1")]); - expect(captured.err).toContain("1 of the file's user(s) are not in this instance"); + expect(captured.err).toContain("1 of the file's users is not in this instance"); }); test("does nothing when none of the migration's users are present", async () => { @@ -399,7 +399,7 @@ describe("deleteMigration", () => { stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); await expect(deleteMigration({ secretKey: "sk_test_x" })).rejects.toThrow( - /permanently deletes 2 user\(s\) and cannot prompt here/, + /permanently deletes 2 users and cannot prompt here/, ); expect(deleteCalls()).toHaveLength(0); }); diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 8b14abe55..db2791205 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -31,6 +31,7 @@ import { } from "../../lib/errors.ts"; import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; import { log } from "../../lib/log.ts"; +import { NEXT_STEPS } from "../../lib/next-steps.ts"; import { confirm } from "../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; @@ -130,7 +131,7 @@ export async function findMigratedUsers(options: { const batches = batch(options.externalIds, EXTERNAL_ID_BATCH); for (const [index, ids] of batches.entries()) { - options.spinner?.update(`Finding migrated users: batch ${index + 1}/${batches.length}`); + options.spinner?.update(`Finding migrated users: batch ${index + 1}/${batches.length}...`); const params = new URLSearchParams(); params.set("limit", String(EXTERNAL_ID_BATCH)); @@ -181,7 +182,7 @@ export async function deleteMigratedUsers(options: { const progress = () => spinner?.update( - `Deleting users: [${processed}/${users.length}] (${deleted} deleted, ${failed} failed)`, + `Deleting users: [${processed}/${users.length}] (${deleted} deleted, ${failed} failed)...`, ); // A failure on one user must not abort the rest: a half-undone migration @@ -264,7 +265,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise { + await withGutter("Undoing a migration", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); const limits = resolveLimits(secretKey); @@ -277,15 +278,13 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise findMigratedUsers({ externalIds, secretKey, spinner }), - "Search complete", + const users = await withSpinner("Finding migrated users...", (spinner) => + findMigratedUsers({ externalIds, secretKey, spinner }), ); if (users.length === 0) { log.info( - `None of the ${externalIds.length} user(s) in ${file} are in ${target ?? "this instance"}. Nothing to delete.`, + `None of the ${externalIds.length} user${externalIds.length === 1 ? "" : "s"} in ${file} are in ${target ?? "this instance"}. Nothing to delete.`, ); return; } @@ -297,7 +296,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise deleteMigratedUsers({ users, secretKey, limits, dateTime, spinner }), - "Deletion complete", + const summary = await withSpinner(`Deleting users: [0/${users.length}]...`, (spinner) => + deleteMigratedUsers({ users, secretKey, limits, dateTime, spinner }), ); - log.raw(formatSummary(summary, logFile)); + log.info(formatSummary(summary, logFile)); + + setNextSteps(NEXT_STEPS.MIGRATE_DELETE); if (summary.failed > 0) process.exitCode = 1; }); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index 58ce5add2..ceef2915a 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -1,4 +1,5 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -290,7 +291,15 @@ describe("exportAuth0", () => { test("names the command that consumes the file", async () => { stubAuth0([[auth0User(0)], []]); - await exportAuth0({ ...CREDENTIALS }); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + await exportAuth0({ ...CREDENTIALS }); + } finally { + setMode(originalMode); + } expect(captured.err).toContain( "migrate run --transformer auth0 --file exports/auth0-export.json", ); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 3d90d661c..bf1657267 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -219,7 +219,7 @@ export async function fetchAllAuth0Users(options: { for (let page = 0; ; page++) { const { users, total } = await fetchAuth0Page(options.credentials, options.token, page); all.push(...users); - options.spinner?.update(`Fetching users from Auth0: ${all.length} so far`); + options.spinner?.update(`Fetching users from Auth0: ${all.length} so far...`); if (users.length < PAGE_SIZE) break; @@ -316,30 +316,30 @@ export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0Exp export async function exportAuth0(options: ExportAuth0Options): Promise { const credentials = await resolveAuth0Credentials(options); - await withGutter("Exporting users from Auth0", async () => { + await withGutter("Exporting users from Auth0", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); log.info(`Exporting from ${credentials.domain}.`); - const token = await withSpinner("Authenticating with Auth0", () => + const token = await withSpinner("Authenticating with Auth0...", () => fetchAuth0Token(credentials), ); - const users = await withSpinner( - "Fetching users from Auth0", - (spinner) => fetchAllAuth0Users({ credentials, token, spinner }), - "Users fetched", + const users = await withSpinner("Fetching users from Auth0...", (spinner) => + fetchAllAuth0Users({ credentials, token, spinner }), ); const { users: exported, coverage } = buildAuth0Export(users, dateTime); const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("auth0")); - reportExport({ - platform: "auth0", - userCount: exported.length, - outputPath, - coverage, - transformerKey: "auth0", - }); + setNextSteps( + reportExport({ + platform: "auth0", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "auth0", + }), + ); if (exported.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index d8183d981..a78df15de 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -113,24 +113,26 @@ export async function exportAuthJs(options: DbExportOptions): Promise { hint: "Postgres, MySQL or a SQLite file — whichever your Auth.js adapter uses.", }); - await withGutter("Exporting users from Auth.js", async () => { + await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); - const { rows, table } = await withSpinner("Reading the user table", () => + const { rows, table } = await withSpinner("Reading the user table...", () => withDbClient(dbUrl, "authjs", fetchAuthJsUsers), ); - log.info(`Read ${rows.length} row(s) from ${table}.`); + log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); const { users, coverage } = buildAuthJsExport(rows, dateTime); const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("authjs")); - reportExport({ - platform: "authjs", - userCount: users.length, - outputPath, - coverage, - transformerKey: "authjs", - }); + setNextSteps( + reportExport({ + platform: "authjs", + userCount: users.length, + outputPath, + coverage, + transformerKey: "authjs", + }), + ); if (users.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index e2dcad93e..85ab190db 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -160,10 +160,10 @@ export async function exportBetterAuth(options: DbExportOptions): Promise hint: "Postgres, MySQL or a SQLite file — whichever your Better Auth install uses.", }); - await withGutter("Exporting users from Better Auth", async () => { + await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); - const { rows, plugins } = await withSpinner("Reading the user table", () => + const { rows, plugins } = await withSpinner("Reading the user table...", () => withDbClient(dbUrl, "betterauth", async (client) => { const plugins = await detectPluginColumns(client); const rows = await client.query(buildBetterAuthQuery(client, plugins)); @@ -180,12 +180,14 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const { users, coverage } = buildBetterAuthExport(rows, dateTime); const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("betterauth")); - reportExport({ - platform: "betterauth", - userCount: users.length, - outputPath, - coverage, - transformerKey: "betterauth", - }); + setNextSteps( + reportExport({ + platform: "betterauth", + userCount: users.length, + outputPath, + coverage, + transformerKey: "betterauth", + }), + ); }); } diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 12a3062db..9ce14d656 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -1,4 +1,5 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -240,12 +241,20 @@ describe("exportClerk", () => { expect(written).toHaveLength(1); expect(written[0]?.id).toBe("u1"); expect(captured.err).toContain("Field coverage"); - expect(captured.err).toContain("Exported 1 user(s)"); + expect(captured.err).toContain("Exported 1 user"); }); test("names the command that consumes the file", async () => { stubPages([[user()], []]); - await exportClerk({ secretKey: "sk_test_x" }); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + await exportClerk({ secretKey: "sk_test_x" }); + } finally { + setMode(originalMode); + } expect(captured.err).toContain( "migrate run --transformer clerk --file exports/clerk-export.json", ); @@ -278,4 +287,29 @@ describe("exportClerk", () => { JSON.parse(fs.readFileSync(path.join(workDir, "exports", "clerk-export.json"), "utf-8")), ).toEqual([]); }); + + test("an empty export warns but does not suggest importing it", async () => { + stubPages([[]]); + const originalMode = getMode(); + setMode("human"); + try { + await exportClerk({ secretKey: "sk_test_x" }); + } finally { + setMode(originalMode); + } + + expect(captured.err).toContain("No users found to export"); + expect(captured.err).not.toContain("Next steps"); + expect(captured.err).not.toContain("migrate run --transformer"); + }); + + test("agent mode suppresses the Next steps block", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("Exported 1 user"); + expect(captured.err).not.toContain("Next steps"); + expect(captured.err).not.toContain("migrate run --transformer"); + }); }); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 37c10d802..a49fa581e 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -172,7 +172,7 @@ export async function fetchAllClerkUsers(options: { const page = Array.isArray(response.body) ? (response.body as BapiUser[]) : []; all.push(...page); - options.spinner?.update(`Fetching users from Clerk: ${all.length} so far`); + options.spinner?.update(`Fetching users from Clerk: ${all.length} so far...`); // A short page means the end; anything else would loop forever on an // instance whose size happens to be a multiple of the page size. @@ -229,29 +229,29 @@ export async function exportClerk(options: ExportClerkOptions): Promise { } const secretKeyOption = options.secretKey ?? options.clerkSecretKey; - await withGutter("Exporting users from Clerk", async () => { + await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); const dateTime = getDateTimeStamp(); log.info(`Exporting from ${target ?? "the resolved instance"}.`); - const users = await withSpinner( - "Fetching users from Clerk", - (spinner) => fetchAllClerkUsers({ secretKey, spinner }), - "Users fetched", + const users = await withSpinner("Fetching users from Clerk...", (spinner) => + fetchAllClerkUsers({ secretKey, spinner }), ); const { users: exported, coverage } = buildClerkExport(users, dateTime); const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("clerk")); - reportExport({ - platform: "clerk", - userCount: exported.length, - outputPath, - coverage, - transformerKey: "clerk", - }); + setNextSteps( + reportExport({ + platform: "clerk", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "clerk", + }), + ); if (exported.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 15116d255..08816b69a 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -213,7 +213,7 @@ describe("authjs export", () => { const written = JSON.parse(fs.readFileSync(path.join(workDir, "authjs.json"), "utf-8")); expect(written).toHaveLength(2); - expect(captured.err).toContain("Read 2 row(s) from"); + expect(captured.err).toContain("Read 2 rows from"); expect(captured.err).toContain("stores no passwords"); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 3772e35de..137d0502d 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -1,4 +1,5 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -414,7 +415,15 @@ describe("exportFirebase", () => { test("names the command that consumes the file", async () => { stubFirebase([[fbUser(0)]], { signIn: {} }); - await exportFirebase({ serviceAccount: "./sa.json" }); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + await exportFirebase({ serviceAccount: "./sa.json" }); + } finally { + setMode(originalMode); + } expect(captured.err).toContain( "migrate run --transformer firebase --file exports/firebase-export.json", ); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 0491227ae..9c70a410b 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -255,7 +255,7 @@ export async function fetchAllFirebaseUsers(options: { const body = (await response.json()) as { users?: FirebaseUser[]; nextPageToken?: string }; all.push(...(body.users ?? [])); - options.spinner?.update(`Fetching users from Firebase: ${all.length} so far`); + options.spinner?.update(`Fetching users from Firebase: ${all.length} so far...`); pageToken = body.nextPageToken; } while (pageToken); @@ -424,28 +424,30 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { + await withGutter("Exporting users from Firebase", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); log.info(`Exporting from the ${account.project_id} project.`); - const token = await withSpinner("Authenticating with Google", () => fetchAccessToken(account)); + const token = await withSpinner("Authenticating with Google...", () => + fetchAccessToken(account), + ); - const users = await withSpinner( - "Fetching users from Firebase", - (spinner) => fetchAllFirebaseUsers({ account, token, spinner }), - "Users fetched", + const users = await withSpinner("Fetching users from Firebase...", (spinner) => + fetchAllFirebaseUsers({ account, token, spinner }), ); const { users: exported, coverage } = buildFirebaseExport(users, dateTime); const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("firebase")); - reportExport({ - platform: "firebase", - userCount: exported.length, - outputPath, - coverage, - transformerKey: "firebase", - }); + setNextSteps( + reportExport({ + platform: "firebase", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "firebase", + }), + ); const passwordCount = coverage.find((entry) => entry.label.includes("password"))?.count ?? 0; const hashConfig = passwordCount > 0 ? await fetchHashConfig(account, token) : null; diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index aa25ea50f..e880d90d3 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -13,6 +13,7 @@ import fs from "node:fs"; import path from "node:path"; import { dim, green, yellow } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; +import { NEXT_STEPS } from "../../../lib/next-steps.ts"; /** Where an export lands when `--output` is not given. */ export function defaultOutputPath(platform: string): string { @@ -36,13 +37,13 @@ export type CoverageField = { label: string; count: number }; /** * How complete an export is, per field. * - * ● every user, ○ some, dim ○ none. The point is to see *before* importing + * ✓ every user, ! some, dim ✗ none. The point is to see *before* importing * that, say, only 3 of 400 users have a password — which changes what the * migration means. */ export function formatFieldCoverage(fields: CoverageField[], total: number): string[] { return fields.map(({ label, count }) => { - const icon = count === total ? green("●") : count > 0 ? yellow("○") : dim("○"); + const icon = count === total ? green("✓") : count > 0 ? yellow("!") : dim("✗"); return ` ${icon} ${dim(`${count}/${total} ${label}`)}`; }); } @@ -56,12 +57,18 @@ export type ExportSummary = { transformerKey: string; }; -/** Reports the coverage table and the exact command that consumes the file. */ -export function reportExport(summary: ExportSummary): void { +/** + * Reports the coverage table. + * + * @returns The next steps for the caller to hand to `setNextSteps`, so the + * suggested import command closes the gutter like every other command's. + * Empty when nothing was exported — there is nothing to import. + */ +export function reportExport(summary: ExportSummary): readonly string[] { log.blank(); if (summary.userCount === 0) { log.warn(`No users found to export. Wrote an empty file to ${summary.outputPath}.`); - return; + return []; } log.info("Field coverage"); @@ -70,12 +77,11 @@ export function reportExport(summary: ExportSummary): void { } log.blank(); - log.success(`Exported ${summary.userCount} user(s) to ${summary.outputPath}`); - log.info( - dim( - `Next: clerk migrate run --transformer ${summary.transformerKey} --file ${relativeIfInside(summary.outputPath)}`, - ), + log.success( + `Exported ${summary.userCount} user${summary.userCount === 1 ? "" : "s"} to ${summary.outputPath}`, ); + + return NEXT_STEPS.MIGRATE_EXPORT(summary.transformerKey, relativeIfInside(summary.outputPath)); } /** Shortens a path for display when it sits under the working directory. */ diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 585ebf9d9..80ff3785e 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -111,23 +111,25 @@ export async function exportSupabase(options: DbExportOptions): Promise { hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", }); - await withGutter("Exporting users from Supabase", async () => { + await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); - const rows = await withSpinner("Reading auth.users", () => + const rows = await withSpinner("Reading auth.users...", () => withDbClient(dbUrl, "supabase", fetchSupabaseUsers), ); const { users, coverage } = buildSupabaseExport(rows, dateTime); const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("supabase")); - reportExport({ - platform: "supabase", - userCount: users.length, - outputPath, - coverage, - transformerKey: "supabase", - }); + setNextSteps( + reportExport({ + platform: "supabase", + userCount: users.length, + outputPath, + coverage, + transformerKey: "supabase", + }), + ); if (users.length > 0) { log.info( diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 749142cf4..80402b6d9 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -299,7 +299,7 @@ export async function importUsers(options: ImportUsersOptions): Promise spinner?.update( - `Importing users: [${processed}/${total}] (${successful} succeeded, ${failed} failed)`, + `Importing users: [${processed}/${total}] (${successful} succeeded, ${failed} failed)...`, ); const recordFailure = (userId: string, message: string, code: string) => { diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts index 353823415..1db17fe3e 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -190,7 +190,7 @@ function renderItem(item: ReadinessItem, total: number): string { return ` ${green("✓")} ${item.label} — ${dim(`enabled in Clerk — ${coverage}`)}`; } // Settings unavailable: state coverage without claiming anything about Clerk. - return ` ${yellow("○")} ${item.label} — ${dim(`${coverage} — check it is enabled in Clerk`)}`; + return ` ${yellow("!")} ${item.label} — ${dim(`${coverage} — check it is enabled in Clerk`)}`; } /** Renders the report for a human, as lines. */ @@ -210,7 +210,7 @@ export function formatReadinessReport(report: ReadinessReport): string[] { if (report.settingsUnavailable) { lines.push( "", - ` ${yellow("○")} ${dim("Could not read this instance's settings, so the checks below are coverage only.")}`, + ` ${yellow("!")} ${dim("Could not read this instance's settings, so the checks below are coverage only.")}`, ` ${dim(` Verify your settings at ${DASHBOARD_URL}`)}`, ); } diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 330d70645..1698a2517 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -263,7 +263,7 @@ describe("run", () => { const created = requests.filter((r) => r.url.endsWith("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/v1/users")); expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u1"]); - expect(captured.err).toContain("skipping 1 user(s) without a password"); + expect(captured.err).toContain("skipping 1 user without a password"); }); test("--resume-after skips everyone up to and including that ID", async () => { @@ -279,7 +279,7 @@ describe("run", () => { await run(baseOptions); expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); - expect(captured.err).toContain("1 user(s) failed validation"); + expect(captured.err).toContain("1 user failed validation"); }); test("warns that --clerk-secret-key is deprecated but still honours it", async () => { @@ -750,7 +750,7 @@ describe("run", () => { await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); expect(created()).toEqual(["sb_email", "sb_both"]); - expect(captured.err).toContain("skipping 1 user(s)"); + expect(captured.err).toContain("skipping 1 user "); expect(captured.err).toContain("discord: 1"); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 9b6e9bdf0..7167b854b 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -14,6 +14,7 @@ import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command import { bold, dim, green, red, yellow } from "../../lib/color.ts"; import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; import { log } from "../../lib/log.ts"; +import { NEXT_STEPS } from "../../lib/next-steps.ts"; import { confirm } from "../../lib/prompts.ts"; import { withGutter, withSpinner } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; @@ -270,7 +271,7 @@ async function skipDisabledProviderUsers( return users; } - const settings = await withSpinner("Checking enabled providers", () => + const settings = await withSpinner("Checking enabled providers...", () => fetchInstanceSettings(secretKey), ); const enabled = settings ? enabledSocialProviders(settings) : null; @@ -300,7 +301,7 @@ async function skipDisabledProviderUsers( .map(([provider, count]) => `${provider}: ${count}`) .join(", "); log.warn( - `--skip-unsupported-providers: skipping ${excludedIds.size} user(s) whose only provider is not enabled in Clerk (${breakdown}).`, + `--skip-unsupported-providers: skipping ${excludedIds.size} user${excludedIds.size === 1 ? "" : "s"} whose only provider is not enabled in Clerk (${breakdown}).`, ); return users.filter((user) => !excludedIds.has(user.userId)); @@ -328,7 +329,7 @@ async function showReadinessReport(input: { }): Promise { if (input.skipReport) return; - const settings = await withSpinner("Checking instance settings", () => + const settings = await withSpinner("Checking instance settings...", () => fetchInstanceSettings(input.secretKey), ); @@ -443,7 +444,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const { transformer, file } = validateRunOptions(options); const firebaseHashConfig = await resolveFirebaseHashConfig(options); - await withGutter("Migrating users to Clerk", async () => { + await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); const limits = resolveLimits(secretKey); @@ -451,9 +452,8 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const logFile = getLogFilePath("migration", dateTime); const { users: loaded, validationFailed } = await withSpinner( - `Loading users from ${file}`, + `Loading users from ${file}...`, () => loadUsersFromFile(file, transformer, dateTime, { context: { firebaseHashConfig } }), - "Users loaded", ); let users = applyResumeAfter(loaded, options.resumeAfter); @@ -469,14 +469,16 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const withPassword = users.filter((user) => Boolean(user.password)); const dropped = users.length - withPassword.length; if (dropped > 0) { - log.info(`--require-password: skipping ${dropped} user(s) without a password.`); + log.info( + `--require-password: skipping ${dropped} user${dropped === 1 ? "" : "s"} without a password.`, + ); } users = withPassword; } if (validationFailed > 0) { log.warn( - `${validationFailed} user(s) failed validation and will be skipped. See ${logFile}.`, + `${validationFailed} user${validationFailed === 1 ? "" : "s"} failed validation and will be skipped. See ${logFile}.`, ); } @@ -493,9 +495,12 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ); } + // `target` already carries the instance's environment ("My App + // (development)"), so the detected type is only worth spelling out when + // there is no app context to name — an explicit `--secret-key`. log.info( - `Importing ${users.length} user(s) via the ${transformer} transformer into ` + - `${target ?? "the resolved instance"} (${limits.instanceType}).`, + `Importing ${users.length} user${users.length === 1 ? "" : "s"} via the ${transformer} transformer into ` + + `${target ?? `the resolved instance (${limits.instanceType})`}.`, ); await showReadinessReport({ @@ -509,7 +514,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (!options.yes && isHuman() && !isAgent()) { const proceed = await confirm({ - message: `Import ${users.length} user(s)?`, + message: `Import ${users.length} user${users.length === 1 ? "" : "s"}?`, default: false, }); if (!proceed) throwUserAbort(); @@ -523,22 +528,23 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ...(options.skipUnsupportedProviders ? { skipUnsupportedProviders: true } : {}), }); - const summary = await withSpinner( - `Importing users: [0/${users.length}]`, - (spinner) => - importUsers({ - users, - secretKey, - limits, - dateTime, - skipPasswordRequirement: !options.requirePassword, - validationFailed, - spinner, - }), - "Import complete", + const summary = await withSpinner(`Importing users: [0/${users.length}]...`, (spinner) => + importUsers({ + users, + secretKey, + limits, + dateTime, + skipPasswordRequirement: !options.requirePassword, + validationFailed, + spinner, + }), ); - log.raw(formatSummary(summary, logFile)); + log.info(formatSummary(summary, logFile)); + + // Offered even when some users failed: a partial import is exactly when + // reading the log and knowing how to undo it matters most. + setNextSteps(NEXT_STEPS.MIGRATE_DONE); if (summary.failed > 0) process.exitCode = 1; }); diff --git a/packages/cli-core/src/commands/migrate/settings/clear.ts b/packages/cli-core/src/commands/migrate/settings/clear.ts index 933d1e540..cb588e0eb 100644 --- a/packages/cli-core/src/commands/migrate/settings/clear.ts +++ b/packages/cli-core/src/commands/migrate/settings/clear.ts @@ -53,6 +53,8 @@ export async function clear(options: SettingsClearOptions = {}): Promise { if (hadConfig) log.success("Cleared the saved transformer and file."); if (dropped.length > 0) { - log.success(`Removed ${dropped.length} credential(s) from ${MIGRATE_ENV_FILE}.`); + log.success( + `Removed ${dropped.length} credential${dropped.length === 1 ? "" : "s"} from ${MIGRATE_ENV_FILE}.`, + ); } } diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index e447ddd2e..613fe1042 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -71,6 +71,16 @@ export const NEXT_STEPS = { "Run `clerk apps list` to see your other applications", "Run `clerk config pull` to inspect the live configuration of this instance", ], + MIGRATE_DONE: [ + "Run `clerk migrate logs list` to inspect the import log", + "Run `clerk migrate delete` to undo this migration", + ], + MIGRATE_DELETE: ["Run `clerk migrate logs list` to inspect the deletion log"], + // The only parameterized entry: a suggested import is worthless unless it + // names the transformer that reads this export and the file just written. + MIGRATE_EXPORT: (transformerKey: string, file: string) => [ + `Run \`clerk migrate run --transformer ${transformerKey} --file ${file}\` to import them`, + ], } as const; /** diff --git a/packages/cli-core/src/lib/spinner.ts b/packages/cli-core/src/lib/spinner.ts index b668c35c1..707ba4d16 100644 --- a/packages/cli-core/src/lib/spinner.ts +++ b/packages/cli-core/src/lib/spinner.ts @@ -107,7 +107,9 @@ export async function withGutter( let nextSteps: readonly string[] | undefined; const controls: GutterControls = { setNextSteps(steps) { - nextSteps = steps; + // Empty is ignored rather than stored: `outro([])` would render the + // "Next steps" header with no bullets under it. Matches printNextSteps. + if (steps.length > 0) nextSteps = steps; }, }; From 68c7057f399da7e2fcb45bfc60631f39ddfe46a5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:14:06 -0400 Subject: [PATCH 011/141] feat(prompts): advertise select-all in the multiselect footer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `MultiSelectPrompt` has always bound `a` to toggle every option, but clack's instruction footer never listed it and takes no override — so a genuinely useful key stayed undiscoverable unless each call site spelled it out in its own message, which is worse: it is a property of the prompt, not of any one question. `MULTISELECT_INSTRUCTIONS` is the only seam, and it is read fresh on every render. Inserted second-to-last so `Enter: confirm` stays where readers expect it. `i` (invert) is left out deliberately: it is rarely what anyone wants, and a four-item legend stops being scannable. The test lives outside `prompts.test.ts`, which mocks the whole module. What is worth checking is that the real clack export is still a live array read at render time — an upgrade that froze it, replaced it, or rendered a copy would drop `a: all` silently and nothing else in the suite would notice. --- .../src/lib/prompts-instructions.test.ts | 29 +++++++++++++++++++ packages/cli-core/src/lib/prompts.ts | 18 ++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 packages/cli-core/src/lib/prompts-instructions.test.ts diff --git a/packages/cli-core/src/lib/prompts-instructions.test.ts b/packages/cli-core/src/lib/prompts-instructions.test.ts new file mode 100644 index 000000000..b159c1a2e --- /dev/null +++ b/packages/cli-core/src/lib/prompts-instructions.test.ts @@ -0,0 +1,29 @@ +/** + * The multiselect footer hack, checked against the real @clack/prompts. + * + * Kept out of `prompts.test.ts`, which mocks the whole module — the one thing + * worth verifying here is that the real export is still a live array clack + * reads at render time. A clack upgrade that froze it, replaced it, or rendered + * a copy would drop `a: all` from the legend silently, and nothing else in the + * suite would notice. + */ + +import { test, expect } from "bun:test"; +import { MULTISELECT_INSTRUCTIONS } from "@clack/prompts"; + +// Importing for the module-level side effect is the point. +await import("./prompts.ts"); + +const legend = () => MULTISELECT_INSTRUCTIONS.join(" • ").replaceAll(/\[[0-9;]*m/g, ""); + +test("the multiselect legend advertises select-all", () => { + expect(legend()).toContain("a: all"); +}); + +test("confirm stays last, where readers expect it", () => { + expect(legend().endsWith("Enter: confirm")).toBe(true); +}); + +test("the keys clack actually binds are the ones named", () => { + expect(legend()).toBe("↑/↓ to navigate • Space: select • a: all • Enter: confirm"); +}); diff --git a/packages/cli-core/src/lib/prompts.ts b/packages/cli-core/src/lib/prompts.ts index 533924e89..ed8a466c5 100644 --- a/packages/cli-core/src/lib/prompts.ts +++ b/packages/cli-core/src/lib/prompts.ts @@ -7,16 +7,34 @@ import { confirm as clackConfirm, isCancel, + MULTISELECT_INSTRUCTIONS, text as clackText, password as clackPassword, multiselect as clackMultiselect, type Option as ClackOption, } from "@clack/prompts"; import { editAsync } from "external-editor"; +import { dim } from "./color.ts"; import { throwUserAbort } from "./errors.ts"; import { ttyContext } from "./listage.ts"; import { log } from "./log.ts"; +/** + * Advertise select-all in the multiselect footer. + * + * `MultiSelectPrompt` binds `a` to toggle every option (and `i` to invert), but + * clack's instruction footer has never listed them and takes no override — the + * array below is the only seam, and it is read fresh on every render. So a + * genuinely useful key stays undiscoverable unless each call site spells it out + * in its own message, which is worse: it is a property of the prompt, not of + * any one question. + * + * Inserted second-to-last so `Enter: confirm` stays where readers expect it. + * `i` is left out deliberately — inverting is rarely what anyone wants, and a + * four-item legend stops being scannable. + */ +MULTISELECT_INSTRUCTIONS.splice(MULTISELECT_INSTRUCTIONS.length - 1, 0, `${dim("a:")} all`); + type ValidationResult = string | Error | true | undefined; type Validate = (value: string | undefined) => ValidationResult | Promise; type SyncValidate = (value: string | undefined) => string | Error | undefined; From e9a66929f0c50c421583c4801952e9bf39184b00 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:14:54 -0400 Subject: [PATCH 012/141] feat(migrate): offer to fix the settings the readiness report flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The report already knew which instance settings would cost users; acting on it meant leaving the CLI for the dashboard. A human run now offers one selectable change per flagged row, before the import confirmation, and writes the selection as a single `PATCH` of the instance config document — the same document `clerk config patch` writes. These are offers, not corrections. A flagged setting is not a wrong setting: an instance that genuinely requires an email address is configured exactly as its owner intended, and fixing the export may well be the right answer. Nothing is preselected, and selecting nothing continues to the import with the instance untouched. The redraw after a write is computed from the write, not from a second settings fetch. Clerk's Frontend API is eventually consistent, so a `/v1/environment` read issued this soon after routinely still reports the pre-write settings and would redraw every row the operator just cleared. The offer then repeats while anything is still flagged: applying one change routinely leaves others worth making, so reaching the second never costs a second run of the command. Email and phone take two writes rather than one — they are verifiable attributes, and Clerk rejects one that is on with no way to verify it, while switching it off empties `verification_strategies`. To make the offer answerable, the report itself now leads with **outcomes** rather than per-field coverage: each user is classified once, into the worst outcome that applies to them, so the ✗/⚠/✓ totals add up to the file. A required identifier rejects a user outright; a required password does not, because the import sends `skip_password_requirement`. The field rows below no longer restate user counts, which read as contradicting that block. "If you import them, this applies to them too" names what is masked behind a rejection. A user who is not being created cannot lose a field, so a setting affecting only rejected users costs nothing today — right up until the requirement rejecting them is relaxed, at which point all of it lands at once. Surfacing it up front collapses apply → re-check → discover → apply into one decision. Stands down with a warning rather than a failed run when the instance cannot be resolved, and for keyless applications, whose Backend API has no route for any of these settings. --- .../cli-core/src/commands/migrate/README.md | 161 +++++++++- .../migrate/lib/modify-settings.test.ts | 303 ++++++++++++++++++ .../commands/migrate/lib/modify-settings.ts | 191 +++++++++++ .../commands/migrate/lib/readiness.test.ts | 187 ++++++++++- .../src/commands/migrate/lib/readiness.ts | 258 ++++++++++++++- .../commands/migrate/run-interactive.test.ts | 264 ++++++++++++++- .../cli-core/src/commands/migrate/run.test.ts | 2 +- packages/cli-core/src/commands/migrate/run.ts | 167 ++++++++-- 8 files changed, 1469 insertions(+), 64 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/modify-settings.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index f74ab3606..3c5b180a3 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -40,8 +40,9 @@ platform and file from the last run so a repeat migration is mostly pressing enter. Anything already passed as a flag is not asked for. Firebase's hash parameters are never pre-filled — see [below](#--firebase--firebase). -Then it prints the [Migration Readiness report](#migration-readiness-report) -and waits for confirmation. Declining writes nothing to Clerk. +Then it prints the [Migration Readiness report](#migration-readiness-report), +offers to [change whatever it flagged](#changing-the-flagged-settings), and +waits for confirmation. Declining writes nothing to Clerk. **Agent mode never prompts.** `clerk migrate` with no flags exits with a usage error naming exactly what to pass: @@ -695,35 +696,163 @@ lecture" and should not pay for the two extra round-trips. Agent runs without It cross-references the file against the destination instance's live settings (BAPI `/v1/domains` → that instance's Frontend API `/v1/environment`) and -flags the two failure modes a migration otherwise discovers halfway through: - -- **Required in Clerk, missing from the file.** Those users fail one at a time, - mid-import, after earlier users already exist. -- **Present in the file, disabled in Clerk.** Social providers users actually - signed up with, or an identifier the instance has switched off. +answers the two questions worth answering before writing anything: **who won't +be imported**, and **who will arrive incomplete**. ``` Migration readiness - 120 users ready to import + 120 users in this file 3 failed validation and will be skipped + ✗ 12 users will not be imported + 12 have no email, which this instance requires + If you import them, this applies to them too: + 12 have a phone, which this instance is not set up to store + ⚠ 20 users will be imported, but not everything they carry + 14 have no password, which this instance requires — they will have to reset it to sign in + 6 have a username, which this instance is not set up to store + ✓ 88 users will be imported in full + Identifiers - ⚠ Email — required in Clerk, but 12 users lack it — 108/120 users - ✓ Username — enabled in Clerk — all users + ⚠ Email — required in Clerk, and not every user has one — 108/120 users + ⚠ Username — not enabled in Clerk — 6/120 users Social connections ✓ Google — enabled in Clerk — 40/120 users ⚠ Discord — not enabled in Clerk — 12/120 users -⚠ 2 settings need attention +⚠ 3 settings need attention ``` +### The two blocks + +**The outcome block** classifies each user **once**, into the worst outcome that +applies to them, so its three totals add up to the file. This matters: per-field +coverage cannot answer "how many won't be imported", because the users missing +an email and the users missing a password overlap by an amount only a per-user +pass knows. A user rejected for their missing email is not also counted under +the missing password they happen to share. + +**"If you import them, this applies to them too"** is the part that stops the +settings interacting invisibly. A user who is not being created cannot lose a +field, so a setting that only affects rejected users costs nothing _today_ and +would otherwise never be mentioned — right up until the operator relaxes the +requirement rejecting them, at which point all of it lands at once. Naming it +up front is what turns + +> make email optional → re-check → discover the phones are being dropped → +> enable phone → re-check + +into a single decision with both offers visible. It is also why a setting can +be flagged in the section rows while contributing nothing to the ✗/⚠/✓ totals. + +**The section rows below** are the other question — per-field coverage against +each setting — and deliberately do not restate user counts, which would read as +contradicting the block above. + +### Which settings cost what + +| Setting | Consequence | +| ------------------------------------------ | --------------------------------------------------------------------------------------------- | +| Identifier (email/phone/username) required | **Not imported.** `POST /v1/users` enforces the sign-up identifier requirements. | +| Password required, user has none | **Imported without a password.** The import sends `skip_password_requirement`, so the user is | +| | created and has to reset their password before they can sign in with one. | +| Attribute disabled in Clerk | **Imported without that field.** The instance has nowhere to put it. | +| Social provider disabled | **Imported**, but that sign-in method is unavailable to them. | + +Social rows are not part of the per-user outcome counts: which providers a user +signed up with lives in the raw export rather than the transformed user, so it +cannot be attributed per user. Their coverage row still names them. + If the instance settings cannot be read — the secret key is rejected, or FAPI is unreachable — the report degrades to a coverage-only listing with a note. Nothing is flagged in that case: "could not read" is not the same as "switched off", and treating it as such would raise alarms about settings that are perfectly fine. +### Changing the flagged settings + +When the report flags anything, a human run offers one selectable change per +flagged row before the import confirmation, so acting on the report does not +mean leaving the CLI for the dashboard: + +``` +Update this instance's settings first? (enter to skip) + ◻ Make Email optional at sign-up + ◻ Enable Discord sign-in + ↑/↓ to navigate • Space: select • a: all • Enter: confirm +``` + +**Nothing is preselected** — relaxing an instance's sign-up requirements is a +real decision, not a default — and selecting nothing continues to the import +prompt with the instance untouched, which is what "enter to skip" is there to +say. + +`a: all` is added to clack's legend in `lib/prompts.ts`: `MultiSelectPrompt` +has always bound `a` to toggle everything (and `i` to invert), but clack's +footer never listed them and takes no override, so the key was undiscoverable. +It applies to every multiselect in the CLI, because it is a property of the +prompt rather than of any one question. + +These are offers, not corrections: **a flagged setting is not a wrong setting.** +An instance that genuinely requires an email address is configured exactly as +its owner intended, and the right answer may well be to fix the export instead. + +Whatever is selected becomes a single `PATCH` of the instance config document, +the same document `clerk config patch` writes. The report is then redrawn so +the confirmation that follows is against the settings the write established. + +**The offer repeats while anything is still flagged.** A redraw is another +decision point, not a receipt: applying one change routinely leaves others +worth making, and each round re-offers only what is left. It ends when the +report has nothing flagged, when the operator selects nothing, or when there is +nothing offerable for the rows that remain — so reaching the second change +never costs a second run of the command. + +The redraw is computed from the write, **not** from a second settings fetch. +Clerk's Frontend API is eventually consistent, so a `/v1/environment` read +issued this soon after the config write routinely still reports the pre-write +settings — which would redraw the report with every row the operator just +cleared still flagged. The Platform API accepting the write is the +authoritative statement of what took, exactly as `clerk config patch` treats +it (see that command's [round-trip verification](../config/README.md#round-trip-verification) +notes for the same reasoning). + +The config leaves each option writes are not shown in the prompt — internal +detail an operator cannot act on — but they are fixed and listed here: + +| Flagged row | Change offered | +| ------------------------------- | --------------------------------------------------------------------------------- | +| Email/Phone/Username — required | `auth_.required_for_sign_up → false` | +| Email — disabled | `auth_email.used_for_sign_up → true` + `verification_strategies → ["email_code"]` | +| Phone — disabled | `auth_phone.used_for_sign_up → true` + `verification_strategies → ["phone_code"]` | +| Username — disabled | `auth_username.used_for_sign_up → true` | +| Password — required / disabled | `auth_password.required → false` / `auth_password.enabled → true` | +| First/Last name | `user_model..required → false` / `user_model..enabled → true` | +| Social provider — disabled | `connection_oauth_.enabled → true` | + +`used_for_sign_up` is the enable field that matters: `POST /v1/users` validates +an import against the instance's sign-up requirements, not its sign-in +strategies. + +**Email and phone take two writes, not one.** They are _verifiable_ attributes, +and Clerk rejects one that is on with no way to verify it: + +``` +422 phone_number: verifiable attributes need to have at least one verification +``` + +Switching the attribute off empties `verification_strategies`, so whatever +turns it back on has to put a strategy back in the same request. Username, +password and the name fields are not verifiable and take one write each. + +The offer is skipped entirely for `-y` and in agent mode, both of which say +"don't prompt". It also stands down, with a warning rather than a failed run, +when the instance to configure cannot be resolved (a bare `--secret-key` in an +unlinked directory) or when it is a **keyless** application — the Backend API +those are reachable through has no route for any of these settings, so +`clerk auth login` is the way in. + ## Artifacts Both are written relative to the **current working directory**, not to the @@ -784,6 +913,14 @@ NDJSON is. The original `.log` stays put. | `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | | `GET` | `/v1/domains` | Readiness report and `--skip-unsupported-providers` — resolves the Frontend API host | +The readiness report also reads the instance's Frontend API +`GET /v1/environment` (bootstrapping a dev browser first on development +instances), and its settings-change offer writes through the Platform API: + +| Method | Path | Used by | +| ------- | ----------------------------------------------------------------- | ------------------------------------------------------ | +| `PATCH` | `/v1/platform/applications/{appID}/instances/{instanceID}/config` | Applying the settings changes selected from the report | + Two exports talk to their own platform rather than to Clerk: | Method | Path | Used by | diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts new file mode 100644 index 000000000..e9b55e8e0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts @@ -0,0 +1,303 @@ +import { describe, expect, test } from "bun:test"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import type { FieldAnalysis } from "./analysis.ts"; +import { buildReadinessReport } from "./readiness.ts"; +import { applyChanges, buildChangePayload, buildSettingChanges } from "./modify-settings.ts"; + +/** Instance settings carrying only the attributes and providers a test names. */ +function settings(config: { + attributes?: Record; + social?: Record; +}): UserSettingsJSON { + return { + attributes: Object.fromEntries( + Object.entries(config.attributes ?? {}).map(([name, value]) => [ + name, + { enabled: value.enabled, required: value.required ?? false }, + ]), + ), + social: config.social ?? {}, + } as unknown as UserSettingsJSON; +} + +function analysis(overrides: Partial & { totalUsers: number }): FieldAnalysis { + return { + identifiers: { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: overrides.totalUsers, + ...overrides.identifiers, + }, + fieldCounts: overrides.fieldCounts ?? {}, + totalUsers: overrides.totalUsers, + }; +} + +/** + * Changes are built from a real report rather than hand-written rows, so a row + * whose `key` stops matching the path table fails here instead of silently + * dropping out of the offer. + */ +function changesFor(input: Parameters[0]) { + return buildSettingChanges(buildReadinessReport(input).blocking); +} + +describe("what gets offered", () => { + test("a required field not every user has is offered as a relaxation", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + + expect(changes).toEqual([ + { + id: "email_address", + label: "Make Email optional at sign-up", + section: "identifiers", + kind: "relax", + writes: [{ path: ["auth_email", "required_for_sign_up"], value: false }], + }, + ]); + }); + + test("a field the file carries but Clerk has switched off is offered as an enable", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { username: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { username: { enabled: false } } }), + }); + + expect(changes).toEqual([ + { + id: "username", + label: "Enable Username", + section: "identifiers", + kind: "enable", + writes: [{ path: ["auth_username", "used_for_sign_up"], value: true }], + }, + ]); + }); + + /** + * Clerk refuses a verifiable attribute that is on with no way to verify it — + * `422 phone_number: verifiable attributes need to have at least one + * verification` — and switching the attribute off empties the strategies, so + * every enable that turned one off has to put one back. + */ + test.each([ + ["phone_number", "auth_phone", "phone_code"], + ["email_address", "auth_email", "email_code"], + ])("enabling %s also restores its verification strategy", (attribute, group, strategy) => { + const carriesIt = attribute === "phone_number" ? { verifiedPhones: 2 } : { verifiedEmails: 2 }; + + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { ...carriesIt, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { [attribute]: { enabled: false } } }), + }); + + expect(changes[0]?.writes).toEqual([ + { path: [group, "used_for_sign_up"], value: true }, + { path: [group, "verification_strategies"], value: [strategy] }, + ]); + expect(buildChangePayload(changes)).toEqual({ + [group]: { used_for_sign_up: true, verification_strategies: [strategy] }, + }); + }); + + // Not verifiable, so no strategy to restore — one write is the whole change. + test("enabling username takes a single write", () => { + const changes = changesFor({ + analysis: analysis({ totalUsers: 2, identifiers: { username: 2 } as never }), + settings: settings({ attributes: { username: { enabled: false } } }), + }); + expect(changes[0]?.writes).toHaveLength(1); + }); + + test("a disabled social provider is offered under Clerk's own strategy name", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_x: { enabled: false } }, + }), + // Supabase calls it `twitter`; the config document calls it `oauth_x`. + providerCounts: { twitter: 2 }, + }); + + expect(changes).toEqual([ + { + id: "twitter", + label: "Enable Twitter (X) sign-in", + section: "social", + kind: "enable", + writes: [{ path: ["connection_oauth_x", "enabled"], value: true }], + }, + ]); + }); + + // "Could not read" is not "switched off", so nothing is flagged and nothing + // is offered — the report already degrades to a coverage-only listing. + test("nothing is offered when the instance settings could not be read", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 1, hasAnyIdentifier: 2 } as never, + }), + settings: null, + }); + + expect(changes).toEqual([]); + }); + + test("nothing is offered when every field is already configured", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + + expect(changes).toEqual([]); + }); +}); + +describe("the payload", () => { + test("collapses changes that share a parent into one branch", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 3 } as never, + fieldCounts: { firstName: 2, lastName: 1 }, + }), + settings: settings({ + attributes: { + email_address: { enabled: true }, + first_name: { enabled: true, required: true }, + last_name: { enabled: true, required: true }, + }, + }), + }); + + expect(buildChangePayload(changes)).toEqual({ + user_model: { first_name: { required: false }, last_name: { required: false } }, + }); + }); + + test("carries only the changes it is given", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 3 } as never, + fieldCounts: { password: 1 }, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + password: { enabled: true, required: true }, + }, + }), + }); + expect(changes.map((change) => change.id)).toEqual(["email_address", "password"]); + + expect(buildChangePayload(changes.filter((change) => change.id === "password"))).toEqual({ + auth_password: { required: false }, + }); + }); + + test("is empty when nothing was selected", () => { + expect(buildChangePayload([])).toEqual({}); + }); +}); + +/** + * The redraw after a write comes from `applyChanges`, not a second fetch: + * Clerk's Frontend API is eventually consistent, so re-reading straight after + * the patch returns the pre-write settings and redraws every row just cleared. + */ +describe("the settings after a write", () => { + /** The two fields a change touches, as `settings()` above builds them. */ + const attr = (value: { enabled: boolean; required: boolean }) => + value as unknown as UserSettingsJSON["attributes"]["email_address"]; + + test("drops the requirement a relaxation removed", () => { + const before = settings({ attributes: { email_address: { enabled: true, required: true } } }); + const input = { + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, + }), + settings: before, + }; + + const after = applyChanges(before, changesFor(input)); + + expect(after?.attributes.email_address).toEqual(attr({ enabled: true, required: false })); + // The report is rebuilt from this, so the row must stop being flagged. + expect(buildReadinessReport({ ...input, settings: after }).blocking).toEqual([]); + }); + + test("turns on what an enable switched on", () => { + const before = settings({ attributes: { username: { enabled: false } } }); + const changes = changesFor({ + analysis: analysis({ totalUsers: 2, identifiers: { username: 2 } as never }), + settings: before, + }); + + expect(applyChanges(before, changes)?.attributes.username).toEqual( + attr({ enabled: true, required: false }), + ); + }); + + test("enables a provider under Clerk's strategy name, not the source platform's", () => { + const before = settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_x: { enabled: false } }, + }); + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: before, + providerCounts: { twitter: 2 }, + }); + + expect(applyChanges(before, changes)?.social).toEqual({ + oauth_x: { enabled: true }, + } as unknown as UserSettingsJSON["social"]); + }); + + test("leaves the settings it was given untouched", () => { + const before = settings({ attributes: { email_address: { enabled: true, required: true } } }); + const changes = changesFor({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, + }), + settings: before, + }); + + applyChanges(before, changes); + + expect(before.attributes.email_address).toMatchObject({ required: true }); + }); + + test("passes null through — unreadable settings flag nothing to change", () => { + expect(applyChanges(null, [])).toBeNull(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts new file mode 100644 index 000000000..8136f6ca1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts @@ -0,0 +1,191 @@ +/** + * Turning a flagged Migration Readiness row into the instance-config change + * that would stop it being flagged. + * + * The report already knows which settings will cost users; without this the + * only way to act on it is to leave the CLI, find the setting in the dashboard, + * and come back. Each change is a single leaf in the Platform API's config + * document, so they compose into one `PATCH` however many the operator picks. + * + * These are offers, not corrections. A flagged setting is not a wrong setting — + * an instance that genuinely requires an email address is configured exactly as + * its owner intended, and the right answer may well be to fix the export + * instead. Nothing here is preselected and nothing is applied unasked. + * + * Only the two verdicts `buildReadinessReport` produces are mapped: "required + * in Clerk" (relax the requirement) and "not enabled in Clerk" (turn it on). A + * row this file has no path for is simply not offered — the report still names + * it and still points at the dashboard. + */ + +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { toClerkStrategy } from "./clerk-config.ts"; +import type { ReadinessItem, ReadinessSection } from "./readiness.ts"; + +/** One leaf of the config document, and what to set it to. */ +export type SettingWrite = { path: string[]; value: boolean | string[] }; + +/** One offered change: what it says, and the config leaves it writes. */ +export type SettingChange = { + /** Stable identity for the multiselect, and for tests. */ + id: string; + label: string; + section: ReadinessSection; + /** `enable` turns something on; `relax` drops a requirement. */ + kind: "enable" | "relax"; + writes: SettingWrite[]; +}; + +type ChangeWrites = { enable: SettingWrite[]; relax: SettingWrite[] }; + +/** + * Where each attribute lives in the config document. `used_for_sign_up` is the + * enable field that matters here: `POST /v1/users` validates an import against + * the instance's sign-up requirements, not its sign-in strategies. + * + * Email and phone are **verifiable** attributes, so enabling one takes two + * writes rather than one. Clerk rejects a verifiable attribute that is on with + * no way to verify it — `422 phone_number: verifiable attributes need to have + * at least one verification` — and switching the attribute off empties + * `verification_strategies`, so whatever turns it back on has to put a strategy + * back. Username, password and the name fields are not verifiable and take one + * write each. + */ +const ATTRIBUTE_WRITES: Record = { + email_address: { + enable: [ + { path: ["auth_email", "used_for_sign_up"], value: true }, + { path: ["auth_email", "verification_strategies"], value: ["email_code"] }, + ], + relax: [{ path: ["auth_email", "required_for_sign_up"], value: false }], + }, + phone_number: { + enable: [ + { path: ["auth_phone", "used_for_sign_up"], value: true }, + { path: ["auth_phone", "verification_strategies"], value: ["phone_code"] }, + ], + relax: [{ path: ["auth_phone", "required_for_sign_up"], value: false }], + }, + username: { + enable: [{ path: ["auth_username", "used_for_sign_up"], value: true }], + relax: [{ path: ["auth_username", "required_for_sign_up"], value: false }], + }, + password: { + enable: [{ path: ["auth_password", "enabled"], value: true }], + relax: [{ path: ["auth_password", "required"], value: false }], + }, + first_name: { + enable: [{ path: ["user_model", "first_name", "enabled"], value: true }], + relax: [{ path: ["user_model", "first_name", "required"], value: false }], + }, + last_name: { + enable: [{ path: ["user_model", "last_name", "enabled"], value: true }], + relax: [{ path: ["user_model", "last_name", "required"], value: false }], + }, +}; + +/** `github` → `connection_oauth_github`, via Clerk's own strategy name. */ +function socialPath(provider: string): string[] { + return [`connection_oauth_${toClerkStrategy(provider).replace(/^oauth_/, "")}`, "enabled"]; +} + +function changeFor(item: ReadinessItem): SettingChange | undefined { + // Required-but-not-universal is the only verdict that relaxes rather than + // enables; every other flagged row is something switched off in Clerk. + const relax = item.clerkRequired === true; + + if (item.section === "social") { + // A provider has no "required" in Clerk, so there is nothing to relax. + if (relax) return undefined; + return { + id: item.key, + label: `Enable ${item.label} sign-in`, + section: item.section, + kind: "enable", + writes: [{ path: socialPath(item.key), value: true }], + }; + } + + const writes = ATTRIBUTE_WRITES[item.key]; + if (!writes) return undefined; + + return { + id: item.key, + label: relax ? `Make ${item.label} optional at sign-up` : `Enable ${item.label}`, + section: item.section, + kind: relax ? "relax" : "enable", + writes: relax ? writes.relax : writes.enable, + }; +} + +/** The changes offerable for a report's flagged rows, in report order. */ +export function buildSettingChanges(flagged: ReadinessItem[]): SettingChange[] { + return flagged.map(changeFor).filter((change): change is SettingChange => change !== undefined); +} + +/** + * Collapses the chosen changes into one config payload. + * + * Changes share parents — `first_name` and `last_name` both write `user_model` + * — so leaves are written into a shared tree rather than merged after the fact. + */ +export function buildChangePayload(changes: SettingChange[]): Record { + const payload: Record = {}; + + for (const write of changes.flatMap((change) => change.writes)) { + let node = payload; + for (const key of write.path.slice(0, -1)) { + node = (node[key] ??= {}) as Record; + } + node[write.path[write.path.length - 1] as string] = write.value; + } + + return payload; +} + +/** + * The instance's settings as they stand once `changes` have been written. + * + * Deliberately not a re-read. Clerk's Frontend API is eventually consistent, so + * a `/v1/environment` fetch issued straight after the config write routinely + * still reports the pre-write settings — which would redraw the report with + * every row it just cleared still flagged. The Platform API answering the write + * is the authoritative statement of what took, exactly as `clerk config patch` + * treats it. + * + * @param settings - `null` passes through: when the settings could not be read + * nothing is ever flagged, so there is nothing to have changed. + */ +export function applyChanges( + settings: UserSettingsJSON | null, + changes: SettingChange[], +): UserSettingsJSON | null { + if (!settings) return null; + + const next = structuredClone(settings); + + for (const change of changes) { + if (change.section === "social") { + const social = next.social as Record; + const strategy = toClerkStrategy(change.id); + social[strategy] = { ...social[strategy], enabled: true }; + continue; + } + + const attributes = next.attributes as Record; + attributes[change.id] = + change.kind === "enable" + ? { + ...attributes[change.id], + enabled: true, + required: attributes[change.id]?.required ?? false, + } + : { + ...attributes[change.id], + enabled: attributes[change.id]?.enabled ?? true, + required: false, + }; + } + + return next; +} diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts index cdac7d866..842e9c4b5 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import type { UserSettingsJSON } from "../../../lib/fapi.ts"; -import type { FieldAnalysis } from "./analysis.ts"; +import { analyzeFields, type FieldAnalysis } from "./analysis.ts"; import { buildReadinessReport, formatReadinessReport, type ReadinessItem } from "./readiness.ts"; /** Instance settings carrying only the attributes and providers a test names. */ @@ -97,7 +97,9 @@ describe("required in Clerk but missing from the file", () => { const email = item(report, "Email"); expect(email?.blocking).toBe(true); - expect(email?.detail).toContain("3 users lack it"); + expect(email?.detail).toContain("required in Clerk"); + // A required identifier is the one verdict Clerk refuses the user over. + expect(email?.consequence).toBe("rejects"); expect(report.blocking).toHaveLength(1); }); @@ -126,20 +128,23 @@ describe("required in Clerk but missing from the file", () => { expect(report.blocking).toHaveLength(0); }); - test("uses the singular form for a single missing user", () => { + // The import sends `skip_password_requirement`, so a required password costs + // the user their password rather than their whole account. + test("a required password drops rather than rejects", () => { const report = buildReadinessReport({ analysis: analysis({ - totalUsers: 2, - identifiers: { verifiedEmails: 1, hasAnyIdentifier: 2, username: 2 } as never, + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + fieldCounts: { password: 1 }, }), settings: settings({ attributes: { - email_address: { enabled: true, required: true }, - username: { enabled: true }, + email_address: { enabled: true }, + password: { enabled: true, required: true }, }, }), }); - expect(item(report, "Email")?.detail).toContain("1 user lacks it"); + expect(item(report, "Password")?.consequence).toBe("drops"); }); }); @@ -255,6 +260,168 @@ describe("file-level totals", () => { }); }); +/** + * The counts an operator actually decides on. Built from the users themselves, + * because per-field coverage cannot answer them: the users missing an email and + * the users missing a password overlap by an amount only a per-user pass knows. + */ +describe("what the settings mean for these users", () => { + const REQUIRE_EMAIL_AND_PASSWORD = settings({ + attributes: { + email_address: { enabled: true, required: true }, + password: { enabled: true, required: true }, + }, + }); + + /** Two with everything, two with no email, one with an email but no password. */ + const USERS = [ + { userId: "a", email: "a@x.dev", password: "hash" }, + { userId: "b", email: "b@x.dev", password: "hash" }, + { userId: "c", username: "c" }, + { userId: "d", username: "d" }, + { userId: "e", email: "e@x.dev" }, + ] as never; + + const outcomes = () => + buildReadinessReport({ + analysis: analyzeFields(USERS), + users: USERS, + settings: REQUIRE_EMAIL_AND_PASSWORD, + }).outcomes; + + test("the three totals account for every user exactly once", () => { + const result = outcomes(); + expect(result).toMatchObject({ rejected: 2, incomplete: 1, complete: 2 }); + expect((result?.rejected ?? 0) + (result?.incomplete ?? 0) + (result?.complete ?? 0)).toBe(5); + }); + + // The file has three users without a password, but two of them are already + // rejected for the email — counting them twice would overstate the damage. + test("a rejected user is not also counted as incomplete", () => { + expect(outcomes()?.incompleteReasons).toEqual([ + { + label: "Password", + count: 1, + detail: expect.stringContaining("1 has no password, which this instance requires"), + }, + ]); + }); + + test("names why the rejected users are rejected", () => { + expect(outcomes()?.rejectedReasons).toEqual([ + { label: "Email", count: 2, detail: "2 have no email, which this instance requires" }, + ]); + }); + + /** + * The rejected users lose nothing today — they are not being created. But the + * moment the operator relaxes the requirement rejecting them (one of the + * changes on offer) every masked setting lands at once. Surfacing it here is + * what saves an apply → re-check → discover → apply → re-check loop. + */ + describe("what is masked behind a rejection", () => { + // b and c have no email, so both are rejected; b also carries a phone the + // instance is not set up to store. Exactly the shape the supabase sample + // hits: every phone belongs to a user who has no email. + const MASKED_USERS = [ + { userId: "a", email: "a@x.dev" }, + { userId: "b", username: "b", phone: "+15551234567" }, + { userId: "c", username: "c" }, + ] as never; + + const report = (attributes: Record) => + buildReadinessReport({ + analysis: analyzeFields(MASKED_USERS), + users: MASKED_USERS, + settings: settings({ attributes }), + }); + + const REQUIRE_EMAIL_PHONE_OFF = { + email_address: { enabled: true, required: true }, + phone_number: { enabled: false }, + username: { enabled: true }, + }; + + test("counts a setting that only bites once the rejected users get in", () => { + const outcomes = report(REQUIRE_EMAIL_PHONE_OFF).outcomes; + + expect(outcomes).toMatchObject({ rejected: 2, incomplete: 0, complete: 1 }); + expect(outcomes?.maskedReasons).toEqual([ + { + label: "Phone", + count: 1, + detail: "1 has a phone, which this instance is not set up to store", + }, + ]); + }); + + test("keeps it out of the incomplete count, which is about users being imported", () => { + expect(report(REQUIRE_EMAIL_PHONE_OFF).outcomes?.incompleteReasons).toEqual([]); + }); + + test("renders it under the rejected group", () => { + const output = formatReadinessReport(report(REQUIRE_EMAIL_PHONE_OFF)).join("\n"); + + expect(output).toContain("If you import them, this applies to them too:"); + expect(output).toContain("1 has a phone, which this instance is not set up to store"); + }); + + // Enabling phone is the other change on offer, and it empties the block — + // which is the check that the two offers really do interact this way. + test("is empty once the masked setting is no longer a problem", () => { + const outcomes = report({ + email_address: { enabled: true, required: true }, + phone_number: { enabled: true }, + username: { enabled: true }, + }).outcomes; + + expect(outcomes).toMatchObject({ rejected: 2 }); + expect(outcomes?.maskedReasons).toEqual([]); + }); + }); + + test("a disabled attribute costs the users who carry it, not the ones who don't", () => { + const users = [ + { userId: "a", email: "a@x.dev", username: "a" }, + { userId: "b", email: "b@x.dev" }, + ] as never; + + const result = buildReadinessReport({ + analysis: analyzeFields(users), + users, + settings: settings({ + attributes: { email_address: { enabled: true }, username: { enabled: false } }, + }), + }).outcomes; + + expect(result).toMatchObject({ rejected: 0, incomplete: 1, complete: 1 }); + expect(result?.incompleteReasons[0]?.detail).toContain("not set up to store"); + }); + + test("is omitted when the caller passes no users", () => { + const report = buildReadinessReport({ + analysis: analyzeFields(USERS), + settings: REQUIRE_EMAIL_AND_PASSWORD, + }); + expect(report.outcomes).toBeUndefined(); + }); + + test("renders each outcome with the reasons behind it", () => { + const output = formatReadinessReport( + buildReadinessReport({ + analysis: analyzeFields(USERS), + users: USERS, + settings: REQUIRE_EMAIL_AND_PASSWORD, + }), + ).join("\n"); + + expect(output).toContain("2 users will not be imported"); + expect(output).toContain("1 user will be imported, but not everything they carry"); + expect(output).toContain("2 users will be imported in full"); + expect(output).toContain("they will have to reset it to sign in"); + }); +}); + describe("rendering", () => { const blocked = () => buildReadinessReport({ @@ -273,7 +440,7 @@ describe("rendering", () => { test("leads with the counts an operator needs before confirming", () => { const output = formatReadinessReport(blocked()).join("\n"); - expect(output).toContain("10 users ready to import"); + expect(output).toContain("10 users in this file"); expect(output).toContain("2 failed validation"); expect(output).toContain("2 without any identifier"); }); @@ -281,7 +448,7 @@ describe("rendering", () => { test("names the blocking rows and points at the dashboard", () => { const output = formatReadinessReport(blocked()).join("\n"); expect(output).toContain("1 setting needs attention"); - expect(output).toContain("3 users lack it"); + expect(output).toContain("required in Clerk, and not every user has one"); expect(output).toContain("dashboard.clerk.com"); }); diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts index 1db17fe3e..e476da4a1 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -16,10 +16,11 @@ import type { UserSettingsJSON } from "../../../lib/fapi.ts"; import { bold, dim, green, red, yellow } from "../../../lib/color.ts"; // Pure attribute lookups, shared with the `users` create wizard. import { isEnabled, isRequired, type AttributeName } from "../../users/interactive/attributes.ts"; -import type { FieldAnalysis } from "./analysis.ts"; +import type { User } from "../types.ts"; +import { hasValue, type FieldAnalysis } from "./analysis.ts"; import { providerLabel, toClerkStrategy } from "./clerk-config.ts"; -const DASHBOARD_URL = "https://dashboard.clerk.com/~/user-authentication"; +export const DASHBOARD_URL = "https://dashboard.clerk.com/~/user-authentication"; export type ReadinessSection = "identifiers" | "auth" | "social" | "model"; @@ -32,16 +33,68 @@ export type ReadinessSection = "identifiers" | "auth" | "social" | "model"; */ export type ReadinessItem = { label: string; + /** + * What the row is about, machine-side: an {@link AttributeName} for every + * section but `social`, and the source platform's provider key for that one. + * `label` is for humans; this is what `modify-settings.ts` looks up. + */ + key: string; section: ReadinessSection; /** Users in the file that carry this field or provider. */ userCount: number; clerkEnabled: boolean | null; clerkRequired: boolean | null; blocking: boolean; + /** + * What this row costs the users it affects. + * + * - `rejects` — Clerk refuses the user outright. Only a required identifier + * does this: `POST /v1/users` enforces the instance's sign-up identifier + * requirements, and a user carrying none of them has nothing to be created + * under. + * - `drops` — the user is created, but this piece of them is not. A required + * password is in this group rather than `rejects` because the import sends + * `skip_password_requirement` (see `import-users.ts`), so the user lands + * without one and has to reset it before they can sign in that way. + */ + consequence?: "rejects" | "drops"; /** Why it blocks — omitted when it does not. */ detail?: string; }; +/** One reason users are affected, and how many of them it affects. */ +export type OutcomeReason = { label: string; count: number; detail: string }; + +/** + * What the settings mean for the users in the file, counted per user rather + * than per field. + * + * Per-field coverage cannot answer "how many users will not be imported" — + * the users missing an email and the users missing a username overlap by an + * unknown amount. Each user is classified once, into the worst outcome that + * applies to them, so the three totals add up to the file. + */ +export type ImportOutcomes = { + rejected: number; + rejectedReasons: OutcomeReason[]; + /** + * What *else* affects the rejected users — surfaced now rather than after + * they become importable. + * + * A user who is not being created cannot lose a field, so these settings cost + * nothing today and would otherwise go unmentioned. But the moment the + * operator relaxes the requirement rejecting them, every one of these lands. + * Reporting it only afterwards turns one decision into a apply → re-check → + * discover → apply → re-check loop, which is exactly what the report exists + * to prevent. + */ + maskedReasons: OutcomeReason[]; + incomplete: number; + incompleteReasons: OutcomeReason[]; + /** Imported with everything the file carries for them. */ + complete: number; +}; + export type ReadinessReport = { totalUsers: number; /** Users with no identifier at all; they cannot be imported under any settings. */ @@ -52,6 +105,8 @@ export type ReadinessReport = { blocking: ReadinessItem[]; /** True when the instance settings could not be read. */ settingsUnavailable: boolean; + /** Omitted when the caller passed no users to classify. */ + outcomes?: ImportOutcomes; }; type BuildInput = { @@ -61,8 +116,21 @@ type BuildInput = { validationFailed?: number; /** Source-platform provider key → user count. Supabase exports only. */ providerCounts?: Record; + /** + * The users themselves, for the per-user outcome counts. Optional so callers + * that only need the coverage rows (and tests working from a synthetic + * {@link FieldAnalysis}) do not have to supply them. + */ + users?: User[]; }; +/** + * Identifiers Clerk creates a user *under*. A required one that a user does not + * carry leaves nothing to create them with, so the API refuses them — which is + * why these are the only attributes whose consequence is `rejects`. + */ +const IDENTIFIER_ATTRIBUTES = new Set(["email_address", "phone_number", "username"]); + /** An identifier or user-model row, with its blocking verdict. */ function buildAttributeItem( label: string, @@ -81,15 +149,18 @@ function buildAttributeItem( if (required === true && missing > 0) { return { label, + key: attribute, section, userCount, clerkEnabled: enabled, clerkRequired: required, blocking: true, - detail: - missing === 1 - ? "required in Clerk, but 1 user lacks it" - : `required in Clerk, but ${missing} users lack it`, + consequence: IDENTIFIER_ATTRIBUTES.has(attribute) ? "rejects" : "drops", + // How many users this costs is the outcome block's job. Restating it here + // reads as a contradiction, because that block counts each user once and + // this row counts the field — a user missing both an email and a password + // appears in both rows but only in the first outcome. + detail: "required in Clerk, and not every user has one", }; } @@ -97,17 +168,20 @@ function buildAttributeItem( if (enabled === false && userCount > 0) { return { label, + key: attribute, section, userCount, clerkEnabled: enabled, clerkRequired: required, blocking: true, + consequence: "drops", detail: "not enabled in Clerk", }; } return { label, + key: attribute, section, userCount, clerkEnabled: enabled, @@ -153,22 +227,135 @@ export function buildReadinessReport(input: BuildInput): ReadinessReport { : null; items.push({ label: providerLabel(provider), + key: provider, section: "social", userCount: count, clerkEnabled: enabled, clerkRequired: null, blocking: enabled === false, - ...(enabled === false ? { detail: "not enabled in Clerk" } : {}), + ...(enabled === false + ? { consequence: "drops" as const, detail: "not enabled in Clerk" } + : {}), }); } + const blocking = items.filter((item) => item.blocking); + return { totalUsers: total, withoutIdentifier: total - analysis.identifiers.hasAnyIdentifier, validationFailed, items, - blocking: items.filter((item) => item.blocking), + blocking, settingsUnavailable: settings === null, + ...(input.users ? { outcomes: countOutcomes(input.users, blocking) } : {}), + }; +} + +/** Whether a user carries the field an attribute row is about. */ +const CARRIES: Record) => boolean> = { + email_address: (u) => + hasValue(u.email) || hasValue(u.emailAddresses) || hasValue(u.unverifiedEmailAddresses), + phone_number: (u) => + hasValue(u.phone) || hasValue(u.phoneNumbers) || hasValue(u.unverifiedPhoneNumbers), + username: (u) => hasValue(u.username), + password: (u) => hasValue(u.password), + first_name: (u) => hasValue(u.firstName), + last_name: (u) => hasValue(u.lastName), +}; + +/** Which users a flagged row actually affects: the ones missing it, or carrying it. */ +function affects(item: ReadinessItem, user: Record): boolean { + const carries = CARRIES[item.key]; + if (!carries) return false; + // A required row costs the users without it; a disabled row costs the ones with it. + return item.clerkRequired === true ? !carries(user) : carries(user); +} + +/** + * One reason line: how many users, what they are missing or carrying, and what + * the instance does about it. Count first, because that is what is being + * decided on. + */ +function describe(item: ReadinessItem, count: number): string { + const noun = item.label.toLowerCase(); + const have = count === 1 ? "has" : "have"; + + if (item.clerkRequired === true) { + const consequence = item.key === "password" ? " — they will have to reset it to sign in" : ""; + return `${count} ${have} no ${noun}, which this instance requires${consequence}`; + } + return `${count} ${have} a ${noun}, which this instance is not set up to store`; +} + +function toReasons(counts: Map): OutcomeReason[] { + return [...counts].map(([label, { item, count }]) => ({ + label, + count, + detail: describe(item, count), + })); +} + +/** + * Classifies every user into exactly one outcome, worst first. + * + * Social rows are left out: which providers a user signed up with lives in the + * raw export rather than the transformed `User`, so they cannot be counted per + * user here. Their coverage row still names them. + */ +function countOutcomes(users: User[], blocking: ReadinessItem[]): ImportOutcomes { + const rejecting = blocking.filter((item) => item.consequence === "rejects"); + const dropping = blocking.filter( + (item) => item.consequence === "drops" && item.section !== "social", + ); + + type Tally = Map; + const rejectedBy: Tally = new Map(); + const droppedBy: Tally = new Map(); + const maskedBy: Tally = new Map(); + let rejected = 0; + let incomplete = 0; + let complete = 0; + + const tally = (into: Tally, item: ReadinessItem) => { + const entry = into.get(item.label) ?? { item, count: 0 }; + entry.count++; + into.set(item.label, entry); + }; + + for (const entry of users) { + const user = entry as unknown as Record; + const gaps = dropping.filter((item) => affects(item, user)); + + const refusals = rejecting.filter((item) => affects(item, user)); + if (refusals.length > 0) { + rejected++; + for (const item of refusals) tally(rejectedBy, item); + // Their gaps are still tallied, into a separate bucket. Dropping them + // here is what makes the settings interact invisibly: relaxing the + // requirement that rejects these users lets them in, and only then does + // whatever else affects them show up — a second round trip to learn + // something that was knowable now. + for (const item of gaps) tally(maskedBy, item); + continue; + } + + if (gaps.length === 0) { + complete++; + continue; + } + + incomplete++; + for (const item of gaps) tally(droppedBy, item); + } + + return { + rejected, + rejectedReasons: toReasons(rejectedBy), + maskedReasons: toReasons(maskedBy), + incomplete, + incompleteReasons: toReasons(droppedBy), + complete, }; } @@ -193,11 +380,59 @@ function renderItem(item: ReadinessItem, total: number): string { return ` ${yellow("!")} ${item.label} — ${dim(`${coverage} — check it is enabled in Clerk`)}`; } +const users = (count: number) => `${count} user${count === 1 ? "" : "s"}`; + +/** + * The three outcomes, each with the reasons behind it. + * + * This is the part of the report that answers "so what": which users the + * instance will refuse, which will arrive with something missing, and why. + * Per-field coverage lives further down and is a different question. + */ +function renderOutcomes(outcomes: ImportOutcomes): string[] { + const lines: string[] = []; + + const group = ( + symbol: string, + colour: (text: string) => string, + headline: string, + reasons: OutcomeReason[], + ) => { + lines.push(` ${colour(symbol)} ${colour(headline)}`); + for (const reason of reasons) lines.push(` ${dim(reason.detail)}`); + }; + + if (outcomes.rejected > 0) { + group("✗", red, `${users(outcomes.rejected)} will not be imported`, outcomes.rejectedReasons); + + // Named here rather than left for a second run of the report: these are the + // settings that start costing something the moment the rejection above is + // lifted, and lifting it is one of the changes on offer. + if (outcomes.maskedReasons.length > 0) { + lines.push(` ${dim("If you import them, this applies to them too:")}`); + for (const reason of outcomes.maskedReasons) lines.push(` ${dim(reason.detail)}`); + } + } + if (outcomes.incomplete > 0) { + group( + "⚠", + yellow, + `${users(outcomes.incomplete)} will be imported, but not everything they carry`, + outcomes.incompleteReasons, + ); + } + if (outcomes.complete > 0) { + lines.push(` ${green("✓")} ${green(`${users(outcomes.complete)} will be imported in full`)}`); + } + + return lines; +} + /** Renders the report for a human, as lines. */ export function formatReadinessReport(report: ReadinessReport): string[] { const lines: string[] = [bold("Migration readiness")]; - lines.push(` ${report.totalUsers} user${report.totalUsers === 1 ? "" : "s"} ready to import`); + lines.push(` ${users(report.totalUsers)} in this file`); if (report.validationFailed > 0) { lines.push(` ${yellow(`${report.validationFailed} failed validation and will be skipped`)}`); } @@ -207,6 +442,11 @@ export function formatReadinessReport(report: ReadinessReport): string[] { ); } + if (report.outcomes) { + const outcomeLines = renderOutcomes(report.outcomes); + if (outcomeLines.length > 0) lines.push("", ...outcomeLines); + } + if (report.settingsUnavailable) { lines.push( "", diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index c5355da20..44198a6f7 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -14,23 +14,46 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { getMode, setMode, type Mode } from "../../mode.ts"; -import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; +import { keylessTargetStubs, listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; +import type { InstanceTarget } from "../../lib/keyless-target.ts"; const mockSelect = mock(async () => "clerk" as unknown); const mockText = mock(async () => "export.json" as unknown); +type MultiselectConfig = { options: { value: string; label: string; hint?: string }[] }; +const mockMultiselect = mock(async (_config: MultiselectConfig) => [] as unknown[]); let confirmAnswer = true; let originalMode: Mode; +const ACCOUNT_TARGET: InstanceTarget = { + kind: "account", + ctx: { + appId: "app_1", + appLabel: "Migration Test", + instanceId: "ins_1", + instanceLabel: "development", + }, + label: "Migration Test (development)", +}; +let instanceTarget: InstanceTarget | Error = ACCOUNT_TARGET; + mock.module("../../lib/listage.ts", () => ({ ...listageStubs, select: (...args: unknown[]) => mockSelect(...(args as [])), })); +mock.module("../../lib/keyless-target.ts", () => ({ + ...keylessTargetStubs, + resolveInstanceTarget: async () => { + if (instanceTarget instanceof Error) throw instanceTarget; + return instanceTarget; + }, +})); + // Every export of the real module must appear here — a missing one is a link // error at import time, which takes down the whole file rather than one prompt. mock.module("../../lib/prompts.ts", () => ({ confirm: async () => confirmAnswer, - multiselect: async () => [], + multiselect: (...args: unknown[]) => mockMultiselect(...(args as [MultiselectConfig])), text: (...args: unknown[]) => mockText(...(args as [])), password: async () => "", editor: async () => "{}", @@ -57,11 +80,17 @@ const EXPORT = [ const baseOptions = { transformer: "clerk", file: "export.json", secretKey: "sk_test_x" }; +let originalPlatformKey: string | undefined; + beforeAll(() => { originalMode = getMode(); setMode("human"); originalCwd = process.cwd(); originalFetch = globalThis.fetch; + // Pinned rather than inherited: the settings-fix write goes through the + // Platform API, and CI has neither a `.env.local` nor a login session. + originalPlatformKey = process.env.CLERK_PLATFORM_API_KEY; + process.env.CLERK_PLATFORM_API_KEY = "ak_test"; workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-config-")); _setConfigDir(configDir); @@ -71,6 +100,8 @@ beforeAll(() => { afterAll(() => { setMode(originalMode); globalThis.fetch = originalFetch; + if (originalPlatformKey === undefined) delete process.env.CLERK_PLATFORM_API_KEY; + else process.env.CLERK_PLATFORM_API_KEY = originalPlatformKey; _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); @@ -80,10 +111,13 @@ afterAll(() => { beforeEach(() => { requests = []; confirmAnswer = true; + instanceTarget = ACCOUNT_TARGET; mockSelect.mockReset(); mockText.mockReset(); + mockMultiselect.mockReset(); mockSelect.mockResolvedValue("clerk"); mockText.mockResolvedValue("export.json"); + mockMultiselect.mockResolvedValue([]); fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); @@ -94,8 +128,16 @@ afterEach(() => { process.exitCode = 0; }); +type StubSettings = { attributes?: object; social?: object } | null; + +let currentSettings: StubSettings = null; +/** What the instance reports once a config PATCH lands, when a test sets one. */ +let settingsAfterFix: StubSettings = null; + /** Stubs BAPI plus the FAPI environment lookup the readiness report needs. */ -function stubInstanceSettings(settings: { attributes?: object; social?: object } | null) { +function stubInstanceSettings(settings: StubSettings) { + currentSettings = settings; + settingsAfterFix = null; globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { const url = input.toString(); requests.push({ @@ -104,13 +146,17 @@ function stubInstanceSettings(settings: { attributes?: object; social?: object } body: init?.body ? JSON.parse(init.body as string) : null, }); if (url.endsWith("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/v1/domains")) { - if (!settings) return new Response("nope", { status: 500 }); + if (!currentSettings) return new Response("nope", { status: 500 }); return Response.json({ data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], }); } if (url.includes("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/v1/dev_browser")) return Response.json({ token: "jwt" }); - if (url.includes("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/v1/environment")) return Response.json({ user_settings: settings }); + if (url.includes("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/v1/environment")) return Response.json({ user_settings: currentSettings }); + if (url.endsWith("/instances/ins_1/config")) { + if (settingsAfterFix) currentSettings = settingsAfterFix; + return Response.json({ config_version: "v1_patched" }); + } return Response.json({ id: "user_created" }); }) as unknown as typeof fetch; } @@ -151,7 +197,7 @@ describe("the readiness report", () => { test("renders before the confirmation", async () => { await run(baseOptions); expect(captured.err).toContain("Migration readiness"); - expect(captured.err).toContain("2 users ready to import"); + expect(captured.err).toContain("2 users in this file"); }); // The whole point of the report: seeing what will go wrong, then backing out @@ -190,7 +236,10 @@ describe("the readiness report", () => { await run(baseOptions); - expect(captured.err).toContain("1 user lacks it"); + // The outcome block is the point: one user has no email, and an instance + // that requires one will refuse them. + expect(captured.err).toContain("1 user will not be imported"); + expect(captured.err).toContain("1 has no email, which this instance requires"); expect(captured.err).toContain("1 setting needs attention"); }); @@ -212,6 +261,207 @@ describe("the readiness report", () => { }); }); +describe("fixing the instance's settings from the report", () => { + /** An export whose second user has no email, against a required-email instance. */ + function blockedOnRequiredEmail() { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", username: "bob" }, + ]), + ); + } + + /** Two flagged rows: email required with a user lacking it, username switched off. */ + function blockedOnTwoSettings() { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: false }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev", username: "alice" }, + { id: "u2", username: "bob" }, + ]), + ); + } + + const patched = () => requests.filter((r) => r.url.endsWith("/instances/ins_1/config")); + const offered = (round = 0) => mockMultiselect.mock.calls[round]?.[0]?.options ?? []; + + // Plain labels only: the config leaf each one writes is internal detail an + // operator cannot act on and does not need to read. + test("offers one change per blocking row, named in plain terms", async () => { + blockedOnRequiredEmail(); + + await run(baseOptions); + + expect(offered()).toEqual([ + { value: "email_address", label: "Make Email optional at sign-up" }, + ]); + }); + + test("does not ask when nothing is blocking", async () => { + await run(baseOptions); + + expect(mockMultiselect).not.toHaveBeenCalled(); + expect(created()).toHaveLength(2); + }); + + // Relaxing an instance's sign-up requirements is a real decision, so nothing + // is preselected and an empty answer must leave the instance untouched. + test("selecting nothing changes nothing and continues to the import", async () => { + blockedOnRequiredEmail(); + mockMultiselect.mockResolvedValue([]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(created()).toHaveLength(2); + }); + + test("selecting a change patches the instance and re-renders the report", async () => { + blockedOnRequiredEmail(); + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(1); + expect(patched()[0]).toMatchObject({ + method: "PATCH", + body: { auth_email: { required_for_sign_up: false } }, + }); + expect(captured.err).toContain("Updated 1 setting"); + // The redraw clears the row that was just fixed, so the confirmation that + // follows is against the settings the write established. + expect(captured.err).toContain("Every field in this file is configured in Clerk"); + expect(created()).toHaveLength(2); + }); + + /** + * The redraw must not re-read the Frontend API. It is eventually consistent, + * so a fetch this soon after the write returns the pre-write settings and + * redraws the report with every row the operator just cleared still flagged. + */ + test("redraws from the write rather than re-reading stale settings", async () => { + blockedOnRequiredEmail(); + // Anything read back now would still say "required" — as it did in practice. + settingsAfterFix = { + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }; + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(requests.filter((r) => r.url.includes("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/v1/environment"))).toHaveLength(1); + expect(captured.err).toContain("Every field in this file is configured in Clerk"); + // Flagged in the first report, and only there — the redraw is clean even + // though a re-read at this moment would still have reported it. + expect(captured.err.split("setting needs attention")).toHaveLength(2); + }); + + // A keyless application is only reachable through the Backend API, which has + // no route for any of these settings — saying so beats a confusing rejection. + test("stands down for a keyless application and still imports", async () => { + blockedOnRequiredEmail(); + instanceTarget = { + kind: "keyless", + keyless: { secretKey: "sk_test_x", source: ".env" }, + label: "keyless", + }; + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(captured.err).toContain("clerk auth login"); + expect(created()).toHaveLength(2); + }); + + /** + * Each redraw is another decision point, not a receipt. Applying one change + * routinely leaves others still worth making, and an operator should not have + * to re-run the whole command to reach them. + */ + describe("offering again while anything is still flagged", () => { + test("re-offers what is left, without the change already applied", async () => { + blockedOnTwoSettings(); + mockMultiselect.mockResolvedValueOnce(["email_address"]); + mockMultiselect.mockResolvedValueOnce(["username"]); + + await run(baseOptions); + + expect(offered(0).map((option) => option.value)).toEqual(["email_address", "username"]); + expect(offered(1).map((option) => option.value)).toEqual(["username"]); + expect(patched()).toHaveLength(2); + expect(patched()[1]).toMatchObject({ + body: { auth_username: { used_for_sign_up: true } }, + }); + }); + + test("stops once nothing is flagged, rather than asking again", async () => { + blockedOnTwoSettings(); + mockMultiselect.mockResolvedValueOnce(["email_address"]); + mockMultiselect.mockResolvedValueOnce(["username"]); + + await run(baseOptions); + + expect(mockMultiselect).toHaveBeenCalledTimes(2); + expect(captured.err).toContain("Every field in this file is configured in Clerk"); + expect(created()).toHaveLength(2); + }); + + test("stops when the operator skips, leaving the rest flagged", async () => { + blockedOnTwoSettings(); + mockMultiselect.mockResolvedValueOnce(["email_address"]); + mockMultiselect.mockResolvedValueOnce([]); + + await run(baseOptions); + + expect(mockMultiselect).toHaveBeenCalledTimes(2); + expect(patched()).toHaveLength(1); + expect(created()).toHaveLength(2); + }); + + // A selection naming nothing on offer is the same as no selection, and must + // not become an empty PATCH. + test("sends nothing when the selection matches no offered change", async () => { + blockedOnRequiredEmail(); + mockMultiselect.mockResolvedValue(["not_a_real_change"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(created()).toHaveLength(2); + }); + }); + + test("warns instead of failing the run when the instance cannot be resolved", async () => { + blockedOnRequiredEmail(); + instanceTarget = new Error("not linked"); + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(captured.err).toContain("nothing was changed"); + expect(created()).toHaveLength(2); + }); +}); + describe("guards that still apply interactively", () => { test("the dev-instance 500-user cap", async () => { fs.writeFileSync( diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 1698a2517..4cd88959d 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -409,7 +409,7 @@ describe("run", () => { await run({ ...baseOptions, yes: false }); expect(captured.err).toContain("Migration readiness"); - expect(captured.err).toContain("1 user lacks it"); + expect(captured.err).toContain("1 user will not be imported"); // The report was printed before the first POST /v1/users. const reportIndex = requests.findIndex((r) => r.url.includes("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/v1/environment")); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 7167b854b..712aa756b 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -13,11 +13,13 @@ import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; import { bold, dim, green, red, yellow } from "../../lib/color.ts"; import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; +import { resolveInstanceTarget, type InstanceTarget } from "../../lib/keyless-target.ts"; import { log } from "../../lib/log.ts"; import { NEXT_STEPS } from "../../lib/next-steps.ts"; -import { confirm } from "../../lib/prompts.ts"; +import { confirm, multiselect } from "../../lib/prompts.ts"; import { withGutter, withSpinner } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; +import { writeInstanceConfig } from "../config/io.ts"; import { importUsers } from "./import-users.ts"; import { analyzeFields } from "./lib/analysis.ts"; import { findMigrateEnvValue } from "./lib/env-file.ts"; @@ -26,7 +28,18 @@ import { fetchInstanceSettings, toClerkStrategy, } from "./lib/clerk-config.ts"; -import { buildReadinessReport, formatReadinessReport } from "./lib/readiness.ts"; +import { + buildReadinessReport, + DASHBOARD_URL, + formatReadinessReport, + type ReadinessReport, +} from "./lib/readiness.ts"; +import { + applyChanges, + buildChangePayload, + buildSettingChanges, + type SettingChange, +} from "./lib/modify-settings.ts"; import { DEV_USER_LIMIT, resolveLimits } from "./lib/instance.ts"; import { getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; import { saveSettings } from "./lib/settings.ts"; @@ -307,32 +320,19 @@ async function skipDisabledProviderUsers( return users.filter((user) => !excludedIds.has(user.userId)); } -/** - * Prints the Migration Readiness report: what the file contains, cross- - * referenced against what the destination instance accepts. - * - * Rendered immediately before the confirmation prompt, so declining that - * prompt aborts with nothing written to Clerk. - * - * Skipped only for `-y`, which says "don't ask, don't lecture" and should not - * pay for two extra network round-trips. Agent mode still gets it: an agent - * driving a migration can act on "this field is required and 40 users lack it" - * exactly as a human would. - */ -async function showReadinessReport(input: { +type ReportInput = { users: User[]; file: string; transformer: string; secretKey: string; validationFailed: number; - skipReport: boolean; -}): Promise { - if (input.skipReport) return; - - const settings = await withSpinner("Checking instance settings...", () => - fetchInstanceSettings(input.secretKey), - ); +}; +/** + * Everything the report needs except the instance's settings — the half that + * comes from the file, and so does not change when the instance does. + */ +async function readFileSide(input: ReportInput) { // Only Supabase exports record per-user providers, so only they can be // cross-referenced against the instance's social connections. let providerCounts: Record | undefined; @@ -344,18 +344,134 @@ async function showReadinessReport(input: { } } - const report = buildReadinessReport({ + return { analysis: analyzeFields(input.users), - settings, validationFailed: input.validationFailed, providerCounts, - }); + }; +} +function printReport(report: ReadinessReport): void { log.blank(); for (const line of formatReadinessReport(report)) log.info(line); log.blank(); } +/** + * Offers to change the instance's settings, one selectable change per flagged + * row. + * + * Without this the report names something the operator has to leave the CLI to + * act on. Nothing is preselected and selecting nothing continues to the import + * prompt unchanged: a flagged setting is not a wrong setting, and relaxing an + * instance's sign-up requirements is a real decision rather than a default. + * + * @returns The changes that were written, so the caller can redraw the report. + */ +async function offerSettingChanges( + report: ReadinessReport, + options: MigrateRunOptions, +): Promise { + const changes = buildSettingChanges(report.blocking); + if (changes.length === 0) return []; + + // Navigation keys are in the prompt's own footer; what that footer cannot say + // is that selecting nothing is a valid answer rather than an unfinished one. + const chosen = await multiselect({ + message: "Update this instance's settings first? (enter to skip)", + options: changes.map((change) => ({ value: change.id, label: change.label })), + initialValues: [], + required: false, + }); + // Filtered before anything is resolved or sent: a selection that matches no + // offered change is the same as no selection, and must not become an empty + // PATCH. + const applied = changes.filter((change) => chosen.includes(change.id)); + if (applied.length === 0) return []; + + // Resolved here rather than up front: an operator who selects nothing should + // not pay for a Platform API round-trip, and a target that cannot be resolved + // (a bare `--secret-key` against an unlinked directory) should not fail the + // whole run before the report has even been offered. + let target: InstanceTarget; + try { + target = await resolveInstanceTarget({ app: options.app, instance: options.instance }); + } catch (error) { + log.warn( + "Could not resolve which instance to configure, so nothing was changed. " + + "Link a project with `clerk link`, or pass `--app `.", + ); + log.debug(`migrate: settings change target unresolved: ${String(error)}`); + return []; + } + + // The Backend API a keyless application is reachable through has no route for + // any of these settings — `config patch` rejects the same payload by name. + if (target.kind === "keyless") { + log.warn( + "These settings need an account to change. Run `clerk auth login` to claim this application, " + + `then re-run, or update them at ${DASHBOARD_URL}.`, + ); + return []; + } + + await withSpinner(`Updating settings on ${target.label}...`, () => + writeInstanceConfig(target, buildChangePayload(applied), { + method: "PATCH", + failureContext: "Failed to update instance settings", + }), + ); + log.success(`Updated ${applied.length} setting${applied.length === 1 ? "" : "s"}.`); + + return applied; +} + +/** + * Prints the Migration Readiness report: what the file contains, cross- + * referenced against what the destination instance accepts. + * + * Rendered immediately before the confirmation prompt, so declining that + * prompt aborts with nothing written to Clerk. + * + * Skipped only for `-y`, which says "don't ask, don't lecture" and should not + * pay for two extra network round-trips. Agent mode still gets it: an agent + * driving a migration can act on "10 users will not be imported, because email + * is required" exactly as a human would — but not the prompt, which needs one. + */ +async function showReadinessReport( + input: ReportInput & { skipReport: boolean; options: MigrateRunOptions }, +): Promise { + if (input.skipReport) return; + + let settings = await withSpinner("Checking instance settings...", () => + fetchInstanceSettings(input.secretKey), + ); + const fileSide = { ...(await readFileSide(input)), users: input.users }; + + let report = buildReadinessReport({ ...fileSide, settings }); + printReport(report); + + if (!isHuman() || isAgent()) return; + + // Every redraw is another decision point, not a receipt. Applying one change + // routinely leaves others still worth making — and can surface consequences + // that were masked behind the row just cleared — so the offer repeats for as + // long as the report has something to offer. + while (report.blocking.length > 0) { + const applied = await offerSettingChanges(report, input.options); + // Nothing selected, nothing offerable, or nowhere to write it: the operator + // has said their piece and the import prompt is next. + if (applied.length === 0) return; + + // Redrawn from the write, not from a re-read. Clerk's Frontend API is + // eventually consistent, so fetching settings again here routinely returns + // the pre-write ones and redraws every row the operator just cleared. + settings = applyChanges(settings, applied); + report = buildReadinessReport({ ...fileSide, settings }); + printReport(report); + } +} + /** * Fills in a missing `--transformer`/`--file` interactively, or explains what * to pass. @@ -510,6 +626,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { secretKey, validationFailed, skipReport: Boolean(options.yes), + options, }); if (!options.yes && isHuman() && !isAgent()) { From 18cef5e4440ca47aabb0bb42d07df4555ebf48b5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:26:06 -0400 Subject: [PATCH 013/141] feat(migrate): document `clerk migrate` rather than `clerk migrate run` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run` is registered `isDefault`, so `clerk migrate --transformer clerk --file users.json` has always worked and is the shorter spelling. Every example, error message, next-step line and README invocation now uses it. `clerk migrate run` stays addressable — scripts and older docs use it — but nothing points there. The group's own help follows `clerk config`: options stay on the subcommands, and `migrate --help` is a list of subcommands plus examples covering each one. `transformers` had no examples block at all; it does now. Two fixes this turned up: - A partial `CLERK_FIREBASE_*` set left in `.env.clerk-migrate` failed *every* subsequent run, including a Supabase one that never asked for Firebase. That was a regression from routing those values through the env file: previously only explicit flags could trigger the all-four-or-nothing check. A partial set that came from saved config is now warned about and ignored; a partial set that came from flags still fails, because there the user did ask. - `readme.test.ts` resolved a documented command to its group and read only that group's options, so every `clerk migrate --transformer …` example looked like it used a flag the binary rejects. It now follows the default subcommand, the same way Commander does. `migrate delete`'s description said "in this directory"; the record it reads has been keyed by project since the `.settings` removal. --- .../cli-core/src/commands/migrate/README.md | 32 +++++----- .../cli-core/src/commands/migrate/delete.ts | 2 +- .../src/commands/migrate/export/auth0.test.ts | 4 +- .../src/commands/migrate/export/clerk.test.ts | 8 +-- .../src/commands/migrate/export/clerk.ts | 2 +- .../commands/migrate/export/firebase.test.ts | 2 +- .../src/commands/migrate/export/firebase.ts | 2 +- .../src/commands/migrate/export/index.ts | 2 +- .../cli-core/src/commands/migrate/index.ts | 59 ++++++++++++++----- .../src/commands/migrate/readme.test.ts | 18 +++++- .../cli-core/src/commands/migrate/run.test.ts | 17 +++++- packages/cli-core/src/commands/migrate/run.ts | 34 ++++++++--- .../migrate/settings/settings.test.ts | 1 + .../cli-core/src/commands/migrate/wizard.ts | 2 +- packages/cli-core/src/lib/config.ts | 2 +- packages/cli-core/src/lib/next-steps.ts | 2 +- 16 files changed, 130 insertions(+), 59 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 3c5b180a3..5f49a5b10 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -5,7 +5,7 @@ Clerk instance. ## Targeting And Auth -`migrate run` resolves its Backend API key through the CLI's standard chain: +`clerk migrate` resolves its Backend API key through the CLI's standard chain: | Flag | Description | | ------------------------ | ---------------------------------------------------------------- | @@ -26,7 +26,7 @@ defaults and the hard development-instance cap below. ### `clerk migrate` (interactive) -Bare `clerk migrate` dispatches to `migrate run`, which walks a human through +Bare `clerk migrate` walks a human through the migration instead of demanding flags — mirroring how bare `clerk deploy` dispatches to `deploy run`. @@ -52,13 +52,13 @@ error naming exactly what to pass: Pass --transformer and --file . ``` -### `clerk migrate run` +### `clerk migrate` Reads an exported user file, maps it onto Clerk's user schema, validates every record, and creates the users through the Backend API. ```sh -clerk migrate run -y --transformer clerk --file users.json +clerk migrate -y --transformer clerk --file users.json ``` | Flag | Description | @@ -121,7 +121,7 @@ limit — the run fails before any request is sent. ### `clerk migrate export` Gets users **out** of a source platform, so there is something to feed -`migrate run`. +`clerk migrate`. ```sh clerk migrate export # pick a platform @@ -158,7 +158,7 @@ other path flag here. | `--client-secret ` | `auth0` | Machine-to-machine application client secret | `export clerk` also takes the targeting flags — it reads from a Clerk instance, -so it resolves a key exactly the way `migrate run` does. +so it resolves a key exactly the way `clerk migrate` does. After each export you get a field-coverage table — which Clerk-relevant fields were present on how many users — so you know the data is thin _before_ you @@ -173,7 +173,7 @@ Field coverage Exported 3 users to /project/exports/clerk-export.json └ Next steps - → Run `clerk migrate run --transformer clerk --file exports/clerk-export.json` to import them + → Run `clerk migrate --transformer clerk --file exports/clerk-export.json` to import them ``` Every export also writes `logs/export-.log`, so `migrate logs list` @@ -259,7 +259,7 @@ prints the exact import command: ``` Password hash parameters Read from the project. Import with: - clerk migrate run -y --transformer firebase --file exports/firebase-export.json \ + clerk migrate -y --transformer firebase --file exports/firebase-export.json \ --firebase-signer-key "…" --firebase-salt-separator "…" \ --firebase-rounds 8 --firebase-mem-cost 14 ``` @@ -297,7 +297,7 @@ returning the first thousand would read as "that is everyone". ### `clerk migrate delete` -The undo for a bad migration. Deletes the users a previous `clerk migrate run` +The undo for a bad migration. Deletes the users a previous `clerk migrate` created in this directory, matched by the `external_id` the import stamped on each one. @@ -306,7 +306,7 @@ clerk migrate delete # confirms first clerk migrate delete -y # non-interactive ``` -Takes the same targeting flags as `migrate run` (`--secret-key`, `--app`, +Takes the same targeting flags as `clerk migrate` (`--secret-key`, `--app`, `--instance`). Flat rather than under a noun group: it is the one command in this tree that @@ -462,7 +462,7 @@ firebase-signer-key aVer…3456 .env.clerk-migrate Firebase base64 sig firebase-rounds — not set Firebase scrypt rounds ``` -Setting names are kebab-case and identical to the `migrate run` flag each one +Setting names are kebab-case and identical to the `clerk migrate` flag each one backs, so `firebase-signer-key` here is `--firebase-signer-key` there rather than a second spelling to learn. The description column carries the prose. @@ -500,7 +500,7 @@ so the output is safe to paste into an issue. Migrating from a platform with no built-in, without recompiling the CLI: ```sh -clerk migrate run --transformer-file ./my-platform.ts --file users.json +clerk migrate --transformer-file ./my-platform.ts --file users.json ``` The file lives in **your** project, not in the CLI, and is imported at runtime. @@ -572,7 +572,7 @@ alongside each digest. Find them in the Firebase console under **Authentication → Users → (⋮) → Password hash parameters**. ```sh -clerk migrate run -y -t firebase -f users.json \ +clerk migrate -y -t firebase -f users.json \ --firebase-signer-key --firebase-salt-separator \ --firebase-rounds 8 --firebase-mem-cost 14 ``` @@ -905,9 +905,9 @@ NDJSON is. The original `.log` stays put. | Method | Path | Used by | | -------- | -------------------------- | ------------------------------------------------------------------------------------ | -| `POST` | `/v1/users` | `migrate run` — creates each user | -| `POST` | `/v1/email_addresses` | `migrate run` — attaches additional emails | -| `POST` | `/v1/phone_numbers` | `migrate run` — attaches additional phones | +| `POST` | `/v1/users` | `clerk migrate` — creates each user | +| `POST` | `/v1/email_addresses` | `clerk migrate` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `clerk migrate` — attaches additional phones | | `GET` | `/v1/users?external_id=…` | `migrate delete` — finds this migration's users, 100 IDs a call | | `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | | `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index db2791205..39394ceff 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -74,7 +74,7 @@ export async function resolveMigrationToUndo(): Promise<{ file: string; key: str if (!settings.file || !settings.transformer) { throw new CliError( - "No migration to undo: this project has no record of a previous `clerk migrate run`.\n" + + "No migration to undo: this project has no record of a previous `clerk migrate`.\n" + "Run `clerk migrate delete` from the project you migrated from.", { code: ERROR_CODE.FILE_NOT_FOUND }, ); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index ceef2915a..9746f79cb 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -300,9 +300,7 @@ describe("exportAuth0", () => { } finally { setMode(originalMode); } - expect(captured.err).toContain( - "migrate run --transformer auth0 --file exports/auth0-export.json", - ); + expect(captured.err).toContain("migrate --transformer auth0 --file exports/auth0-export.json"); }); test("--output controls the destination", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 9ce14d656..b628a8408 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -255,9 +255,7 @@ describe("exportClerk", () => { } finally { setMode(originalMode); } - expect(captured.err).toContain( - "migrate run --transformer clerk --file exports/clerk-export.json", - ); + expect(captured.err).toContain("migrate --transformer clerk --file exports/clerk-export.json"); }); test("--output controls the destination, relative to the working directory", async () => { @@ -300,7 +298,7 @@ describe("exportClerk", () => { expect(captured.err).toContain("No users found to export"); expect(captured.err).not.toContain("Next steps"); - expect(captured.err).not.toContain("migrate run --transformer"); + expect(captured.err).not.toContain("migrate --transformer"); }); test("agent mode suppresses the Next steps block", async () => { @@ -310,6 +308,6 @@ describe("exportClerk", () => { expect(captured.err).toContain("Exported 1 user"); expect(captured.err).not.toContain("Next steps"); - expect(captured.err).not.toContain("migrate run --transformer"); + expect(captured.err).not.toContain("migrate --transformer"); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index a49fa581e..564bcabd2 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -5,7 +5,7 @@ * onto `bapiRequest` instead of `@clerk/backend` so it shares the CLI's auth * resolution, `--verbose` request tracing and error taxonomy. * - * The output feeds `clerk migrate run --transformer clerk` unedited, which is + * The output feeds `clerk migrate --transformer clerk` unedited, which is * what makes development → production a two-command operation. * * **Passwords do not come out of this endpoint.** Clerk never returns password diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 137d0502d..1de54a3c8 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -425,7 +425,7 @@ describe("exportFirebase", () => { setMode(originalMode); } expect(captured.err).toContain( - "migrate run --transformer firebase --file exports/firebase-export.json", + "migrate --transformer firebase --file exports/firebase-export.json", ); }); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 9c70a410b..aedb8fb9c 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -397,7 +397,7 @@ export function formatHashConfigGuidance( bold("Password hash parameters"), "Read from the project. Import with:", dim( - ` clerk migrate run -y --transformer firebase --file ${outputPath} \\\n` + + ` clerk migrate -y --transformer firebase --file ${outputPath} \\\n` + ` --firebase-signer-key "${config.signerKey}" \\\n` + ` --firebase-salt-separator "${config.saltSeparator}" \\\n` + ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 2ac69b5b3..773e07f22 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -84,7 +84,7 @@ const DB_PLATFORMS = [ export function registerMigrateExport(migrateCommand: Command<[], Record>): void { const exportCommand = migrateCommand .command("export") - .description("Export users from a source platform, ready for `migrate run`") + .description("Export users from a source platform, ready for `clerk migrate`") .setExamples([ { command: "clerk migrate export", description: "Pick a platform interactively" }, { diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index a1a56e1c7..b17195af4 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -16,19 +16,37 @@ export function registerMigrate(program: Program): void { .command("migrate") .description("Migrate users into Clerk from another auth provider") .setExamples([ + { command: "clerk migrate", description: "Walk through a migration interactively" }, { - command: "clerk migrate", - description: "Walk through a migration interactively", + command: "clerk migrate -y --transformer clerk --file users.json", + description: "Import users from a Clerk export", }, { - command: "clerk migrate run -y --transformer clerk --file users.json", - description: "Import users from a Clerk export", + command: "clerk migrate -y -t supabase -f users.json --skip-unsupported-providers", + description: "Skip Supabase users whose provider is not enabled", + }, + { + command: "clerk migrate export supabase", + description: "Export users from Supabase, ready to import", }, + { command: "clerk migrate settings", description: "Show what a run here would pick up" }, + { + command: "clerk migrate settings set firebase-signer-key abc123", + description: "Save a credential to .env.clerk-migrate", + }, + { command: "clerk migrate logs", description: "List the local migration logs" }, + { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { command: "clerk migrate delete", description: "Undo the last migration" }, ]); - // `isDefault` so bare `clerk migrate` runs the wizard, mirroring how bare - // `clerk deploy` dispatches to `deploy run`. Not hidden: unlike deploy's, - // this subcommand is documented and carries every flag. + // `isDefault` so `clerk migrate` is the whole command: bare, it runs the + // wizard; with flags, they fall through to here. `run` stays addressable + // because scripts and older docs use it, but `clerk migrate` is the spelling + // every example gives. + // + // The flags stay here rather than on `migrate`, matching how `config` keeps + // its own on `pull`/`patch`/`put` — a group's help is a list of subcommands + // and examples, not a merge of everything underneath it. migrateCommand .command("run", { isDefault: true }) .description("Import users from an exported JSON or CSV file") @@ -51,10 +69,10 @@ export function registerMigrate(program: Program): void { ) .option("--firebase-signer-key ", "Firebase base64 signer key") .option("--firebase-salt-separator ", "Firebase base64 salt separator") - .option("--firebase-rounds ", "Firebase scrypt rounds", (value) => + .option("--firebase-rounds ", "Firebase scrypt rounds", (value: string) => parseIntegerOption(value, "--firebase-rounds", { min: 1 }), ) - .option("--firebase-mem-cost ", "Firebase scrypt memory cost", (value) => + .option("--firebase-mem-cost ", "Firebase scrypt memory cost", (value: string) => parseIntegerOption(value, "--firebase-mem-cost", { min: 1 }), ) .option("-y, --yes", "Skip the confirmation prompt") @@ -64,19 +82,19 @@ export function registerMigrate(program: Program): void { .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") .setExamples([ { - command: "clerk migrate run -y --transformer clerk --file users.json", + command: "clerk migrate -y --transformer clerk --file users.json", description: "Import a Clerk Dashboard export", }, { - command: "clerk migrate run -y -t clerk -f users.csv --require-password", + command: "clerk migrate -y -t clerk -f users.csv --require-password", description: "Import only the users that carry a password digest", }, { - command: "clerk migrate run -y -t clerk -f users.json -r user_2x9k", + command: "clerk migrate -y -t clerk -f users.json -r user_2x9k", description: "Resume a partial migration after the last imported user", }, { - command: "clerk migrate run -y -t supabase -f users.json --skip-unsupported-providers", + command: "clerk migrate -y -t supabase -f users.json --skip-unsupported-providers", description: "Skip Supabase users whose only provider is not enabled in Clerk", }, ]) @@ -88,7 +106,7 @@ export function registerMigrate(program: Program): void { // destroys data in Clerk, and it is worth keeping short and prominent. migrateCommand .command("delete") - .description("Delete the users created by the last migration in this directory") + .description("Delete the users created by the last migration for this project") .option("-y, --yes", "Skip the confirmation prompt") .option("--secret-key ", "Backend API secret key to use") .option("--clerk-secret-key ", "Deprecated alias for --secret-key") @@ -111,7 +129,18 @@ export function registerMigrate(program: Program): void { // need a command rather than only appearing in the interactive picker. const transformersCommand = migrateCommand .command("transformers") - .description("Inspect the available source-platform transformers"); + .description("Inspect the available source-platform transformers") + .setExamples([ + { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { + command: "clerk migrate transformers list --json", + description: "Machine-readable, including each one's ID field", + }, + { + command: "clerk migrate transformers list --transformer-file ./my-transformer.ts", + description: "Include one you wrote", + }, + ]); transformersCommand .command("list", { isDefault: true }) diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index ef5a05f85..f41aec567 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -60,10 +60,26 @@ function resolve(tokens: string[]): { command: Command; rest: string[] } { return { command, rest: tokens.slice(index) }; } +/** + * The flags a command accepts, including those of a default subcommand. + * + * `clerk migrate` carries no options of its own — `run` is registered + * `isDefault`, so Commander hands it everything after the group name. The + * documented spelling is `clerk migrate --transformer …`, and this has to see + * the same flags Commander does or every such example reads as unsupported. + */ function flagsOf(command: Command): string[] { - return command.options.flatMap( + const own = command.options.flatMap( (option) => [option.short, option.long].filter(Boolean) as string[], ); + + const defaultChild = command.commands.find( + // Commander records the default subcommand on the parent, not the child. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (child) => child.name() === (command as any)._defaultCommandName, + ); + + return defaultChild ? [...own, ...flagsOf(defaultChild)] : own; } /** Every command under `migrate`, so no subcommand escapes the flag sweep. */ diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 4cd88959d..ad30e0e03 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -98,6 +98,7 @@ describe("resolveFirebaseHashConfig", () => { }); describe("environment fallback", () => { + const captured = useCaptureLog(); const ENV = { CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER", CLERK_FIREBASE_SALT_SEPARATOR: "Bw==", @@ -142,9 +143,21 @@ describe("resolveFirebaseHashConfig", () => { }); }); - test("still demands the full set when the environment supplies only part", async () => { + // Stale saved config, not an instruction: a signer key left over from a + // Firebase migration must not fail the Supabase run that follows it. The + // flag path stays strict — see "rejects a set missing %s" above. + test("ignores a partial set rather than failing a run that never asked for it", async () => { setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); - await expect(resolveFirebaseHashConfig({})).rejects.toThrow(/--firebase-salt-separator/); + + expect(await resolveFirebaseHashConfig({})).toBeUndefined(); + expect(captured.err).toContain("Ignoring an incomplete Firebase hash configuration"); + }); + + test("still fails when a flag supplied part of the set", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); + await expect(resolveFirebaseHashConfig({ firebaseRounds: 8 })).rejects.toThrow( + /--firebase-salt-separator/, + ); }); // An empty var is how a shell spells "unset", and treating it as set would diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 712aa756b..1d8b4c16d 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -1,13 +1,15 @@ /** - * `clerk migrate run` — non-interactive user import. + * `clerk migrate` — the user import itself. * * Ported from the standalone migration-tool's `src/migrate/cli.ts` * (`runNonInteractive`), with auth moved onto the CLI's standard secret-key * resolution chain and every failure raised as a `CliError` instead of * `console.error` + `process.exit`. * - * The interactive wizard that a bare `clerk migrate` will launch is a separate - * command; this path is the one an agent or a script drives. + * Registered as the `run` subcommand and marked default, so `clerk migrate` and + * `clerk migrate run` both land here. Whatever the flags did not supply is + * filled in by `wizard.ts` for a human, or raised as a usage error naming the + * missing flags for an agent, which cannot answer a prompt. */ import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; @@ -122,6 +124,7 @@ async function withFirebaseEnv(options: MigrateRunOptions): Promise { + const fromFlags = FIREBASE_FLAGS.filter(([key]) => rawOptions[key] !== undefined); const options = await withFirebaseEnv(rawOptions); const provided = FIREBASE_FLAGS.filter(([key]) => options[key] !== undefined); @@ -131,6 +134,20 @@ export async function resolveFirebaseHashConfig( const missing = FIREBASE_FLAGS.filter(([key]) => options[key] === undefined).map( ([, flag]) => flag, ); + + // A partial set nobody asked for on this command line is stale saved + // config, not an instruction: a `CLERK_FIREBASE_SIGNER_KEY` left in + // `.env.clerk-migrate` after a Firebase migration must not fail the + // Supabase run that follows it. Warned rather than dropped silently, + // because on a Firebase run it is the reason passwords will not import. + if (fromFlags.length === 0) { + log.warn( + `Ignoring an incomplete Firebase hash configuration (no ${missing.join(", ")}). ` + + "Run `clerk migrate settings` to see what is set.", + ); + return undefined; + } + throwUsageError( `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", @@ -167,8 +184,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: - "clerk migrate run -y --transformer-file ./my-transformer.ts --file users.json", + command: "clerk migrate -y --transformer-file ./my-transformer.ts --file users.json", description: "Import with a custom transformer", }, ], @@ -190,7 +206,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate run -y --transformer clerk --file users.json", + command: "clerk migrate -y --transformer clerk --file users.json", description: "Import a Clerk export", }, ], @@ -208,7 +224,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate run -y --transformer clerk --file users.json", + command: "clerk migrate -y --transformer clerk --file users.json", description: "Import a Clerk export", }, ], @@ -530,11 +546,11 @@ async function applyCustomTransformer(options: MigrateRunOptions): Promise { await list(); + // eslint-disable-next-line no-control-regex const plain = captured.err.replaceAll(/\u001B\[\d+m/g, ""); const columnOf = (description: string) => plain diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index d5b556edb..d855e9b26 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -151,7 +151,7 @@ export function throwAgentFlagsRequired(missing: { transformer: boolean; file: b undefined, [ { - command: `clerk migrate run -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, + command: `clerk migrate -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, description: "Run non-interactively", }, ], diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index 8139d9ca0..f77a9602e 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -50,7 +50,7 @@ interface RelayEntry { token: string; } -/** What `clerk migrate run` last imported for a project, and how. */ +/** What `clerk migrate` last imported for a project, and how. */ interface MigrationEntry { transformer?: string; file?: string; diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index 613fe1042..41d415f90 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -79,7 +79,7 @@ export const NEXT_STEPS = { // The only parameterized entry: a suggested import is worthless unless it // names the transformer that reads this export and the file just written. MIGRATE_EXPORT: (transformerKey: string, file: string) => [ - `Run \`clerk migrate run --transformer ${transformerKey} --file ${file}\` to import them`, + `Run \`clerk migrate --transformer ${transformerKey} --file ${file}\` to import them`, ], } as const; From 85afd339e6476ae5e335bc47356d0df50f14a4fe Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:34:13 -0400 Subject: [PATCH 014/141] fix(migrate): read the Firebase hash parameters only when the transformer is firebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `migrate run` is one command serving every platform, so a `CLERK_FIREBASE_SIGNER_KEY` left in `.env.clerk-migrate` after a Firebase migration was in scope for whatever ran next. A complete leftover set was resolved and passed along on a Supabase import; a partial one failed that import outright, naming four `--firebase-*` flags the user had not used and did not need. The gate now sits before the lookup rather than being a filter after it: any transformer but `firebase` returns immediately, without reading the environment, the env files, or even its own flags. Nothing downstream misused the value — only the Firebase transformer reads it off `TransformContext` — but resolving it at all is what let stale config warn and fail unrelated runs. The previous fix only covered the partial case, and did it transformer-blind. Moved to `lib/firebase-hash.ts` so the wizard can resolve after the platform is picked without importing from `run.ts`, which imports the wizard. That also keeps the interactive path: choosing Firebase with all four already set skips the prompt, choosing anything else never looks. The per-platform export commands need no equivalent gate — `migrate export auth0` reads `AUTH0_*` and nothing else, because there the command *is* the platform. `migrate run` is the only one that spans them. --- .../migrate/lib/firebase-hash.test.ts | 172 ++++++++++++++++++ .../src/commands/migrate/lib/firebase-hash.ts | 119 ++++++++++++ .../cli-core/src/commands/migrate/run.test.ts | 114 +----------- packages/cli-core/src/commands/migrate/run.ts | 110 +---------- .../cli-core/src/commands/migrate/wizard.ts | 24 ++- 5 files changed, 317 insertions(+), 222 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/firebase-hash.ts diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts new file mode 100644 index 000000000..ccf5fa75e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts @@ -0,0 +1,172 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { resolveFirebaseHashConfig } from "./firebase-hash.ts"; + +const captured = useCaptureLog(); + +const ALL_FLAGS = { + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, +}; + +const ENV = { + CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER", + CLERK_FIREBASE_SALT_SEPARATOR: "Bw==", + CLERK_FIREBASE_ROUNDS: "8", + CLERK_FIREBASE_MEM_COST: "14", +}; + +let workDir: string; +let originalCwd: string; + +const setEnv = (vars: Partial) => Object.assign(process.env, vars); + +beforeEach(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-fbhash-"))); + process.chdir(workDir); +}); + +afterEach(() => { + for (const name of Object.keys(ENV)) delete process.env[name]; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("gating on the transformer", () => { + // `migrate run` is one command for every platform, so a signer key left in + // .env.clerk-migrate after a Firebase migration is in scope for whatever runs + // next unless the transformer says otherwise. + test.each([["clerk"], ["supabase"], ["auth0"], ["authjs"], ["betterauth"]])( + "reads nothing for the %s transformer", + async (transformer) => { + setEnv(ENV); + expect(await resolveFirebaseHashConfig({}, transformer)).toBeUndefined(); + }, + ); + + test("stays silent about a complete set on another platform's run", async () => { + setEnv(ENV); + await resolveFirebaseHashConfig({}, "supabase"); + expect(captured.err).toBe(""); + }); + + // The case that regressed: half a set used to fail every later run. + test("stays silent about a partial set on another platform's run", async () => { + fs.writeFileSync(path.join(workDir, ".env.clerk-migrate"), "CLERK_FIREBASE_SIGNER_KEY=left\n"); + + expect(await resolveFirebaseHashConfig({}, "supabase")).toBeUndefined(); + expect(captured.err).toBe(""); + }); + + test("ignores even explicit flags when the platform is not firebase", async () => { + expect(await resolveFirebaseHashConfig(ALL_FLAGS, "supabase")).toBeUndefined(); + }); + + test("resolves nothing before the platform is known", async () => { + setEnv(ENV); + expect(await resolveFirebaseHashConfig({}, undefined)).toBeUndefined(); + }); +}); + +describe("on a firebase run", () => { + test("builds the config from flags", async () => { + expect(await resolveFirebaseHashConfig(ALL_FLAGS, "firebase")).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + test("falls back to the environment", async () => { + setEnv(ENV); + expect((await resolveFirebaseHashConfig({}, "firebase"))?.base64_signer_key).toBe("ENV_SIGNER"); + }); + + test("reads .env.clerk-migrate when the variable is not exported", async () => { + fs.writeFileSync( + path.join(workDir, ".env.clerk-migrate"), + Object.entries(ENV) + .map(([key, value]) => `${key}=${value}`) + .join("\n"), + ); + + expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); + }); + + test("prefers a flag over the environment", async () => { + setEnv(ENV); + expect((await resolveFirebaseHashConfig(ALL_FLAGS, "firebase"))?.base64_signer_key).toBe( + "SIGNER", + ); + }); + + test("fills only the gaps the flags left", async () => { + setEnv({ CLERK_FIREBASE_ROUNDS: "8", CLERK_FIREBASE_MEM_COST: "14" }); + + expect( + await resolveFirebaseHashConfig( + { firebaseSignerKey: "SIGNER", firebaseSaltSeparator: "Bw==" }, + "firebase", + ), + ).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + // A digest built from a partial set is well-formed but verifies against + // nothing, so every migrated user would silently fail to sign in. + test.each([ + ["firebaseSignerKey", "--firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost"], + ] as const)("rejects a flag set missing %s, naming it", async (omit, flag) => { + const partial = { ...ALL_FLAGS }; + delete (partial as Record)[omit]; + + await expect(resolveFirebaseHashConfig(partial, "firebase")).rejects.toThrow(new RegExp(flag)); + }); + + test("names every missing flag at once", async () => { + await expect( + resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" }, "firebase"), + ).rejects.toThrow(/--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/); + }); + + // Saved config is a leftover, not an instruction — but on a Firebase import + // it is the reason the passwords will not come across, so it is said aloud. + test("warns and continues when only saved config is partial", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); + + expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); + expect(captured.err).toContain("Ignoring an incomplete Firebase hash configuration"); + }); + + test("still fails when a flag supplied part of the set", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); + + await expect(resolveFirebaseHashConfig({ firebaseRounds: 8 }, "firebase")).rejects.toThrow( + /--firebase-salt-separator/, + ); + }); + + // An empty variable is how a shell spells "unset". + test("ignores an empty variable", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "" }); + expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); + }); + + test("returns nothing when neither flags nor the environment supply a config", async () => { + expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts new file mode 100644 index 000000000..836d374ea --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts @@ -0,0 +1,119 @@ +/** + * Firebase's four scrypt parameters: where they come from, and when they are + * looked for at all. + * + * **Only read when the transformer is `firebase`.** `migrate run` is one + * command serving every platform, so a `CLERK_FIREBASE_SIGNER_KEY` left in + * `.env.clerk-migrate` after a Firebase migration is in scope for the Supabase + * run that follows it unless something says otherwise. Nothing downstream would + * misuse it — only the Firebase transformer reads the config off + * {@link TransformContext} — but resolving it means a stale or partial set can + * warn, or fail, a run that never mentioned Firebase. So the gate is here, + * before the lookup, rather than a filter after it. + * + * The per-platform export commands need no such gate: `migrate export auth0` + * reads `AUTH0_*` and nothing else, because the command itself is the platform. + * This is the only place one command spans them all. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { findMigrateEnvValue } from "./env-file.ts"; +import type { FirebaseHashConfig } from "../types.ts"; + +/** The `--firebase-*` flags, and the variable each falls back to. */ +export const FIREBASE_FLAGS = [ + ["firebaseSignerKey", "--firebase-signer-key", "CLERK_FIREBASE_SIGNER_KEY"], + ["firebaseSaltSeparator", "--firebase-salt-separator", "CLERK_FIREBASE_SALT_SEPARATOR"], + ["firebaseRounds", "--firebase-rounds", "CLERK_FIREBASE_ROUNDS"], + ["firebaseMemCost", "--firebase-mem-cost", "CLERK_FIREBASE_MEM_COST"], +] as const; + +const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); + +export type FirebaseHashFlags = { + firebaseSignerKey?: string; + firebaseSaltSeparator?: string; + firebaseRounds?: number; + firebaseMemCost?: number; +}; + +/** + * Overlays the `CLERK_FIREBASE_*` values onto whichever flags were not passed. + * + * Resolved through {@link findMigrateEnvValue}: the environment first, then + * `.env.clerk-migrate`, then the app's own `.env` files. The signer key is a + * Firebase secret, so it is never written to the CLI's config — + * `.env.clerk-migrate` is gitignored on creation. + */ +async function withFirebaseEnv(flags: FirebaseHashFlags): Promise { + const merged: FirebaseHashFlags = { ...flags }; + for (const [key, , envVar] of FIREBASE_FLAGS) { + if (merged[key] !== undefined) continue; + const located = await findMigrateEnvValue([envVar]); + if (!located || located.value.trim() === "") continue; + // A non-numeric round count is left to fail the flag's own validation + // rather than silently becoming NaN. + (merged as Record)[key] = FIREBASE_NUMERIC.has(key) + ? Number(located.value) + : located.value; + } + return merged; +} + +/** + * Resolves the four parameters from flags, then the `CLERK_FIREBASE_*` + * variables, then the project's env files. + * + * The four are required as a set: a digest built from a partial set is + * well-formed but verifies against nothing, so every migrated user would fail + * to sign in with no error at import time. How a partial set is treated depends + * on where it came from — flags are an instruction, saved config is not. + * + * @param transformer - The platform being migrated. Anything but `firebase` + * returns immediately, without reading the environment. + * @returns The config, or `undefined` when none was supplied — which is fine + * for an export that carries no password hashes. + */ +export async function resolveFirebaseHashConfig( + flags: FirebaseHashFlags, + transformer: string | undefined, +): Promise { + if (transformer !== "firebase") return undefined; + + const fromFlags = FIREBASE_FLAGS.filter(([key]) => flags[key] !== undefined); + const resolved = await withFirebaseEnv(flags); + const provided = FIREBASE_FLAGS.filter(([key]) => resolved[key] !== undefined); + + if (provided.length === 0) return undefined; + + if (provided.length < FIREBASE_FLAGS.length) { + const missing = FIREBASE_FLAGS.filter(([key]) => resolved[key] === undefined).map( + ([, flag]) => flag, + ); + + // Saved config is a leftover, not an instruction: half a set in + // `.env.clerk-migrate` should not fail the run, but on a Firebase import it + // is the reason the passwords will not come across, so it is said out loud. + if (fromFlags.length === 0) { + log.warn( + `Ignoring an incomplete Firebase hash configuration (no ${missing.join(", ")}). ` + + "Run `clerk migrate settings` to see what is set.", + ); + return undefined; + } + + throwUsageError( + `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + + "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + "https://clerk.com/docs/guides/development/migrating/firebase", + ); + } + + return { + base64_signer_key: resolved.firebaseSignerKey as string, + base64_salt_separator: resolved.firebaseSaltSeparator as string, + rounds: resolved.firebaseRounds as number, + mem_cost: resolved.firebaseMemCost as number, + }; +} diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index ad30e0e03..9d9272f95 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -8,7 +8,7 @@ import { useCaptureLog } from "../../test/lib/stubs.ts"; import { getLogDir } from "./lib/logger.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; import { loadSettings } from "./lib/settings.ts"; -import { applyResumeAfter, resolveFirebaseHashConfig, run, validateRunOptions } from "./run.ts"; +import { applyResumeAfter, run, validateRunOptions } from "./run.ts"; import type { User } from "./types.ts"; let workDir: string; @@ -61,118 +61,6 @@ describe("validateRunOptions", () => { }); }); -describe("resolveFirebaseHashConfig", () => { - const ALL = { - firebaseSignerKey: "SIGNER", - firebaseSaltSeparator: "Bw==", - firebaseRounds: 8, - firebaseMemCost: 14, - }; - - test("builds the config when all four flags are present", async () => { - expect(await resolveFirebaseHashConfig(ALL)).toEqual({ - base64_signer_key: "SIGNER", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }); - }); - - // A digest built from a partial set is well-formed but verifies against - // nothing, so every migrated user would silently fail to sign in. - test.each([ - ["firebaseSignerKey", "--firebase-signer-key"], - ["firebaseSaltSeparator", "--firebase-salt-separator"], - ["firebaseRounds", "--firebase-rounds"], - ["firebaseMemCost", "--firebase-mem-cost"], - ] as const)("rejects a set missing %s, naming the flag", async (omit, flag) => { - const partial = { ...ALL }; - delete (partial as Record)[omit]; - await expect(resolveFirebaseHashConfig(partial)).rejects.toThrow(new RegExp(flag)); - }); - - test("names every missing flag at once", async () => { - await expect(resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" })).rejects.toThrow( - /--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/, - ); - }); - - describe("environment fallback", () => { - const captured = useCaptureLog(); - const ENV = { - CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER", - CLERK_FIREBASE_SALT_SEPARATOR: "Bw==", - CLERK_FIREBASE_ROUNDS: "8", - CLERK_FIREBASE_MEM_COST: "14", - }; - - afterEach(() => { - for (const name of Object.keys(ENV)) delete process.env[name]; - }); - - const setEnv = (vars: Partial) => Object.assign(process.env, vars); - - test("builds the config when no flag is passed", async () => { - setEnv(ENV); - expect(await resolveFirebaseHashConfig({})).toEqual({ - base64_signer_key: "ENV_SIGNER", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }); - }); - - test("prefers a flag over the environment", async () => { - setEnv(ENV); - expect((await resolveFirebaseHashConfig(ALL))?.base64_signer_key).toBe("SIGNER"); - }); - - // Half from the environment and half from flags is still a complete set. - test("fills only the gaps the flags left", async () => { - setEnv({ CLERK_FIREBASE_ROUNDS: "8", CLERK_FIREBASE_MEM_COST: "14" }); - expect( - await resolveFirebaseHashConfig({ - firebaseSignerKey: "SIGNER", - firebaseSaltSeparator: "Bw==", - }), - ).toEqual({ - base64_signer_key: "SIGNER", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }); - }); - - // Stale saved config, not an instruction: a signer key left over from a - // Firebase migration must not fail the Supabase run that follows it. The - // flag path stays strict — see "rejects a set missing %s" above. - test("ignores a partial set rather than failing a run that never asked for it", async () => { - setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); - - expect(await resolveFirebaseHashConfig({})).toBeUndefined(); - expect(captured.err).toContain("Ignoring an incomplete Firebase hash configuration"); - }); - - test("still fails when a flag supplied part of the set", async () => { - setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); - await expect(resolveFirebaseHashConfig({ firebaseRounds: 8 })).rejects.toThrow( - /--firebase-salt-separator/, - ); - }); - - // An empty var is how a shell spells "unset", and treating it as set would - // demand the other three for a config nobody asked for. - test("ignores an empty variable", async () => { - setEnv({ CLERK_FIREBASE_SIGNER_KEY: "" }); - expect(await resolveFirebaseHashConfig({})).toBeUndefined(); - }); - }); - - test("returns nothing when neither flags nor the environment supply a config", async () => { - expect(await resolveFirebaseHashConfig({})).toBeUndefined(); - }); -}); - describe("applyResumeAfter", () => { test("returns everything when no ID is given", () => { expect(applyResumeAfter(users("a", "b"), undefined)).toHaveLength(2); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 1d8b4c16d..bdcd865e9 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -24,7 +24,7 @@ import { isAgent, isHuman } from "../../mode.ts"; import { writeInstanceConfig } from "../config/io.ts"; import { importUsers } from "./import-users.ts"; import { analyzeFields } from "./lib/analysis.ts"; -import { findMigrateEnvValue } from "./lib/env-file.ts"; +import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebase-hash.ts"; import { enabledSocialProviders, fetchInstanceSettings, @@ -54,7 +54,7 @@ import { import { fileExists, getFileType, loadUsersFromFile } from "./lib/transform.ts"; import { loadCustomTransformer } from "./transformers/load-custom.ts"; import { registerCustomTransformer, transformerKeys } from "./transformers/registry.ts"; -import type { FirebaseHashConfig, ImportSummary, User } from "./types.ts"; +import type { ImportSummary, User } from "./types.ts"; import { runWizard, throwAgentFlagsRequired } from "./wizard.ts"; export type MigrateRunOptions = { @@ -72,96 +72,7 @@ export type MigrateRunOptions = { transformerFile?: string; /** Supabase: drop users whose only social provider is disabled in Clerk. */ skipUnsupportedProviders?: boolean; - firebaseSignerKey?: string; - firebaseSaltSeparator?: string; - firebaseRounds?: number; - firebaseMemCost?: number; -}; - -const FIREBASE_FLAGS = [ - ["firebaseSignerKey", "--firebase-signer-key", "CLERK_FIREBASE_SIGNER_KEY"], - ["firebaseSaltSeparator", "--firebase-salt-separator", "CLERK_FIREBASE_SALT_SEPARATOR"], - ["firebaseRounds", "--firebase-rounds", "CLERK_FIREBASE_ROUNDS"], - ["firebaseMemCost", "--firebase-mem-cost", "CLERK_FIREBASE_MEM_COST"], -] as const; - -const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); - -/** - * Overlays the `CLERK_FIREBASE_*` values onto whichever flags were not passed. - * - * Resolved through {@link findMigrateEnvValue}: the environment first, then - * `.env.clerk-migrate`, then the app's own `.env` files. The signer key is a - * Firebase secret, so it is never written to the CLI's config — - * `.env.clerk-migrate` is gitignored on creation. - */ -async function withFirebaseEnv(options: MigrateRunOptions): Promise { - const merged = { ...options }; - for (const [key, , envVar] of FIREBASE_FLAGS) { - if (merged[key] !== undefined) continue; - const located = await findMigrateEnvValue([envVar]); - if (!located || located.value.trim() === "") continue; - // A non-numeric round count is left to fail the flag's own validation - // rather than silently becoming NaN. - (merged as Record)[key] = FIREBASE_NUMERIC.has(key) - ? Number(located.value) - : located.value; - } - return merged; -} - -/** - * Resolves Firebase's four hash parameters from flags, falling back to the - * `CLERK_FIREBASE_*` environment variables and the project's `.env` files. - * - * The four are required as a set: a digest built from a partial set is - * well-formed but verifies against nothing, so every migrated user would fail - * to sign in with no error at import time. - * - * @returns The config, or `undefined` when none was supplied — which is fine - * for an export that carries no password hashes. - */ -export async function resolveFirebaseHashConfig( - rawOptions: MigrateRunOptions, -): Promise { - const fromFlags = FIREBASE_FLAGS.filter(([key]) => rawOptions[key] !== undefined); - const options = await withFirebaseEnv(rawOptions); - const provided = FIREBASE_FLAGS.filter(([key]) => options[key] !== undefined); - - if (provided.length === 0) return undefined; - - if (provided.length < FIREBASE_FLAGS.length) { - const missing = FIREBASE_FLAGS.filter(([key]) => options[key] === undefined).map( - ([, flag]) => flag, - ); - - // A partial set nobody asked for on this command line is stale saved - // config, not an instruction: a `CLERK_FIREBASE_SIGNER_KEY` left in - // `.env.clerk-migrate` after a Firebase migration must not fail the - // Supabase run that follows it. Warned rather than dropped silently, - // because on a Firebase run it is the reason passwords will not import. - if (fromFlags.length === 0) { - log.warn( - `Ignoring an incomplete Firebase hash configuration (no ${missing.join(", ")}). ` + - "Run `clerk migrate settings` to see what is set.", - ); - return undefined; - } - - throwUsageError( - `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + - "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", - "https://clerk.com/docs/guides/development/migrating/firebase", - ); - } - - return { - base64_signer_key: options.firebaseSignerKey as string, - base64_salt_separator: options.firebaseSaltSeparator as string, - rounds: options.firebaseRounds as number, - mem_cost: options.firebaseMemCost as number, - }; -} +} & FirebaseHashFlags; /** * Validates the flags a run needs before anything is read or sent. @@ -504,14 +415,11 @@ async function resolveMissingOptions(options: MigrateRunOptions): Promise { const secretKeyOption = options.secretKey ?? options.clerkSecretKey; const { transformer, file } = validateRunOptions(options); - const firebaseHashConfig = await resolveFirebaseHashConfig(options); + const firebaseHashConfig = await resolveFirebaseHashConfig(options, transformer); await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index d855e9b26..ff6e7173e 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -14,6 +14,7 @@ import { throwUsageError } from "../../lib/errors.ts"; import { select } from "../../lib/listage.ts"; import { log } from "../../lib/log.ts"; import { text } from "../../lib/prompts.ts"; +import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebase-hash.ts"; import { loadSettings } from "./lib/settings.ts"; import { fileExists, getFileType } from "./lib/transform.ts"; import { transformers } from "./transformers/registry.ts"; @@ -112,11 +113,13 @@ async function askNumber(label: string): Promise { * * @param provided - Flags the caller already supplied; those are not asked for. */ -export async function runWizard(provided: { - transformer?: string; - file?: string; - firebaseHashConfig?: FirebaseHashConfig; -}): Promise { +export async function runWizard( + provided: { + transformer?: string; + file?: string; + firebaseHashConfig?: FirebaseHashConfig; + } & FirebaseHashFlags, +): Promise { const saved = await loadSettings(); const transformer = provided.transformer ?? (await pickTransformer(saved.transformer)); @@ -124,9 +127,14 @@ export async function runWizard(provided: { let firebaseHashConfig = provided.firebaseHashConfig; if (transformer === "firebase" && !firebaseHashConfig) { - // Never prefilled: the signer key is a secret the CLI does not keep. A - // repeat run supplies it through `--firebase-*` or `CLERK_FIREBASE_*`, - // which short-circuits this prompt entirely. + // Looked up here rather than before the picker: until the platform is + // chosen there is no reason to read Firebase's variables at all, and a + // migration from anywhere else must not see them. + firebaseHashConfig = await resolveFirebaseHashConfig(provided, "firebase"); + } + if (transformer === "firebase" && !firebaseHashConfig) { + // Prompted, never prefilled: the signer key is a secret the CLI does not + // keep, so there is nothing to offer back. firebaseHashConfig = await askFirebaseHashConfig(); } From 3a667d791f469b275610495e63eb02707f0c9a8d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:28:29 -0400 Subject: [PATCH 015/141] chore: ignore local migration exports and service account keys `clerk migrate export` writes real user records to ./exports, and the Firebase export is driven by a service account key people download into the checkout. Neither belongs in the repository, and both are one `git add -A` away from it. --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index c5f69f393..9ab506561 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,7 @@ test/e2e/.har # Local planning/spec docs docs/superpowers/ + +# Local migration exports and the credentials that produced them +exports/ +*service-account*.json From 829d97985fb578cf3b83eb3ad1b24a89f37daa6d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:28:38 -0400 Subject: [PATCH 016/141] fix(config): persist the migration entry to disk `setMigrationEntry` mutated the in-memory config and returned without writing it, so nothing recorded what the last import did. `clerk migrate delete` reads that entry to find the users to undo, and with it never written the undo path had nothing to work from. --- packages/cli-core/src/lib/config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index 6d2faf342..c93b359e1 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -238,6 +238,7 @@ export async function setMigrationEntry(key: string, entry: MigrationEntry): Pro const config = await readConfig(); if (!config.migrations) config.migrations = {}; config.migrations[key] = entry; + await writeConfig(config); } /** Persistent random machine id for telemetry. Generated on first use. */ From 9a96503a3977b2b845ce841d794acee745a8caa9 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:29:51 -0400 Subject: [PATCH 017/141] refactor(migrate)!: rename `migrate run` to `migrate import`, drop --clerk-secret-key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run` was registered `isDefault`, so `clerk migrate` on its own meant "import". That reads fine until `migrate export` sits beside it: one direction is implied by the bare group name and the other has to be spelled out. Both are named now, and bare `clerk migrate` prints help. The `--clerk-secret-key` alias goes with it. It was carried over from the standalone migration tool, but `migrate` ships new in this CLI — there is no released spelling to stay compatible with, so there is nothing to deprecate. --- .changeset/migrate-cli.md | 2 +- .../cli-core/src/commands/migrate/README.md | 80 ++++++++++--------- .../cli-core/src/commands/migrate/delete.ts | 12 +-- .../src/commands/migrate/export/clerk.ts | 14 +--- .../src/commands/migrate/export/firebase.ts | 4 +- .../src/commands/migrate/export/index.ts | 3 +- .../src/commands/migrate/index.test.ts | 42 ++++------ .../cli-core/src/commands/migrate/index.ts | 28 +++---- .../src/commands/migrate/lib/clerk-config.ts | 2 +- .../migrate/lib/firebase-hash.test.ts | 2 +- .../src/commands/migrate/lib/firebase-hash.ts | 2 +- .../src/commands/migrate/lib/retry.ts | 2 +- .../src/commands/migrate/readme.test.ts | 8 +- .../commands/migrate/run-interactive.test.ts | 4 +- .../cli-core/src/commands/migrate/run.test.ts | 7 -- packages/cli-core/src/commands/migrate/run.ts | 35 ++++---- .../src/commands/migrate/settings/registry.ts | 2 +- .../migrate/settings/settings.test.ts | 2 +- .../commands/migrate/transformers/registry.ts | 2 +- .../cli-core/src/commands/migrate/wizard.ts | 6 +- packages/cli-core/src/lib/config.ts | 2 +- packages/cli-core/src/lib/next-steps.ts | 2 +- 22 files changed, 116 insertions(+), 147 deletions(-) diff --git a/.changeset/migrate-cli.md b/.changeset/migrate-cli.md index 42864a475..6ba3096b0 100644 --- a/.changeset/migrate-cli.md +++ b/.changeset/migrate-cli.md @@ -2,4 +2,4 @@ "clerk": minor --- -Add `clerk migrate` for importing users, exporting from supported auth providers, reviewing migration logs, undoing a migration, and extending imports with custom transformers. +Add `clerk migrate` for importing users with `migrate import`, exporting from supported auth providers, reviewing migration logs, undoing a migration, and extending imports with custom transformers. diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 5f49a5b10..53dcc2b44 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -5,14 +5,14 @@ Clerk instance. ## Targeting And Auth -`clerk migrate` resolves its Backend API key through the CLI's standard chain: +`clerk migrate import` resolves its Backend API key through the CLI's standard +chain: -| Flag | Description | -| ------------------------ | ---------------------------------------------------------------- | -| `--secret-key ` | Use a specific Backend API secret key directly | -| `--clerk-secret-key ` | **Deprecated** alias for `--secret-key`; warns and keeps working | -| `--app ` | Target an application directly, even outside a linked project | -| `--instance ` | Target `dev`, `prod`, or a full instance ID | +| Flag | Description | +| -------------------- | ------------------------------------------------------------- | +| `--secret-key ` | Use a specific Backend API secret key directly | +| `--app ` | Target an application directly, even outside a linked project | +| `--instance ` | Target `dev`, `prod`, or a full instance ID | Resolution order: `--secret-key` → `--app` + Platform API lookup → `CLERK_SECRET_KEY` → the keyless project's own key → a linked project profile @@ -24,14 +24,18 @@ defaults and the hard development-instance cap below. ## Commands -### `clerk migrate` (interactive) +`clerk migrate` on its own is a group name, not a command: it prints its help +and lists the subcommands below. The direction is always spelled out — +`migrate import` moves users **into** Clerk, `migrate export` gets them **out** +of a source platform — so neither is implied by the group. -Bare `clerk migrate` walks a human through -the migration instead of demanding flags — mirroring how bare `clerk deploy` -dispatches to `deploy run`. +### `clerk migrate import` (interactive) + +Bare `clerk migrate import` walks a human through the import instead of +demanding flags. ```sh -clerk migrate +clerk migrate import ``` It picks the transformer from a list built off the registry, asks for the file, @@ -44,21 +48,21 @@ Then it prints the [Migration Readiness report](#migration-readiness-report), offers to [change whatever it flagged](#changing-the-flagged-settings), and waits for confirmation. Declining writes nothing to Clerk. -**Agent mode never prompts.** `clerk migrate` with no flags exits with a usage -error naming exactly what to pass: +**Agent mode never prompts.** `clerk migrate import` with no flags exits with a +usage error naming exactly what to pass: ``` -`clerk migrate` is interactive and cannot prompt in agent mode. +`clerk migrate import` is interactive and cannot prompt in agent mode. Pass --transformer and --file . ``` -### `clerk migrate` +### `clerk migrate import` Reads an exported user file, maps it onto Clerk's user schema, validates every record, and creates the users through the Backend API. ```sh -clerk migrate -y --transformer clerk --file users.json +clerk migrate import -y --transformer clerk --file users.json ``` | Flag | Description | @@ -75,8 +79,8 @@ clerk migrate -y --transformer clerk --file users.json | `--firebase-mem-cost ` | Firebase scrypt memory cost | | `-y, --yes` | Skip the confirmation prompt | -Plus the targeting flags from the table above: `--secret-key`, -`--clerk-secret-key`, `--app` and `--instance`. +Plus the targeting flags from the table above: `--secret-key`, `--app` and +`--instance`. `--transformer` and `--file` are required. Omitting either fails with a usage error that names the valid values. @@ -121,7 +125,7 @@ limit — the run fails before any request is sent. ### `clerk migrate export` Gets users **out** of a source platform, so there is something to feed -`clerk migrate`. +`clerk migrate import`. ```sh clerk migrate export # pick a platform @@ -259,7 +263,7 @@ prints the exact import command: ``` Password hash parameters Read from the project. Import with: - clerk migrate -y --transformer firebase --file exports/firebase-export.json \ + clerk migrate import -y --transformer firebase --file exports/firebase-export.json \ --firebase-signer-key "…" --firebase-salt-separator "…" \ --firebase-rounds 8 --firebase-mem-cost 14 ``` @@ -297,16 +301,16 @@ returning the first thousand would read as "that is everyone". ### `clerk migrate delete` -The undo for a bad migration. Deletes the users a previous `clerk migrate` -created in this directory, matched by the `external_id` the import stamped on -each one. +The undo for a bad migration. Deletes the users a previous +`clerk migrate import` created in this directory, matched by the `external_id` +the import stamped on each one. ```sh clerk migrate delete # confirms first clerk migrate delete -y # non-interactive ``` -Takes the same targeting flags as `clerk migrate` (`--secret-key`, `--app`, +Takes the same targeting flags as `clerk migrate import` (`--secret-key`, `--app`, `--instance`). Flat rather than under a noun group: it is the one command in this tree that @@ -462,7 +466,7 @@ firebase-signer-key aVer…3456 .env.clerk-migrate Firebase base64 sig firebase-rounds — not set Firebase scrypt rounds ``` -Setting names are kebab-case and identical to the `clerk migrate` flag each one +Setting names are kebab-case and identical to the `clerk migrate import` flag each one backs, so `firebase-signer-key` here is `--firebase-signer-key` there rather than a second spelling to learn. The description column carries the prose. @@ -500,7 +504,7 @@ so the output is safe to paste into an issue. Migrating from a platform with no built-in, without recompiling the CLI: ```sh -clerk migrate --transformer-file ./my-platform.ts --file users.json +clerk migrate import --transformer-file ./my-platform.ts --file users.json ``` The file lives in **your** project, not in the CLI, and is imported at runtime. @@ -572,7 +576,7 @@ alongside each digest. Find them in the Firebase console under **Authentication → Users → (⋮) → Password hash parameters**. ```sh -clerk migrate -y -t firebase -f users.json \ +clerk migrate import -y -t firebase -f users.json \ --firebase-signer-key --firebase-salt-separator \ --firebase-rounds 8 --firebase-mem-cost 14 ``` @@ -859,13 +863,13 @@ Both are written relative to the **current working directory**, not to the CLI's config directory, because they describe "which file am I migrating" rather than "which project is linked here". -| Path | Contents | -| -------------------------------------- | --------------------------------------------------------------------- | -| `./logs/migration-.log` | NDJSON: one line per user, plus validation failures and retry notices | -| `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | -| `./logs/export-.log` | NDJSON: one line per exported user | -| `./exports/-export.json` | The export itself, unless `--output` says otherwise | -| `./.env.clerk-migrate` | Migration credentials, written by `settings set` and gitignored | +| Path | Contents | +| ------------------------------------------ | --------------------------------------------------------------------- | +| `./logs/migration-.log` | NDJSON: one line per user, plus validation failures and retry notices | +| `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | +| `./logs/export-.log` | NDJSON: one line per exported user | +| `./exports/-export-.json` | The export itself, unless `--output` says otherwise | +| `./.env.clerk-migrate` | Migration credentials, written by `settings set` and gitignored | The transformer and file of the last run are **not** written here. They go to the `migrations` section of the CLI's own config file, keyed by project the @@ -905,9 +909,9 @@ NDJSON is. The original `.log` stays put. | Method | Path | Used by | | -------- | -------------------------- | ------------------------------------------------------------------------------------ | -| `POST` | `/v1/users` | `clerk migrate` — creates each user | -| `POST` | `/v1/email_addresses` | `clerk migrate` — attaches additional emails | -| `POST` | `/v1/phone_numbers` | `clerk migrate` — attaches additional phones | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | | `GET` | `/v1/users?external_id=…` | `migrate delete` — finds this migration's users, 100 IDs a call | | `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | | `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 39394ceff..06781d3cd 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -50,7 +50,6 @@ const EXTERNAL_ID_BATCH = 100; export type MigrateDeleteOptions = { yes?: boolean; secretKey?: string; - clerkSecretKey?: string; app?: string; instance?: string; }; @@ -74,7 +73,7 @@ export async function resolveMigrationToUndo(): Promise<{ file: string; key: str if (!settings.file || !settings.transformer) { throw new CliError( - "No migration to undo: this project has no record of a previous `clerk migrate`.\n" + + "No migration to undo: this project has no record of a previous `clerk migrate import`.\n" + "Run `clerk migrate delete` from the project you migrated from.", { code: ERROR_CODE.FILE_NOT_FOUND }, ); @@ -258,16 +257,11 @@ function formatSummary(summary: DeleteSummary, logFile: string): string { } export async function deleteMigration(options: MigrateDeleteOptions): Promise { - if (options.clerkSecretKey) { - log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); - } - const secretKeyOption = options.secretKey ?? options.clerkSecretKey; - const { file, key } = await resolveMigrationToUndo(); await withGutter("Undoing a migration", async ({ setNextSteps }) => { - const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); - const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const limits = resolveLimits(secretKey); const dateTime = getDateTimeStamp(); const logFile = getLogFilePath("user-deletion", dateTime); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 564bcabd2..a9ff2f062 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -5,7 +5,7 @@ * onto `bapiRequest` instead of `@clerk/backend` so it shares the CLI's auth * resolution, `--verbose` request tracing and error taxonomy. * - * The output feeds `clerk migrate --transformer clerk` unedited, which is + * The output feeds `clerk migrate import --transformer clerk` unedited, which is * what makes development → production a two-command operation. * * **Passwords do not come out of this endpoint.** Clerk never returns password @@ -28,7 +28,6 @@ const PAGE_SIZE = 500; export type ExportClerkOptions = { output?: string; secretKey?: string; - clerkSecretKey?: string; app?: string; instance?: string; }; @@ -67,7 +66,7 @@ type IdentifierWithId = BapiIdentifier & { id?: string }; /** * Splits identifiers into verified and unverified, primary first. * - * The primary has to lead: `migrate run` puts the first entry on + * The primary has to lead: `migrate import` puts the first entry on * `POST /v1/users` and attaches the rest afterwards, so a reordered list would * silently change which address the user signs in with. */ @@ -224,14 +223,9 @@ export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExpo } export async function exportClerk(options: ExportClerkOptions): Promise { - if (options.clerkSecretKey) { - log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); - } - const secretKeyOption = options.secretKey ?? options.clerkSecretKey; - await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { - const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); - const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const dateTime = getDateTimeStamp(); log.info(`Exporting from ${target ?? "the resolved instance"}.`); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index aedb8fb9c..8106840f1 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -371,7 +371,7 @@ export function buildFirebaseExport(users: FirebaseUser[], dateTime: string) { }; } -/** The exact `migrate run` invocation, with the project's own parameters. */ +/** The exact `migrate import` invocation, with the project's own parameters. */ export function formatHashConfigGuidance( config: HashConfig | null, outputPath: string, @@ -397,7 +397,7 @@ export function formatHashConfigGuidance( bold("Password hash parameters"), "Read from the project. Import with:", dim( - ` clerk migrate -y --transformer firebase --file ${outputPath} \\\n` + + ` clerk migrate import -y --transformer firebase --file ${outputPath} \\\n` + ` --firebase-signer-key "${config.signerKey}" \\\n` + ` --firebase-salt-separator "${config.saltSeparator}" \\\n` + ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 773e07f22..a0067a2a7 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -84,7 +84,7 @@ const DB_PLATFORMS = [ export function registerMigrateExport(migrateCommand: Command<[], Record>): void { const exportCommand = migrateCommand .command("export") - .description("Export users from a source platform, ready for `clerk migrate`") + .description("Export users from a source platform, ready for `clerk migrate import`") .setExamples([ { command: "clerk migrate export", description: "Pick a platform interactively" }, { @@ -104,7 +104,6 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Where to write the export, relative to the current directory") .option("--secret-key ", "Backend API secret key to use") - .option("--clerk-secret-key ", "Deprecated alias for --secret-key") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") .setExamples([ diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index 07454ad98..cb24f0384 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -18,24 +18,15 @@ describe("registerMigrate", () => { expect(migrate?.description()).toContain("Migrate users"); }); - test("registers the run subcommand", () => { - expect(findCommand(["migrate", "run"])).toBeDefined(); + test("registers the import subcommand", () => { + expect(findCommand(["migrate", "import"])).toBeDefined(); }); - // Bare `clerk migrate` dispatches to `migrate run`, mirroring how bare - // `clerk deploy` dispatches to `deploy run`. - test("makes run the default subcommand, so bare `clerk migrate` starts the wizard", () => { - const run = findCommand(["migrate", "run"]); - expect(run as unknown as { _defaultCommandName?: unknown }).toBeDefined(); + // The direction is never implied: `import` and `export` are siblings, so a + // default would make one of them the meaning of the bare group name. + test("leaves migrate with no default subcommand", () => { const migrate = findCommand(["migrate"]) as unknown as { _defaultCommandName?: string }; - expect(migrate._defaultCommandName).toBe("run"); - }); - - test("keeps run visible in help, unlike deploy's hidden default", () => { - expect(findCommand(["migrate", "run"])?.parent?.commands.map((c) => c.name())).toContain("run"); - expect( - (findCommand(["migrate", "run"]) as unknown as { _hidden?: boolean })._hidden, - ).toBeFalsy(); + expect(migrate._defaultCommandName).toBeFalsy(); }); test.each([ @@ -50,11 +41,10 @@ describe("registerMigrate", () => { "--firebase-mem-cost", "--yes", "--secret-key", - "--clerk-secret-key", "--app", "--instance", - ])("migrate run accepts %s", (flag) => { - const flags = findCommand(["migrate", "run"])?.options.map((option) => option.long); + ])("migrate import accepts %s", (flag) => { + const flags = findCommand(["migrate", "import"])?.options.map((option) => option.long); expect(flags).toContain(flag); }); @@ -120,10 +110,10 @@ describe("registerMigrate", () => { test("documents the default output location in help", () => { expect(findCommand(["migrate", "export", "clerk"])?.description()).toContain( - "./exports/clerk-export.json", + "./exports/clerk-export-.json", ); expect(findCommand(["migrate", "export", "auth0"])?.description()).toContain( - "./exports/auth0-export.json", + "./exports/auth0-export-.json", ); }); @@ -140,8 +130,8 @@ describe("registerMigrate", () => { ); }); - test("migrate run accepts --transformer-file", () => { - expect(findCommand(["migrate", "run"])?.options.map((o) => o.long)).toContain( + test("migrate import accepts --transformer-file", () => { + expect(findCommand(["migrate", "import"])?.options.map((o) => o.long)).toContain( "--transformer-file", ); }); @@ -153,7 +143,7 @@ describe("registerMigrate", () => { expect(findCommand(["migrate", "delete"])?.description()).toContain("last migration"); }); - test.each(["--yes", "--secret-key", "--clerk-secret-key", "--app", "--instance"])( + test.each(["--yes", "--secret-key", "--app", "--instance"])( "migrate delete accepts %s", (flag) => { expect(findCommand(["migrate", "delete"])?.options.map((o) => o.long)).toContain(flag); @@ -191,7 +181,9 @@ describe("registerMigrate", () => { }); test("constrains --transformer to the registered transformers, for validation and completion", () => { - const option = findCommand(["migrate", "run"])?.options.find((o) => o.long === "--transformer"); + const option = findCommand(["migrate", "import"])?.options.find( + (o) => o.long === "--transformer", + ); // Tracks the registry so adding a platform needs no edit here. expect(option?.argChoices).toEqual(transformerKeys()); }); @@ -202,7 +194,7 @@ describe("registerMigrate", () => { ["-r", "--resume-after"], ["-y", "--yes"], ])("exposes %s as the short form of %s", (short, long) => { - const option = findCommand(["migrate", "run"])?.options.find((o) => o.long === long); + const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === long); expect(option?.short).toBe(short); }); }); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index b17195af4..e006cb6dd 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -14,15 +14,15 @@ const migrate = { run, delete: deleteMigration, transformersList }; export function registerMigrate(program: Program): void { const migrateCommand = program .command("migrate") - .description("Migrate users into Clerk from another auth provider") + .description("Migrate users into Clerk from another auth provider or another Clerk instance") .setExamples([ - { command: "clerk migrate", description: "Walk through a migration interactively" }, + { command: "clerk migrate import", description: "Walk through an import interactively" }, { - command: "clerk migrate -y --transformer clerk --file users.json", + command: "clerk migrate import -y --transformer clerk --file users.json", description: "Import users from a Clerk export", }, { - command: "clerk migrate -y -t supabase -f users.json --skip-unsupported-providers", + command: "clerk migrate import -y -t supabase -f users.json --skip-unsupported-providers", description: "Skip Supabase users whose provider is not enabled", }, { @@ -39,16 +39,16 @@ export function registerMigrate(program: Program): void { { command: "clerk migrate delete", description: "Undo the last migration" }, ]); - // `isDefault` so `clerk migrate` is the whole command: bare, it runs the - // wizard; with flags, they fall through to here. `run` stays addressable - // because scripts and older docs use it, but `clerk migrate` is the spelling - // every example gives. + // Named, not `isDefault`. `import` and `export` are the two directions this + // group moves users in, and neither is implied by the bare group name — a + // default would make `clerk migrate --file users.json` mean "import" while + // its sibling has to be spelled out. Bare `clerk migrate` prints help. // // The flags stay here rather than on `migrate`, matching how `config` keeps // its own on `pull`/`patch`/`put` — a group's help is a list of subcommands // and examples, not a merge of everything underneath it. migrateCommand - .command("run", { isDefault: true }) + .command("import") .description("Import users from an exported JSON or CSV file") .addOption( createOption( @@ -77,24 +77,23 @@ export function registerMigrate(program: Program): void { ) .option("-y, --yes", "Skip the confirmation prompt") .option("--secret-key ", "Backend API secret key to use") - .option("--clerk-secret-key ", "Deprecated alias for --secret-key") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") .setExamples([ { - command: "clerk migrate -y --transformer clerk --file users.json", + command: "clerk migrate import -y --transformer clerk --file users.json", description: "Import a Clerk Dashboard export", }, { - command: "clerk migrate -y -t clerk -f users.csv --require-password", + command: "clerk migrate import -y -t clerk -f users.csv --require-password", description: "Import only the users that carry a password digest", }, { - command: "clerk migrate -y -t clerk -f users.json -r user_2x9k", + command: "clerk migrate import -y -t clerk -f users.json -r user_2x9k", description: "Resume a partial migration after the last imported user", }, { - command: "clerk migrate -y -t supabase -f users.json --skip-unsupported-providers", + command: "clerk migrate import -y -t supabase -f users.json --skip-unsupported-providers", description: "Skip Supabase users whose only provider is not enabled in Clerk", }, ]) @@ -109,7 +108,6 @@ export function registerMigrate(program: Program): void { .description("Delete the users created by the last migration for this project") .option("-y, --yes", "Skip the confirmation prompt") .option("--secret-key ", "Backend API secret key to use") - .option("--clerk-secret-key ", "Deprecated alias for --secret-key") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") .setExamples([ diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts index 7b6cd7dfa..58bb6dceb 100644 --- a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -4,7 +4,7 @@ * * Ported from the standalone migration-tool's `src/lib/clerk.ts`, rewritten * onto the CLI's own primitives: the FAPI host comes from BAPI `/v1/domains` - * — a secret key is all `migrate run` is given — and the settings come from + * — a secret key is all `migrate import` is given — and the settings come from * `lib/fapi.ts` rather than a bespoke fetch. */ diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts index ccf5fa75e..45c65c452 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts @@ -39,7 +39,7 @@ afterEach(() => { }); describe("gating on the transformer", () => { - // `migrate run` is one command for every platform, so a signer key left in + // `migrate import` is one command for every platform, so a signer key left in // .env.clerk-migrate after a Firebase migration is in scope for whatever runs // next unless the transformer says otherwise. test.each([["clerk"], ["supabase"], ["auth0"], ["authjs"], ["betterauth"]])( diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts index 836d374ea..10b06653d 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts @@ -2,7 +2,7 @@ * Firebase's four scrypt parameters: where they come from, and when they are * looked for at all. * - * **Only read when the transformer is `firebase`.** `migrate run` is one + * **Only read when the transformer is `firebase`.** `migrate import` is one * command serving every platform, so a `CLERK_FIREBASE_SIGNER_KEY` left in * `.env.clerk-migrate` after a Firebase migration is in scope for the Supabase * run that follows it unless something says otherwise. Nothing downstream would diff --git a/packages/cli-core/src/commands/migrate/lib/retry.ts b/packages/cli-core/src/commands/migrate/lib/retry.ts index f9d1fc158..a01de49a2 100644 --- a/packages/cli-core/src/commands/migrate/lib/retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/retry.ts @@ -1,5 +1,5 @@ /** - * Rate-limit backoff, shared by `migrate run` and `migrate delete`. + * Rate-limit backoff, shared by `migrate import` and `migrate delete`. * * Both walk the whole user set through BAPI and hit the same limits, so they * back off identically rather than approximately: extracting this is what diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index f41aec567..be0bc5028 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -63,10 +63,10 @@ function resolve(tokens: string[]): { command: Command; rest: string[] } { /** * The flags a command accepts, including those of a default subcommand. * - * `clerk migrate` carries no options of its own — `run` is registered - * `isDefault`, so Commander hands it everything after the group name. The - * documented spelling is `clerk migrate --transformer …`, and this has to see - * the same flags Commander does or every such example reads as unsupported. + * `migrate logs` and `migrate transformers` register their `list` `isDefault`, + * so Commander hands it everything after the group name. The documented + * spelling is `clerk migrate logs --json`, and this has to see the same flags + * Commander does or every such example reads as unsupported. */ function flagsOf(command: Command): string[] { const own = command.options.flatMap( diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 44198a6f7..9d29d32f1 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -1,5 +1,5 @@ /** - * The human-mode half of `migrate run`: the wizard fills in missing flags, the + * The human-mode half of `migrate import`: the wizard fills in missing flags, the * readiness report renders, and declining the confirmation writes nothing. * * Kept in its own file because `mock.module` registrations are process-lifetime, @@ -164,7 +164,7 @@ function stubInstanceSettings(settings: StubSettings) { const created = () => requests.filter((r) => r.url.endsWith("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/v1/users")); describe("the wizard fills in missing flags", () => { - test("bare `clerk migrate` prompts for the transformer and file, then imports", async () => { + test("bare `clerk migrate import` prompts for the transformer and file, then imports", async () => { await run({ secretKey: "sk_test_x" }); expect(mockSelect).toHaveBeenCalledTimes(1); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 9d9272f95..7fa811d46 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -183,13 +183,6 @@ describe("run", () => { expect(captured.err).toContain("1 user failed validation"); }); - test("warns that --clerk-secret-key is deprecated but still honours it", async () => { - await run({ ...baseOptions, secretKey: undefined, clerkSecretKey: "sk_test_x" }); - - expect(captured.err).toContain("--clerk-secret-key is deprecated"); - expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); - }); - test("refuses to exceed the development-instance user limit", async () => { fs.writeFileSync( path.join(workDir, "export.json"), diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index bdcd865e9..e37cde385 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -1,15 +1,15 @@ /** - * `clerk migrate` — the user import itself. + * `clerk migrate import` — the user import itself. * * Ported from the standalone migration-tool's `src/migrate/cli.ts` * (`runNonInteractive`), with auth moved onto the CLI's standard secret-key * resolution chain and every failure raised as a `CliError` instead of * `console.error` + `process.exit`. * - * Registered as the `run` subcommand and marked default, so `clerk migrate` and - * `clerk migrate run` both land here. Whatever the flags did not supply is - * filled in by `wizard.ts` for a human, or raised as a usage error naming the - * missing flags for an agent, which cannot answer a prompt. + * Registered as the `import` subcommand. The exported handler keeps the name + * `run` because `import` is a reserved word. Whatever the flags did not supply + * is filled in by `wizard.ts` for a human, or raised as a usage error naming + * the missing flags for an agent, which cannot answer a prompt. */ import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; @@ -64,8 +64,6 @@ export type MigrateRunOptions = { requirePassword?: boolean; yes?: boolean; secretKey?: string; - /** Deprecated alias for `--secret-key`, kept for existing prompts and docs. */ - clerkSecretKey?: string; app?: string; instance?: string; /** Path to a user-authored transformer, for a platform with no built-in. */ @@ -95,7 +93,8 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate -y --transformer-file ./my-transformer.ts --file users.json", + command: + "clerk migrate import -y --transformer-file ./my-transformer.ts --file users.json", description: "Import with a custom transformer", }, ], @@ -117,7 +116,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate -y --transformer clerk --file users.json", + command: "clerk migrate import -y --transformer clerk --file users.json", description: "Import a Clerk export", }, ], @@ -135,7 +134,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate -y --transformer clerk --file users.json", + command: "clerk migrate import -y --transformer clerk --file users.json", description: "Import a Clerk export", }, ], @@ -404,7 +403,7 @@ async function showReadinessReport( * to pass. * * Agent mode is the CLI's existing non-interactive signal, so an agent that - * runs bare `clerk migrate` gets a usage error naming the flags rather than a + * runs bare `clerk migrate import` gets a usage error naming the flags rather than a * prompt it cannot answer. */ async function resolveMissingOptions(options: MigrateRunOptions): Promise { @@ -454,11 +453,12 @@ async function applyCustomTransformer(options: MigrateRunOptions): Promise { - if (rawOptions.clerkSecretKey) { - log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); - } - rawOptions = await applyCustomTransformer(rawOptions); const options = await resolveMissingOptions(rawOptions); - const secretKeyOption = options.secretKey ?? options.clerkSecretKey; const { transformer, file } = validateRunOptions(options); const firebaseHashConfig = await resolveFirebaseHashConfig(options, transformer); await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { - const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); - const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const limits = resolveLimits(secretKey); const dateTime = getDateTimeStamp(); const logFile = getLogFilePath("migration", dateTime); diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index e034ec60d..8ec0fe982 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -20,7 +20,7 @@ export interface SettingDef { /** * What the user types: `clerk migrate settings set `. * - * Kebab-case, and identical to the `migrate run` flag it backs. A setting and + * Kebab-case, and identical to the `migrate import` flag it backs. A setting and * its flag are the same knob reached two ways, so `firebase-signer-key` here * and `--firebase-signer-key` there must not drift into two spellings the * user has to learn separately. Sentence-case prose belongs in diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index 91d982076..42e4f7c1e 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -139,7 +139,7 @@ describe("list", () => { ); }); - // The names are kebab-case because they mirror the `migrate run` flags; the + // The names are kebab-case because they mirror the `migrate import` flags; the // description column is what makes the list readable. test("explains each setting in prose", async () => { await list(); diff --git a/packages/cli-core/src/commands/migrate/transformers/registry.ts b/packages/cli-core/src/commands/migrate/transformers/registry.ts index 6279577a0..df9448595 100644 --- a/packages/cli-core/src/commands/migrate/transformers/registry.ts +++ b/packages/cli-core/src/commands/migrate/transformers/registry.ts @@ -1,7 +1,7 @@ /** * Transformer registry. * - * `migrate run` reads this array to resolve `--transformer` and to list the + * `migrate import` reads this array to resolve `--transformer` and to list the * valid choices in help output and tab-completion. * * To add a platform: create `transformers/.ts` exporting a diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index ff6e7173e..122173046 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -1,5 +1,5 @@ /** - * The interactive path behind a bare `clerk migrate`. + * The interactive path behind a bare `clerk migrate import`. * * Ported from the standalone migration-tool's `src/migrate/cli.ts` interactive * flow. The platform and file are pre-filled from the previous run, so a repeat @@ -154,12 +154,12 @@ export function throwAgentFlagsRequired(missing: { transformer: boolean; file: b ].filter(Boolean); throwUsageError( - `\`clerk migrate\` is interactive and cannot prompt in agent mode. Pass ${flags.join(" and ")}.`, + `\`clerk migrate import\` is interactive and cannot prompt in agent mode. Pass ${flags.join(" and ")}.`, undefined, undefined, [ { - command: `clerk migrate -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, + command: `clerk migrate import -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, description: "Run non-interactively", }, ], diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index c93b359e1..08e0295d1 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -50,7 +50,7 @@ interface RelayEntry { token: string; } -/** What `clerk migrate` last imported for a project, and how. */ +/** What `clerk migrate import` last imported for a project, and how. */ interface MigrationEntry { transformer?: string; file?: string; diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index 41d415f90..a1b53e1d7 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -79,7 +79,7 @@ export const NEXT_STEPS = { // The only parameterized entry: a suggested import is worthless unless it // names the transformer that reads this export and the file just written. MIGRATE_EXPORT: (transformerKey: string, file: string) => [ - `Run \`clerk migrate --transformer ${transformerKey} --file ${file}\` to import them`, + `Run \`clerk migrate import --transformer ${transformerKey} --file ${file}\` to import them`, ], } as const; From 35c2b0974489acf8709765137f1a1ed527877a89 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:31:56 -0400 Subject: [PATCH 018/141] feat(migrate): timestamp export filenames and ask where to save MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two exports of the same platform used to write the same path, so the second silently overwrote the first. Filenames now carry a local `YYYYMMDD-HHmm` stamp, and every export settles its destination *before* it starts — one prompt, prefilled with the proposed path, so Enter accepts it. Asked up front on purpose: coming back to a long export stalled on a prompt, with every user held in memory and nothing on disk, is the worse half of that trade. `--output` is an answer already given, and agent mode takes the proposal without asking. --- .../cli-core/src/commands/migrate/README.md | 22 +++-- .../src/commands/migrate/export/auth0.test.ts | 25 ++++-- .../src/commands/migrate/export/auth0.ts | 6 +- .../src/commands/migrate/export/authjs.ts | 6 +- .../src/commands/migrate/export/betterauth.ts | 6 +- .../src/commands/migrate/export/clerk.test.ts | 35 +++++--- .../src/commands/migrate/export/clerk.ts | 6 +- .../commands/migrate/export/firebase.test.ts | 25 ++++-- .../src/commands/migrate/export/firebase.ts | 6 +- .../src/commands/migrate/export/index.ts | 18 ++-- .../commands/migrate/export/shared.test.ts | 82 +++++++++++++++++++ .../src/commands/migrate/export/shared.ts | 52 +++++++++++- .../src/commands/migrate/export/supabase.ts | 6 +- 13 files changed, 250 insertions(+), 45 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/export/shared.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 53dcc2b44..c448e8359 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -148,9 +148,21 @@ with what a database export needs. | `betterauth` | Better Auth database | `--transformer betterauth` | | `firebase` | Firebase Identity Toolkit | `--transformer firebase` | -Exports land at `./exports/-export.json` unless `--output` says -otherwise. `--output` resolves against the **current directory**, like every -other path flag here. +Every export asks where to save the file before it starts, proposing +`./exports/-export-.json`. Press enter to take it, +or type over it to save somewhere else — the proposal is prefilled, so it is +one prompt rather than a confirm and a path question. + +The stamp is ISO 8601 basic format in local time, to the minute: it goes in a +name people read off the screen and tab-complete, and it means a second export +never silently overwrites the first. + +`--output` answers that prompt up front and skips it, as does agent mode, which +takes the proposed path. `--output` resolves against the **current directory**, +like every other path flag here. + +The question comes before any users are fetched, so a long export can be left +unattended rather than stalling on a prompt with everything held in memory. | Flag | Platforms | Description | | -------------------------- | ---------------------------------- | -------------------------------------------- | @@ -175,9 +187,9 @@ Field coverage ! 1/3 have a username ! 2/3 have a password (not exportable — see below) -Exported 3 users to /project/exports/clerk-export.json +Exported 3 users to /project/exports/clerk-export-20260817-1432.json └ Next steps - → Run `clerk migrate --transformer clerk --file exports/clerk-export.json` to import them + → Run `clerk migrate import --transformer clerk --file exports/clerk-export-20260817-1432.json` to import them ``` Every export also writes `logs/export-.log`, so `migrate logs list` diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index 9746f79cb..aae8e7906 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -42,6 +42,9 @@ afterAll(() => { }); beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); requests = []; fs.rmSync(getLogDir(), { recursive: true, force: true }); fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); @@ -276,15 +279,25 @@ describe("buildAuth0Export", () => { }); }); +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + describe("exportAuth0", () => { test("writes the default path and reports coverage", async () => { stubAuth0([[auth0User(0)], []]); await exportAuth0({ ...CREDENTIALS }); - const written = JSON.parse( - fs.readFileSync(path.join(workDir, "exports", "auth0-export.json"), "utf-8"), - ) as Record[]; + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^auth0-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; expect(written[0]?.user_id).toBe("auth0|a0"); expect(captured.err).toContain("Field coverage"); }); @@ -296,11 +309,13 @@ describe("exportAuth0", () => { const originalMode = getMode(); setMode("human"); try { - await exportAuth0({ ...CREDENTIALS }); + // --output answers the destination prompt, which human mode would + // otherwise stop on. + await exportAuth0({ ...CREDENTIALS, output: "exports/mine.json" }); } finally { setMode(originalMode); } - expect(captured.err).toContain("migrate --transformer auth0 --file exports/auth0-export.json"); + expect(captured.err).toContain("migrate import --transformer auth0 --file exports/mine.json"); }); test("--output controls the destination", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index bf1657267..478b944d7 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -23,7 +23,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spin import { isAgent, isHuman } from "../../../mode.ts"; import { findMigrateEnvValue } from "../lib/env-file.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; const PAGE_SIZE = 100; @@ -316,6 +316,8 @@ export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0Exp export async function exportAuth0(options: ExportAuth0Options): Promise { const credentials = await resolveAuth0Credentials(options); + const destination = await resolveOutputPath("auth0", options.output); + await withGutter("Exporting users from Auth0", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); log.info(`Exporting from ${credentials.domain}.`); @@ -329,7 +331,7 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { ); const { users: exported, coverage } = buildAuth0Export(users, dateTime); - const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("auth0")); + const outputPath = writeExportOutput(exported, destination); setNextSteps( reportExport({ diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index a78df15de..1f74223a4 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -15,7 +15,7 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { log } from "../../../lib/log.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; /** Table names to try, in order. Prisma capitalizes; Drizzle does not. */ @@ -113,6 +113,8 @@ export async function exportAuthJs(options: DbExportOptions): Promise { hint: "Postgres, MySQL or a SQLite file — whichever your Auth.js adapter uses.", }); + const destination = await resolveOutputPath("authjs", options.output); + await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); @@ -122,7 +124,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); const { users, coverage } = buildAuthJsExport(rows, dateTime); - const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("authjs")); + const outputPath = writeExportOutput(users, destination); setNextSteps( reportExport({ diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 85ab190db..006af76f9 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -19,7 +19,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; /** Columns a Better Auth plugin adds to the user table. */ @@ -160,6 +160,8 @@ export async function exportBetterAuth(options: DbExportOptions): Promise hint: "Postgres, MySQL or a SQLite file — whichever your Better Auth install uses.", }); + const destination = await resolveOutputPath("betterauth", options.output); + await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); @@ -178,7 +180,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise ); const { users, coverage } = buildBetterAuthExport(rows, dateTime); - const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("betterauth")); + const outputPath = writeExportOutput(users, destination); setNextSteps( reportExport({ diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index b628a8408..75e1f0565 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -33,6 +33,9 @@ afterAll(() => { }); beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); requests = []; fs.rmSync(getLogDir(), { recursive: true, force: true }); fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); @@ -74,7 +77,7 @@ describe("mapClerkUserToExport", () => { }); }); - // `migrate run` puts the first entry on POST /v1/users and attaches the rest + // `migrate import` puts the first entry on POST /v1/users and attaches the rest // afterwards, so a reordered list would change which address signs the user in. test("keeps the primary identifier out of the additional list", () => { const mapped = mapClerkUserToExport( @@ -229,15 +232,25 @@ describe("buildClerkExport", () => { }); }); +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + describe("exportClerk", () => { test("writes the default path and reports coverage", async () => { stubPages([[user({ id: "u1", first_name: "Ada" })], []]); await exportClerk({ secretKey: "sk_test_x" }); - const written = JSON.parse( - fs.readFileSync(path.join(workDir, "exports", "clerk-export.json"), "utf-8"), - ) as Record[]; + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^clerk-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; expect(written).toHaveLength(1); expect(written[0]?.id).toBe("u1"); expect(captured.err).toContain("Field coverage"); @@ -251,11 +264,13 @@ describe("exportClerk", () => { const originalMode = getMode(); setMode("human"); try { - await exportClerk({ secretKey: "sk_test_x" }); + // --output answers the destination prompt, which human mode would + // otherwise stop on. + await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); } finally { setMode(originalMode); } - expect(captured.err).toContain("migrate --transformer clerk --file exports/clerk-export.json"); + expect(captured.err).toContain("migrate import --transformer clerk --file exports/mine.json"); }); test("--output controls the destination, relative to the working directory", async () => { @@ -264,7 +279,7 @@ describe("exportClerk", () => { await exportClerk({ secretKey: "sk_test_x", output: "somewhere/mine.json" }); expect(fs.existsSync(path.join(workDir, "somewhere", "mine.json"))).toBe(true); - expect(fs.existsSync(path.join(workDir, "exports", "clerk-export.json"))).toBe(false); + expect(fs.existsSync(path.join(workDir, "exports"))).toBe(false); }); // Silence here would be the worst outcome: the operator finds out when @@ -281,9 +296,7 @@ describe("exportClerk", () => { await exportClerk({ secretKey: "sk_test_x" }); expect(captured.err).toContain("No users found to export"); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, "exports", "clerk-export.json"), "utf-8")), - ).toEqual([]); + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toEqual([]); }); test("an empty export warns but does not suggest importing it", async () => { @@ -291,7 +304,7 @@ describe("exportClerk", () => { const originalMode = getMode(); setMode("human"); try { - await exportClerk({ secretKey: "sk_test_x" }); + await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); } finally { setMode(originalMode); } diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index a9ff2f062..fc3875339 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -20,7 +20,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { retryOn429 } from "../lib/retry.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; /** BAPI's maximum page size for `GET /v1/users`. */ const PAGE_SIZE = 500; @@ -223,6 +223,8 @@ export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExpo } export async function exportClerk(options: ExportClerkOptions): Promise { + const destination = await resolveOutputPath("clerk", options.output); + await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); @@ -235,7 +237,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { ); const { users: exported, coverage } = buildClerkExport(users, dateTime); - const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("clerk")); + const outputPath = writeExportOutput(exported, destination); setNextSteps( reportExport({ diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 1de54a3c8..5754bf901 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -65,6 +65,9 @@ afterAll(() => { }); beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); requests = []; delete process.env.FIREBASE_AUTH_EMULATOR_HOST; fs.rmSync(getLogDir(), { recursive: true, force: true }); @@ -397,6 +400,13 @@ describe("formatHashConfigGuidance", () => { }); }); +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + describe("exportFirebase", () => { test("exports end to end and reports coverage", async () => { stubFirebase([[fbUser(0), fbUser(1)]], { @@ -405,9 +415,12 @@ describe("exportFirebase", () => { await exportFirebase({ serviceAccount: "./sa.json" }); - const written = JSON.parse( - fs.readFileSync(path.join(workDir, "exports", "firebase-export.json"), "utf-8"), - ) as Record[]; + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^firebase-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; expect(written).toHaveLength(2); expect(captured.err).toContain("Field coverage"); expect(captured.err).toContain("demo-fb project"); @@ -420,12 +433,14 @@ describe("exportFirebase", () => { const originalMode = getMode(); setMode("human"); try { - await exportFirebase({ serviceAccount: "./sa.json" }); + // --output answers the destination prompt, which human mode would + // otherwise stop on. + await exportFirebase({ serviceAccount: "./sa.json", output: "exports/mine.json" }); } finally { setMode(originalMode); } expect(captured.err).toContain( - "migrate --transformer firebase --file exports/firebase-export.json", + "migrate import --transformer firebase --file exports/mine.json", ); }); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 8106840f1..acb413e6d 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -30,7 +30,7 @@ import { loggedFetch } from "../../../lib/fetch.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; /** Identity Toolkit's maximum for `accounts:batchGet`. */ const PAGE_SIZE = 1000; @@ -424,6 +424,8 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { const dateTime = getDateTimeStamp(); log.info(`Exporting from the ${account.project_id} project.`); @@ -437,7 +439,7 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise.json)", + ) .option("-o, --output ", "Where to write the export, relative to the current directory") .option("--secret-key ", "Backend API secret key to use") .option("--app ", "Application ID to target (works from any directory)") @@ -109,7 +111,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json", }, { command: "clerk migrate export clerk --instance prod --output prod-users.json", @@ -122,7 +124,9 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)", + ) .option("--domain ", "Auth0 tenant domain, e.g. my-tenant.us.auth0.com") .option("--client-id ", "Machine-to-machine application client ID") .option("--client-secret ", "Machine-to-machine application client secret") @@ -144,7 +148,9 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)", + ) .option("--service-account ", "Path to a service account key JSON file") .option("-o, --output ", "Where to write the export, relative to the current directory") .setExamples([ @@ -162,7 +168,9 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)`, + ) .option("--db-url ", "Postgres, MySQL or SQLite connection string") .option("-o, --output ", "Where to write the export, relative to the current directory") .setExamples([ diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts new file mode 100644 index 000000000..23b387db1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -0,0 +1,82 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +const mockText = mock(); +mock.module("../../../lib/prompts.ts", () => ({ + text: (...args: unknown[]) => mockText(...args), +})); + +let human = true; +mock.module("../../../mode.ts", () => ({ + isHuman: () => human, + isAgent: () => !human, + getMode: () => (human ? "human" : "agent"), + setMode: () => {}, +})); + +const { defaultOutputPath, outputStamp, resolveOutputPath } = await import("./shared.ts"); + +beforeEach(() => { + human = true; + mockText.mockReset(); +}); + +describe("outputStamp", () => { + // Local time, and no seconds: this ends up in a filename someone reads off + // the screen and types back. + test("stamps to the minute", () => { + expect(outputStamp(new Date(2026, 7, 17, 14, 32, 59))).toBe("20260817-1432"); + }); + + test("pads single-digit months, days, hours and minutes", () => { + expect(outputStamp(new Date(2026, 0, 3, 9, 5, 0))).toBe("20260103-0905"); + }); +}); + +describe("defaultOutputPath", () => { + test("names the platform and the stamp, under exports/", () => { + expect(defaultOutputPath("clerk", new Date(2026, 7, 17, 14, 32))).toBe( + "exports/clerk-export-20260817-1432.json", + ); + }); + + // Two exports of the same platform an hour apart must not collide. + test("gives two runs different names", () => { + expect(defaultOutputPath("auth0", new Date(2026, 7, 17, 14, 32))).not.toBe( + defaultOutputPath("auth0", new Date(2026, 7, 17, 15, 32)), + ); + }); +}); + +describe("resolveOutputPath", () => { + test("--output is an answer already given", async () => { + expect(await resolveOutputPath("clerk", "somewhere/mine.json")).toBe("somewhere/mine.json"); + expect(mockText).not.toHaveBeenCalled(); + }); + + // One prompt, not a confirm plus a path question: the proposal is prefilled, + // so enter accepts it and typing replaces it. + test("prefills the proposed path so enter accepts it", async () => { + mockText.mockImplementation(async (config: { default: string }) => config.default); + + const chosen = await resolveOutputPath("clerk"); + + expect(chosen).toMatch(/^exports\/clerk-export-\d{8}-\d{4}\.json$/); + expect(mockText).toHaveBeenCalledTimes(1); + expect(mockText.mock.calls[0]?.[0]).toMatchObject({ message: "Save the export to:" }); + }); + + test("takes a path typed over the proposal, trimmed", async () => { + mockText.mockResolvedValue(" ../elsewhere/users.json "); + + expect(await resolveOutputPath("firebase")).toBe("../elsewhere/users.json"); + }); + + test("agent mode takes the proposed path without asking", async () => { + human = false; + + expect(await resolveOutputPath("supabase")).toMatch( + /^exports\/supabase-export-\d{8}-\d{4}\.json$/, + ); + expect(mockText).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index e880d90d3..7f968d845 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -14,10 +14,58 @@ import path from "node:path"; import { dim, green, yellow } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { NEXT_STEPS } from "../../../lib/next-steps.ts"; +import { text } from "../../../lib/prompts.ts"; +import { isHuman } from "../../../mode.ts"; + +/** + * `YYYYMMDD-HHmm`, local time — ISO 8601 basic format, minus seconds. + * + * Basic throughout rather than `2026-08-17-1954`, which mixes the extended + * date form with the basic time form and leaves the trailing group looking + * like a fourth date component. One separator, and it sorts lexically. + * + * Seconds are dropped on purpose. This lands in a filename people read off the + * screen, type back and tab-complete, and two exports of the same platform + * inside one minute is not an accident anyone has by surprise. + * + * Local rather than UTC because the only reader is the person who just ran the + * command, deciding which of two files is the one they meant. + */ +export function outputStamp(now: Date = new Date()): string { + const pad = (value: number) => String(value).padStart(2, "0"); + const date = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}`; + return `${date}-${pad(now.getHours())}${pad(now.getMinutes())}`; +} /** Where an export lands when `--output` is not given. */ -export function defaultOutputPath(platform: string): string { - return path.join("exports", `${platform}-export.json`); +export function defaultOutputPath(platform: string, now?: Date): string { + return path.join("exports", `${platform}-export-${outputStamp(now)}.json`); +} + +/** + * Settles where the file lands, before the export runs. + * + * Asked up front rather than at write time so a long export can be left + * unattended — coming back to a stalled prompt with every user held in memory + * and nothing on disk is the worse half of that trade. + * + * One prompt, not a confirm followed by a path prompt: the proposed path is + * prefilled, so Enter accepts it and typing replaces it. + * + * `--output` is an answer already given, and agent mode has nobody to ask. + */ +export async function resolveOutputPath(platform: string, output?: string): Promise { + if (output) return output; + + const proposed = defaultOutputPath(platform); + if (!isHuman()) return proposed; + + const chosen = await text({ + message: "Save the export to:", + default: proposed, + validate: (value) => (value?.trim() ? undefined : "A path is required"), + }); + return chosen.trim(); } /** diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 80ff3785e..2436c95c5 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -14,7 +14,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; /** @@ -111,6 +111,8 @@ export async function exportSupabase(options: DbExportOptions): Promise { hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", }); + const destination = await resolveOutputPath("supabase", options.output); + await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); @@ -119,7 +121,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { ); const { users, coverage } = buildSupabaseExport(rows, dateTime); - const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("supabase")); + const outputPath = writeExportOutput(users, destination); setNextSteps( reportExport({ From 5a1618981fcee9dc102ae324f6ee8631da83fbe7 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:32:07 -0400 Subject: [PATCH 019/141] feat(migrate): URL-encode credentials pasted into a connection string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dashboards hand out `postgres://user:[YOUR-PASSWORD]@host/db` and people paste their real password in verbatim. A `#`, `@` or `/` in it makes the whole string unparseable, here and later inside `Bun.SQL` — and the prompt is masked, so the paste that failed is not even visible to check. `normalizeConnectionString` percent-encodes the userinfo when the raw string will not parse, splitting on the LAST `@` so an unencoded one inside the password does not end the userinfo early. Strings that already parse are returned untouched, so a correctly encoded password is never double-encoded. --- .../migrate/export/db-exports.test.ts | 43 +++++++++++- .../src/commands/migrate/export/db-options.ts | 68 +++++++++++++++---- 2 files changed, 95 insertions(+), 16 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 08816b69a..62097f2b2 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -25,7 +25,11 @@ import { PLUGIN_COLUMNS, } from "./betterauth.ts"; import { buildSupabaseExport } from "./supabase.ts"; -import { looksLikeConnectionString, resolveDbUrl } from "./db-options.ts"; +import { + looksLikeConnectionString, + normalizeConnectionString, + resolveDbUrl, +} from "./db-options.ts"; /** A cwd with no `.env` files, so these tests exercise only the injected env. */ const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); @@ -104,6 +108,33 @@ describe("looksLikeConnectionString", () => { }); }); +describe("normalizeConnectionString", () => { + test("encodes a password pasted in raw", () => { + const raw = "postgres://postgres:aB#c%92^d@db.example.supabase.co:5432/postgres"; + const normalized = normalizeConnectionString(raw); + + expect(looksLikeConnectionString(normalized)).toBe(true); + expect(decodeURIComponent(new URL(normalized).password)).toBe("aB#c%92^d"); + expect(new URL(normalized).hostname).toBe("db.example.supabase.co"); + }); + + test("encodes an unencoded @ in the password", () => { + const normalized = normalizeConnectionString("postgres://u:p@ss@host:5432/db"); + + expect(decodeURIComponent(new URL(normalized).password)).toBe("p@ss"); + expect(new URL(normalized).hostname).toBe("host"); + }); + + test("leaves an already-valid string alone", () => { + const encoded = "postgres://u:p%40ss@host:5432/db"; + expect(normalizeConnectionString(encoded)).toBe(encoded); + }); + + test("leaves non-URL forms alone", () => { + expect(normalizeConnectionString(" ./db.sqlite ")).toBe("./db.sqlite"); + }); +}); + describe("resolveDbUrl", () => { const config = { platform: "authjs" as const, envVar: "AUTHJS_DB_URL", prompt: "url" }; @@ -120,6 +151,16 @@ describe("resolveDbUrl", () => { ).toBe("mysql://u:p@h/db"); }); + test("encodes a raw password passed to the flag", async () => { + const url = await resolveDbUrl( + { dbUrl: "postgres://u:p#ss@host:5432/db" }, + config, + NO_ENV_FILES, + {}, + ); + expect(decodeURIComponent(new URL(url).password)).toBe("p#ss"); + }); + test("rejects a flag that is not a connection string, naming the encoding trap", async () => { await expect(resolveDbUrl({ dbUrl: "not a url" }, config, NO_ENV_FILES, {})).rejects.toThrow( /URL-encode it/, diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 999e6d5be..e2957697e 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -27,22 +27,60 @@ type ResolveConfig = { hint?: string; }; +const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2):\/\//i; + +/** + * True when the string parses as a URL with a host. + * + * A hostname is required: `postgres://` alone parses as a valid URL, and + * accepting it only defers the failure into the driver. + */ +function parsesAsUrl(value: string): boolean { + try { + return new URL(value).hostname.length > 0; + } catch { + return false; + } +} + +/** + * Percent-encodes the credentials when the raw string will not parse as a URL. + * + * Dashboards hand out `postgres://user:[YOUR-PASSWORD]@host/db` and people + * paste their real password in verbatim. A `#`, `@`, `/` or `^` in it makes the + * whole string unparseable — here and later inside `Bun.SQL` — so encode it for + * them rather than bouncing a paste they cannot even see (the prompt is + * masked). Strings that already parse are returned untouched, so a password + * that was correctly encoded is never double-encoded. + */ +export function normalizeConnectionString(value: string): string { + const trimmed = value.trim(); + if (!URL_SCHEME.test(trimmed) || parsesAsUrl(trimmed)) return trimmed; + + // Greedy up to the LAST `@`: everything before it is userinfo, so an + // unencoded `@` inside the password does not split the string early. + const match = /^([a-z0-9+]+:\/\/)(.*)@([^@]*)$/i.exec(trimmed); + if (!match) return trimmed; + + const [, scheme = "", userinfo = "", rest = ""] = match; + const separator = userinfo.indexOf(":"); + const user = separator === -1 ? userinfo : userinfo.slice(0, separator); + const secret = separator === -1 ? undefined : userinfo.slice(separator + 1); + const credentials = + secret === undefined + ? encodeURIComponent(user) + : `${encodeURIComponent(user)}:${encodeURIComponent(secret)}`; + + const encoded = `${scheme}${credentials}@${rest}`; + return parsesAsUrl(encoded) ? encoded : trimmed; +} + /** True for something that could plausibly be a connection string. */ export function looksLikeConnectionString(value: string): boolean { const trimmed = value.trim(); if (!trimmed) return false; - if (/^(postgresql|postgres|mysql|mysql2):\/\//i.test(trimmed)) { - try { - // A hostname is required: `postgres://` alone parses as a valid URL, and - // accepting it only defers the failure into the driver. - return new URL(trimmed).hostname.length > 0; - } catch { - // A password with an unencoded `@` or `#` is the usual cause, and it is - // worth saying so rather than failing later inside the driver. - return false; - } - } + if (URL_SCHEME.test(trimmed)) return parsesAsUrl(trimmed); return ( trimmed.startsWith("file:") || /\.(sqlite3?|db)$/i.test(trimmed) || trimmed.startsWith("./") @@ -61,7 +99,7 @@ export async function resolveDbUrl( cwd: string = process.cwd(), env: Record = process.env, ): Promise { - const fromFlag = options.dbUrl?.trim(); + const fromFlag = options.dbUrl ? normalizeConnectionString(options.dbUrl) : undefined; if (fromFlag) { if (!looksLikeConnectionString(fromFlag)) { throwUsageError( @@ -73,7 +111,7 @@ export async function resolveDbUrl( } const located = await findMigrateEnvValue([config.envVar], cwd, env); - const fromEnv = located?.value.trim(); + const fromEnv = located ? normalizeConnectionString(located.value) : undefined; if (fromEnv) { if (looksLikeConnectionString(fromEnv)) return fromEnv; // Falling through silently would make the prompt look unexplained. @@ -100,12 +138,12 @@ export async function resolveDbUrl( const answer = await passwordPrompt({ message: config.prompt, validate: (value) => - looksLikeConnectionString(value ?? "") + looksLikeConnectionString(normalizeConnectionString(value ?? "")) ? undefined : "Expected postgres://…, mysql://… or a SQLite file path", }); - return answer.trim(); + return normalizeConnectionString(answer); } /** Describes the target for the run's opening line, credentials removed. */ From 8f374a90c76c191cf9cde5b80a68fe016257ba9a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:32:24 -0400 Subject: [PATCH 020/141] feat(migrate): choose the Clerk export source from a flat instance list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every other resolver in the CLI answers "where do I operate?" with the linked project, silently. For an export that default is actively dangerous: the linked instance is normally the migration's *destination*, so taking it without asking is how a run exports an instance and imports it straight back into itself. So a resolved instance is no longer taken silently — the account's instances are offered, one flat row each (`my-app - Production instance (ins_…)`) rather than an application picker followed by an instance picker. An application is not what an export reads from; an instance is, and dev and prod are different user pools. The resolved application's instances lead the list, so taking one is still a single Enter. `--secret-key` still names an instance outright and runs unquestioned. --- .../cli-core/src/commands/migrate/README.md | 23 +- .../migrate/export/clerk-source.test.ts | 211 ++++++++++++++++++ .../commands/migrate/export/clerk-source.ts | 173 ++++++++++++++ .../src/commands/migrate/export/clerk.ts | 17 +- .../src/commands/migrate/export/index.ts | 2 +- 5 files changed, 419 insertions(+), 7 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/export/clerk-source.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/clerk-source.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index c448e8359..b3c440bae 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -174,7 +174,28 @@ unattended rather than stalling on a prompt with everything held in memory. | `--client-secret ` | `auth0` | Machine-to-machine application client secret | `export clerk` also takes the targeting flags — it reads from a Clerk instance, -so it resolves a key exactly the way `clerk migrate` does. +so it resolves a key the same way `clerk migrate import` does, with one extra +step. The linked project is usually the migration's _destination_, so taking it +as the source without asking is how a run exports an instance and imports it +back into itself. Instead: + +- `--secret-key ` names the source instance outright and runs unquestioned. +- Anything resolved on your behalf — the linked project, a keyless app — is + never taken silently. A picker of every **instance** on your account opens + instead — one flat row each, `my-app - Production instance (ins_…)`, not an + application picker followed by an instance picker — with the resolved + application's instances listed **first** so taking one is still a single + Enter. Only when there are no instances to offer does it stop and list + `--secret-key`, `--app`/`--instance` and `clerk link` instead. +- With nothing to resolve at all (no link, no key, no flags), that same picker + opens directly, rather than an error about an unlinked directory. + +The picker has no "create a new application" choice, unlike `clerk link`'s — a +new application has no users to export. Rows are searchable by what they show, +so typing an application name, `production`, or an instance id all narrow it. + +In agent mode the resolved instance is used without a prompt; pass +`--secret-key` or `--app`/`--instance` to be explicit. After each export you get a field-coverage table — which Clerk-relevant fields were present on how many users — so you know the data is thin _before_ you diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts new file mode 100644 index 000000000..0b7fa6e46 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -0,0 +1,211 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +const mockDescribeBapiTarget = mock(); +const mockResolveBapiSecretKey = mock(); +mock.module("../../../lib/bapi-command.ts", () => ({ + describeBapiTarget: (...args: unknown[]) => mockDescribeBapiTarget(...args), + resolveBapiSecretKey: (...args: unknown[]) => mockResolveBapiSecretKey(...args), +})); + +const mockResolveProfile = mock(); +mock.module("../../../lib/config.ts", () => ({ + resolveProfile: (...args: unknown[]) => mockResolveProfile(...args), +})); + +const mockFetchApps = mock(); +mock.module("../../../lib/app-picker.ts", () => ({ + fetchAppsTolerantly: (...args: unknown[]) => mockFetchApps(...args), +})); + +const mockSearch = mock(); +mock.module("../../../lib/listage.ts", () => ({ + search: (...args: unknown[]) => mockSearch(...args), +})); + +const mockResolveUsersInstanceContext = mock(); +mock.module("../../users/interactive/instance-context.ts", () => ({ + resolveUsersInstanceContext: (...args: unknown[]) => mockResolveUsersInstanceContext(...args), +})); + +let human = true; +mock.module("../../../mode.ts", () => ({ + isHuman: () => human, + isAgent: () => !human, + getMode: () => (human ? "human" : "agent"), + setMode: () => {}, +})); + +const { resolveClerkSource } = await import("./clerk-source.ts"); + +const captured = useCaptureLog(); + +beforeEach(() => { + human = true; + mockDescribeBapiTarget.mockReset(); + mockResolveBapiSecretKey.mockReset(); + mockResolveProfile.mockReset(); + mockResolveProfile.mockResolvedValue(undefined); + mockResolveUsersInstanceContext.mockReset(); + mockFetchApps.mockReset(); + mockSearch.mockReset(); + delete process.env.CLERK_SECRET_KEY; +}); + +/** The linked-project case: something resolved, nobody asked for it. */ +function stubResolved(target: string | undefined, secretKey = "sk_test_resolved") { + mockDescribeBapiTarget.mockResolvedValue(target); + mockResolveBapiSecretKey.mockResolvedValue(secretKey); +} + +describe("resolveClerkSource", () => { + test("--secret-key names the instance outright and is never questioned", async () => { + stubResolved(undefined, "sk_test_explicit"); + + const source = await resolveClerkSource({ secretKey: "sk_test_explicit" }); + + expect(source).toEqual({ secretKey: "sk_test_explicit", target: undefined }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // Exporting the instance that is about to be imported *into* is the failure + // this whole module exists to prevent, so a resolved instance is offered as + // one choice among the account's applications rather than taken silently. + test("offers every instance, flat, with the linked application's first", async () => { + stubResolved("my-app (development)"); + mockResolveProfile.mockResolvedValue({ profile: { appId: "app_2" } }); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "my-app", + instances: [ + { instance_id: "ins_1d", environment_type: "development" }, + { instance_id: "ins_1p", environment_type: "production" }, + ], + }, + { + application_id: "app_2", + name: "other-app", + instances: [{ instance_id: "ins_2d", environment_type: "development" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + mockResolveUsersInstanceContext.mockResolvedValue({ + secretKey: "sk_live_other", + appLabel: "my-app", + instanceLabel: "production", + }); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_live_other", target: "my-app (production)" }); + // One row per instance, not per application: dev and prod are different + // user pools, and exporting the wrong one is silent. + const { message, source: listSource } = mockSearch.mock.calls[0]![0]; + expect(message).toBe("What Clerk instance do you want to export users from?"); + expect(listSource("")).toEqual([ + { + name: "other-app - Development instance (ins_2d)", + value: { app: "app_2", instance: "ins_2d" }, + }, + { + name: "my-app - Development instance (ins_1d)", + value: { app: "app_1", instance: "ins_1d" }, + }, + { + name: "my-app - Production instance (ins_1p)", + value: { app: "app_1", instance: "ins_1p" }, + }, + ]); + // Both halves are handed on, so the secret-key lookup runs against exactly + // the instance that was chosen and nothing prompts a second time. + expect(mockResolveUsersInstanceContext).toHaveBeenCalledWith({ + app: "app_1", + instance: "ins_1p", + }); + expect(captured.err).toBe(""); + }); + + // The list is searched by its rendered label, so an application id typed from + // a dashboard URL still finds its instances. + test("filters on the rendered label", async () => { + stubResolved("my-app (development)"); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "my-app", + instances: [{ instance_id: "ins_1p", environment_type: "production" }], + }, + { + application_id: "app_2", + name: "other-app", + instances: [{ instance_id: "ins_2d", environment_type: "development" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + mockResolveUsersInstanceContext.mockResolvedValue({ secretKey: "sk_live_other" }); + + await resolveClerkSource({}); + + const { source: listSource } = mockSearch.mock.calls[0]![0]; + expect(listSource("ins_2d")).toEqual([ + { + name: "other-app - Development instance (ins_2d)", + value: { app: "app_2", instance: "ins_2d" }, + }, + ]); + expect(listSource("production")).toHaveLength(1); + }); + + // An empty list is not a picker. PLAPI being degraded looks the same as an + // account with no applications, and neither one has an instance to offer. + test("no instances to offer falls back to the flags", async () => { + stubResolved("my-app (development)"); + // An application with no instances is not an offer either. + mockFetchApps.mockResolvedValue([{ application_id: "app_1", name: "my-app", instances: [] }]); + + await expect(resolveClerkSource({})).rejects.toBeInstanceOf(UserAbortError); + + expect(mockSearch).not.toHaveBeenCalled(); + expect(captured.err).toContain("--secret-key"); + expect(captured.err).toContain("--app"); + expect(captured.err).toContain("clerk link"); + }); + + test("agent mode takes the resolved instance without prompting", async () => { + human = false; + stubResolved("my-app (production)"); + + const source = await resolveClerkSource({}); + + expect(source.secretKey).toBe("sk_test_resolved"); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("an unlinked directory picks an application instead of failing", async () => { + mockDescribeBapiTarget.mockRejectedValue( + new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }), + ); + mockResolveUsersInstanceContext.mockResolvedValue({ + secretKey: "sk_test_picked", + appLabel: "other-app", + instanceLabel: "production", + }); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_test_picked", target: "other-app (production)" }); + // The picker just asked which application; asking again is noise. + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("an explicit --app that fails to resolve surfaces the error, not the picker", async () => { + const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); + mockDescribeBapiTarget.mockRejectedValue(failure); + + await expect(resolveClerkSource({ app: "app_123" })).rejects.toThrow(failure); + + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts new file mode 100644 index 000000000..0754114d5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -0,0 +1,173 @@ +/** + * Which Clerk instance `migrate export clerk` reads *from*. + * + * Every other resolver in the CLI answers "where do I operate?" with the linked + * project, silently. For an export that default is actively dangerous: the + * linked instance is normally the migration's *destination*, so taking it + * without asking is how a run ends up exporting an instance and importing it + * straight back into itself. + * + * So the source is resolved in three tiers: + * + * 1. `--secret-key` names an instance outright — it runs unquestioned. + * 2. Anything the CLI resolved on the user's behalf (the linked project, a + * keyless app) is not taken silently: every instance on the account is + * offered, with the resolved application's instances first so "yes, that + * one" is still a single Enter. + * 3. Nothing to resolve at all — no link, no key, no flags — offers those same + * instances, the trade `users list` makes, rather than failing on an + * unlinked directory. + */ + +import { fetchAppsTolerantly } from "../../../lib/app-picker.ts"; +import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; +import { resolveProfile } from "../../../lib/config.ts"; +import { CliError, ERROR_CODE, throwUserAbort } from "../../../lib/errors.ts"; +import { search } from "../../../lib/listage.ts"; +import type { ApplicationInstance } from "../../../lib/plapi.ts"; +import { log } from "../../../lib/log.ts"; +import { isHuman } from "../../../mode.ts"; +import { resolveUsersInstanceContext } from "../../users/interactive/instance-context.ts"; + +/** e.g. `Development instance`. Unknown environment types print as-is. */ +function instanceLabel(instance: ApplicationInstance): string { + const type = instance.environment_type; + if (!type) return "instance"; + return `${type.charAt(0).toUpperCase()}${type.slice(1)} instance`; +} + +export type ResolveClerkSourceOptions = { + secretKey?: string; + app?: string; + instance?: string; + cwd?: string; +}; + +export type ClerkExportSource = { + secretKey: string; + /** + * Human-readable target, e.g. `my-app (production)`. Absent when + * `--secret-key` (or `CLERK_SECRET_KEY`) named the instance directly, since + * a bare key carries no application context to describe. + */ + target?: string; +}; + +/** {@link ClerkExportSource} plus whether the user already chose it out loud. */ +type ResolvedSource = ClerkExportSource & { chosen: boolean }; + +async function resolveSource(options: ResolveClerkSourceOptions): Promise { + try { + return { + target: await describeBapiTarget(options), + secretKey: await resolveBapiSecretKey(options), + // Flags and env keys are a choice the user typed; the linked project is + // one they made for some other purpose, possibly months ago. + chosen: Boolean(options.secretKey), + }; + } catch (error) { + const hasExplicitTarget = + Boolean(options.secretKey) || + Boolean(options.app) || + Boolean(options.instance) || + Boolean(process.env.CLERK_SECRET_KEY); + + if ( + !isHuman() || + hasExplicitTarget || + !(error instanceof CliError) || + error.code !== ERROR_CODE.NO_SECRET_KEY + ) { + throw error; + } + + const ctx = await resolveUsersInstanceContext({}); + return { + secretKey: ctx.secretKey, + target: ctx.appLabel ? `${ctx.appLabel} (${ctx.instanceLabel})` : undefined, + // The picker just asked. Confirming the answer to a question the user + // answered one prompt ago is noise. + chosen: true, + }; + } +} + +/** + * Offers every instance on the account, flat — one row per instance rather than + * an application picker followed by an instance picker. + * + * An application is not what an export reads from; an instance is. Picking + * "Migration Test" and then "development" is two questions with one answer, and + * it hides the thing that actually matters — dev and prod are different user + * pools, and exporting the wrong one is silent. + * + * Deliberately not `pickOrCreateApp`: its "+ Create a new application" choice + * makes sense when you are choosing somewhere to *write*, and no sense at all + * as an export source — a brand-new application has no users in it. + * + * @param currentAppId the application the CLI resolved on the user's behalf. + * Its instances lead the list, because they are the likeliest answer. + * @returns undefined when there is nothing to offer, so the caller can fall + * back to telling the user which flags to pass instead of showing an empty + * list. `fetchAppsTolerantly` returns empty on a degraded PLAPI, not just on + * an account with no applications. + */ +async function pickInstance(currentAppId?: string): Promise { + const apps = await fetchAppsTolerantly(); + + const ordered = currentAppId + ? [ + ...apps.filter((app) => app.application_id === currentAppId), + ...apps.filter((app) => app.application_id !== currentAppId), + ] + : apps; + + const choices = ordered.flatMap((app) => + (app.instances ?? []).map((instance) => ({ + name: `${app.name || app.application_id} - ${instanceLabel(instance)} (${instance.instance_id})`, + value: { app: app.application_id, instance: instance.instance_id }, + })), + ); + if (choices.length === 0) return undefined; + + const picked = await search<{ app: string; instance: string }>({ + message: "What Clerk instance do you want to export users from?", + source: (term) => + term + ? choices.filter((choice) => choice.name.toLowerCase().includes(term.toLowerCase())) + : choices, + }); + + // Both halves are passed on, so the secret-key lookup runs against exactly + // the instance that was chosen and nothing prompts a second time. + const ctx = await resolveUsersInstanceContext(picked); + return { + secretKey: ctx.secretKey, + target: ctx.appLabel ? `${ctx.appLabel} (${ctx.instanceLabel})` : undefined, + }; +} + +/** The application the CLI resolved on the user's behalf, if it knows one. */ +async function currentAppId(options: ResolveClerkSourceOptions): Promise { + if (options.app) return options.app; + const resolved = await resolveProfile(options.cwd ?? process.cwd()).catch(() => undefined); + return resolved?.profile.appId; +} + +export async function resolveClerkSource( + options: ResolveClerkSourceOptions, +): Promise { + const { chosen, ...source } = await resolveSource(options); + if (chosen || !source.target || !isHuman()) return source; + + const picked = await pickInstance(await currentAppId(options)); + if (picked) return picked; + + log.info( + "Export from a different instance with one of:\n" + + " `--secret-key ` — the source instance's secret key\n" + + " `--app --instance ` — another application on your account\n" + + " `clerk link` — link this directory to a different application first", + ); + throwUserAbort(); +} diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index fc3875339..ec4a633be 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -15,11 +15,11 @@ */ import { bapiRequest } from "../../../lib/bapi.ts"; -import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { retryOn429 } from "../lib/retry.ts"; +import { resolveClerkSource } from "./clerk-source.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; /** BAPI's maximum page size for `GET /v1/users`. */ @@ -223,17 +223,24 @@ export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExpo } export async function exportClerk(options: ExportClerkOptions): Promise { + // Resolved before the gutter opens, the way `export auth0` resolves its + // credentials: confirming the source is a question about whether to run at + // all, not a step of the run. + const source = await resolveClerkSource({ + secretKey: options.secretKey, + app: options.app, + instance: options.instance, + }); + const destination = await resolveOutputPath("clerk", options.output); await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { - const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); - const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const dateTime = getDateTimeStamp(); - log.info(`Exporting from ${target ?? "the resolved instance"}.`); + log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); const users = await withSpinner("Fetching users from Clerk...", (spinner) => - fetchAllClerkUsers({ secretKey, spinner }), + fetchAllClerkUsers({ secretKey: source.secretKey, spinner }), ); const { users: exported, coverage } = buildClerkExport(users, dateTime); diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index f9f99dea6..b11ea8814 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -111,7 +111,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json", + description: "Prompts for the source instance and where to save the file", }, { command: "clerk migrate export clerk --instance prod --output prod-users.json", From f0777861ab7aadbf12734284b69dcfd6a538b44f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:32:36 -0400 Subject: [PATCH 021/141] feat(migrate): prompt for the Firebase service account key `export firebase` without `--service-account` exited with a usage error, which is a dead end in the interactive picker: choose Firebase, get told to re-run with a flag. It now prompts, the way `export supabase` prompts for its connection string. The answer can be a path to the downloaded file *or* the key's JSON pasted whole, so a key kept in a password manager or a CI secret never has to be written to disk. Prompted as a password, since the JSON carries a private key. Agent mode has nobody to ask, so it still names the flag. --- .../cli-core/src/commands/migrate/README.md | 16 ++- .../commands/migrate/export/firebase.test.ts | 27 +++- .../src/commands/migrate/export/firebase.ts | 128 +++++++++++++----- 3 files changed, 132 insertions(+), 39 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index b3c440bae..51dcd957d 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -284,10 +284,18 @@ clerk migrate export firebase --service-account ./service-account.json ``` Needs a service account key from **Project settings → Service accounts → -Generate new private key**, with the Firebase Authentication Admin role. The -file is validated before anything reaches the network, so downloading the web -app config by mistake fails in a second with the right console page named -rather than after an auth round-trip. Key material never appears in output. +Generate new private key**, with the Firebase Authentication Admin role. + +Without `--service-account` you are prompted for it, the way `export supabase` +prompts for its connection string. The answer can be a path to the downloaded +file _or_ the key's JSON pasted whole, so a key kept in a password manager or a +CI secret never has to be written to disk. The prompt is masked, since the key +carries a private key. Agent mode cannot prompt, so it names the flag instead. + +Either way the key is validated before anything reaches the network, so +downloading the web app config by mistake fails in a second with the right +console page named rather than after an auth round-trip. Key material never +appears in output. Firebase's scrypt is a modified variant, so a digest is worthless without the project's four hash parameters. The export **reads them from the project** and diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 5754bf901..4f9c4ac25 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -13,6 +13,7 @@ import { fetchAllFirebaseUsers, fetchHashConfig, formatHashConfigGuidance, + loadServiceAccount, mapFirebaseUserToExport, readServiceAccount, signServiceAccountJwt, @@ -111,6 +112,28 @@ const fbUser = (i: number, overrides: Record = {}) => ({ ...overrides, }); +describe("loadServiceAccount", () => { + // The prompt takes either, so a key pasted out of a password manager never + // has to be written to disk first. + test("accepts the key JSON pasted whole", () => { + expect(loadServiceAccount(` ${JSON.stringify(account)} `).project_id).toBe("demo-fb"); + }); + + test("accepts a path to the key file", () => { + expect(loadServiceAccount("./sa.json").project_id).toBe("demo-fb"); + }); + + test("rejects a paste that is not valid JSON", () => { + expect(() => loadServiceAccount('{"project_id":')).toThrow(/pasted key is not valid JSON/); + }); + + test("rejects a paste missing a required field", () => { + expect(() => loadServiceAccount('{"type":"service_account"}')).toThrow( + /The pasted key is not a usable service account key: "project_id" is missing/, + ); + }); +}); + describe("readServiceAccount", () => { test("reads a valid key file", () => { expect(readServiceAccount("./sa.json").project_id).toBe("demo-fb"); @@ -450,7 +473,9 @@ describe("exportFirebase", () => { expect(fs.existsSync(path.join(workDir, "fb.json"))).toBe(true); }); - test("requires --service-account, before anything is read", async () => { + // Human runs get a prompt instead; an agent has nobody to ask, so it is told + // which flag to pass. + test("agent mode names the flag rather than prompting", async () => { await expect(exportFirebase({})).rejects.toThrow(/needs a service account key file/); }); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index acb413e6d..6facdd770 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -28,6 +28,8 @@ import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; import { bold, dim } from "../../../lib/color.ts"; import { loggedFetch } from "../../../lib/fetch.ts"; import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt } from "../../../lib/prompts.ts"; +import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; @@ -55,12 +57,43 @@ export type ServiceAccount = { }; /** - * Reads and validates a service-account key file. + * Validates already-parsed JSON as a service-account key. * * Every failure names the field, because the usual causes are downloading the * wrong JSON from the console (a web app config rather than a service account) * or pasting a key with its newlines mangled. + * + * @param label how to refer to the source in an error — a file name, or + * "the pasted key" when it came from the prompt. */ +function validateServiceAccount(parsed: unknown, label: string): ServiceAccount { + const account = parsed as Partial & { type?: string }; + const invalid = (problem: string): never => { + throw new CliError(`${label} is not a usable service account key: ${problem}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + }; + + if (account.type && account.type !== "service_account") { + invalid( + `its "type" is "${account.type}", not "service_account". Download a private key from ` + + "Project settings → Service accounts → Generate new private key.", + ); + } + for (const field of ["project_id", "client_email", "private_key"] as const) { + if (typeof account[field] !== "string" || account[field].length === 0) { + invalid(`"${field}" is missing`); + } + } + if (!account.private_key?.includes("PRIVATE KEY")) { + invalid('"private_key" does not look like a PEM key — check its newlines survived copying'); + } + + return account as ServiceAccount; +} + +/** Reads and validates a service-account key file. */ export function readServiceAccount(file: string): ServiceAccount { const resolved = path.resolve(process.cwd(), file); @@ -81,30 +114,71 @@ export function readServiceAccount(file: string): ServiceAccount { }); } - const account = parsed as Partial & { type?: string }; - const invalid = (problem: string): never => { - throw new CliError(`${file} is not a usable service account key: ${problem}`, { - code: ERROR_CODE.USAGE_ERROR, + return validateServiceAccount(parsed, file); +} + +/** + * Accepts what the prompt accepts: a path to the downloaded key file, or the + * key's JSON pasted in whole. Console downloads land as a file, but a key + * copied out of a password manager or CI secret never touches disk. + */ +export function loadServiceAccount(source: string): ServiceAccount { + const trimmed = source.trim(); + if (!trimmed.startsWith("{")) return readServiceAccount(trimmed); + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch (error) { + throw new CliError(`The pasted key is not valid JSON: ${(error as Error).message}`, { + code: ERROR_CODE.INVALID_JSON, docsUrl: DOCS_URL, }); - }; + } - if (account.type && account.type !== "service_account") { - invalid( - `its "type" is "${account.type}", not "service_account". Download a private key from ` + - "Project settings → Service accounts → Generate new private key.", + return validateServiceAccount(parsed, "The pasted key"); +} + +/** + * Resolves the key: the flag, then a prompt — the shape `export supabase` uses + * for its connection string. Prompted as a password: the JSON carries a private + * key, and a path typed blind is short enough to survive being masked. + */ +async function resolveServiceAccount(options: ExportFirebaseOptions): Promise { + if (options.serviceAccount) return loadServiceAccount(options.serviceAccount); + + if (!isHuman()) { + throwUsageError( + "`clerk migrate export firebase` needs a service account key file and cannot prompt here. " + + "Pass --service-account .", + DOCS_URL, + undefined, + [ + { + command: "clerk migrate export firebase --service-account ./service-account.json", + description: "Export using a downloaded service account key", + }, + ], ); } - for (const field of ["project_id", "client_email", "private_key"] as const) { - if (typeof account[field] !== "string" || account[field].length === 0) { - invalid(`"${field}" is missing`); - } - } - if (!account.private_key?.includes("PRIVATE KEY")) { - invalid('"private_key" does not look like a PEM key — check its newlines survived copying'); - } - return account as ServiceAccount; + log.info( + dim("Firebase console → Project settings → Service accounts → Generate new private key."), + ); + + const answer = await passwordPrompt({ + message: "Path to the service account key file, or paste the key JSON", + validate: (value) => { + try { + loadServiceAccount(value ?? ""); + return undefined; + } catch (error) { + return error instanceof CliError ? error.message : String(error); + } + }, + }); + + return loadServiceAccount(answer); } function base64Url(input: string | Uint8Array): string { @@ -406,23 +480,9 @@ export function formatHashConfigGuidance( } export async function exportFirebase(options: ExportFirebaseOptions): Promise { - if (!options.serviceAccount) { - throwUsageError( - "`clerk migrate export firebase` needs a service account key file. Pass --service-account .", - DOCS_URL, - undefined, - [ - { - command: "clerk migrate export firebase --service-account ./service-account.json", - description: "Export using a downloaded service account key", - }, - ], - ); - } - // Read and validate before anything reaches the network, so a wrong file // fails in a second rather than after an auth round-trip. - const account = readServiceAccount(options.serviceAccount); + const account = await resolveServiceAccount(options); const destination = await resolveOutputPath("firebase", options.output); From f0ba767c55a3ab189b531cbd4f83b3871e932a94 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:32:36 -0400 Subject: [PATCH 022/141] refactor(migrate): restyle `clerk migrate transformers list` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A section heading and a leading sentence, matching how `--help` lays out its own sections, with each description wrapped rather than run off the edge. Width is capped at 80 columns, not merely measured, so two runs of the same command lay out the same way on different terminals. A backticked span is never broken across lines: `log.info` pairs backticks per line, so a split span leaves an unmatched backtick on each and colours the wrong half of both. No gutter — this reads a static registry, it does not run anything — and no dimmed text, which the descriptions are the whole point of. --- .../cli-core/src/commands/migrate/README.md | 22 +++++ .../migrate/transformers/list.test.ts | 48 +++++++--- .../src/commands/migrate/transformers/list.ts | 94 ++++++++++++++----- 3 files changed, 131 insertions(+), 33 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 51dcd957d..8d8e97e4c 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -485,6 +485,28 @@ clerk migrate transformers list --transformer-file ./my-transformer.ts | `--json` | Output as JSON | | `--transformer-file ` | Also list a transformer you wrote | +Each entry prints its key, the platform label, and what the transformer assumes +about the export — wrapped to the terminal, capped at 80 columns so two runs of +the same command lay out the same way. A backticked span is never broken across +lines. There is no intro/outro gutter: this reads a static registry rather than +running anything. + +``` +A transformer maps one platform's export onto the fields Clerk imports. Pass the +one your export came from as `--transformer `. + +Transformers: + clerk Clerk + Migrate between Clerk instances (e.g. development to production, or to + another Clerk application). Export your users from the Clerk Dashboard + first. + + … + +6 built-in transformers +Migrating from something else? Write a transformer and pass --transformer-file. +``` + `--json` gives an agent the same data, including which source field each transformer maps to `userId`. diff --git a/packages/cli-core/src/commands/migrate/transformers/list.test.ts b/packages/cli-core/src/commands/migrate/transformers/list.test.ts index 26179ee9f..51c57eca2 100644 --- a/packages/cli-core/src/commands/migrate/transformers/list.test.ts +++ b/packages/cli-core/src/commands/migrate/transformers/list.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import { getMode, setMode, type Mode } from "../../../mode.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { list } from "./list.ts"; +import { list, wrapText } from "./list.ts"; import { transformers } from "./registry.ts"; const captured = useCaptureLog(); @@ -43,11 +43,15 @@ describe("human output", () => { expect(captured.err).toContain(transformer.label); }); - // `log.info` auto-highlights backticked spans, so the rendered description - // carries colour codes the source string does not. + // Two normalizations: `log.info` auto-highlights backticked spans, so the + // rendered description carries colour codes the source string does not, and + // descriptions are wrapped to the terminal width across several indented + // lines. Collapsing whitespace compares the words, not the layout. + const collapse = (value: string) => stripAnsi(value).replace(/\s+/g, " "); + test.each([...transformers])("includes the $key description", async (transformer) => { await list(); - expect(stripAnsi(captured.err)).toContain(transformer.description); + expect(collapse(captured.err)).toContain(collapse(transformer.description)); }); test("counts the built-ins", async () => { @@ -128,19 +132,41 @@ describe("human-mode frame", () => { setMode(originalMode); }); - test("wraps its output in an intro/outro gutter", async () => { + // Reading a static registry is not a run: there is no progress to bracket, + // and the gutter's `│` would sit in front of every wrapped line. + test("prints no intro/outro gutter", async () => { await list(); - expect(captured.err).toContain("┌"); - expect(captured.err).toContain("Listing transformers"); - expect(captured.err).toContain("└"); - expect(captured.err).toContain("Done"); + expect(captured.err).not.toContain("┌"); + expect(captured.err).not.toContain("└"); + expect(stripAnsi(captured.err)).toContain("Transformers:"); }); - test("--json stays outside the gutter, on stdout only", async () => { + test("--json stays on stdout only", async () => { await list({ json: true }); expect(() => JSON.parse(captured.out)).not.toThrow(); - expect(captured.err).not.toContain("┌"); + expect(captured.err).toBe(""); + }); +}); + +describe("wrapText", () => { + test("breaks on whitespace within the width", () => { + expect(wrapText("one two three four", 9)).toEqual(["one two", "three", "four"]); + }); + + // `log.info` pairs backticks per line, so a span split across two lines + // leaves one unmatched backtick on each and colours the wrong half of both. + test("never breaks inside a backticked span", () => { + const lines = wrapText("Assumes an export of `SELECT id, name FROM users`.", 30); + + expect(lines).toContain("`SELECT id, name FROM users`."); + for (const line of lines) { + expect((line.match(/`/g) ?? []).length % 2).toBe(0); + } + }); + + test("gives an over-long word its own line rather than dropping it", () => { + expect(wrapText("short supercalifragilistic", 8)).toEqual(["short", "supercalifragilistic"]); }); }); diff --git a/packages/cli-core/src/commands/migrate/transformers/list.ts b/packages/cli-core/src/commands/migrate/transformers/list.ts index 37da1e0a5..993ee4838 100644 --- a/packages/cli-core/src/commands/migrate/transformers/list.ts +++ b/packages/cli-core/src/commands/migrate/transformers/list.ts @@ -6,9 +6,8 @@ * A compiled binary's users have neither, so the list is a command. */ -import { bold, cyan, dim } from "../../../lib/color.ts"; +import { bold, cyan } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; -import { withGutter } from "../../../lib/spinner.ts"; import type { TransformerRegistryEntry } from "../types.ts"; import { loadCustomTransformer } from "./load-custom.ts"; import { transformers } from "./registry.ts"; @@ -32,6 +31,47 @@ function toJson(entries: Listed[]) { })); } +/** + * Capped, not just measured: a description that rewrapped differently on every + * terminal makes two runs of the same command look like different output. 80 is + * the same width `--help` lays itself out at. + */ +const MAX_WIDTH = 80; + +function outputWidth(): number { + return Math.min(process.stderr.columns || MAX_WIDTH, MAX_WIDTH); +} + +/** + * A run of non-space characters, except that a backticked span counts as one + * character run even when it contains spaces. Keeps `SELECT a, b FROM users` + * whole: `log.info` pairs backticks per line, so a span broken across two lines + * leaves an unmatched backtick on each and colours the wrong half of both. + */ +const WORD = /(?:`[^`]*`|\S)+/g; + +/** + * Wraps on whitespace. Safe to measure raw because the backtick spans + * `log.info` highlights keep their backticks — the colour it adds is invisible + * to width, and nothing here is coloured before wrapping. + */ +export function wrapText(text: string, width: number): string[] { + const lines: string[] = []; + let line = ""; + + for (const word of text.match(WORD) ?? []) { + if (!line) line = word; + else if (line.length + 1 + word.length <= width) line += ` ${word}`; + else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + + return lines; +} + export async function list(options: TransformersListOptions = {}): Promise { const entries: Listed[] = transformers.map((entry) => ({ ...entry, builtIn: true })); @@ -45,26 +85,36 @@ export async function list(options: TransformersListOptions = {}): Promise return; } - await withGutter("Listing transformers", async () => { - for (const entry of entries) { - const suffix = entry.builtIn ? "" : ` ${dim(`(custom — ${entry.source})`)}`; - log.info(`${cyan(bold(entry.key))} ${entry.label}${suffix}`); - log.info(` ${dim(entry.description)}`); - log.info(""); - } + const width = outputWidth(); - const custom = entries.length - transformers.length; - log.info( - dim( - `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + - (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), - ), - ); - - if (custom === 0) { - log.info( - dim("Migrating from something else? Write a transformer and pass --transformer-file."), - ); + // No gutter: this reads a static registry, it does not run anything. The + // frame belongs on `migrate import`, where there is progress to bracket. + for (const line of wrapText( + "A transformer maps one platform's export onto the fields Clerk imports. " + + "Pass the one your export came from as `--transformer `.", + width, + )) { + log.info(line); + } + log.blank(); + + log.info(bold("Transformers:")); + for (const entry of entries) { + const suffix = entry.builtIn ? "" : ` (custom — ${entry.source})`; + log.info(` ${cyan(bold(entry.key))} ${entry.label}${suffix}`); + for (const line of wrapText(entry.description, width - 4)) { + log.info(` ${line}`); } - }); + log.blank(); + } + + const custom = entries.length - transformers.length; + log.info( + `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + + (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), + ); + + if (custom === 0) { + log.info("Migrating from something else? Write a transformer and pass --transformer-file."); + } } From aa62b62162b0e572351955c75fc67b2ad363ea24 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 8 Sep 2026 11:31:29 -0400 Subject: [PATCH 023/141] refactor(migrate): name log files after the commands that write them `migrate import` now writes `import-.log` and `migrate delete` writes `delete-.log`, so a listing points at the command behind each line. The old `migration-` and `user-deletion-` names, written by the standalone tool and earlier CLI builds, still classify and convert. `migrate logs list` leads with the filename (what `logs convert` and `logs clean` talk about), renders the UTC stamp in the reader's own zone, prints the log directory relative to the cwd, and closes with a fixed legend of every kind rather than only the ones present. --- .../cli-core/src/commands/migrate/README.md | 51 ++++++-- .../src/commands/migrate/delete.test.ts | 2 +- .../cli-core/src/commands/migrate/delete.ts | 2 +- .../src/commands/migrate/import-users.test.ts | 2 +- .../commands/migrate/lib/log-files.test.ts | 45 +++---- .../src/commands/migrate/lib/log-files.ts | 24 +++- .../src/commands/migrate/lib/logger.test.ts | 6 +- .../src/commands/migrate/lib/logger.ts | 34 +++-- .../src/commands/migrate/logs/list.ts | 91 +++++++++++--- .../migrate/logs/logs-interactive.test.ts | 46 +++---- .../src/commands/migrate/logs/logs.test.ts | 119 +++++++++++------- .../cli-core/src/commands/migrate/run.test.ts | 2 +- packages/cli-core/src/commands/migrate/run.ts | 2 +- 13 files changed, 283 insertions(+), 143 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 8d8e97e4c..bd601ca00 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -380,7 +380,7 @@ Rate limiting and 429 retries are literally the same code path as the import A failure on one user is logged and the rest continue: a half-undone migration with no record of which half is far worse than a reported failure. Every -attempt lands in a timestamped `logs/user-deletion-.log`, carrying +attempt lands in a timestamped `logs/delete-.log`, carrying both the source ID and the Clerk ID. The command exits non-zero if any deletion failed. @@ -399,12 +399,12 @@ clerk migrate logs # defaults to list clerk migrate logs list --json clerk migrate logs clean -y clerk migrate logs convert --all -clerk migrate logs convert migration-2026-01-01T12-00-00.log +clerk migrate logs convert import-2026-01-01T12-00-00.log ``` | Subcommand | Takes | Description | | -------------- | ------------------ | ----------------------------------------------- | -| `logs list` | `--json` | Type, timestamp, size and entry count per file | +| `logs list` | `--json` | File, type, date, size and entry count per file | | `logs clean` | `-y, --yes` | Delete the `.log` files in `./logs/` | | `logs convert` | `[file…]`, `--all` | NDJSON → a JSON array, written as `.json` | @@ -414,15 +414,42 @@ All three read the directory through one shared enumerator, which is what makes #### `logs list` The default, because listing is read-only and therefore safe to run by -accident. Reports each file's type, timestamp, size and entry count, newest +accident. Reports each file's name, type, date, size and entry count, newest first; `--json` gives an agent the same data without parsing NDJSON. ``` -TYPE TIMESTAMP SIZE ENTRIES -migration 2026-02-01T09-14-22 4.1 KB 120 -deletion 2026-01-30T17-02-51 612 B 18 +Each log represents a user export, user import, or a user delete run. +Each log consists of a single NDJSON entry per user. + +FILE TYPE DATE SIZE ENTRIES +import-2026-02-01T09-14-22.log import Feb 1, 2026 at 4:14 AM 4.1 KB 120 +delete-2026-01-30T17-02-51.log delete Jan 30, 2026 at 12:02 PM 612 B 18 + +2 log files in ./logs + +Log types: + export One entry per user pulled from the source platform. + import One entry per user created in Clerk, with any error. + delete One entry per user removed from Clerk, with any error. ``` +A kind is the name of the command that wrote it — `migrate import` writes +`import-.log` — so a listing points straight at the run behind each +line. The legend is fixed rather than derived from what happens to be present, +because "what else could be here" is the other half of the question. + +The older `migration-` and `user-deletion-` names, written by the standalone +tool and by earlier CLI builds, still classify as `import` and `delete`, so a +directory of old logs lists and converts unchanged. + +The filename leads, because it is what `logs convert` and `logs clean` talk +about. The date column renders the filename's UTC stamp in the reader's own +zone — "which run was that" is a question about local time; `--json` keeps the +raw stamp. + +The directory is printed relative (`./logs`) when it sits under the current +directory and absolute when it does not, so the path can be pasted either way. + Says so plainly when `./logs/` is empty or absent. #### `logs clean` @@ -444,7 +471,7 @@ A malformed line is reported with its line number and skipped, and the remaining entries still convert: ``` -migration-2026-01-01T12-00-00.log:2 is not valid JSON and was skipped — … +import-2026-01-01T12-00-00.log:2 is not valid JSON and was skipped — … 1 malformed line skipped. ``` @@ -928,9 +955,9 @@ rather than "which project is linked here". | Path | Contents | | ------------------------------------------ | --------------------------------------------------------------------- | -| `./logs/migration-.log` | NDJSON: one line per user, plus validation failures and retry notices | -| `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | | `./logs/export-.log` | NDJSON: one line per exported user | +| `./logs/import-.log` | NDJSON: one line per user, plus validation failures and retry notices | +| `./logs/delete-.log` | NDJSON: one line per `migrate delete` attempt | | `./exports/-export-.json` | The export itself, unless `--output` says otherwise | | `./.env.clerk-migrate` | Migration credentials, written by `settings set` and gitignored | @@ -959,8 +986,8 @@ long append-only stream, and that format is the one that survives it: Which is also why it greps usefully without any tooling: ```sh -grep '"status":"success"' logs/migration-2026-01-01T12-00-00.log | wc -l -grep '"userId":"user_123"' logs/migration-2026-01-01T12-00-00.log +grep '"status":"success"' logs/import-2026-01-01T12-00-00.log | wc -l +grep '"userId":"user_123"' logs/import-2026-01-01T12-00-00.log ``` The trade-off is that spreadsheets, databases and most JSON tooling want an diff --git a/packages/cli-core/src/commands/migrate/delete.test.ts b/packages/cli-core/src/commands/migrate/delete.test.ts index 01711ec97..3ae3131ac 100644 --- a/packages/cli-core/src/commands/migrate/delete.test.ts +++ b/packages/cli-core/src/commands/migrate/delete.test.ts @@ -373,7 +373,7 @@ describe("deleteMigration", () => { const logs = fs.readdirSync(getLogDir()); expect(logs).toHaveLength(1); - expect(logs[0]).toMatch(/^user-deletion-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + expect(logs[0]).toMatch(/^delete-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); }); test("leaves users the migration did not create alone", async () => { diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 06781d3cd..32e7d4c41 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -264,7 +264,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise { const logEntries = () => fs - .readFileSync(getLogFilePath("migration", DATE_TIME), "utf-8") + .readFileSync(getLogFilePath("import", DATE_TIME), "utf-8") .trim() .split("\n") .map((line) => JSON.parse(line) as Record); diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.test.ts b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts index 1af83c1f0..cc7587453 100644 --- a/packages/cli-core/src/commands/migrate/lib/log-files.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts @@ -33,14 +33,17 @@ function writeLog(name: string, entries: unknown[]): string { describe("classifyLogFile", () => { test.each([ - ["migration-2026-01-01T12-00-00.log", "migration", "2026-01-01T12-00-00"], - ["user-deletion-2026-01-01T12-00-00.log", "deletion", "2026-01-01T12-00-00"], + ["import-2026-01-01T12-00-00.log", "import", "2026-01-01T12-00-00"], + ["delete-2026-01-01T12-00-00.log", "delete", "2026-01-01T12-00-00"], ["export-2026-01-01T12-00-00.log", "export", "2026-01-01T12-00-00"], + // Written by the standalone tool and by earlier CLI builds. + ["migration-2026-01-01T12-00-00.log", "import", "2026-01-01T12-00-00"], + ["user-deletion-2026-01-01T12-00-00.log", "delete", "2026-01-01T12-00-00"], ])("%s is a %s log from %s", (name, kind, timestamp) => { expect(classifyLogFile(name)).toEqual({ kind: kind as never, timestamp }); }); - test.each([["random.log"], ["migration.log"], ["notes.txt"]])( + test.each([["random.log"], ["import.log"], ["notes.txt"]])( "%s is unrecognized rather than a parse failure", (name) => { expect(classifyLogFile(name)).toEqual({ kind: "unknown", timestamp: "" }); @@ -60,12 +63,12 @@ describe("listLogFiles", () => { }); test("reports kind, timestamp, size and entry count per file", () => { - writeLog("migration-2026-01-01T12-00-00.log", [{ userId: "u1" }, { userId: "u2" }]); + writeLog("import-2026-01-01T12-00-00.log", [{ userId: "u1" }, { userId: "u2" }]); const [file] = listLogFiles(); expect(file).toMatchObject({ - name: "migration-2026-01-01T12-00-00.log", - kind: "migration", + name: "import-2026-01-01T12-00-00.log", + kind: "import", timestamp: "2026-01-01T12-00-00", entryCount: 2, }); @@ -73,11 +76,11 @@ describe("listLogFiles", () => { }); test("ignores files that are not logs", () => { - writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); - fs.writeFileSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"), "[]"); + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); + fs.writeFileSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"), "[]"); fs.writeFileSync(path.join(getLogDir(), "notes.txt"), "hi"); - expect(listLogFiles().map((file) => file.name)).toEqual(["migration-2026-01-01T12-00-00.log"]); + expect(listLogFiles().map((file) => file.name)).toEqual(["import-2026-01-01T12-00-00.log"]); }); test("ignores subdirectories", () => { @@ -86,9 +89,9 @@ describe("listLogFiles", () => { }); test("returns the newest run first", () => { - writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); - writeLog("migration-2026-03-01T12-00-00.log", [{ a: 1 }]); - writeLog("migration-2026-02-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-03-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-02-01T12-00-00.log", [{ a: 1 }]); expect(listLogFiles().map((file) => file.timestamp)).toEqual([ "2026-03-01T12-00-00", @@ -97,21 +100,21 @@ describe("listLogFiles", () => { ]); }); - // Sorting on the filename would put every "user-deletion-" ahead of every + // Sorting on the filename would put every "import-" ahead of every // "migration-", regardless of when the runs actually happened. test("orders by timestamp across log kinds, not by the name's prefix", () => { - writeLog("user-deletion-2026-01-30T17-02-51.log", [{ a: 1 }]); - writeLog("migration-2026-02-01T09-14-22.log", [{ a: 1 }]); + writeLog("import-2026-01-30T17-02-51.log", [{ a: 1 }]); + writeLog("export-2026-02-01T09-14-22.log", [{ a: 1 }]); - expect(listLogFiles().map((file) => file.kind)).toEqual(["migration", "deletion"]); + expect(listLogFiles().map((file) => file.kind)).toEqual(["export", "import"]); }); test("sorts unrecognized names last", () => { writeLog("something-else.log", [{ a: 1 }]); - writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); expect(listLogFiles().map((file) => file.name)).toEqual([ - "migration-2026-01-01T12-00-00.log", + "import-2026-01-01T12-00-00.log", "something-else.log", ]); }); @@ -130,15 +133,15 @@ describe("listLogFiles", () => { describe("findLogFile", () => { beforeEach(() => { - writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); }); test("finds a log by name", () => { - expect(findLogFile("migration-2026-01-01T12-00-00.log")?.entryCount).toBe(1); + expect(findLogFile("import-2026-01-01T12-00-00.log")?.entryCount).toBe(1); }); test("accepts a path and matches on the basename", () => { - expect(findLogFile("./logs/migration-2026-01-01T12-00-00.log")?.entryCount).toBe(1); + expect(findLogFile("./logs/import-2026-01-01T12-00-00.log")?.entryCount).toBe(1); }); test("returns nothing for a name that is not there", () => { diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.ts b/packages/cli-core/src/commands/migrate/lib/log-files.ts index 1498ddd36..af87c6712 100644 --- a/packages/cli-core/src/commands/migrate/lib/log-files.ts +++ b/packages/cli-core/src/commands/migrate/lib/log-files.ts @@ -10,15 +10,27 @@ import fs from "node:fs"; import path from "node:path"; import { getLogDir } from "./logger.ts"; -/** The run that produced a log file, read from its filename prefix. */ -export type LogKind = "migration" | "deletion" | "export" | "unknown"; +/** + * The run that produced a log file, read from its filename prefix. + * + * The kinds are the command names — `migrate import` writes `import-*.log` — + * so a listing points straight at the command that produced each line. + */ +export type LogKind = "export" | "import" | "delete" | "unknown"; -const FILENAME_PATTERN = /^(migration|user-deletion|export)-(.+)\.log$/; +const FILENAME_PATTERN = /^(export|import|delete|migration|user-deletion)-(.+)\.log$/; +/** + * `migration-` and `user-deletion-` are the names the standalone tool and + * earlier CLI builds wrote. They still classify, so a directory of older logs + * lists and converts rather than reading as "unknown". + */ const KIND_BY_PREFIX: Record = { - migration: "migration", - "user-deletion": "deletion", export: "export", + import: "import", + delete: "delete", + migration: "import", + "user-deletion": "delete", }; export type LogFile = { @@ -84,7 +96,7 @@ export function listLogFiles(): LogFile[] { // Sort on the timestamp, not the filename: the kind prefix sorts first in a // filename comparison, which would interleave a run from January ahead of one - // from March purely because "user-deletion" > "migration". Timestamps are + // from March purely because "import" > "export". Timestamps are // ISO-ish and zero-padded, so lexical order is chronological. Names without // one sort last, then alphabetically. return files.sort( diff --git a/packages/cli-core/src/commands/migrate/lib/logger.test.ts b/packages/cli-core/src/commands/migrate/lib/logger.test.ts index db973f00a..eb153218d 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.test.ts @@ -35,7 +35,7 @@ beforeEach(() => { function readEntries(): Record[] { return fs - .readFileSync(getLogFilePath("migration", DATE_TIME), "utf-8") + .readFileSync(getLogFilePath("import", DATE_TIME), "utf-8") .trim() .split("\n") .map((line) => JSON.parse(line) as Record); @@ -47,8 +47,8 @@ describe("log file paths", () => { }); test("replaces the timestamp's colons so the name is valid on Windows", () => { - expect(path.basename(getLogFilePath("migration", DATE_TIME))).toBe( - "migration-2026-01-01T12-00-00.log", + expect(path.basename(getLogFilePath("import", DATE_TIME))).toBe( + "import-2026-01-01T12-00-00.log", ); }); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.ts b/packages/cli-core/src/commands/migrate/lib/logger.ts index 72cd98372..54d0d54a3 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.ts @@ -28,6 +28,20 @@ export function getLogDir(): string { return path.join(process.cwd(), "logs"); } +/** + * The log directory the way the user would type it from here. + * + * Relative (`./logs`) when it sits under the current directory, absolute when + * it does not — a path the reader can paste either way, without a home + * directory's worth of prefix on the common case. + */ +export function displayLogDir(): string { + const dir = getLogDir(); + const relative = path.relative(process.cwd(), dir); + if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) return dir; + return `.${path.sep}${relative}`; +} + /** Absolute path of the log file a run with this timestamp writes to. */ export function getLogFilePath(logFile: string, dateTime: string): string { // Colons are illegal in Windows filenames, and the timestamp is an ISO string. @@ -59,13 +73,13 @@ export function errorLogger(payload: ErrorPayload, dateTime: string): void { status: payload.status, error: err.longMessage ?? err.message, }; - appendToLogFile(getLogFilePath("migration", dateTime), entry); + appendToLogFile(getLogFilePath("import", dateTime), entry); } } /** Writes a user that failed schema validation before any API call. */ export function validationLogger(payload: ValidationErrorPayload, dateTime: string): void { - appendToLogFile(getLogFilePath("migration", dateTime), { + appendToLogFile(getLogFilePath("import", dateTime), { userId: payload.userId, status: "fail" as const, error: payload.error, @@ -76,25 +90,25 @@ export function validationLogger(payload: ValidationErrorPayload, dateTime: stri /** Writes the outcome of one import attempt. */ export function importLogger(entry: ImportLogEntry, dateTime: string): void { - appendToLogFile(getLogFilePath("migration", dateTime), entry); + appendToLogFile(getLogFilePath("import", dateTime), entry); } /** * Writes the outcome of one deletion attempt. * - * A separate `user-deletion-` file rather than another line in the migration - * log: undoing a migration is its own run, and mixing the two would make - * "what did this import do" unanswerable after an undo. + * A separate `delete-` file rather than another line in the import log: undoing + * a migration is its own run, and mixing the two would make "what did this + * import do" unanswerable after an undo. */ export function deleteLogger(entry: DeleteLogEntry, dateTime: string): void { - appendToLogFile(getLogFilePath("user-deletion", dateTime), entry); + appendToLogFile(getLogFilePath("delete", dateTime), entry); } /** * Writes the outcome of exporting one user. * - * Its own `export-` file for the same reason deletions get theirs: an export - * is a distinct run, and `migrate logs list` reports each kind separately. + * Its own `export-` file for the same reason deletes get theirs: an export is a + * distinct run, and `migrate logs list` reports each kind separately. */ export function exportLogger(entry: ExportLogEntry, dateTime: string): void { appendToLogFile(getLogFilePath("export", dateTime), entry); @@ -109,6 +123,6 @@ export function deleteErrorLogger(payload: ErrorPayload, dateTime: string): void status: payload.status, error: err.longMessage ?? err.message, }; - appendToLogFile(getLogFilePath("user-deletion", dateTime), entry); + appendToLogFile(getLogFilePath("delete", dateTime), entry); } } diff --git a/packages/cli-core/src/commands/migrate/logs/list.ts b/packages/cli-core/src/commands/migrate/logs/list.ts index 1ef7f9405..ae0f53928 100644 --- a/packages/cli-core/src/commands/migrate/logs/list.ts +++ b/packages/cli-core/src/commands/migrate/logs/list.ts @@ -6,11 +6,20 @@ * an agent a read-only way to inspect a migration without parsing NDJSON. */ -import { cyan, dim } from "../../../lib/color.ts"; +import { bold, cyan, dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { withGutter } from "../../../lib/spinner.ts"; -import { formatSize, listLogFiles, type LogFile } from "../lib/log-files.ts"; -import { getLogDir } from "../lib/logger.ts"; +import { formatSize, listLogFiles, type LogFile, type LogKind } from "../lib/log-files.ts"; +import { displayLogDir } from "../lib/logger.ts"; + +/** Every kind a log file can be, and what one entry in it records. */ +const KIND_LEGEND: Record, string> = { + export: "One entry per user pulled from the source platform.", + import: "One entry per user created in Clerk, with any error.", + delete: "One entry per user removed from Clerk, with any error.", +}; + +const legendWidth = Math.max(...Object.keys(KIND_LEGEND).map((kind) => kind.length)) + 2; export type LogsListOptions = { json?: boolean; @@ -27,6 +36,22 @@ function toJson(files: LogFile[]) { })); } +/** + * The filename stamp as a date a human reads at a glance. + * + * The stamp is UTC (`getDateTimeStamp` is an ISO string with its colons swapped + * for filename-legal dashes), so it is parsed as UTC and rendered in the + * viewer's own zone — "which run was that" is a question about local time. + * Returns the raw stamp for anything unparseable rather than printing + * "Invalid Date". + */ +export function formatTimestamp(stamp: string): string { + if (!stamp) return ""; + const date = new Date(`${stamp.replace(/T(\d{2})-(\d{2})-(\d{2})$/, "T$1:$2:$3")}Z`); + if (Number.isNaN(date.getTime())) return stamp; + return date.toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short" }); +} + export async function list(options: LogsListOptions = {}): Promise { const files = listLogFiles(); @@ -37,32 +62,64 @@ export async function list(options: LogsListOptions = {}): Promise { await withGutter("Listing migration logs", async () => { if (files.length === 0) { - log.info(`No migration logs in ${getLogDir()}.`); + log.info(`No migration logs in ${displayLogDir()}.`); return; } - const kindWidth = Math.max(...files.map((file) => file.kind.length), "TYPE".length) + 2; - const timeWidth = - Math.max(...files.map((file) => file.timestamp.length), "TIMESTAMP".length) + 2; - const sizeWidth = Math.max(...files.map((file) => formatSize(file.sizeBytes).length), 4) + 2; + const rows = files.map((file) => ({ + name: file.name, + kind: file.kind, + when: formatTimestamp(file.timestamp), + size: formatSize(file.sizeBytes), + entries: String(file.entryCount), + })); + + const width = (header: string, pick: (row: (typeof rows)[number]) => string) => + Math.max(header.length, ...rows.map((row) => pick(row).length)) + 2; + + /** + * Pads to the visible width, then colours. Colouring first would count the + * ANSI escape bytes towards the width and pull every later column left. + */ + const column = (text: string, size: number, paint: (value: string) => string) => + paint(text) + " ".repeat(Math.max(0, size - text.length)); + + const nameWidth = width("FILE", (row) => row.name); + const kindWidth = width("TYPE", (row) => row.kind); + const whenWidth = width("DATE", (row) => row.when); + const sizeWidth = width("SIZE", (row) => row.size); + + log.info("Each log represents a user export, user import, or a user delete run."); + log.info("Each log consists of a single NDJSON entry per user."); + log.blank(); log.info( - dim("TYPE".padEnd(kindWidth)) + - dim("TIMESTAMP".padEnd(timeWidth)) + + dim("FILE".padEnd(nameWidth)) + + dim("TYPE".padEnd(kindWidth)) + + dim("DATE".padEnd(whenWidth)) + dim("SIZE".padEnd(sizeWidth)) + dim("ENTRIES"), ); - for (const file of files) { + for (const row of rows) { log.info( - cyan(file.kind.padEnd(kindWidth)) + - (file.timestamp || dim("—")).padEnd(timeWidth) + - dim(formatSize(file.sizeBytes).padEnd(sizeWidth)) + - String(file.entryCount), + column(row.name, nameWidth, cyan) + + row.kind.padEnd(kindWidth) + + column(row.when || "—", whenWidth, row.when ? (value) => value : dim) + + column(row.size, sizeWidth, dim) + + row.entries, ); } - log.info(""); - log.info(dim(`${files.length} log file${files.length === 1 ? "" : "s"} in ${getLogDir()}`)); + log.blank(); + log.info(`${files.length} log file${files.length === 1 ? "" : "s"} in ${displayLogDir()}`); + log.blank(); + + // A listing only shows the kinds that happen to be present, so the legend + // is fixed: it also answers "what else could be here". + log.info(bold("Log types:")); + for (const [kind, description] of Object.entries(KIND_LEGEND)) { + log.info(` ${cyan(bold(kind))}${" ".repeat(legendWidth - kind.length)}${description}`); + } }); } diff --git a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts index 4a7cf9eb1..72da1cffc 100644 --- a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts @@ -44,8 +44,8 @@ const captured = useCaptureLog(); let workDir: string; let originalCwd: string; -const MIGRATION = "migration-2026-01-01T12-00-00.log"; -const DELETION = "user-deletion-2026-02-01T12-00-00.log"; +const IMPORT = "import-2026-01-01T12-00-00.log"; +const DELETE = "delete-2026-02-01T12-00-00.log"; beforeAll(() => { originalMode = getMode(); @@ -80,8 +80,8 @@ function writeLog(name: string, entries: unknown[]): void { describe("logs clean", () => { test("prompts before deleting anything", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); await clean(); @@ -93,7 +93,7 @@ describe("logs clean", () => { // Deleting on a stray enter would be the wrong default for a destructive // command sitting next to `clerk migrate delete`. test("defaults the prompt to no", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await clean(); @@ -101,16 +101,16 @@ describe("logs clean", () => { }); test("declining leaves every file in place", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); mockConfirm.mockResolvedValue(false); await expect(clean()).rejects.toThrow(UserAbortError); - expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); }); test("-y skips the prompt entirely", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await clean({ yes: true }); @@ -126,40 +126,40 @@ describe("logs clean", () => { describe("logs convert", () => { test("offers a multiselect when given neither files nor --all", async () => { - writeLog(MIGRATION, [{ a: 1 }, { b: 2 }]); - writeLog(DELETION, [{ a: 1 }]); - mockMultiselect.mockResolvedValue([MIGRATION]); + writeLog(IMPORT, [{ a: 1 }, { b: 2 }]); + writeLog(DELETE, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([IMPORT]); await convert(); const options = mockMultiselect.mock.calls[0]?.[0]?.options; - expect(options?.map((option) => option.value)).toEqual([DELETION, MIGRATION]); + expect(options?.map((option) => option.value)).toEqual([DELETE, IMPORT]); expect(options?.[1]?.hint).toBe("2 entries"); }); test("converts only what was selected", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ a: 1 }]); - mockMultiselect.mockResolvedValue([MIGRATION]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([IMPORT]); await convert(); expect(fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json"))).toEqual([ - "migration-2026-01-01T12-00-00.json", + "import-2026-01-01T12-00-00.json", ]); }); test("selecting nothing aborts without writing", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); mockMultiselect.mockResolvedValue([]); await expect(convert()).rejects.toThrow(UserAbortError); - expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); }); test("does not prompt when --all was passed", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await convert({ all: true }); @@ -168,9 +168,9 @@ describe("logs convert", () => { }); test("does not prompt when files were named", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); - await convert({ files: [MIGRATION] }); + await convert({ files: [IMPORT] }); expect(mockMultiselect).not.toHaveBeenCalled(); }); @@ -180,7 +180,7 @@ describe("logs convert", () => { // with `└ Failed`. Declining a prompt is not a failure, so the two must not swap. describe("cancelling inside the gutter", () => { test("declining the logs clean confirm closes with Paused, not Failed", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); mockConfirm.mockResolvedValue(false); await expect(clean()).rejects.toThrow(UserAbortError); @@ -190,7 +190,7 @@ describe("cancelling inside the gutter", () => { }); test("selecting nothing in the logs convert multiselect closes with Paused", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); mockMultiselect.mockResolvedValue([]); await expect(convert()).rejects.toThrow(UserAbortError); diff --git a/packages/cli-core/src/commands/migrate/logs/logs.test.ts b/packages/cli-core/src/commands/migrate/logs/logs.test.ts index d0db18fb6..2e80ce5d5 100644 --- a/packages/cli-core/src/commands/migrate/logs/logs.test.ts +++ b/packages/cli-core/src/commands/migrate/logs/logs.test.ts @@ -8,10 +8,13 @@ import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { clean } from "./clean.ts"; import { convert } from "./convert.ts"; -import { list } from "./list.ts"; +import { formatTimestamp, list } from "./list.ts"; const captured = useCaptureLog(); +const ANSI_ESCAPE_PATTERN = new RegExp(String.raw`\u001b\[[0-9;]*m`, "g"); +const stripAnsi = (value: string) => value.replace(ANSI_ESCAPE_PATTERN, ""); + let workDir: string; let originalCwd: string; @@ -39,8 +42,8 @@ function writeLog(name: string, entries: unknown[]): void { ); } -const MIGRATION = "migration-2026-01-01T12-00-00.log"; -const DELETION = "user-deletion-2026-02-01T12-00-00.log"; +const IMPORT = "import-2026-01-01T12-00-00.log"; +const DELETE = "delete-2026-02-01T12-00-00.log"; describe("logs list", () => { test("says so plainly when there is no logs directory", async () => { @@ -54,42 +57,66 @@ describe("logs list", () => { expect(captured.err).toContain("No migration logs in"); }); - test("reports type, timestamp, size and entry count", async () => { - writeLog(MIGRATION, [{ userId: "u1" }, { userId: "u2" }, { userId: "u3" }]); + test("reports file, type, date, size and entry count", async () => { + writeLog(IMPORT, [{ userId: "u1" }, { userId: "u2" }, { userId: "u3" }]); await list(); + expect(captured.err).toContain("FILE"); expect(captured.err).toContain("TYPE"); - expect(captured.err).toContain("TIMESTAMP"); + expect(captured.err).toContain("DATE"); expect(captured.err).toContain("SIZE"); expect(captured.err).toContain("ENTRIES"); - expect(captured.err).toContain("migration"); - expect(captured.err).toContain("2026-01-01T12-00-00"); + expect(captured.err).toContain(IMPORT); + expect(captured.err).toContain("import"); + expect(captured.err).toContain(formatTimestamp("2026-01-01T12-00-00")); expect(captured.err).toMatch(/\bB\b/); expect(captured.err).toContain("3"); }); + // The filename stamp is for sorting and for `logs convert`; the column a + // human scans should read like a date. + test("shows the date rendered, not the raw filename stamp", async () => { + writeLog(IMPORT, [{ userId: "u1" }]); + + await list(); + + const dateColumn = stripAnsi(captured.err) + .split("\n") + .find((line) => line.includes(IMPORT)); + expect(dateColumn?.replace(IMPORT, "")).not.toContain("2026-01-01T12-00-00"); + }); + + test("reports the log directory relative to the current directory", async () => { + writeLog(IMPORT, [{ userId: "u1" }]); + + await list(); + + expect(stripAnsi(captured.err)).toContain(`1 log file in .${path.sep}logs`); + expect(captured.err).not.toContain(getLogDir()); + }); + test("lists every log kind", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); await list(); - expect(captured.err).toContain("migration"); - expect(captured.err).toContain("deletion"); + expect(captured.err).toContain("import"); + expect(captured.err).toContain("delete"); expect(captured.err).toContain("2 log files"); }); test("--json emits a machine-readable listing on stdout", async () => { - writeLog(MIGRATION, [{ userId: "u1" }]); + writeLog(IMPORT, [{ userId: "u1" }]); await list({ json: true }); const parsed = JSON.parse(captured.out) as Record[]; expect(parsed).toHaveLength(1); expect(parsed[0]).toMatchObject({ - name: MIGRATION, - kind: "migration", + name: IMPORT, + kind: "import", timestamp: "2026-01-01T12-00-00", entry_count: 1, }); @@ -109,22 +136,22 @@ describe("logs clean", () => { // Tests run non-TTY, which is the same signal an agent gives. test("refuses without -y when it cannot prompt, and explains", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await expect(clean()).rejects.toThrow(/cannot prompt here.*Pass -y/s); - expect(fs.existsSync(path.join(getLogDir(), MIGRATION))).toBe(true); + expect(fs.existsSync(path.join(getLogDir(), IMPORT))).toBe(true); }); test("names how many files are at stake when it refuses", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); await expect(clean()).rejects.toThrow(/2 log files/); }); test("-y deletes the log files and reports the count", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); await clean({ yes: true }); @@ -133,12 +160,12 @@ describe("logs clean", () => { }); test("leaves converted JSON output alone", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - fs.writeFileSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"), "[]"); + writeLog(IMPORT, [{ a: 1 }]); + fs.writeFileSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"), "[]"); await clean({ yes: true }); - expect(fs.readdirSync(getLogDir())).toEqual(["migration-2026-01-01T12-00-00.json"]); + expect(fs.readdirSync(getLogDir())).toEqual(["import-2026-01-01T12-00-00.json"]); }); }); @@ -149,42 +176,42 @@ describe("logs convert", () => { }); test("writes a JSON array alongside the original, leaving it intact", async () => { - writeLog(MIGRATION, [{ userId: "u1" }, { userId: "u2" }]); + writeLog(IMPORT, [{ userId: "u1" }, { userId: "u2" }]); - await convert({ files: [MIGRATION] }); + await convert({ files: [IMPORT] }); - const output = path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"); + const output = path.join(getLogDir(), "import-2026-01-01T12-00-00.json"); expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([ { userId: "u1" }, { userId: "u2" }, ]); - expect(fs.existsSync(path.join(getLogDir(), MIGRATION))).toBe(true); + expect(fs.existsSync(path.join(getLogDir(), IMPORT))).toBe(true); expect(captured.err).toContain("Originals left in place"); }); test("--all converts every log file", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ b: 2 }]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ b: 2 }]); await convert({ all: true }); const written = fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json")); expect(written.sort()).toEqual([ - "migration-2026-01-01T12-00-00.json", - "user-deletion-2026-02-01T12-00-00.json", + "delete-2026-02-01T12-00-00.json", + "import-2026-01-01T12-00-00.json", ]); }); test("accepts a path and resolves it against ./logs/", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); - await convert({ files: [`./logs/${MIGRATION}`] }); + await convert({ files: [`./logs/${IMPORT}`] }); - expect(fs.existsSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"))).toBe(true); + expect(fs.existsSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"))).toBe(true); }); test("fails clearly on a file that is not there", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await expect(convert({ files: ["migration-nope.log"] })).rejects.toThrow(CliError); }); @@ -192,26 +219,26 @@ describe("logs convert", () => { // Silently dropping the line would leave a JSON array that looks complete. test("reports a malformed line by number and converts the rest", async () => { fs.mkdirSync(getLogDir(), { recursive: true }); - fs.writeFileSync(path.join(getLogDir(), MIGRATION), '{"a":1}\n{"b":\n{"c":3}\n'); + fs.writeFileSync(path.join(getLogDir(), IMPORT), '{"a":1}\n{"b":\n{"c":3}\n'); - await convert({ files: [MIGRATION] }); + await convert({ files: [IMPORT] }); - expect(captured.err).toContain(`${MIGRATION}:2`); + expect(captured.err).toContain(`${IMPORT}:2`); expect(captured.err).toContain("1 malformed line skipped"); - const output = path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"); + const output = path.join(getLogDir(), "import-2026-01-01T12-00-00.json"); expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([{ a: 1 }, { c: 3 }]); }); test("refuses without a target when it cannot prompt, naming the alternatives", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await expect(convert()).rejects.toThrow(/cannot prompt here/); - expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); }); test("reports the entry count per converted file", async () => { - writeLog(MIGRATION, [{ a: 1 }, { b: 2 }, { c: 3 }]); + writeLog(IMPORT, [{ a: 1 }, { b: 2 }, { c: 3 }]); await convert({ all: true }); @@ -232,7 +259,7 @@ describe("human-mode frame", () => { }); test("logs list wraps its output in an intro/outro gutter", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await list(); @@ -243,7 +270,7 @@ describe("human-mode frame", () => { }); test("--json stays outside the gutter, on stdout only", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await list({ json: true }); @@ -252,7 +279,7 @@ describe("human-mode frame", () => { }); test("a failure inside logs convert closes with Failed and still throws", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await expect(convert({ files: ["nope.log"] })).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 7fa811d46..049a553ca 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -144,7 +144,7 @@ describe("run", () => { const logs = fs.readdirSync(getLogDir()); expect(logs).toHaveLength(1); - expect(logs[0]).toMatch(/^migration-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + expect(logs[0]).toMatch(/^import-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); const entries = fs .readFileSync(path.join(getLogDir(), logs[0] as string), "utf-8") diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index e37cde385..ca0576022 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -484,7 +484,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const limits = resolveLimits(secretKey); const dateTime = getDateTimeStamp(); - const logFile = getLogFilePath("migration", dateTime); + const logFile = getLogFilePath("import", dateTime); const { users: loaded, validationFailed } = await withSpinner( `Loading users from ${file}...`, From b1fec90e5f400a86e789fd100bde706f60f69a92 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 8 Sep 2026 11:31:37 -0400 Subject: [PATCH 024/141] refactor: move the redaction placeholder into constants `[REDACTED]` was local to `lib/users.ts`; `clerk migrate settings` needs the same string so a withheld credential reads identically wherever the CLI declines to show one. --- packages/cli-core/src/lib/constants.ts | 12 ++++++++++++ packages/cli-core/src/lib/users.ts | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/lib/constants.ts b/packages/cli-core/src/lib/constants.ts index f88439df2..6505ab035 100644 --- a/packages/cli-core/src/lib/constants.ts +++ b/packages/cli-core/src/lib/constants.ts @@ -55,3 +55,15 @@ export const NPM_REGISTRY_URL = "https://registry.npmjs.org/"; /** Event ingestion endpoint (telemetry-service worker → BigQuery). */ export const DEFAULT_TELEMETRY_ENDPOINT = "https://clerk-telemetry.com/v1/event"; export const TELEMETRY_TIMEOUT_MS = 1000; + +// ── Redaction ───────────────────────────────────────────────────────────── + +/** + * What a withheld secret displays as, everywhere the CLI shows one. + * + * Square brackets rather than a mask or a truncation: a row of dots or a + * head-and-tail (`aVer…3456`) reads as a value, and the reader has to work out + * that it is not one. Used by `clerk users create --dry-run` and by + * `clerk migrate settings`. + */ +export const REDACTED = "[REDACTED]"; diff --git a/packages/cli-core/src/lib/users.ts b/packages/cli-core/src/lib/users.ts index d43a767e6..5d9345f18 100644 --- a/packages/cli-core/src/lib/users.ts +++ b/packages/cli-core/src/lib/users.ts @@ -1,8 +1,8 @@ import { bapiRequest } from "./bapi.ts"; +import { REDACTED } from "./constants.ts"; import { ERROR_CODE, throwUsageError } from "./errors.ts"; const USERS_INVALID_JSON_MESSAGE = "User payload must be a JSON object."; -const REDACTED = "[REDACTED]"; const DIRECT_REDACT_KEYS = new Set(["password", "code"]); const OBJECT_REDACT_KEYS = new Set(["private_metadata", "unsafe_metadata"]); From f9c0b0826b6e0e46a4e980e615d7d394c4b663d7 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 8 Sep 2026 11:31:50 -0400 Subject: [PATCH 025/141] feat(migrate): accept Firebase's own variable names and restyle settings list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Firebase hands its scrypt parameters over as `base64_signer_key`, `rounds` and friends, and every guide — Clerk's own standalone script included — tells you to paste them into `.env` under those names. Those spellings, and their `FIREBASE_` prefixed forms, now resolve as aliases behind the `CLERK_FIREBASE_*` variables, read from one registry shared by the listing and the import. `migrate settings list` gains the orientation lines, count and next-steps block the CLI's other listings carry, attributes an environment value to the env file it actually came from (Bun loads `.env.local` before the CLI runs, so "`ROUNDS` env var" named nothing the reader could edit), names the alias alongside the file, and withholds credentials as `[REDACTED]` rather than a head-and-tail truncation. --- .../cli-core/src/commands/migrate/README.md | 95 ++++++++++++++----- .../src/commands/migrate/lib/env-file.test.ts | 54 ++++++++++- .../src/commands/migrate/lib/env-file.ts | 43 ++++++++- .../migrate/lib/firebase-hash.test.ts | 39 ++++++++ .../src/commands/migrate/lib/firebase-hash.ts | 23 +++-- .../src/commands/migrate/readme.test.ts | 13 ++- .../src/commands/migrate/settings/list.ts | 47 +++++++-- .../src/commands/migrate/settings/registry.ts | 45 +++++++-- .../migrate/settings/settings.test.ts | 62 ++++++++++-- packages/cli-core/src/lib/dotenv.ts | 6 +- packages/cli-core/src/lib/next-steps.ts | 4 + 11 files changed, 363 insertions(+), 68 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index bd601ca00..018cd76d6 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -540,36 +540,65 @@ transformer maps to `userId`. ### `clerk migrate settings` What a run in this directory would pick up, and where each value comes from. +Listing is the default, because it is the read-only one: a bare `clerk migrate +settings` shows, never changes. ```sh clerk migrate settings # list +clerk migrate settings list --json clerk migrate settings set transformer firebase -clerk migrate settings set firebase-signer-key abc123… +clerk migrate settings set firebase-signer-key abc123 clerk migrate settings clear -y ``` -``` -SETTING VALUE SOURCE DESCRIPTION -transformer firebase clerk config Source platform the export came from -file users.json clerk config Export file to import users from -firebase-signer-key aVer…3456 .env.clerk-migrate Firebase base64 signer key -firebase-rounds — not set Firebase scrypt rounds -``` +| Subcommand | Takes | Description | +| ----------------------------- | ---------------- | --------------------------------------------------------- | +| `settings list` | `--json` | Every setting, its value and the source it resolved from | +| `settings set ` | ` ` | Change one setting | +| `settings clear` | `-y, --yes` | Forget this project's settings and delete its credentials | -Setting names are kebab-case and identical to the `clerk migrate import` flag each one -backs, so `firebase-signer-key` here is `--firebase-signer-key` there rather -than a second spelling to learn. The description column carries the prose. +Setting names are kebab-case and identical to the `clerk migrate import` flag +each one backs, so `firebase-signer-key` here is `--firebase-signer-key` there +rather than a second spelling to learn. The description column carries the +prose. The source column is the point. A migration reads from flags, the environment, two of the app's env files and the CLI's config, so when a run picks up a stale -value the question is never "what is it" but "which of those won". +value the question is never "what is it" but "which of those won". A value that +arrived under one of the accepted aliases names the variable alongside the file. + +It names a **file** wherever there is one to name. Bun loads `.env`/`.env.local` +into the environment before the CLI runs, so a value a developer typed into +`.env.local` would otherwise be reported as "`ROUNDS` env var" — true, and no +help to someone asking which file to edit. Attribution is by value: a file +holding the same key with a _different_ value lost to something exported in the +shell, and that row keeps saying `ROUNDS env var`, because that is exactly the +case this column exists to catch. + +A setting with no value leaves the column empty rather than filling it with a +placeholder — the source column already reads `not set` on that row, and the +blank is what makes the settings that do have a value stand out. + +It closes on next steps naming the two commands that change what it just +showed — the same block `clerk mcp list` and `clerk whoami` end on, and human +only. The full command surface stays in `--help`. -| Command | Description | -| ----------------------------- | ----------------------------------------------------- | -| `settings` / `settings list` | Show every setting, its value and its source | -| `settings list --json` | The same, machine-readable | -| `settings set ` | Change one setting | -| `settings clear [-y]` | Forget this project's settings and delete its secrets | +``` +A migration run in this directory picks these up unless a flag overrides them. +Each setting is named after the `clerk migrate import` flag it stands in for. + +SETTING VALUE SOURCE DESCRIPTION +transformer firebase clerk config Source platform the export came from +file users.json clerk config Export file to import users from +firebase-signer-key [REDACTED] .env.clerk-migrate Firebase base64 signer key +firebase-rounds 8 .env.local (ROUNDS) Firebase scrypt rounds +firebase-mem-cost 14 MEM_COST env var Firebase scrypt memory cost + +4 of 7 settings set. Credentials are shown redacted. + + → Run `clerk migrate settings set ` to change one + → Run `clerk migrate settings clear` to forget them all, credentials included +``` #### Where each setting is kept @@ -586,8 +615,11 @@ being migrated and does not belong in the file its developers read daily. The CLI adds it to `.gitignore` the first time it writes it, and deletes it when `settings clear` removes the last value. -Credentials are redacted wherever they are displayed, including under `--json`, -so the output is safe to paste into an issue. +Credentials are withheld wherever they are displayed, including under `--json`, +so the output is safe to paste into an issue. They display as `[REDACTED]` — +the same thing `clerk users create --dry-run` prints for a password — rather +than a truncation like `aVer…3456`: the source column already says which value +is in play, and a partial secret is one the reader has to recognise as partial. ### Custom transformers (`--transformer-file`) @@ -681,12 +713,23 @@ that file is not a secret store. To avoid re-passing all four on every run, set them once with [`clerk migrate settings`](#clerk-migrate-settings), or export them yourself: -| Variable | Flag | -| ------------------------------- | --------------------------- | -| `CLERK_FIREBASE_SIGNER_KEY` | `--firebase-signer-key` | -| `CLERK_FIREBASE_SALT_SEPARATOR` | `--firebase-salt-separator` | -| `CLERK_FIREBASE_ROUNDS` | `--firebase-rounds` | -| `CLERK_FIREBASE_MEM_COST` | `--firebase-mem-cost` | +| Flag | Variable | Also accepted | +| --------------------------- | ------------------------------- | --------------------------------------------------------- | +| `--firebase-signer-key` | `CLERK_FIREBASE_SIGNER_KEY` | `FIREBASE_BASE64_SIGNER_KEY`, `BASE64_SIGNER_KEY` | +| `--firebase-salt-separator` | `CLERK_FIREBASE_SALT_SEPARATOR` | `FIREBASE_BASE64_SALT_SEPARATOR`, `BASE64_SALT_SEPARATOR` | +| `--firebase-rounds` | `CLERK_FIREBASE_ROUNDS` | `FIREBASE_ROUNDS`, `ROUNDS` | +| `--firebase-mem-cost` | `CLERK_FIREBASE_MEM_COST` | `FIREBASE_MEM_COST`, `MEM_COST` | + +The unprefixed names are what Firebase itself calls these (`base64_signer_key`, +`rounds`) and what every guide, Clerk's own standalone migration script +included, tells you to paste into `.env`. Someone who followed one has the +values the import needs, spelled the way the source platform spells them, so +they are read rather than reported as missing. + +They are a fallback, not a synonym: a `CLERK_FIREBASE_*` variable wins wherever +both exist, and `clerk migrate settings` names the variable it read alongside +the file — `ROUNDS` is generic enough to mean something else in an app that was +never a Firebase project, and that should be visible rather than silent. Resolution order is flag, then exported variable, then `.env.clerk-migrate`, then the app's `.env.local`/`.env`. The sources can be mixed as long as all four diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.test.ts b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts index 7c190a9d7..855cc9274 100644 --- a/packages/cli-core/src/commands/migrate/lib/env-file.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts @@ -71,7 +71,11 @@ describe("findMigrateEnvValue", () => { await writeMigrateEnvValues({ CLERK_FIREBASE_SIGNER_KEY: "from-file" }, workDir); const located = await findMigrateEnvValue(["CLERK_FIREBASE_SIGNER_KEY"], workDir, {}); - expect(located).toEqual({ value: "from-file", source: MIGRATE_ENV_FILE }); + expect(located).toEqual({ + value: "from-file", + name: "CLERK_FIREBASE_SIGNER_KEY", + source: MIGRATE_ENV_FILE, + }); }); test("beats the app's own .env.local", async () => { @@ -89,12 +93,58 @@ describe("findMigrateEnvValue", () => { const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { CLERK_FIREBASE_ROUNDS: "99", }); - expect(located).toEqual({ value: "99", source: "CLERK_FIREBASE_ROUNDS env var" }); + expect(located).toEqual({ + value: "99", + name: "CLERK_FIREBASE_ROUNDS", + source: "CLERK_FIREBASE_ROUNDS env var", + }); }); test("returns nothing when the setting is absent everywhere", async () => { expect(await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, {})).toBeUndefined(); }); + + // Bun loads `.env.local` into process.env before the CLI runs, so a value a + // developer put in a file arrives looking like an exported variable. Naming + // the variable answers nothing — the question is which file to edit. + describe("attributing an environment value to the file it came from", () => { + test("names the file when it holds the same value", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "8", + }); + expect(located?.source).toBe(".env.local"); + }); + + // The one case the source column exists for: the file lost, so naming it + // would point at the value that is not being used. + test("keeps the variable when the file holds a different value", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "99", + }); + expect(located?.source).toBe("CLERK_FIREBASE_ROUNDS env var"); + }); + + test("prefers the file the runtime would have loaded last", async () => { + fs.writeFileSync(path.join(workDir, ".env"), "CLERK_FIREBASE_ROUNDS=8\n"); + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "8", + }); + expect(located?.source).toBe(".env.local"); + }); + + test("keeps the variable when no file holds it at all", async () => { + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "8", + }); + expect(located?.source).toBe("CLERK_FIREBASE_ROUNDS env var"); + }); + }); }); describe("clearMigrateEnvValues", () => { diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.ts b/packages/cli-core/src/commands/migrate/lib/env-file.ts index 4d7a61f0a..2c15c9163 100644 --- a/packages/cli-core/src/commands/migrate/lib/env-file.ts +++ b/packages/cli-core/src/commands/migrate/lib/env-file.ts @@ -31,6 +31,36 @@ export const MIGRATE_ENV_FILE = ".env.clerk-migrate"; /** Lowest priority first: the migration's own file overrides the app's. */ const MIGRATE_ENV_FILES = [".env", ".env.local", MIGRATE_ENV_FILE] as const; +/** + * The project env file a value in the environment actually came from, if any. + * + * Bun loads `.env`, `.env.local` and friends into `process.env` before the CLI + * runs, so a variable a developer wrote into `.env.local` reaches + * {@link findEnvValue} as an environment variable and gets reported as one. + * That is true but useless: "`ROUNDS` env var" does not tell anyone which of + * their files to edit. + * + * Attribution is by value, not by presence. A file that holds the same key with + * a *different* value lost to something exported in the shell, and saying + * `.env.local` there would name the file that is not winning — the one case + * this column exists to catch. Highest-priority file first, matching the order + * the runtime loaded them in. + */ +async function fileHolding( + cwd: string, + { name, value }: LocatedEnvValue, +): Promise { + for (const envFile of [...MIGRATE_ENV_FILES].reverse()) { + const file = Bun.file(join(cwd, envFile)); + if (!(await file.exists())) continue; + + for (const line of parseEnvFile(await file.text())) { + if (line.type === "entry" && line.key === name && line.value === value) return envFile; + } + } + return undefined; +} + /** Resolves a migration setting: environment first, then the project's env files. */ export async function findMigrateEnvValue( names: string[], @@ -38,8 +68,17 @@ export async function findMigrateEnvValue( env: Record = process.env, ): Promise { const located = await findEnvValue(cwd, names, { env, files: MIGRATE_ENV_FILES }); - if (located) log.debug(`migrate: ${names[0]} from ${located.source}`); - return located; + if (!located) return undefined; + + // `findEnvValue` reports the environment before it reads a file, so a value + // the runtime loaded out of `.env.local` is credited to the variable rather + // than to the file the user would edit. Put the file back. + const source = located.source.endsWith(" env var") + ? ((await fileHolding(cwd, located)) ?? located.source) + : located.source; + + log.debug(`migrate: ${names[0]} from ${source}`); + return { ...located, source }; } /** diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts index 45c65c452..ec5d89fb2 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts @@ -170,3 +170,42 @@ describe("on a firebase run", () => { expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); }); }); + +// Firebase names these `base64_signer_key`, `rounds` and friends, and that is +// how every guide — Clerk's own standalone script included — tells you to write +// them into `.env`. A project that followed one has the values already. +describe("the names Firebase itself uses", () => { + const writeEnvLocal = (contents: string) => + fs.writeFileSync(path.join(workDir, ".env.local"), contents); + + test("reads a set written under the unprefixed names", async () => { + writeEnvLocal("BASE64_SIGNER_KEY=SIGNER\nBASE64_SALT_SEPARATOR=Bw==\nROUNDS=8\nMEM_COST=14\n"); + + expect(await resolveFirebaseHashConfig({}, "firebase")).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + test("reads a set written under the FIREBASE_ prefix", async () => { + writeEnvLocal( + "FIREBASE_BASE64_SIGNER_KEY=SIGNER\nFIREBASE_BASE64_SALT_SEPARATOR=Bw==\n" + + "FIREBASE_ROUNDS=8\nFIREBASE_MEM_COST=14\n", + ); + + expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); + }); + + // The alias is a fallback, not a synonym: `ROUNDS` in an app's own env file + // is not necessarily about Firebase at all. + test("prefers the prefixed variable in the same file", async () => { + writeEnvLocal( + "ROUNDS=99\nCLERK_FIREBASE_ROUNDS=8\nBASE64_SIGNER_KEY=SIGNER\n" + + "BASE64_SALT_SEPARATOR=Bw==\nMEM_COST=14\n", + ); + + expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts index 10b06653d..d32fc3c58 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts @@ -18,15 +18,16 @@ import { throwUsageError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; +import { envNames, findSetting } from "../settings/registry.ts"; import { findMigrateEnvValue } from "./env-file.ts"; import type { FirebaseHashConfig } from "../types.ts"; -/** The `--firebase-*` flags, and the variable each falls back to. */ +/** The `--firebase-*` flags, and the setting each falls back to. */ export const FIREBASE_FLAGS = [ - ["firebaseSignerKey", "--firebase-signer-key", "CLERK_FIREBASE_SIGNER_KEY"], - ["firebaseSaltSeparator", "--firebase-salt-separator", "CLERK_FIREBASE_SALT_SEPARATOR"], - ["firebaseRounds", "--firebase-rounds", "CLERK_FIREBASE_ROUNDS"], - ["firebaseMemCost", "--firebase-mem-cost", "CLERK_FIREBASE_MEM_COST"], + ["firebaseSignerKey", "--firebase-signer-key", "firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator", "firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds", "firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost", "firebase-mem-cost"], ] as const; const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); @@ -39,18 +40,22 @@ export type FirebaseHashFlags = { }; /** - * Overlays the `CLERK_FIREBASE_*` values onto whichever flags were not passed. + * Overlays the saved environment values onto whichever flags were not passed. * - * Resolved through {@link findMigrateEnvValue}: the environment first, then + * The variables come from the settings registry — `CLERK_FIREBASE_*` and the + * unprefixed names Firebase itself uses — so `clerk migrate settings` and the + * import read exactly the same set. Resolved through + * {@link findMigrateEnvValue}: the environment first, then * `.env.clerk-migrate`, then the app's own `.env` files. The signer key is a * Firebase secret, so it is never written to the CLI's config — * `.env.clerk-migrate` is gitignored on creation. */ async function withFirebaseEnv(flags: FirebaseHashFlags): Promise { const merged: FirebaseHashFlags = { ...flags }; - for (const [key, , envVar] of FIREBASE_FLAGS) { + for (const [key, , settingName] of FIREBASE_FLAGS) { if (merged[key] !== undefined) continue; - const located = await findMigrateEnvValue([envVar]); + const setting = findSetting(settingName); + const located = setting && (await findMigrateEnvValue(envNames(setting))); if (!located || located.value.trim() === "") continue; // A non-numeric round count is left to fail the flag's own validation // rather than silently becoming NaN. diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index be0bc5028..4887b0b64 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -32,9 +32,16 @@ function documentedCommands(markdown: string): string[] { // Line continuations first: the Firebase example spans three lines. for (const line of block.replace(/\\\n\s*/g, " ").split("\n")) { const start = line.indexOf("clerk migrate"); - // A command never contains a backtick or a `#`; the sample error output - // that quotes `clerk migrate` mid-sentence does. - if (start !== -1) found.add(line.slice(start).split(/[`#]/)[0]!.trim()); + // A command never contains a backtick, a `#`, or a run of two spaces; + // the sample error output that quotes `clerk migrate` mid-sentence does, + // and so does a pasted listing whose descriptions sit in a padded column. + if (start !== -1) + found.add( + line + .slice(start) + .split(/[`#]|\s{2,}/)[0]! + .trim(), + ); } } diff --git a/packages/cli-core/src/commands/migrate/settings/list.ts b/packages/cli-core/src/commands/migrate/settings/list.ts index f5e9e43ec..049264958 100644 --- a/packages/cli-core/src/commands/migrate/settings/list.ts +++ b/packages/cli-core/src/commands/migrate/settings/list.ts @@ -6,13 +6,20 @@ * environment, two of the app's env files and the CLI's config; when a run uses * a stale value, the question is never "what is it" but "which of those is * winning". Credentials are redacted, so this is safe to paste into an issue. + * + * Laid out like the CLI's other listings — `migrate logs list` and `migrate + * transformers list`: a line or two of orientation, the table, then a count. + * It closes with next steps, the way `mcp list` and `whoami` do, because a + * listing is where someone lands before they know what to type. Those are for + * humans; the full command surface stays in `--help`. */ import { cyan, dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; +import { NEXT_STEPS, printNextSteps } from "../../../lib/next-steps.ts"; import { findMigrateEnvValue } from "../lib/env-file.ts"; import { loadSettings } from "../lib/settings.ts"; -import { displayValue, SETTINGS, type SettingDef } from "./registry.ts"; +import { displayValue, envNames, SETTINGS, type SettingDef } from "./registry.ts"; export type SettingsListOptions = { json?: boolean; @@ -24,6 +31,21 @@ interface ResolvedSetting { source?: string; } +/** + * Names the variable as well as the file when an alias supplied the value. + * + * `.env.local` alone would be a half-answer for a setting that has four + * accepted spellings: the reader has to know *which* line in that file the run + * is reading before they can change it. An exported variable already carries + * its name in `source`. + */ +function describeSource(setting: SettingDef, located: { name: string; source: string }): string { + if (located.name === setting.envVar || located.source.startsWith(located.name)) { + return located.source; + } + return `${located.source} (${located.name})`; +} + async function resolveAll(): Promise { const saved = await loadSettings(); @@ -36,8 +58,10 @@ async function resolveAll(): Promise { : { setting, value: String(value), source: "clerk config" }; } - const located = await findMigrateEnvValue([setting.envVar as string]); - return located ? { setting, value: located.value, source: located.source } : { setting }; + const located = await findMigrateEnvValue(envNames(setting)); + return located + ? { setting, value: located.value, source: describeSource(setting, located) } + : { setting }; }), ); } @@ -73,10 +97,13 @@ export async function list(options: SettingsListOptions = {}): Promise { return; } + // An unset value leaves the column empty rather than filling it with a + // placeholder: the source column already reads "not set" on the same row, and + // an empty cell is what makes the settings that do have a value stand out. const cells = resolved.map(({ setting, value, source }) => ({ setting, name: setting.name, - value: value === undefined ? "—" : displayValue(setting, value), + value: value === undefined ? "" : displayValue(setting, value), unset: value === undefined, source: source ?? "not set", })); @@ -88,6 +115,10 @@ export async function list(options: SettingsListOptions = {}): Promise { const valueWidth = width("VALUE", (c) => c.value); const sourceWidth = width("SOURCE", (c) => c.source); + log.info("A migration run in this directory picks these up unless a flag overrides them."); + log.info("Each setting is named after the `clerk migrate import` flag it stands in for."); + log.blank(); + log.info( column("SETTING", nameWidth, dim) + column("VALUE", valueWidth, dim) + @@ -98,12 +129,16 @@ export async function list(options: SettingsListOptions = {}): Promise { for (const cell of cells) { log.info( column(cell.name, nameWidth, cyan) + - column(cell.value, valueWidth, cell.unset ? dim : (value) => value) + + column(cell.value, valueWidth, (value) => value) + column(cell.source, sourceWidth, dim) + dim(cell.setting.description), ); } + const set = cells.filter((cell) => !cell.unset).length; log.blank(); - log.info(dim("Credentials are shown redacted. `clerk migrate settings set `.")); + log.info(`${set} of ${cells.length} settings set. Credentials are shown redacted.`); + log.blank(); + + printNextSteps(NEXT_STEPS.MIGRATE_SETTINGS); } diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index 8ec0fe982..2b403da46 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -14,6 +14,8 @@ * this table rather than each keeping their own idea of what exists. */ +import { REDACTED } from "../../../lib/constants.ts"; + export type SettingStore = "config" | "env"; export interface SettingDef { @@ -31,6 +33,22 @@ export interface SettingDef { description: string; /** For `env` settings, the variable read at run time. */ envVar?: string; + /** + * Other variables accepted for the same setting, read only when + * {@link envVar} is absent. + * + * Firebase hands its four scrypt parameters over as `base64_signer_key`, + * `rounds` and friends, and every guide — including Clerk's own standalone + * migration script — tells the reader to paste them into `.env` under those + * names. Someone who did that has the values the CLI needs, spelled the way + * the source platform spells them, and a listing that reports "not set" is + * wrong about the project rather than strict about it. + * + * Prefixed names still win, and the listing names the variable it read, so a + * generic `ROUNDS` that means something else in the app is visible rather + * than silent. + */ + envAliases?: string[]; /** For `config` settings, the key on the saved migration entry. */ configKey?: "transformer" | "file" | "skipUnsupportedProviders"; /** Redact when displaying — the value is a credential. */ @@ -71,6 +89,7 @@ export const SETTINGS: SettingDef[] = [ name: "firebase-signer-key", store: "env", envVar: "CLERK_FIREBASE_SIGNER_KEY", + envAliases: ["FIREBASE_BASE64_SIGNER_KEY", "BASE64_SIGNER_KEY"], description: "Firebase base64 signer key", secret: true, }, @@ -78,12 +97,14 @@ export const SETTINGS: SettingDef[] = [ name: "firebase-salt-separator", store: "env", envVar: "CLERK_FIREBASE_SALT_SEPARATOR", + envAliases: ["FIREBASE_BASE64_SALT_SEPARATOR", "BASE64_SALT_SEPARATOR"], description: "Firebase base64 salt separator", }, { name: "firebase-rounds", store: "env", envVar: "CLERK_FIREBASE_ROUNDS", + envAliases: ["FIREBASE_ROUNDS", "ROUNDS"], description: "Firebase scrypt rounds", validate: positiveInteger, }, @@ -91,6 +112,7 @@ export const SETTINGS: SettingDef[] = [ name: "firebase-mem-cost", store: "env", envVar: "CLERK_FIREBASE_MEM_COST", + envAliases: ["FIREBASE_MEM_COST", "MEM_COST"], description: "Firebase scrypt memory cost", validate: positiveInteger, }, @@ -103,17 +125,24 @@ export function findSetting(name: string): SettingDef | undefined { } /** - * Shows enough of a credential to recognise it, never enough to use it. + * Every variable an `env` setting answers to, highest priority first. * - * Anything short enough that head-and-tail would leak most of it is masked - * whole: a 10-character key shown as `abcd…wxyz` has given away 8 of them. + * One list, read by both the listing and the run, so `clerk migrate settings` + * can never show a value the import would ignore. */ -export function redact(value: string): string { - if (value.length < 16) return "•".repeat(8); - return `${value.slice(0, 4)}…${value.slice(-4)}`; +export function envNames(setting: SettingDef): string[] { + return [setting.envVar as string, ...(setting.envAliases ?? [])]; } -/** The display value for a setting: redacted when it is a credential. */ +/** + * The display value for a setting: withheld entirely when it is a credential. + * + * {@link REDACTED} is what `clerk users create --dry-run` already prints for a + * password, so a credential reads the same wherever the CLI declines to show + * one. Head-and-tail (`aVer…3456`) would say *which* key is set, but the source + * column answers that, and a partial value is one the reader has to recognise + * as partial. + */ export function displayValue(setting: SettingDef, value: string): string { - return setting.secret ? redact(value) : value; + return setting.secret ? REDACTED : value; } diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index 42e4f7c1e..f27b84f8e 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -3,12 +3,13 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { _setConfigDir } from "../../../lib/config.ts"; +import { setMode } from "../../../mode.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { MIGRATE_ENV_FILE } from "../lib/env-file.ts"; import { loadSettings, saveSettings } from "../lib/settings.ts"; import { clear } from "./clear.ts"; import { list } from "./list.ts"; -import { redact } from "./registry.ts"; +import { displayValue, findSetting } from "./registry.ts"; import { set } from "./set.ts"; const captured = useCaptureLog(); @@ -44,14 +45,20 @@ afterEach(() => { process.exitCode = 0; }); -describe("redact", () => { - test("shows head and tail of a long value", () => { - expect(redact("aVeryLongSignerKeyValue123456")).toBe("aVer…3456"); - }); +describe("displayValue", () => { + const signerKey = findSetting("firebase-signer-key")!; + + // No part of the value, at any length — the same `[REDACTED]` that + // `clerk users create --dry-run` prints for a password. + test.each([["short"], ["0123456789"], ["aVeryLongSignerKeyValue123456"]])( + "withholds the credential %p entirely", + (value) => { + expect(displayValue(signerKey, value)).toBe("[REDACTED]"); + }, + ); - // Head-and-tail on a short value gives away most of it. - test.each([["short"], ["0123456789"], ["123456789012345"]])("masks %p whole", (value) => { - expect(redact(value)).toBe("••••••••"); + test("shows a setting that is not a credential", () => { + expect(displayValue(findSetting("transformer")!, "firebase")).toBe("firebase"); }); }); @@ -121,7 +128,7 @@ describe("list", () => { await list(); - expect(captured.err).toContain("aVer…3456"); + expect(captured.err).toContain("[REDACTED]"); expect(captured.err).not.toContain("aVeryLongSignerKeyValue123456"); expect(captured.err).toContain("firebase"); }); @@ -135,7 +142,7 @@ describe("list", () => { expect(captured.out).not.toContain("aVeryLongSignerKeyValue123456"); expect(JSON.parse(captured.out)).toContainEqual( - expect.objectContaining({ name: "firebase-signer-key", value: "aVer…3456", secret: true }), + expect.objectContaining({ name: "firebase-signer-key", value: "[REDACTED]", secret: true }), ); }); @@ -181,6 +188,41 @@ describe("list", () => { await list({ json: true }); expect(JSON.parse(captured.out).every((entry: { set: boolean }) => !entry.set)).toBe(true); }); + + // A listing is where someone lands before they know what to type, so it + // closes by naming the two commands that change what it just showed — + // the same next-steps block `mcp list` and `whoami` end on. + test("closes with next steps", async () => { + setMode("human"); + await list(); + setMode("agent"); + + expect(captured.err).toContain("clerk migrate settings set "); + expect(captured.err).toContain("clerk migrate settings clear"); + }); + + test("counts how many are set", async () => { + await set("transformer", "firebase"); + captured.clear(); + + await list(); + + expect(captured.err).toContain("1 of 7 settings set"); + }); + + // Firebase's own names for these, and what every guide tells you to paste + // into `.env`. Reporting "not set" for a value the import would read is the + // listing being wrong about the project rather than strict about it. + test("reads a credential written under the name Firebase uses", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "ROUNDS=8\n"); + captured.clear(); + + await list(); + + fs.rmSync(path.join(workDir, ".env.local")); + // Named alongside the file: `ROUNDS` may mean something else in this app. + expect(captured.err).toContain(".env.local (ROUNDS)"); + }); }); describe("clear", () => { diff --git a/packages/cli-core/src/lib/dotenv.ts b/packages/cli-core/src/lib/dotenv.ts index 16bc28187..a2e0c2ec7 100644 --- a/packages/cli-core/src/lib/dotenv.ts +++ b/packages/cli-core/src/lib/dotenv.ts @@ -48,6 +48,8 @@ export interface FindEnvValueOptions { export interface LocatedEnvValue { value: string; + /** Which of `names` supplied it — the caller may have passed several aliases. */ + name: string; /** Where it came from, for `--verbose` (`CLERK_SECRET_KEY env var`, `.env.local`). */ source: string; } @@ -70,7 +72,7 @@ export async function findEnvValue( for (const name of new Set(names)) { const value = env[name]; - if (value) return { value, source: `${name} env var` }; + if (value) return { value, name, source: `${name} env var` }; } // Priority is by name, not by position: the framework-specific name beats @@ -84,7 +86,7 @@ export async function findEnvValue( for (const line of parseEnvFile(await file.text())) { if (line.type !== "entry" || !line.value) continue; if (names.includes(line.key)) { - foundByName.set(line.key, { value: line.value, source: envFile }); + foundByName.set(line.key, { value: line.value, name: line.key, source: envFile }); } } } diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index a1b53e1d7..fbcf6fc64 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -76,6 +76,10 @@ export const NEXT_STEPS = { "Run `clerk migrate delete` to undo this migration", ], MIGRATE_DELETE: ["Run `clerk migrate logs list` to inspect the deletion log"], + MIGRATE_SETTINGS: [ + "Run `clerk migrate settings set ` to change one", + "Run `clerk migrate settings clear` to forget them all, credentials included", + ], // The only parameterized entry: a suggested import is worthless unless it // names the transformer that reads this export and the file just written. MIGRATE_EXPORT: (transformerKey: string, file: string) => [ From bf4d6ed24f8f797db1e1f5a2738e1155ca159d17 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 10 Sep 2026 23:59:24 -0400 Subject: [PATCH 026/141] fix(cli): name the host when a request cannot connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bun reports every connection-level failure — DNS, refused, no route — as a bare `Error` reading "Unable to connect. Is the computer able to access the url?". It names neither the host nor what wanted it, and the global handler could only render it as `unexpected_error`. Connection failures now surface as a `CliError` naming the host, under the new `network_unreachable` code. Everything else, an aborted request included, is left exactly as thrown. --- packages/cli-core/src/lib/errors.ts | 2 ++ packages/cli-core/src/lib/fetch.test.ts | 26 +++++++++++++++++++++++++ packages/cli-core/src/lib/fetch.ts | 22 ++++++++++++++++++++- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/lib/errors.ts b/packages/cli-core/src/lib/errors.ts index 247e02776..c984ba4d0 100644 --- a/packages/cli-core/src/lib/errors.ts +++ b/packages/cli-core/src/lib/errors.ts @@ -96,6 +96,8 @@ export const ERROR_CODE = { INSTALLER_NOT_FOUND: "installer_not_found", /** The npm registry was unreachable. */ REGISTRY_UNREACHABLE: "registry_unreachable", + /** A request never reached the server — DNS, refused connection, no route. */ + NETWORK_UNREACHABLE: "network_unreachable", /** Production instance was created but came back without a domain. */ DEPLOY_DOMAIN_MISSING: "deploy_domain_missing", /** Local publishable key and secret key address different applications. */ diff --git a/packages/cli-core/src/lib/fetch.test.ts b/packages/cli-core/src/lib/fetch.test.ts index 505131b9e..9ad2432d6 100644 --- a/packages/cli-core/src/lib/fetch.test.ts +++ b/packages/cli-core/src/lib/fetch.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { _resetUserAgentCache, loggedFetch } from "./fetch.ts"; import { _resetInterruptState, abortInFlight, beginInterrupt, interruptSignal } from "./signals.ts"; import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from "./config.ts"; +import { CliError } from "./errors.ts"; const originalFetch = globalThis.fetch; @@ -34,6 +35,31 @@ describe("loggedFetch", () => { expect(init.headers.get("User-Agent")).toBe("Custom/1.0"); }); + test("reports a connection failure as a CliError naming the host", async () => { + globalThis.fetch = mock(async () => { + // Bun's own shape for DNS failures, refused connections and no-route. + const error: NodeJS.ErrnoException = new Error( + "Unable to connect. Is the computer able to access the url?", + ); + error.code = "ConnectionRefused"; + throw error; + }) as unknown as typeof fetch; + + const failure = loggedFetch("https://example.test/x", { tag: "test" }); + await expect(failure).rejects.toThrow(/Could not reach example\.test/); + await expect(failure).rejects.toBeInstanceOf(CliError); + }); + + test("leaves a non-connection failure alone", async () => { + globalThis.fetch = mock(async () => { + throw new DOMException("The operation was aborted.", "AbortError"); + }) as unknown as typeof fetch; + + await expect(loggedFetch("https://example.test/x", { tag: "test" })).rejects.toThrow( + /operation was aborted/, + ); + }); + test("preserves other caller-provided headers", async () => { globalThis.fetch = mock( async () => new Response("ok", { status: 200 }), diff --git a/packages/cli-core/src/lib/fetch.ts b/packages/cli-core/src/lib/fetch.ts index 1d73ea9ff..36289e164 100644 --- a/packages/cli-core/src/lib/fetch.ts +++ b/packages/cli-core/src/lib/fetch.ts @@ -8,6 +8,7 @@ * every network error. See `.claude/rules/debug-logging.md`. */ +import { CliError, ERROR_CODE } from "./errors.ts"; import { log } from "./log.ts"; import { interruptSignal } from "./signals.ts"; import { withNetworkAccess } from "./host-execution.ts"; @@ -74,6 +75,23 @@ function interruptSignalFor( return own ? AbortSignal.any([own, interruptSignal()]) : interruptSignal(); } +/** + * Bun reports every connection-level failure — DNS, refused, no route — as a + * bare `Error` reading "Unable to connect. Is the computer able to access the + * url?", which names neither the host nor what wanted it, and which the global + * handler can only render as `unexpected_error`. Everything else, an aborted + * request included, is left exactly as thrown. + */ +function asConnectionError(error: unknown, url: string): unknown { + if ((error as NodeJS.ErrnoException | null)?.code !== "ConnectionRefused") return error; + + const host = URL.parse(url)?.host ?? url; + return new CliError( + `Could not reach ${host}. Check your network connection (or VPN) and try again.`, + { code: ERROR_CODE.NETWORK_UNREACHABLE }, + ); +} + export async function loggedFetch(url: URL | string, options: LoggedFetchInit): Promise { const { tag, bestEffort, ignoreInterrupt, ...init } = options; const method = init.method ?? "GET"; @@ -85,7 +103,9 @@ export async function loggedFetch(url: URL | string, options: LoggedFetchInit): const response = await withNetworkAccess( { operation: "connect", target: urlStr, label: tag, bestEffort }, async () => fetch(url, { ...init, headers, signal }), - ); + ).catch((error: unknown) => { + throw asConnectionError(error, urlStr); + }); if (!response.ok) { // Clone so the caller can still consume the body for error construction. const body = await response.clone().text(); From 1c029c8c317367f79ec1711abd77265f67fc152c Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 11 Sep 2026 00:01:17 -0400 Subject: [PATCH 027/141] feat(migrate): ask once where migration logs go, and remember it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration logs are the only record of which users landed and which failed, and `migrate delete` reads them to undo a run — so where they go is worth one question, asked before the first log file is written. `import`, `export` and `delete` now start at `startLogging()`, which settles the directory (`CLERK_MIGRATE_LOG_DIR`, then the saved `log-dir`, then `./logs`) and asks a human who has chosen neither. The answer is saved under the new `log-dir` setting, so the question is asked once per project and never again; `-y`, agent mode and a non-TTY take `./logs` and save nothing, leaving the question open for the first interactive run. `logs list|clean|convert` resolve the directory without ever asking: they are read-only, and "where should logs go?" is not a question to put in front of someone who asked to see the logs they already have. `log-dir` is the first setting kept in the config that also answers to an environment variable, so `settings list` checks the environment for a config setting too — a listing that showed the remembered path while the run read another is the one thing the source column exists to prevent. --- .../cli-core/src/commands/migrate/README.md | 48 ++++++-- .../cli-core/src/commands/migrate/delete.ts | 4 +- .../src/commands/migrate/export/auth0.test.ts | 3 +- .../src/commands/migrate/export/auth0.ts | 4 +- .../src/commands/migrate/export/authjs.ts | 4 +- .../src/commands/migrate/export/betterauth.ts | 4 +- .../src/commands/migrate/export/clerk.test.ts | 3 +- .../src/commands/migrate/export/clerk.ts | 4 +- .../commands/migrate/export/firebase.test.ts | 3 +- .../src/commands/migrate/export/firebase.ts | 4 +- .../src/commands/migrate/export/supabase.ts | 4 +- .../migrate/lib/log-dir-prompt.test.ts | 113 ++++++++++++++++++ .../src/commands/migrate/lib/logger.test.ts | 95 +++++++++++++++ .../src/commands/migrate/lib/logger.ts | 110 ++++++++++++++++- .../src/commands/migrate/logs/clean.ts | 3 +- .../src/commands/migrate/logs/convert.ts | 3 +- .../src/commands/migrate/logs/index.ts | 4 +- .../src/commands/migrate/logs/list.ts | 5 +- .../commands/migrate/run-interactive.test.ts | 10 +- packages/cli-core/src/commands/migrate/run.ts | 4 +- .../src/commands/migrate/settings/list.ts | 11 ++ .../src/commands/migrate/settings/registry.ts | 22 +++- .../migrate/settings/settings.test.ts | 2 +- packages/cli-core/src/lib/config.ts | 2 + packages/cli-core/src/test/lib/stubs.ts | 32 ++++- 25 files changed, 462 insertions(+), 39 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 018cd76d6..91a06eebb 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -386,7 +386,8 @@ failed. ### `clerk migrate logs` -Everything that touches the local `./logs/` directory. Noun-verb like every +Everything that touches the local log directory — `./logs` unless the project +says otherwise; see [Where logs go](#where-logs-go). Noun-verb like every other group in the CLI (`config pull`, `users list`), rather than the standalone tool's `clean-logs`/`convert-logs`, which were npm script names. @@ -405,11 +406,36 @@ clerk migrate logs convert import-2026-01-01T12-00-00.log | Subcommand | Takes | Description | | -------------- | ------------------ | ----------------------------------------------- | | `logs list` | `--json` | File, type, date, size and entry count per file | -| `logs clean` | `-y, --yes` | Delete the `.log` files in `./logs/` | +| `logs clean` | `-y, --yes` | Delete the `.log` files in the log directory | | `logs convert` | `[file…]`, `--all` | NDJSON → a JSON array, written as `.json` | All three read the directory through one shared enumerator, which is what makes -`logs list` nearly free. +`logs list` nearly free. All three **resolve** the directory without ever asking +for one: they are read-only, and "where should logs go?" is not a question to +put in front of someone who asked to see the logs they already have. + +#### Where logs go + +`./logs`, relative to the current directory, until the project says otherwise. +Resolution order, highest first: + +| Source | Set by | +| --------------------------- | ----------------------------------------------------- | +| `CLERK_MIGRATE_LOG_DIR` | The shell, `.env`, `.env.local`, `.env.clerk-migrate` | +| `log-dir` in the CLI config | The first-run prompt, or `settings set` | +| `./logs` | The fallback | + +The first time `migrate import`, `migrate export` or `migrate delete` runs +interactively in a project with none of those set, it asks where logs should be +saved and offers `./logs`. The answer is saved under `log-dir`, so it is asked +once per project and never again. `-y`, agent mode and a non-TTY take `./logs` +without asking **and without saving it** — landing on a default is not a choice, +and recording one would retire the question for a human who never saw it. + +Logs are the only record of which users landed and which failed, and +`migrate delete` reads them to undo a run, so where they go is worth the one +question. Change it later with `clerk migrate settings set log-dir `, or +clear it with `clerk migrate settings clear log-dir` to be asked again. #### `logs list` @@ -450,7 +476,7 @@ raw stamp. The directory is printed relative (`./logs`) when it sits under the current directory and absolute when it does not, so the path can be pasted either way. -Says so plainly when `./logs/` is empty or absent. +Says so plainly when the log directory is empty or absent. #### `logs clean` @@ -604,10 +630,16 @@ firebase-mem-cost 14 MEM_COST env var Firebase scrypt mem Two stores, split by what the value **is** rather than by which command wrote it: -| Store | Holds | Why | -| -------------------- | --------------------------------------------------- | ---------------------------------------------------------------- | -| CLI config | `transformer`, `file`, `skip-unsupported-providers` | Project state, not secret, useless outside the CLI | -| `.env.clerk-migrate` | `firebase-*` | Credentials: gitignored on write, and hand-editable for rotation | +| Store | Holds | Why | +| -------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------- | +| CLI config | `transformer`, `file`, `skip-unsupported-providers`, `log-dir` | Project state, not secret, useless outside the CLI | +| `.env.clerk-migrate` | `firebase-*` | Credentials: gitignored on write, and hand-editable for rotation | + +`log-dir` is the one setting that answers to both: it is remembered in the CLI +config, and `CLERK_MIGRATE_LOG_DIR` outranks what is remembered, so a directory +can be pinned for one shell without disturbing the project. The listing's source +column says which is winning, and `settings clear log-dir` clears both — half a +clear would report the setting gone while the next run still read it. `.env.clerk-migrate` is the migration's own file rather than the app's `.env.local`, because a Firebase signer key is of no use to the application diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 32e7d4c41..f80133f76 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -37,7 +37,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../lib/spinner import { isAgent, isHuman } from "../../mode.ts"; import { normalizeErrorMessage } from "./import-users.ts"; import { resolveLimits, type ResolvedLimits } from "./lib/instance.ts"; -import { deleteErrorLogger, deleteLogger, getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; +import { deleteErrorLogger, deleteLogger, startLogging, getLogFilePath } from "./lib/logger.ts"; import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; import { createApiScheduler } from "./lib/scheduler.ts"; import { loadSettings } from "./lib/settings.ts"; @@ -263,7 +263,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise { const destination = await resolveOutputPath("auth0", options.output); await withGutter("Exporting users from Auth0", async ({ setNextSteps }) => { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); log.info(`Exporting from ${credentials.domain}.`); const token = await withSpinner("Authenticating with Auth0...", () => diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 1f74223a4..d378c93c1 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -13,7 +13,7 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { log } from "../../../lib/log.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; @@ -116,7 +116,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { const destination = await resolveOutputPath("authjs", options.output); await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); const { rows, table } = await withSpinner("Reading the user table...", () => withDbClient(dbUrl, "authjs", fetchAuthJsUsers), diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 006af76f9..f2d26c6c6 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -17,7 +17,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; @@ -163,7 +163,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const destination = await resolveOutputPath("betterauth", options.output); await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); const { rows, plugins } = await withSpinner("Reading the user table...", () => withDbClient(dbUrl, "betterauth", async (client) => { diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 75e1f0565..67f825172 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -3,7 +3,7 @@ import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { buildClerkExport, @@ -13,6 +13,7 @@ import { } from "./clerk.ts"; const captured = useCaptureLog(); +useMigrateLogDir(); let workDir: string; let originalCwd: string; diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index ec4a633be..28d5a1e4d 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -17,7 +17,7 @@ import { bapiRequest } from "../../../lib/bapi.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; import { retryOn429 } from "../lib/retry.ts"; import { resolveClerkSource } from "./clerk-source.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; @@ -235,7 +235,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { const destination = await resolveOutputPath("clerk", options.output); await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 4f9c4ac25..d4a440b7e 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { buildFirebaseExport, @@ -21,6 +21,7 @@ import { } from "./firebase.ts"; const captured = useCaptureLog(); +useMigrateLogDir(); let workDir: string; let originalCwd: string; diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 6facdd770..93a6f53af 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -31,7 +31,7 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; /** Identity Toolkit's maximum for `accounts:batchGet`. */ @@ -487,7 +487,7 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); log.info(`Exporting from the ${account.project_id} project.`); const token = await withSpinner("Authenticating with Google...", () => diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 2436c95c5..cf257f35f 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -12,7 +12,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; @@ -114,7 +114,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { const destination = await resolveOutputPath("supabase", options.output); await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); const rows = await withSpinner("Reading auth.users...", () => withDbClient(dbUrl, "supabase", fetchSupabaseUsers), diff --git a/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts b/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts new file mode 100644 index 000000000..766145d24 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts @@ -0,0 +1,113 @@ +/** + * The one path `logger.test.ts` cannot cover: `ensureLogDir` actually asking. + * + * Its own file because `mock.module` registrations last for the process, and + * `bun test --parallel` puts several files in each worker — a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +type TextConfig = { message: string; default?: string; placeholder?: string }; + +let answer = ""; +const mockText = mock(async (_config: TextConfig) => answer); + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../../lib/prompts.ts", () => ({ + text: (...args: unknown[]) => mockText(...(args as [TextConfig])), + confirm: async () => true, + multiselect: async () => [], + password: async () => "", + editor: async () => "{}", +})); + +const { _resetLogDir, ensureLogDir } = await import("./logger.ts"); +const { loadSettings, saveSettings } = await import("./settings.ts"); + +useCaptureLog(); + +let workDir: string; +let configDir: string; +let originalCwd: string; +let originalMode: Mode; +let originalEnv: string | undefined; + +beforeAll(() => { + originalCwd = process.cwd(); + originalMode = getMode(); + originalEnv = process.env.CLERK_MIGRATE_LOG_DIR; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-cfg-")); + _setConfigDir(configDir); + process.chdir(workDir); +}); + +afterAll(() => { + setMode(originalMode); + if (originalEnv === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; + else process.env.CLERK_MIGRATE_LOG_DIR = originalEnv; + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + _resetLogDir(); + delete process.env.CLERK_MIGRATE_LOG_DIR; + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + mockText.mockClear(); + answer = ""; + setMode("human"); +}); + +afterEach(() => _resetLogDir()); + +describe("ensureLogDir asks once", () => { + test("saves the answer, so the next run does not ask", async () => { + answer = "./migration-logs"; + + expect(await ensureLogDir()).toBe(path.join(workDir, "migration-logs")); + expect(await loadSettings()).toMatchObject({ logDir: "./migration-logs" }); + + _resetLogDir(); + expect(await ensureLogDir()).toBe(path.join(workDir, "migration-logs")); + expect(mockText).toHaveBeenCalledTimes(1); + }); + + test("offers ./logs as the default", async () => { + await ensureLogDir(); + expect(mockText.mock.calls[0]?.[0]).toMatchObject({ default: "./logs" }); + }); + + // Enter on the prompt is an answer, not a skip: it settles the question so + // the next run goes straight to importing. + test("treats an empty answer as ./logs and remembers it", async () => { + answer = " "; + + expect(await ensureLogDir()).toBe(path.join(workDir, "logs")); + expect(await loadSettings()).toMatchObject({ logDir: "./logs" }); + }); + + test("leaves the project's other settings alone", async () => { + await saveSettings({ transformer: "firebase", file: "users.json" }); + answer = "./audit"; + + await ensureLogDir(); + + expect(await loadSettings()).toEqual({ + transformer: "firebase", + file: "users.json", + logDir: "./audit", + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.test.ts b/packages/cli-core/src/commands/migrate/lib/logger.test.ts index eb153218d..e9176df9a 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.test.ts @@ -3,13 +3,21 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { + _resetLogDir, + DEFAULT_LOG_DIR, + ensureLogDir, errorLogger, getDateTimeStamp, getLogDir, getLogFilePath, importLogger, + resolveLogDir, validationLogger, } from "./logger.ts"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { MIGRATE_ENV_FILE } from "./env-file.ts"; +import { loadSettings, saveSettings } from "./settings.ts"; const DATE_TIME = "2026-01-01T12:00:00"; @@ -30,6 +38,7 @@ afterAll(() => { }); beforeEach(() => { + _resetLogDir(); fs.rmSync(getLogDir(), { recursive: true, force: true }); }); @@ -108,3 +117,89 @@ describe("log writers", () => { }); }); }); + +describe("resolving the log directory", () => { + let configDir: string; + let originalMode: Mode; + let originalEnv: string | undefined; + + beforeAll(() => { + originalMode = getMode(); + originalEnv = process.env.CLERK_MIGRATE_LOG_DIR; + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-cfg-")); + _setConfigDir(configDir); + }); + + afterAll(() => { + setMode(originalMode); + if (originalEnv === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; + else process.env.CLERK_MIGRATE_LOG_DIR = originalEnv; + _setConfigDir(undefined); + fs.rmSync(configDir, { recursive: true, force: true }); + }); + + beforeEach(() => { + delete process.env.CLERK_MIGRATE_LOG_DIR; + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.rmSync(path.join(workDir, MIGRATE_ENV_FILE), { force: true }); + setMode("agent"); + }); + + test("falls back to ./logs when nothing has chosen one", async () => { + expect(await resolveLogDir()).toBe(path.join(workDir, "logs")); + }); + + test("prefers the saved setting over the default", async () => { + await saveSettings({ logDir: "./audit" }); + expect(await resolveLogDir()).toBe(path.join(workDir, "audit")); + }); + + // A variable exported for one shell is the narrower statement of the two. + test("prefers the environment over the saved setting", async () => { + await saveSettings({ logDir: "./audit" }); + process.env.CLERK_MIGRATE_LOG_DIR = "./from-env"; + + expect(await resolveLogDir()).toBe(path.join(workDir, "from-env")); + }); + + test("reads the migration's own env file", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "CLERK_MIGRATE_LOG_DIR=./from-file\n"); + expect(await resolveLogDir()).toBe(path.join(workDir, "from-file")); + }); + + // Every synchronous write reads the settled value, so resolving is what makes + // the log files land anywhere but ./logs. + test("settles the directory the log writers use", async () => { + await saveSettings({ logDir: "./audit" }); + await resolveLogDir(); + + expect(getLogFilePath("import", DATE_TIME)).toBe( + path.join(workDir, "audit", `import-${DATE_TIME.replace(/:/g, "-")}.log`), + ); + }); + + describe("ensureLogDir", () => { + test("takes the default without saving it when nobody can be asked", async () => { + expect(await ensureLogDir()).toBe(path.join(workDir, DEFAULT_LOG_DIR)); + // Nothing saved: the question stays open for the first interactive run. + expect(await loadSettings()).toEqual({}); + }); + + test("does not ask once the setting is saved", async () => { + await saveSettings({ logDir: "./audit" }); + setMode("human"); + + // Reaching the prompt in a test without a TTY throws, so returning is the + // assertion. + expect(await ensureLogDir()).toBe(path.join(workDir, "audit")); + }); + + test("does not ask when the environment already answers", async () => { + process.env.CLERK_MIGRATE_LOG_DIR = "./from-env"; + setMode("human"); + + expect(await ensureLogDir()).toBe(path.join(workDir, "from-env")); + expect(await loadSettings()).toEqual({}); + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.ts b/packages/cli-core/src/commands/migrate/lib/logger.ts index 54d0d54a3..814fe4914 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.ts @@ -14,6 +14,11 @@ import fs from "node:fs"; import path from "node:path"; import { log } from "../../../lib/log.ts"; +import { text } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { envNames, findSetting } from "../settings/registry.ts"; +import { findMigrateEnvValue } from "./env-file.ts"; +import { loadSettings, saveSettings } from "./settings.ts"; import type { DeleteLogEntry, ErrorLog, @@ -23,9 +28,110 @@ import type { ValidationErrorPayload, } from "../types.ts"; -/** Absolute path of the cwd-relative `logs/` directory. */ +/** Where logs go when nobody has said otherwise. */ +export const DEFAULT_LOG_DIR = "./logs"; + +/** + * The directory settled for this process, once something has settled it. + * + * The log writers are synchronous — a run interrupted with Ctrl-C has to leave + * a complete record of what it already processed — but resolving the directory + * reads the config, the env files and possibly the operator. So resolution + * happens once, up front, and every synchronous write reads the answer from + * here. {@link resolveLogDir} and {@link ensureLogDir} are the only writers. + */ +let settled: string | undefined; + +function remember(dir: string): string { + settled = path.resolve(process.cwd(), dir); + return settled; +} + +/** Forgets the settled directory. Tests only — each one resolves its own. */ +export function _resetLogDir(): void { + settled = undefined; +} + +/** + * Absolute path of the log directory. + * + * Falls back to `./logs` when nothing has resolved yet, so a caller that + * forgets to is wrong about *where*, never broken. + */ export function getLogDir(): string { - return path.join(process.cwd(), "logs"); + return settled ?? path.resolve(process.cwd(), DEFAULT_LOG_DIR); +} + +/** The `log-dir` setting, which owns both the env var and the config key. */ +const LOG_DIR = findSetting("log-dir") as NonNullable>; + +/** + * The directory the operator has already chosen, by either route. + * + * The environment wins over the remembered value, matching every other setting + * the CLI resolves: a variable exported for one shell is the narrower, more + * deliberate statement of the two. + */ +async function chosenLogDir(): Promise { + const located = await findMigrateEnvValue(envNames(LOG_DIR)); + if (located?.value) return located.value; + return (await loadSettings()).logDir; +} + +/** + * Settles the log directory without asking: environment, then the saved + * setting, then `./logs`. + * + * For the read-only log commands. Landing on the default here does not save it + * — an operator who has only ever *listed* logs has still made no choice, and + * recording one on their behalf would skip the question forever. + */ +export async function resolveLogDir(): Promise { + return remember((await chosenLogDir()) ?? DEFAULT_LOG_DIR); +} + +/** + * Settles the log directory, asking a human who has not chosen yet. + * + * Migration logs are the only record of which users landed and which failed, + * and `migrate delete` reads them to undo a run — so where they go is worth one + * question, once per project, before the first thing is written. The answer is + * saved, so it is asked once and never again. + * + * `-y`, agent mode and a non-TTY take the default rather than a prompt they + * cannot answer, and save nothing: the question stays open for the first + * interactive run. + */ +export async function ensureLogDir(): Promise { + const chosen = await chosenLogDir(); + if (chosen) return remember(chosen); + if (!isHuman() || isAgent()) return remember(DEFAULT_LOG_DIR); + + const answer = await text({ + message: "Where should migration logs be saved?", + default: DEFAULT_LOG_DIR, + placeholder: DEFAULT_LOG_DIR, + }); + const dir = answer.trim() || DEFAULT_LOG_DIR; + + await saveSettings({ ...(await loadSettings()), logDir: dir }); + log.info( + `Saving migration logs to ${dir}. Change it with \`clerk migrate settings set log-dir \`.`, + ); + + return remember(dir); +} + +/** + * Settles where this run's logs go, and stamps it. + * + * Every command that writes a log starts here rather than calling + * {@link getDateTimeStamp} directly, so there is no path on which a log file is + * named before its directory has been resolved. + */ +export async function startLogging(): Promise { + await ensureLogDir(); + return getDateTimeStamp(); } /** diff --git a/packages/cli-core/src/commands/migrate/logs/clean.ts b/packages/cli-core/src/commands/migrate/logs/clean.ts index 3c609f5ad..4d17d684d 100644 --- a/packages/cli-core/src/commands/migrate/logs/clean.ts +++ b/packages/cli-core/src/commands/migrate/logs/clean.ts @@ -16,7 +16,7 @@ import { confirm } from "../../../lib/prompts.ts"; import { withGutter } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { listLogFiles } from "../lib/log-files.ts"; -import { getLogDir } from "../lib/logger.ts"; +import { getLogDir, resolveLogDir } from "../lib/logger.ts"; export type LogsCleanOptions = { yes?: boolean; @@ -24,6 +24,7 @@ export type LogsCleanOptions = { export async function clean(options: LogsCleanOptions = {}): Promise { await withGutter("Cleaning migration logs", async () => { + await resolveLogDir(); const files = listLogFiles(); if (files.length === 0) { diff --git a/packages/cli-core/src/commands/migrate/logs/convert.ts b/packages/cli-core/src/commands/migrate/logs/convert.ts index 33adf2bae..4536e70a5 100644 --- a/packages/cli-core/src/commands/migrate/logs/convert.ts +++ b/packages/cli-core/src/commands/migrate/logs/convert.ts @@ -15,7 +15,7 @@ import { multiselect } from "../../../lib/prompts.ts"; import { withGutter } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { findLogFile, listLogFiles, readNdjson, type LogFile } from "../lib/log-files.ts"; -import { getLogDir } from "../lib/logger.ts"; +import { getLogDir, resolveLogDir } from "../lib/logger.ts"; export type LogsConvertOptions = { all?: boolean; @@ -85,6 +85,7 @@ export async function convert(options: LogsConvertOptions = {}): Promise { // The multiselect lives inside the gutter so cancelling it closes with // `└ Paused` rather than leaving a half-drawn frame. await withGutter("Converting migration logs", async () => { + await resolveLogDir(); const targets = await resolveTargets(options); if (targets.length === 0) return; diff --git a/packages/cli-core/src/commands/migrate/logs/index.ts b/packages/cli-core/src/commands/migrate/logs/index.ts index 50279c8ff..d12a5d29b 100644 --- a/packages/cli-core/src/commands/migrate/logs/index.ts +++ b/packages/cli-core/src/commands/migrate/logs/index.ts @@ -32,7 +32,7 @@ export function registerMigrateLogs(migrateCommand: Command<[], Record, string> = { @@ -53,6 +53,9 @@ export function formatTimestamp(stamp: string): string { } export async function list(options: LogsListOptions = {}): Promise { + // Resolve, never ask: listing is read-only, and "where should logs go?" is + // not a question to answer before showing someone the ones they have. + await resolveLogDir(); const files = listLogFiles(); if (options.json) { diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 9d29d32f1..a7e6909c8 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -14,7 +14,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { getMode, setMode, type Mode } from "../../mode.ts"; -import { keylessTargetStubs, listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; +import { + keylessTargetStubs, + listageStubs, + useCaptureLog, + useMigrateLogDir, +} from "../../test/lib/stubs.ts"; import type { InstanceTarget } from "../../lib/keyless-target.ts"; const mockSelect = mock(async () => "clerk" as unknown); @@ -22,6 +27,8 @@ const mockText = mock(async () => "export.json" as unknown); type MultiselectConfig = { options: { value: string; label: string; hint?: string }[] }; const mockMultiselect = mock(async (_config: MultiselectConfig) => [] as unknown[]); let confirmAnswer = true; +/** Every confirmation the run put up, in order — the wording is the assertion. */ +let confirmMessages: string[] = []; let originalMode: Mode; const ACCOUNT_TARGET: InstanceTarget = { @@ -66,6 +73,7 @@ const { loadSettings, saveSettings } = await import("./lib/settings.ts"); const { _setConfigDir } = await import("../../lib/config.ts"); const captured = useCaptureLog(); +useMigrateLogDir(); let workDir: string; let configDir: string; diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index ca0576022..dd8b6b467 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -43,7 +43,7 @@ import { type SettingChange, } from "./lib/modify-settings.ts"; import { DEV_USER_LIMIT, resolveLimits } from "./lib/instance.ts"; -import { getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; +import { startLogging, getLogFilePath } from "./lib/logger.ts"; import { saveSettings } from "./lib/settings.ts"; import { countSocialProviders, @@ -483,7 +483,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const limits = resolveLimits(secretKey); - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); const logFile = getLogFilePath("import", dateTime); const { users: loaded, validationFailed } = await withSpinner( diff --git a/packages/cli-core/src/commands/migrate/settings/list.ts b/packages/cli-core/src/commands/migrate/settings/list.ts index 049264958..747a179f5 100644 --- a/packages/cli-core/src/commands/migrate/settings/list.ts +++ b/packages/cli-core/src/commands/migrate/settings/list.ts @@ -52,6 +52,17 @@ async function resolveAll(): Promise { return Promise.all( SETTINGS.map(async (setting): Promise => { if (setting.store === "config") { + // `log-dir` is remembered in the config but yields to an environment + // value, so the environment has to be checked first here too — a + // listing that shows the remembered path while the run reads another + // is the one thing the source column exists to prevent. + if (setting.envVar) { + const located = await findMigrateEnvValue(envNames(setting)); + if (located) { + return { setting, value: located.value, source: describeSource(setting, located) }; + } + } + const value = saved[setting.configKey as keyof typeof saved]; return value === undefined ? { setting } diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index 2b403da46..7a471e855 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -31,7 +31,14 @@ export interface SettingDef { name: string; store: SettingStore; description: string; - /** For `env` settings, the variable read at run time. */ + /** + * The environment variable this setting is read from at run time. + * + * Required for an `env` setting, which lives nowhere else. A `config` + * setting may also declare one, meaning "remembered here, but an environment + * value wins" — `log-dir` is that shape, so an operator can pin a directory + * per shell without disturbing what the project remembers. + */ envVar?: string; /** * Other variables accepted for the same setting, read only when @@ -50,7 +57,7 @@ export interface SettingDef { */ envAliases?: string[]; /** For `config` settings, the key on the saved migration entry. */ - configKey?: "transformer" | "file" | "skipUnsupportedProviders"; + configKey?: "transformer" | "file" | "skipUnsupportedProviders" | "logDir"; /** Redact when displaying — the value is a credential. */ secret?: boolean; /** Reject a value the run would only fail on later. */ @@ -65,6 +72,9 @@ const positiveInteger = (value: string): string | undefined => { const boolean = (value: string): string | undefined => ["true", "false"].includes(value) ? undefined : "Expected true or false"; +const path = (value: string): string | undefined => + value.trim().length > 0 ? undefined : "Expected a directory path"; + export const SETTINGS: SettingDef[] = [ { name: "transformer", @@ -85,6 +95,14 @@ export const SETTINGS: SettingDef[] = [ description: "Skip users with no provider enabled in Clerk (Supabase)", validate: boolean, }, + { + name: "log-dir", + store: "config", + configKey: "logDir", + envVar: "CLERK_MIGRATE_LOG_DIR", + description: "Directory migration logs are written to", + validate: path, + }, { name: "firebase-signer-key", store: "env", diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index f27b84f8e..946d475c6 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -207,7 +207,7 @@ describe("list", () => { await list(); - expect(captured.err).toContain("1 of 7 settings set"); + expect(captured.err).toContain("1 of 8 settings set"); }); // Firebase's own names for these, and what every guide tells you to paste diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index 08e0295d1..4d84ed82d 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -55,6 +55,8 @@ interface MigrationEntry { transformer?: string; file?: string; skipUnsupportedProviders?: boolean; + /** Where this project's migration logs are written. Absent until chosen. */ + logDir?: string; } interface ClerkConfig { diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index e16e7f131..7cb412560 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -1,7 +1,8 @@ import { Writable } from "node:stream"; -import { afterEach, beforeEach, type spyOn } from "bun:test"; +import { afterAll, afterEach, beforeAll, beforeEach, type spyOn } from "bun:test"; import { type CapturedLogs, setActiveCapture } from "../../lib/log.ts"; import { setUiOutput } from "../../lib/ui.ts"; +import { _resetLogDir } from "../../commands/migrate/lib/logger.ts"; export function capturedOutput(spy: ReturnType): string { return spy.mock.calls.map((c: unknown[]) => c[0]).join("\n"); @@ -251,3 +252,32 @@ type FetchImpl = (input: string | URL | Request, init?: RequestInit) => Promise< export function stubFetch(impl: FetchImpl): void { globalThis.fetch = impl as typeof fetch; } + +/** + * Settles the migration log directory for a whole test file. + * + * `migrate import`, `export` and `delete` ask a human where logs should go the + * first time a project runs one. A test that flips to human mode to exercise + * something else — next steps, a wizard — would stop on that question and, + * where `prompts.ts` is mocked, silently eat the answer meant for another + * prompt. Pinning the environment variable answers it before it is asked, the + * same way an operator who exported one never sees it. + */ +export function useMigrateLogDir(dir = "./logs"): void { + let original: string | undefined; + + beforeAll(() => { + original = process.env.CLERK_MIGRATE_LOG_DIR; + process.env.CLERK_MIGRATE_LOG_DIR = dir; + }); + + // Resolution is cached per process, and each test file runs under its own + // temporary cwd, so the cached absolute path has to go with it. + beforeEach(() => _resetLogDir()); + + afterAll(() => { + if (original === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; + else process.env.CLERK_MIGRATE_LOG_DIR = original; + _resetLogDir(); + }); +} From 9900266d94ccc0e92d9ea664118509eca8ef2929 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 11 Sep 2026 00:01:54 -0400 Subject: [PATCH 028/141] feat(migrate): clear one setting by name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clerk migrate settings clear ` forgets a single setting and leaves the rest of the project alone; with no name it still clears both stores, as before. Both stores are cleared either way, because a setting can sit in either and `log-dir` can sit in both — clearing half of one is worse than clearing none, since the command would report the setting gone while the next run still read it. An `env` value goes under every spelling the setting answers to, so dropping `CLERK_FIREBASE_ROUNDS` no longer leaves a bare `ROUNDS` behind to win the next resolution. `.choices()` rejects an unknown name before the action runs, so the friendly "Unknown setting" errors inside `set.ts` and `clear.ts` were unreachable from the CLI and a one-character miss got back only the list of eight names. The argument's parser now names the near miss first — `logs-dir` suggests `log-dir` — while leaving whether a value is allowed to Commander. --- .../cli-core/src/commands/migrate/README.md | 30 ++++- .../src/commands/migrate/settings/clear.ts | 109 +++++++++++++++--- .../src/commands/migrate/settings/index.ts | 59 ++++++++-- .../src/commands/migrate/settings/registry.ts | 45 ++++++++ .../migrate/settings/settings.test.ts | 77 ++++++++++++- packages/cli-core/src/lib/next-steps.ts | 1 + 6 files changed, 292 insertions(+), 29 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 91a06eebb..1fbf3e1ea 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -574,14 +574,31 @@ clerk migrate settings # list clerk migrate settings list --json clerk migrate settings set transformer firebase clerk migrate settings set firebase-signer-key abc123 -clerk migrate settings clear -y +clerk migrate settings clear firebase-signer-key # forget one +clerk migrate settings clear -y # forget them all ``` -| Subcommand | Takes | Description | -| ----------------------------- | ---------------- | --------------------------------------------------------- | -| `settings list` | `--json` | Every setting, its value and the source it resolved from | -| `settings set ` | ` ` | Change one setting | -| `settings clear` | `-y, --yes` | Forget this project's settings and delete its credentials | +| Subcommand | Takes | Description | +| ----------------------------- | --------------------- | -------------------------------------------------------- | +| `settings list` | `--json` | Every setting, its value and the source it resolved from | +| `settings set ` | ` ` | Change one setting | +| `settings clear [name]` | `[name]`, `-y, --yes` | Forget one setting, or every setting and its credentials | + +`settings clear ` leaves the rest of the project's settings alone. For a +credential it drops every variable the setting answers to, aliases included — +clearing `firebase-rounds` while a bare `ROUNDS` stayed behind in the same file +would report the setting cleared and leave the next run reading the old value. +It only ever edits `.env.clerk-migrate`; a value coming from the app's own env +file or the shell is named in the listing's source column and has to be removed +there. + +A misspelled name gets the closest match back, not just the list: + +``` +$ clerk migrate settings clear logs-dir +error: command-argument value 'logs-dir' is invalid for argument 'name'. + Did you mean "log-dir"? Allowed choices are transformer, file, … +``` Setting names are kebab-case and identical to the `clerk migrate import` flag each one backs, so `firebase-signer-key` here is `--firebase-signer-key` there @@ -623,6 +640,7 @@ firebase-mem-cost 14 MEM_COST env var Firebase scrypt mem 4 of 7 settings set. Credentials are shown redacted. → Run `clerk migrate settings set ` to change one + → Run `clerk migrate settings clear ` to forget one → Run `clerk migrate settings clear` to forget them all, credentials included ``` diff --git a/packages/cli-core/src/commands/migrate/settings/clear.ts b/packages/cli-core/src/commands/migrate/settings/clear.ts index cb588e0eb..23871d90e 100644 --- a/packages/cli-core/src/commands/migrate/settings/clear.ts +++ b/packages/cli-core/src/commands/migrate/settings/clear.ts @@ -1,41 +1,120 @@ /** - * `clerk migrate settings clear` — forget this project's migration settings. + * `clerk migrate settings clear [name]` — forget this project's migration + * settings, or just one of them. * - * Clears both stores by default. The credentials half is the reason this - * command exists: after a migration finishes, a Firebase signer key sitting in - * the repo has no further use, and "delete the file yourself" is a step people - * skip. + * Clears both stores when given no name. The credentials half is the reason + * this command exists: after a migration finishes, a Firebase signer key + * sitting in the repo has no further use, and "delete the file yourself" is a + * step people skip. * * `migrate delete` reads the saved transformer and file to know what to undo, * so clearing is confirmed unless `-y` — an operator who clears and then wants * to undo has no record left to undo from. */ -import { throwUserAbort } from "../../../lib/errors.ts"; +import { throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { confirm } from "../../../lib/prompts.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { clearMigrateEnvValues, MIGRATE_ENV_FILE } from "../lib/env-file.ts"; import { loadSettings, saveSettings } from "../lib/settings.ts"; -import { SETTINGS } from "./registry.ts"; +import type { MigrationEntry } from "../../../lib/config.ts"; +import { envNames, findSetting, SETTING_NAMES, SETTINGS } from "./registry.ts"; export type SettingsClearOptions = { yes?: boolean; }; -const ENV_VARS = SETTINGS.filter((s) => s.store === "env").map((s) => s.envVar as string); +/** + * Every variable the migration settings own, `log-dir`'s included. + * + * Keyed on declaring an `envVar` rather than on `store === "env"`: `log-dir` is + * remembered in the config but still answers to a variable, and a full clear + * that left that variable behind would not have cleared the setting. + */ +const ENV_VARS = SETTINGS.filter((s) => s.envVar).map((s) => s.envVar as string); + +/** + * Warns that clearing this is what `migrate delete` reads to find the users the + * last run created. + */ +function warnAboutUndo(saved: MigrationEntry): void { + if (!saved.file) return; + log.warn( + `\`clerk migrate delete\` uses the saved file (${saved.file}) to identify the users the last run created. ` + + "Clearing it leaves nothing to undo from.", + ); +} + +/** + * Clears one named setting, leaving the rest of the project's settings alone. + * + * Both stores are cleared, because a setting can sit in either and `log-dir` + * can sit in both. Clearing half of one is worse than clearing none: the + * command reports the setting gone while the next run still reads it. + * + * An `env` value goes under every spelling the setting answers to, not just the + * prefixed one — dropping `CLERK_FIREBASE_ROUNDS` while `ROUNDS` stayed in the + * same file would leave the old value winning. + */ +async function clearOne(name: string, options: SettingsClearOptions): Promise { + const setting = findSetting(name); + if (!setting) { + throwUsageError( + `Unknown setting "${name}". Valid names: ${SETTING_NAMES.join(", ")}.`, + undefined, + undefined, + [ + { + command: "clerk migrate settings", + description: "List the settings and their current values", + }, + ], + ); + } + + const saved = await loadSettings(); + + if (!options.yes && isHuman() && !isAgent()) { + // Only the file itself is what `migrate delete` cannot do without; the + // transformer it can be told again. + if (setting.configKey === "file") warnAboutUndo(saved); + const proceed = await confirm({ message: `Clear \`${name}\`?`, default: false }); + if (!proceed) throwUserAbort(); + } + + const cleared: string[] = []; + + if (setting.envVar && (await clearMigrateEnvValues(envNames(setting))).length > 0) { + cleared.push(MIGRATE_ENV_FILE); + } + + const key = setting.configKey as keyof MigrationEntry | undefined; + if (key && saved[key] !== undefined) { + const { [key]: _cleared, ...rest } = saved; + await saveSettings(rest); + cleared.push("this project's settings"); + } + + if (cleared.length === 0) { + log.info( + `\`${name}\` is not set here. A value coming from the app's own env files or the shell has ` + + "to be removed there — run `clerk migrate settings` to see which is supplying it.", + ); + return; + } + + log.success(`Cleared \`${name}\` from ${cleared.join(" and ")}.`); +} + +export async function clear(options: SettingsClearOptions = {}, name?: string): Promise { + if (name !== undefined) return clearOne(name, options); -export async function clear(options: SettingsClearOptions = {}): Promise { const saved = await loadSettings(); const hadConfig = Object.keys(saved).length > 0; if (!options.yes && isHuman() && !isAgent()) { - if (hadConfig && saved.file) { - log.warn( - `\`clerk migrate delete\` uses the saved file (${saved.file}) to identify the users the last run created. ` + - "Clearing it leaves nothing to undo from.", - ); - } + if (hadConfig) warnAboutUndo(saved); const proceed = await confirm({ message: "Clear this project's migration settings?", default: false, diff --git a/packages/cli-core/src/commands/migrate/settings/index.ts b/packages/cli-core/src/commands/migrate/settings/index.ts index 7ce5b6f44..2a9aa0fdf 100644 --- a/packages/cli-core/src/commands/migrate/settings/index.ts +++ b/packages/cli-core/src/commands/migrate/settings/index.ts @@ -1,12 +1,48 @@ -import { createArgument } from "@commander-js/extra-typings"; +import { createArgument, InvalidArgumentError } from "@commander-js/extra-typings"; import type { Command } from "@commander-js/extra-typings"; import { clear } from "./clear.ts"; import { list } from "./list.ts"; -import { SETTING_NAMES } from "./registry.ts"; +import { SETTING_NAMES, suggestSettingName } from "./registry.ts"; import { set } from "./set.ts"; const settings = { clear, list, set }; +/** + * The `` argument both `set` and `clear` take. + * + * `.choices()` is what drives tab-completion and the help output's choice list, + * but it is implemented as a `parseArg` that throws before the action runs — so + * the friendlier "Unknown setting" errors inside `set.ts` and `clear.ts` are + * unreachable from the CLI, and a one-character miss like `logs-dir` gets only + * the full list back. Wrapping that parser keeps the completion metadata and + * puts the near miss first, where a reader scanning eight names would not find + * it. + */ +function settingNameArgument` | `[${string}]`>( + spec: S, + description: string, +) { + const argument = createArgument(spec, description).choices(SETTING_NAMES); + const rejectUnlessAllowed = argument.parseArg; + + // Whether the value is allowed stays Commander's question — asking it here + // too would be a second copy of the rule, free to disagree with the first. + // This only adds to the answer when the answer is no. + argument.parseArg = (value: string, previous: T): T => { + try { + return rejectUnlessAllowed?.(value, previous) as T; + } catch (error) { + const suggestion = suggestSettingName(value); + if (!suggestion) throw error; + throw new InvalidArgumentError( + `Did you mean "${suggestion}"? Allowed choices are ${SETTING_NAMES.join(", ")}.`, + ); + } + }; + + return argument; +} + /** * Registers `settings list|set|clear` under the `migrate` group. * @@ -29,6 +65,10 @@ export function registerMigrateSettings( command: "clerk migrate settings set firebase-signer-key abc123", description: "Save a credential to the gitignored .env.clerk-migrate", }, + { + command: "clerk migrate settings clear firebase-signer-key", + description: "Forget one setting", + }, { command: "clerk migrate settings clear -y", description: "Forget this project's settings" }, ]); @@ -47,7 +87,7 @@ export function registerMigrateSettings( settingsCommand .command("set") .description("Set one setting for this project") - .addArgument(createArgument("", "Setting to change").choices(SETTING_NAMES)) + .addArgument(settingNameArgument("", "Setting to change")) .addArgument(createArgument("", "New value")) .setExamples([ { @@ -63,13 +103,18 @@ export function registerMigrateSettings( settingsCommand .command("clear") - .description("Forget the saved settings and remove the saved credentials") + .description("Forget one saved setting, or every setting and saved credential") + .addArgument(settingNameArgument("[name]", "Setting to clear; omit to clear them all")) .option("-y, --yes", "Skip the confirmation prompt") .setExamples([ - { command: "clerk migrate settings clear", description: "Clear after confirming" }, + { command: "clerk migrate settings clear", description: "Clear everything after confirming" }, + { + command: "clerk migrate settings clear file", + description: "Forget only the remembered export file", + }, { command: "clerk migrate settings clear -y", description: "Clear without prompting" }, ]) - .action((_opts, cmd) => - settings.clear(cmd.optsWithGlobals() as Parameters[0]), + .action((name, _opts, cmd) => + settings.clear(cmd.optsWithGlobals() as Parameters[0], name), ); } diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index 7a471e855..f3eb1ffc7 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -142,6 +142,51 @@ export function findSetting(name: string): SettingDef | undefined { return SETTINGS.find((setting) => setting.name === name); } +/** Levenshtein distance, iterative over a single row. */ +function distance(a: string, b: string): number { + const row = Array.from({ length: b.length + 1 }, (_, i) => i); + + for (let i = 1; i <= a.length; i++) { + let diagonal = row[0] as number; + row[0] = i; + for (let j = 1; j <= b.length; j++) { + const above = row[j] as number; + row[j] = Math.min( + above + 1, + (row[j - 1] as number) + 1, + diagonal + (a[i - 1] === b[j - 1] ? 0 : 1), + ); + diagonal = above; + } + } + + return row[b.length] as number; +} + +/** + * The setting a misspelling was probably reaching for. + * + * Every setting name is a compound of short words — `log-dir`, `firebase-mem-cost` + * — so the misses that matter are a pluralised segment or a transposed pair, + * not a different word entirely. One edit per three characters keeps + * `logs-dir` pointing at `log-dir` without letting an unrelated name match + * something and send the reader off after it. + * + * @returns The closest name within that budget, or `undefined` when nothing is + * close enough to be worth naming. + */ +export function suggestSettingName(name: string): string | undefined { + const budget = Math.max(1, Math.floor(name.length / 3)); + + let best: { name: string; distance: number } | undefined; + for (const candidate of SETTING_NAMES) { + const gap = distance(name, candidate); + if (gap <= budget && (!best || gap < best.distance)) best = { name: candidate, distance: gap }; + } + + return best?.name; +} + /** * Every variable an `env` setting answers to, highest priority first. * diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index 946d475c6..220a6cf35 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -9,7 +9,7 @@ import { MIGRATE_ENV_FILE } from "../lib/env-file.ts"; import { loadSettings, saveSettings } from "../lib/settings.ts"; import { clear } from "./clear.ts"; import { list } from "./list.ts"; -import { displayValue, findSetting } from "./registry.ts"; +import { displayValue, findSetting, suggestSettingName } from "./registry.ts"; import { set } from "./set.ts"; const captured = useCaptureLog(); @@ -250,3 +250,78 @@ describe("clear", () => { expect(envFileContent()).toBe("OTHER=keep\n"); }); }); + +describe("clear ", () => { + test("drops one config setting and keeps the rest", async () => { + await set("transformer", "firebase"); + await set("file", "users.json"); + + await clear({ yes: true }, "file"); + + expect(await loadSettings()).toEqual({ transformer: "firebase" }); + }); + + test("drops one credential and keeps the rest of the env file", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + await set("firebase-rounds", "8"); + + await clear({ yes: true }, "firebase-signer-key"); + + expect(envFileContent()).toContain("CLERK_FIREBASE_ROUNDS=8"); + expect(envFileContent()).not.toContain("CLERK_FIREBASE_SIGNER_KEY"); + }); + + // Clearing only the prefixed name would report success and leave the next run + // reading the alias. + test("drops every spelling the setting answers to", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "ROUNDS=8\nOTHER=keep\n"); + + await clear({ yes: true }, "firebase-rounds"); + + expect(envFileContent()).toBe("OTHER=keep\n"); + }); + + test("says so when the setting was not set here", async () => { + await clear({ yes: true }, "firebase-rounds"); + expect(captured.err).toContain("firebase-rounds"); + expect(captured.err).toContain("is not set here"); + + captured.clear(); + await clear({ yes: true }, "transformer"); + expect(captured.err).toContain("transformer"); + expect(captured.err).toContain("is not set here"); + }); + + // `log-dir` is remembered in the config but yields to an env var, so half a + // clear would report success and leave the run reading the same directory. + test("clears a setting that lives in both stores", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "CLERK_MIGRATE_LOG_DIR=./env-logs\n"); + await saveSettings({ logDir: "./saved-logs", transformer: "firebase" }); + + await clear({ yes: true }, "log-dir"); + + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + expect(await loadSettings()).toEqual({ transformer: "firebase" }); + }); + + test("rejects a name that is not a setting", async () => { + await expect(clear({ yes: true }, "nope")).rejects.toThrow(/Unknown setting "nope"/); + }); +}); + +describe("suggestSettingName", () => { + // `.choices()` rejects before the action runs, so this is the only thing + // standing between a one-character miss and a bare list of eight names. + test.each([ + ["logs-dir", "log-dir"], + ["log_dir", "log-dir"], + ["firebase-round", "firebase-rounds"], + ["transfomer", "transformer"], + ])("%s -> %s", (typo, expected) => { + expect(suggestSettingName(typo)).toBe(expected); + }); + + test.each(["banana", "secret", ""])("says nothing for %p", (unrelated) => { + expect(suggestSettingName(unrelated)).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index fbcf6fc64..999f67d30 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -78,6 +78,7 @@ export const NEXT_STEPS = { MIGRATE_DELETE: ["Run `clerk migrate logs list` to inspect the deletion log"], MIGRATE_SETTINGS: [ "Run `clerk migrate settings set ` to change one", + "Run `clerk migrate settings clear ` to forget one", "Run `clerk migrate settings clear` to forget them all, credentials included", ], // The only parameterized entry: a suggested import is worthless unless it From 0f3574492d5a2f0e24f2dad928d64588961446c0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 11 Sep 2026 00:03:31 -0400 Subject: [PATCH 029/141] fix(migrate): warn instead of refusing when an import may exceed the dev user limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `DEV_USER_LIMIT` was 500 and enforced: an import of more users into a development instance was refused outright. Both halves were wrong. The limit a development instance is created with is 100, Clerk raises it per instance on request, and the real value (`max_allowed_users`) is served by no public API — so the number can never be known to be this instance's, and refusing blocked imports the destination would happily accept. The import now reads the live user count from `GET /v1/users/count`, measures the file against the headroom that implies, and warns when it does not fit — naming what the instance already holds and roughly how many users will be rejected. A human is asked whether to continue; `-y` and agent mode proceed on the warning alone. The final prompt then restates the split ("Import 1 user and expect 1 to fail?") rather than a number the instance will not take. The summary's error breakdown gains notes for the two errors that read as account-level restrictions and are not: blocked SMS countries (a per-instance blocklist, with development instances pointed at Clerk's test numbers and production at the Dashboard setting) and the user quota. Both messages point at "contact support", which is the wrong first move for most readers. After a partial import the next steps now lead with the grep that names which users failed and why, since the breakdown only counts each error. --- .../cli-core/src/commands/migrate/README.md | 18 ++- .../src/commands/migrate/lib/clerk-config.ts | 25 ++++ .../src/commands/migrate/lib/instance.test.ts | 4 +- .../src/commands/migrate/lib/instance.ts | 12 +- .../commands/migrate/run-interactive.test.ts | 61 ++++++-- .../cli-core/src/commands/migrate/run.test.ts | 72 +++++++-- packages/cli-core/src/commands/migrate/run.ts | 141 ++++++++++++++++-- packages/cli-core/src/lib/next-steps.ts | 10 +- 8 files changed, 297 insertions(+), 46 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 1fbf3e1ea..4c70a6517 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -119,8 +119,21 @@ that, assuming ~100ms of API latency. Both are overridable: A non-numeric or non-positive value is ignored in favour of the default. -**Development instances refuse imports over 500 users**, matching Clerk's own -limit — the run fails before any request is sent. +**Development instances warn when an import may exceed their user limit.** New +development instances are created with a 100-user limit; production instances +have none. Before importing, the run reads the instance's current user count +(`GET /v1/users/count`) and warns when the file would take it past 100. + +The run then stops and asks before going ahead. It is a prompt rather than a +hard refusal because the number checked against may not be this instance's: +Clerk raises a development instance's limit on request, and the raised value +(`max_allowed_users`) is not served by BAPI, DAPI or FAPI — so the CLI can show +the live count but never the live limit. Declining aborts before anything is +written to Clerk; `-y` and agent mode proceed on the warning alone. + +Users that do exceed the limit come back in the error breakdown as +`You have reached your limit of N users`, annotated with what a development +instance can do about it. ### `clerk migrate export` @@ -1097,6 +1110,7 @@ NDJSON is. The original `.log` stays put. | `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | | `GET` | `/v1/users?external_id=…` | `migrate delete` — finds this migration's users, 100 IDs a call | | `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | | `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | | `GET` | `/v1/domains` | Readiness report and `--skip-unsupported-providers` — resolves the Frontend API host | diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts index 58bb6dceb..f69ae7fab 100644 --- a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -117,3 +117,28 @@ export async function fetchEnabledSocialProviders(secretKey: string): Promise { + try { + const response = await bapiRequest({ method: "GET", path: "/v1/users/count", secretKey }); + const total = (response.body as { total_count?: unknown })?.total_count; + return typeof total === "number" ? total : null; + } catch (error) { + log.debug( + `migrate: could not read the instance's user count: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return null; + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/instance.test.ts b/packages/cli-core/src/commands/migrate/lib/instance.test.ts index 6f6330f56..ef95b0b92 100644 --- a/packages/cli-core/src/commands/migrate/lib/instance.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/instance.test.ts @@ -35,8 +35,8 @@ describe("default limits", () => { expect(getDefaultConcurrencyLimit(rateLimit)).toBe(expected); }); - test("development instances are capped at 500 users", () => { - expect(DEV_USER_LIMIT).toBe(500); + test("development instances default to 100 users", () => { + expect(DEV_USER_LIMIT).toBe(100); }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/instance.ts b/packages/cli-core/src/commands/migrate/lib/instance.ts index d094c11cf..24f222a9d 100644 --- a/packages/cli-core/src/commands/migrate/lib/instance.ts +++ b/packages/cli-core/src/commands/migrate/lib/instance.ts @@ -6,8 +6,16 @@ * `resolveBapiSecretKey`, and only the two override knobs read the environment. */ -/** Development instances are capped at this many users by Clerk. */ -export const DEV_USER_LIMIT = 500; +/** + * The user limit a development instance is created with. + * + * Only a default: Clerk raises it per instance on request, and the real value + * (`max_allowed_users`) is not served by BAPI, DAPI or FAPI — only by Clerk's + * internal staff API. So this is a number to warn against, never one to refuse + * an import over; the instance in front of you may be allowed far more. + * Production instances have no limit at all. + */ +export const DEV_USER_LIMIT = 100; /** How many times a 429 is retried before the user is recorded as failed. */ export const MAX_RETRIES = 5; diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index a7e6909c8..1c7728c1d 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -59,7 +59,10 @@ mock.module("../../lib/keyless-target.ts", () => ({ // Every export of the real module must appear here — a missing one is a link // error at import time, which takes down the whole file rather than one prompt. mock.module("../../lib/prompts.ts", () => ({ - confirm: async () => confirmAnswer, + confirm: async ({ message }: { message: string }) => { + confirmMessages.push(message); + return confirmAnswer; + }, multiselect: (...args: unknown[]) => mockMultiselect(...(args as [MultiselectConfig])), text: (...args: unknown[]) => mockText(...(args as [])), password: async () => "", @@ -119,6 +122,7 @@ afterAll(() => { beforeEach(() => { requests = []; confirmAnswer = true; + confirmMessages = []; instanceTarget = ACCOUNT_TARGET; mockSelect.mockReset(); mockText.mockReset(); @@ -471,18 +475,51 @@ describe("fixing the instance's settings from the report", () => { }); describe("guards that still apply interactively", () => { - test("the dev-instance 500-user cap", async () => { - fs.writeFileSync( - path.join(workDir, "export.json"), - JSON.stringify( - Array.from({ length: 501 }, (_, i) => ({ - id: `u${i}`, - primary_email_address: `u${i}@x.dev`, - })), - ), - ); + /** Makes `GET /v1/users/count` report an instance with one seat left. */ + function stubNearlyFullInstance(): void { + const inner = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (input.toString().includes("/v1/users/count")) { + return Response.json({ object: "total_count", total_count: 99 }); + } + return inner(input, init); + }) as typeof fetch; + } + + test("the dev-instance user limit, which the operator can agree to import past", async () => { + stubNearlyFullInstance(); + confirmAnswer = true; + + await run(baseOptions); + + expect(captured.err).toContain("100-user limit"); + expect(created()).toHaveLength(2); + }); - await expect(run(baseOptions)).rejects.toThrow(/development instance/); + // Asking "Import 2 users?" after warning that one of them cannot fit is the + // report and the quota disagreeing in the same run. + test("the final prompt restates the quota split rather than the file size", async () => { + stubNearlyFullInstance(); + + await run(baseOptions); + + expect(confirmMessages).toContain("Import 1 user and expect 1 to fail?"); + }); + + test("the final prompt names the whole file when the quota is not in play", async () => { + await run(baseOptions); + + expect(confirmMessages).toContain("Import 2 users?"); + }); + + test("declining the user-limit prompt writes nothing to Clerk", async () => { + stubNearlyFullInstance(); + confirmAnswer = false; + + await expect(run(baseOptions)).rejects.toThrow(UserAbortError); + + // Aborted before the readiness report, so nothing was read from FAPI either. + expect(captured.err).not.toContain("Migration readiness"); expect(created()).toHaveLength(0); }); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 049a553ca..0dd898c03 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -8,7 +8,7 @@ import { useCaptureLog } from "../../test/lib/stubs.ts"; import { getLogDir } from "./lib/logger.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; import { loadSettings } from "./lib/settings.ts"; -import { applyResumeAfter, run, validateRunOptions } from "./run.ts"; +import { applyResumeAfter, explainErrors, run, validateRunOptions } from "./run.ts"; import type { User } from "./types.ts"; let workDir: string; @@ -183,19 +183,35 @@ describe("run", () => { expect(captured.err).toContain("1 user failed validation"); }); - test("refuses to exceed the development-instance user limit", async () => { - fs.writeFileSync( - path.join(workDir, "export.json"), - JSON.stringify( - Array.from({ length: 501 }, (_, i) => ({ - id: `u${i}`, - primary_email_address: `u${i}@x.dev`, - })), - ), - ); + /** Makes `GET /v1/users/count` report an instance that already holds users. */ + function stubUserCount(total: number): void { + const inner = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (input.toString().includes("/v1/users/count")) { + return Response.json({ object: "total_count", total_count: total }); + } + return inner(input, init); + }) as typeof fetch; + } - await expect(run(baseOptions)).rejects.toThrow(/development instance/); - expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(0); + // `baseOptions` passes -y, which has nobody to answer the prompt this warning + // otherwise raises — see run-interactive.test.ts for the prompt itself. + test("warns under -y when an import may exceed the development-instance user limit", async () => { + stubUserCount(99); + + await run(baseOptions); + + expect(captured.err).toContain("100-user limit"); + expect(captured.err).toContain("already holds 99"); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + }); + + test("stays quiet when the instance has room for the whole file", async () => { + stubUserCount(10); + + await run(baseOptions); + + expect(captured.err).not.toContain("100-user limit"); }); test("aborts before any API call when the hasher is unrecognized", async () => { @@ -685,3 +701,33 @@ describe("run", () => { }); }); }); + +describe("explainErrors", () => { + const COUNTRY = + "Phone numbers from this country (France) are currently not supported. For more information, please contact support."; + const QUOTA = + "You have reached your limit of 100 users. If you need more users, please use a Production instance."; + + test("names the development instance as the reason countries are blocked", () => { + const [note] = explainErrors([COUNTRY], "dev"); + expect(note).toContain("Development instances block SMS to most countries"); + expect(note).toContain("test-emails-and-phones"); + }); + + test("sends a production operator to the Dashboard instead of support", () => { + const [note] = explainErrors([COUNTRY], "prod"); + expect(note).toContain("customization/sms/settings"); + expect(note).not.toContain("Development instances"); + }); + + test("explains the user quota only where one applies", () => { + expect(explainErrors([QUOTA], "dev").join(" ")).toContain("development-instance quota"); + // Production has no such quota, and the API's message already names the + // plan upgrade in the one case it does. + expect(explainErrors([QUOTA], "prod")).toEqual([]); + }); + + test("says nothing about errors it does not recognize", () => { + expect(explainErrors(["Something else went wrong."], "dev")).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index dd8b6b467..57e374f35 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -28,6 +28,7 @@ import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebas import { enabledSocialProviders, fetchInstanceSettings, + fetchUserCount, toClerkStrategy, } from "./lib/clerk-config.ts"; import { @@ -42,7 +43,7 @@ import { buildSettingChanges, type SettingChange, } from "./lib/modify-settings.ts"; -import { DEV_USER_LIMIT, resolveLimits } from "./lib/instance.ts"; +import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; import { startLogging, getLogFilePath } from "./lib/logger.ts"; import { saveSettings } from "./lib/settings.ts"; import { @@ -168,7 +169,59 @@ export function applyResumeAfter(users: User[], resumeAfter: string | undefined) return users.slice(index + 1); } -function formatSummary(summary: ImportSummary, logFile: string): string { +/** Where a production instance's operator changes the SMS country blocklist. */ +const SMS_SETTINGS_URL = "https://dashboard.clerk.com/~/customization/sms/settings"; + +/** Clerk's fictional email addresses and phone numbers, for development. */ +const TEST_NUMBERS_URL = "https://clerk.com/docs/guides/development/testing/test-emails-and-phones"; + +/** + * What the API's error messages leave out: whether the operator can do + * something about them, and where. + * + * Both of these read as account-level restrictions and are not. Blocked + * countries are a per-instance SMS blocklist that development instances are + * created with far more of, and the user limit is a development-instance quota + * that production does not have at all — so "contact support", which both + * messages point at, is the wrong first move for most readers. + * + * @returns One note per recognized error family, empty when none apply. + */ +export function explainErrors(errors: Iterable, instanceType: InstanceType): string[] { + const all = [...errors]; + const notes: string[] = []; + + if (all.some((error) => error.includes("Phone numbers from this country"))) { + notes.push( + instanceType === "dev" + ? `Development instances block SMS to most countries by default — this is not a limit on your account. ` + + `Use Clerk's test phone numbers while developing (${TEST_NUMBERS_URL}), and contact support only if ` + + `you need real numbers in a specific country before going to production.` + : `Unblock the countries you need under SMS settings in the Dashboard (${SMS_SETTINGS_URL}). ` + + `Plans without SMS support cannot remove them; contact support if the setting is refused.`, + ); + } + + // Production has no user limit unless a plan imposes one, and the API's own + // message already names the fix ("upgrade to a paid plan") in that case. + if ( + instanceType === "dev" && + all.some((error) => /You have reached your limit of \d+ users/.test(error)) + ) { + notes.push( + `The user limit is a development-instance quota (${DEV_USER_LIMIT} by default). Import into a production ` + + `instance to bring everyone across, or contact support to raise this instance's limit.`, + ); + } + + return notes; +} + +function formatSummary( + summary: ImportSummary, + logFile: string, + instanceType: InstanceType, +): string { const inFile = summary.totalProcessed + summary.validationFailed; const lines = [ `${bold("Total users in file:")} ${inFile}`, @@ -184,12 +237,66 @@ function formatSummary(summary: ImportSummary, logFile: string): string { for (const [error, count] of summary.errorBreakdown) { lines.push(` ${count} user${count === 1 ? "" : "s"}: ${error}`); } + for (const note of explainErrors(summary.errorBreakdown.keys(), instanceType)) { + lines.push("", note); + } } lines.push("", dim(`Log: ${logFile}`)); return lines.join("\n"); } +/** + * Stops an import that looks likely to exhaust a development instance's user + * quota, and asks before letting it through anyway. + * + * A prompt rather than a hard refusal, because the number it checks against + * cannot be trusted to be this instance's: {@link DEV_USER_LIMIT} is only what + * an instance is *created* with, Clerk raises it per instance on request, and + * no public endpoint serves the real value. The existing user count is live; + * the limit it is measured against is not. Refusing outright would block + * imports the destination would happily accept, so the operator — who can ask + * Clerk what their limit is — gets the last word. + * + * `-y` and agent mode proceed on the warning alone, matching the import + * confirmation below: neither has anyone to answer the question. + * + * @returns How many of `incoming` the quota is expected to reject, or `0` when + * the whole file fits. The final import prompt reports the same split, so + * that "yes" is never a bigger number than the instance will accept. + * @throws UserAbortError when the operator declines. + */ +async function confirmDevUserLimit( + incoming: number, + secretKey: string, + yes: boolean, +): Promise { + const existing = await withSpinner("Checking the instance's user count...", async () => + fetchUserCount(secretKey), + ); + const headroom = Math.max(0, DEV_USER_LIMIT - (existing ?? 0)); + if (incoming <= headroom) return 0; + + const rejected = incoming - headroom; + const held = existing === null ? "" : `, and this one already holds ${existing}`; + log.warn( + `Development instances default to a ${DEV_USER_LIMIT}-user limit${held}. About ${rejected} of the ` + + `${incoming} user${incoming === 1 ? "" : "s"} in this file will be rejected with a quota error unless ` + + `Clerk has raised this instance's limit — the limit itself is not readable from the API.\n` + + `Import into a production instance to bring everyone across, or contact support to raise the limit.`, + ); + + if (yes || !isHuman() || isAgent()) return rejected; + + const proceed = await confirm({ + message: `Continue anyway, expecting about ${rejected} user${rejected === 1 ? "" : "s"} to be rejected?`, + default: false, + }); + if (!proceed) throwUserAbort(); + + return rejected; +} + /** * Drops users whose only way into Clerk is a social provider the destination * instance has not enabled. @@ -522,13 +629,10 @@ export async function run(rawOptions: MigrateRunOptions): Promise { return; } - if (limits.instanceType === "dev" && users.length > DEV_USER_LIMIT) { - throw new CliError( - `Cannot import ${users.length} users into a development instance — the limit is ${DEV_USER_LIMIT}.\n` + - "Target a production instance, or reduce the import file.", - { code: ERROR_CODE.USAGE_ERROR }, - ); - } + const quotaRejections = + limits.instanceType === "dev" + ? await confirmDevUserLimit(users.length, secretKey, Boolean(options.yes)) + : 0; // `target` already carries the instance's environment ("My App // (development)"), so the detected type is only worth spelling out when @@ -549,8 +653,15 @@ export async function run(rawOptions: MigrateRunOptions): Promise { }); if (!options.yes && isHuman() && !isAgent()) { + // The readiness report counts the whole file, because settings decide + // what Clerk *accepts*. The quota decides how much of it gets in at all, + // so the last prompt — the one that starts writing — restates that split + // rather than asking about a number the instance will not take. + const importable = users.length - quotaRejections; const proceed = await confirm({ - message: `Import ${users.length} user${users.length === 1 ? "" : "s"}?`, + message: quotaRejections + ? `Import ${importable} user${importable === 1 ? "" : "s"} and expect ${quotaRejections} to fail?` + : `Import ${users.length} user${users.length === 1 ? "" : "s"}?`, default: false, }); if (!proceed) throwUserAbort(); @@ -576,11 +687,15 @@ export async function run(rawOptions: MigrateRunOptions): Promise { }), ); - log.info(formatSummary(summary, logFile)); + log.info(formatSummary(summary, logFile, limits.instanceType)); // Offered even when some users failed: a partial import is exactly when - // reading the log and knowing how to undo it matters most. - setNextSteps(NEXT_STEPS.MIGRATE_DONE); + // reading the log and knowing how to undo it matters most. When users did + // fail, the per-user record of *why* leads, since the breakdown above only + // counts each error and never names who hit it. + setNextSteps( + summary.failed > 0 ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(logFile) : NEXT_STEPS.MIGRATE_DONE, + ); if (summary.failed > 0) process.exitCode = 1; }); diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index 999f67d30..35c34dfdb 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -75,14 +75,20 @@ export const NEXT_STEPS = { "Run `clerk migrate logs list` to inspect the import log", "Run `clerk migrate delete` to undo this migration", ], + // `logs list` only names the file; after a partial import the operator needs + // the failures themselves, which live one line per user in that file. + MIGRATE_DONE_WITH_ERRORS: (logFile: string) => [ + `Run \`grep '"status":"error"' ${logFile}\` to see every user that failed and why`, + "Run `clerk migrate delete` to undo this migration", + ], MIGRATE_DELETE: ["Run `clerk migrate logs list` to inspect the deletion log"], MIGRATE_SETTINGS: [ "Run `clerk migrate settings set ` to change one", "Run `clerk migrate settings clear ` to forget one", "Run `clerk migrate settings clear` to forget them all, credentials included", ], - // The only parameterized entry: a suggested import is worthless unless it - // names the transformer that reads this export and the file just written. + // A suggested import is worthless unless it names the transformer that reads + // this export and the file just written. MIGRATE_EXPORT: (transformerKey: string, file: string) => [ `Run \`clerk migrate import --transformer ${transformerKey} --file ${file}\` to import them`, ], From 0faebbc3abeb5974509d5ec43de5e634df3735f7 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 11 Sep 2026 00:03:42 -0400 Subject: [PATCH 030/141] feat(migrate): sign in and link before an import starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this the first complaint came from deep inside the secret-key chain, which resolves the linked profile before it ever asks for a token — so a signed-out operator in an unlinked directory was told to run `clerk link`, a command that would only turn around and ask them to sign in. Both failures landed after the wizard had already walked them through picking a platform and a file. `migrate import` now checks for somewhere to import *into* first, mirroring `resolveBapiSecretKey`: `--secret-key`, `--app`, `CLERK_SECRET_KEY` and an unclaimed accountless application each name the destination on their own. A human gets the same sign-in-then-link flow `clerk link` already runs; an agent, which can answer neither a browser login nor an application picker, gets an error naming whichever half is missing. --- .../cli-core/src/commands/migrate/run.test.ts | 21 +++++- packages/cli-core/src/commands/migrate/run.ts | 71 ++++++++++++++++++- 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 0dd898c03..f7e5364c2 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -1,10 +1,14 @@ -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { _setConfigDir } from "../../lib/config.ts"; import { CliError } from "../../lib/errors.ts"; -import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +// Every test below names its own `--secret-key`, which short-circuits the +// signed-in check — except the one that asserts what happens without it. +mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); import { getLogDir } from "./lib/logger.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; import { loadSettings } from "./lib/settings.ts"; @@ -126,6 +130,19 @@ describe("run", () => { secretKey: "sk_test_x", }; + test("refuses before the wizard when nobody is signed in", async () => { + const previous = process.env.CLERK_SECRET_KEY; + delete process.env.CLERK_SECRET_KEY; + try { + await expect(run({ transformer: "clerk", file: "export.json", yes: true })).rejects.toThrow( + /Not logged in/, + ); + expect(requests).toHaveLength(0); + } finally { + if (previous !== undefined) process.env.CLERK_SECRET_KEY = previous; + } + }); + test("imports every user in the file end to end", async () => { await run(baseOptions); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 57e374f35..8ef97230b 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -14,8 +14,21 @@ import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; import { bold, dim, green, red, yellow } from "../../lib/color.ts"; -import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; -import { resolveInstanceTarget, type InstanceTarget } from "../../lib/keyless-target.ts"; +import { resolveProfile } from "../../lib/config.ts"; +import { hasAccountCredentials } from "../../lib/credential-store.ts"; +import { + AUTH_ERROR_REASON, + AuthError, + CliError, + ERROR_CODE, + throwUsageError, + throwUserAbort, +} from "../../lib/errors.ts"; +import { + resolveInstanceTarget, + resolveKeylessTarget, + type InstanceTarget, +} from "../../lib/keyless-target.ts"; import { log } from "../../lib/log.ts"; import { NEXT_STEPS } from "../../lib/next-steps.ts"; import { confirm, multiselect } from "../../lib/prompts.ts"; @@ -57,6 +70,8 @@ import { loadCustomTransformer } from "./transformers/load-custom.ts"; import { registerCustomTransformer, transformerKeys } from "./transformers/registry.ts"; import type { ImportSummary, User } from "./types.ts"; import { runWizard, throwAgentFlagsRequired } from "./wizard.ts"; +import { login } from "../auth/login.ts"; +import { link } from "../link/index.ts"; export type MigrateRunOptions = { transformer?: string; @@ -579,7 +594,59 @@ async function applyCustomTransformer(options: MigrateRunOptions): Promise { + // Each of these names the destination instance on its own, with no account + // and no linked directory involved — mirroring resolveBapiSecretKey. + if (options.secretKey || options.app || process.env.CLERK_SECRET_KEY) return; + // An unclaimed accountless application keeps its only secret key on disk. + if (await resolveKeylessTarget({ instance: options.instance })) return; + + const interactive = isHuman() && !isAgent(); + + if (!(await hasAccountCredentials())) { + if (!interactive) { + throw new AuthError({ + reason: AUTH_ERROR_REASON.NOT_LOGGED_IN, + message: + "Not logged in, so there is no Clerk instance to import into. Run `clerk auth login`, then `clerk link`.", + examples: [ + { command: "clerk auth login", description: "Sign in, then re-run the import" }, + { + command: + "clerk migrate import -y --secret-key sk_test_... --transformer clerk --file users.json", + description: "Import without signing in", + }, + ], + }); + } + log.info("Not logged in. Signing in first..."); + await login({ showNextSteps: false }); + } + + // Left to the secret-key chain when non-interactive: its `not_linked` error + // is the one every other command raises, and there is nothing to add to it. + if (interactive && !(await resolveProfile(process.cwd()))) { + log.info("This directory isn't linked to a Clerk application. Linking one first..."); + await link({ skipIfLinked: true }); + } +} + export async function run(rawOptions: MigrateRunOptions): Promise { + await ensureImportTarget(rawOptions); rawOptions = await applyCustomTransformer(rawOptions); const options = await resolveMissingOptions(rawOptions); From 4bf8048c68223df2bb38614cd88086bed53f1723 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 11 Sep 2026 00:03:54 -0400 Subject: [PATCH 031/141] feat(migrate): accept libsql/Turso connection strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--db-url "libsql://app-org.turso.io"` fell through to the SQLite default and `bun:sqlite` tried to open a local file by that name. A libsql URL now routes to the server's HTTP pipeline endpoint instead: `bun:sqlite` only opens local files, and `@libsql/client` ships native optional dependencies that do not survive `bun build --compile`, so the wire protocol is fewer lines than the dependency would be. The client reports itself as `sqlite`, since that is the dialect — nothing downstream branches differently. The token comes from `?authToken=` on the URL, the form the Turso CLI prints, or from `TURSO_AUTH_TOKEN`/`LIBSQL_AUTH_TOKEN`; a self-hosted sqld with auth disabled needs neither. Redaction covers the query parameter as well as userinfo, so a token cannot reach an error message or `--verbose` output, and a 401 is explained rather than left as a bare status. --- .../cli-core/src/commands/migrate/README.md | 30 ++-- .../src/commands/migrate/export/authjs.ts | 2 +- .../src/commands/migrate/export/betterauth.ts | 2 +- .../src/commands/migrate/export/db-options.ts | 11 +- .../src/commands/migrate/export/index.ts | 2 +- .../src/commands/migrate/lib/db.test.ts | 71 +++++++++ .../cli-core/src/commands/migrate/lib/db.ts | 140 +++++++++++++++++- 7 files changed, 235 insertions(+), 23 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 4c70a6517..9a4d001e0 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -177,14 +177,14 @@ like every other path flag here. The question comes before any users are fetched, so a long export can be left unattended rather than stalling on a prompt with everything held in memory. -| Flag | Platforms | Description | -| -------------------------- | ---------------------------------- | -------------------------------------------- | -| `-o, --output ` | all | Where to write the export | -| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL or SQLite connection string | -| `--service-account ` | `firebase` | Path to a service account key JSON file | -| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | -| `--client-id ` | `auth0` | Machine-to-machine application client ID | -| `--client-secret ` | `auth0` | Machine-to-machine application client secret | +| Flag | Platforms | Description | +| -------------------------- | ---------------------------------- | --------------------------------------------------------- | +| `-o, --output ` | all | Where to write the export | +| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL, libsql/Turso or SQLite connection string | +| `--service-account ` | `firebase` | Path to a service account key JSON file | +| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | +| `--client-id ` | `auth0` | Machine-to-machine application client ID | +| `--client-secret ` | `auth0` | Machine-to-machine application client secret | `export clerk` also takes the targeting flags — it reads from a Clerk instance, so it resolves a key the same way `clerk migrate import` does, with one extra @@ -249,13 +249,17 @@ These three read the database directly, over **`--db-url`**: clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" clerk migrate export authjs --db-url "mysql://user:...@127.0.0.1:3306/authjs" clerk migrate export betterauth --db-url "./db.sqlite" +clerk migrate export betterauth --db-url "libsql://app-org.turso.io?authToken=..." # or set TURSO_AUTH_TOKEN ``` -Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`. Both are -built into the runtime, so nothing native ships in the binary — that is the -whole reason the `engines.bun` floor exists. Resolution is `--db-url`, then -`SUPABASE_DB_URL` / `AUTHJS_DB_URL` / `BETTERAUTH_DB_URL`, then a masked prompt, -since a connection string carries the password inline. +Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; +`libsql://` (Turso) over the server's HTTP pipeline endpoint, since `bun:sqlite` +only opens local files and `@libsql/client` ships native optional dependencies. +Nothing native ships in the binary — that is the whole reason the `engines.bun` +floor exists. Resolution is `--db-url`, then `SUPABASE_DB_URL` / `AUTHJS_DB_URL` +/ `BETTERAUTH_DB_URL`, then a masked prompt, since a connection string carries +the password inline. A libsql token comes from `?authToken=` on the URL, or from +`TURSO_AUTH_TOKEN` / `LIBSQL_AUTH_TOKEN`, and is redacted like a password. **Connection strings are redacted everywhere.** Errors show `postgres://***@host/db`, including when the password itself contains an diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index d378c93c1..d9b3f2c72 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -110,7 +110,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { platform: "authjs", envVar: "AUTHJS_DB_URL", prompt: "Auth.js database connection string", - hint: "Postgres, MySQL or a SQLite file — whichever your Auth.js adapter uses.", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Auth.js adapter uses.", }); const destination = await resolveOutputPath("authjs", options.output); diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index f2d26c6c6..348271a6d 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -157,7 +157,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise platform: "betterauth", envVar: "BETTERAUTH_DB_URL", prompt: "Better Auth database connection string", - hint: "Postgres, MySQL or a SQLite file — whichever your Better Auth install uses.", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Better Auth install uses.", }); const destination = await resolveOutputPath("betterauth", options.output); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index e2957697e..3ef77efbe 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -10,7 +10,7 @@ import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isAgent, isHuman } from "../../../mode.ts"; -import { detectDbType, redactConnectionString, type DbPlatform } from "../lib/db.ts"; +import { detectDbType, isLibsqlUrl, redactConnectionString, type DbPlatform } from "../lib/db.ts"; import { findMigrateEnvValue } from "../lib/env-file.ts"; export type DbExportOptions = { @@ -27,7 +27,7 @@ type ResolveConfig = { hint?: string; }; -const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2):\/\//i; +const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2|libsql):\/\//i; /** * True when the string parses as a URL with a host. @@ -103,7 +103,7 @@ export async function resolveDbUrl( if (fromFlag) { if (!looksLikeConnectionString(fromFlag)) { throwUsageError( - `--db-url does not look like a connection string. Expected postgres://…, mysql://… or a SQLite file path.\n` + + `--db-url does not look like a connection string. Expected postgres://…, mysql://…, libsql://… or a SQLite file path.\n` + "If the password contains @, # or /, URL-encode it.", ); } @@ -140,7 +140,7 @@ export async function resolveDbUrl( validate: (value) => looksLikeConnectionString(normalizeConnectionString(value ?? "")) ? undefined - : "Expected postgres://…, mysql://… or a SQLite file path", + : "Expected postgres://…, mysql://…, libsql://… or a SQLite file path", }); return normalizeConnectionString(answer); @@ -148,5 +148,6 @@ export async function resolveDbUrl( /** Describes the target for the run's opening line, credentials removed. */ export function describeTarget(connectionString: string): string { - return `${detectDbType(connectionString)} at ${redactConnectionString(connectionString)}`; + const label = isLibsqlUrl(connectionString) ? "libsql" : detectDbType(connectionString); + return `${label} at ${redactConnectionString(connectionString)}`; } diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index b11ea8814..e72066676 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -171,7 +171,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)`, ) - .option("--db-url ", "Postgres, MySQL or SQLite connection string") + .option("--db-url ", "Postgres, MySQL, libsql/Turso or SQLite connection string") .option("-o, --output ", "Where to write the export, relative to the current directory") .setExamples([ { diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index 3fc969ee5..05ae45928 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -39,6 +39,7 @@ describe("detectDbType", () => { ["mysql://u:p@h/db", "mysql"], ["mysql2://u:p@h/db", "mysql"], ["./db.sqlite", "sqlite"], + ["libsql://app-org.turso.io", "sqlite"], ["file:./db.sqlite", "sqlite"], ["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/abs/path.db", "sqlite"], [" postgres://u:p@h/db ", "postgres"], @@ -52,6 +53,7 @@ describe("redactConnectionString", () => { ["postgres://user:secret@host:5432/db", "postgres://***@host:5432/db"], ["mysql://root:hunter2@127.0.0.1:3306/app", "mysql://***@127.0.0.1:3306/app"], ["postgres://host/db", "postgres://host/db"], + ["libsql://app.turso.io?authToken=secret", "libsql://app.turso.io?authToken=***"], ])("%s -> %s", (input, expected) => { expect(redactConnectionString(input)).toBe(expected); }); @@ -89,6 +91,75 @@ describe("sqlitePath", () => { }); }); +describe("a libsql client", () => { + const originalFetch = globalThis.fetch; + let requests: { url: string; token?: string; body: any }[] = []; + + function stubFetch(result: unknown) { + requests = []; + globalThis.fetch = (async (url: string, init: RequestInit) => { + requests.push({ + url: String(url), + token: (init.headers as Record).authorization, + body: JSON.parse(String(init.body)), + }); + return new Response(JSON.stringify({ results: [result, { type: "ok" }] }), { + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + } + + const okRows = (cols: string[], rows: unknown[][]) => ({ + type: "ok", + response: { type: "execute", result: { cols: cols.map((name) => ({ name })), rows } }, + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + test("posts to the pipeline endpoint with the URL's token and decodes rows", async () => { + stubFetch( + okRows( + ["id", "count", "verified", "missing", "hash"], + [ + [ + { type: "text", value: "u1" }, + { type: "integer", value: "12" }, + { type: "float", value: 1.5 }, + { type: "null" }, + { type: "blob", base64: Buffer.from("hash").toString("base64") }, + ], + ], + ), + ); + + const client = await createDbClient("libsql://app-org.turso.io?authToken=t0ken"); + const rows = await client.query('SELECT * FROM "user" WHERE id = ?', ["u1"]); + await client.close(); + + expect(requests[0]?.url).toBe("https://app-org.turso.io/v2/pipeline"); + expect(requests[0]?.token).toBe("Bearer t0ken"); + expect(requests.at(-1)?.body.requests[0].stmt.args).toEqual([{ type: "text", value: "u1" }]); + expect(rows).toEqual([ + { + id: "u1", + count: 12, + verified: 1.5, + missing: null, + hash: Buffer.from("hash"), + }, + ] as never); + expect(client.dbType).toBe("sqlite"); + }); + + test("reports a server-side error", async () => { + stubFetch({ type: "error", error: { message: "no such table: user" } }); + + await expect(createDbClient("libsql://app-org.turso.io")).rejects.toThrow(CliError); + }); +}); + describe("a sqlite client", () => { test("connects and queries", async () => { const client = await createDbClient(dbPath); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index d048a3348..64191c125 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -36,7 +36,8 @@ export interface DbClient { * * Anything that is not a recognized URL scheme is treated as a SQLite path, * matching how the standalone tool behaved and how users actually pass - * `./db.sqlite`. + * `./db.sqlite`. `libsql://` (Turso) is SQLite too — it only differs in how + * the rows are fetched, so callers that branch on the dialect want "sqlite". */ export function detectDbType(connectionString: string): DbType { const lower = connectionString.trim().toLowerCase(); @@ -45,6 +46,11 @@ export function detectDbType(connectionString: string): DbType { return "sqlite"; } +/** True for a remote libsql/Turso URL, which is read over HTTP rather than opened. */ +export function isLibsqlUrl(connectionString: string): boolean { + return /^libsql:\/\//i.test(connectionString.trim()); +} + /** * Replaces any credentials in a connection string with `***`. * @@ -58,7 +64,10 @@ export function redactConnectionString(connectionString: string): string { // the rest of the password in the message. Everything before the final `@` // is userinfo, so redacting all of it is always safe. // Non-URL forms (SQLite paths) have no `://` and are left alone. - return connectionString.replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@"); + // Turso carries its credential as `?authToken=`, not as userinfo. + return connectionString + .replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@") + .replace(/([?&]authToken=)[^&]*/gi, "$1***"); } /** Strips a `file:` prefix and any URL query, leaving a filesystem path. */ @@ -93,6 +102,120 @@ function bunSqlClient(connectionString: string, dbType: "postgres" | "mysql"): D }; } +/** + * One value in Hrana's wire format, the protocol libsql servers speak. + * + * Integers arrive as strings so 64-bit values survive JSON. + */ +type HranaValue = { type: string; value?: string | number; base64?: string }; + +function decodeHrana(value: HranaValue): unknown { + switch (value.type) { + case "null": + return null; + case "integer": { + const raw = String(value.value ?? "0"); + const asNumber = Number(raw); + // Past 2^53 a number would silently lose digits; ids can get that big. + return Number.isSafeInteger(asNumber) ? asNumber : BigInt(raw); + } + case "float": + return Number(value.value); + case "blob": + // bun:sqlite hands back bytes for a BLOB, so this does too. + return Buffer.from(value.base64 ?? "", "base64"); + default: + return value.value ?? null; + } +} + +function encodeHrana(param: unknown): HranaValue { + if (param === null || param === undefined) return { type: "null" }; + if (typeof param === "bigint") return { type: "integer", value: param.toString() }; + if (typeof param === "boolean") return { type: "integer", value: param ? "1" : "0" }; + if (typeof param === "number") { + return Number.isInteger(param) + ? { type: "integer", value: String(param) } + : { type: "float", value: param }; + } + if (param instanceof Uint8Array) { + return { type: "blob", base64: Buffer.from(param).toString("base64") }; + } + return { type: "text", value: String(param) }; +} + +/** + * Talks to a libsql server (Turso) over its HTTP pipeline endpoint. + * + * `bun:sqlite` opens local files and cannot reach a remote database, and + * `@libsql/client` ships native optional dependencies that do not survive + * `bun build --compile`. The protocol is one POST per statement, so it is + * fewer lines to speak it directly than to carry the dependency. + * + * The token comes from `?authToken=` on the URL — the form the Turso CLI + * prints — or from `TURSO_AUTH_TOKEN`/`LIBSQL_AUTH_TOKEN`. A self-hosted sqld + * with auth disabled needs neither, so a missing token is not an error here. + */ +function libsqlClient( + connectionString: string, + env: Record = process.env, +): DbClient { + const url = new URL(connectionString.trim()); + const token = url.searchParams.get("authToken") || env.TURSO_AUTH_TOKEN || env.LIBSQL_AUTH_TOKEN; + const endpoint = `https://${url.host}/v2/pipeline`; + + return { + dbType: "sqlite", + async query>(query: string, params: unknown[] = []) { + const response = await fetch(endpoint, { + method: "POST", + headers: { + "content-type": "application/json", + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + // `close` keeps every request stateless: no baton to carry forward. + body: JSON.stringify({ + requests: [ + { type: "execute", stmt: { sql: query, args: params.map(encodeHrana) } }, + { type: "close" }, + ], + }), + }); + + if (!response.ok) { + throw new Error(`libsql request failed: ${response.status} ${response.statusText}`.trim()); + } + + const body = (await response.json()) as { + results?: { + type: string; + error?: { message?: string }; + response?: { result?: { cols?: { name?: string }[]; rows?: HranaValue[][] } }; + }[]; + }; + + const first = body.results?.[0]; + if (!first || first.type === "error") { + throw new Error(first?.error?.message ?? "libsql returned no result"); + } + + const cols = first.response?.result?.cols ?? []; + const rows = first.response?.result?.rows ?? []; + return rows.map( + (row) => + Object.fromEntries( + row.map((value, index) => [cols[index]?.name ?? String(index), decodeHrana(value)]), + ) as T, + ); + }, + placeholder: () => "?", + quote: QUOTING.sqlite, + close() { + return Promise.resolve(); + }, + }; +} + function sqliteClient(connectionString: string): DbClient { const database = new Database(sqlitePath(connectionString), { readonly: true }); @@ -124,6 +247,12 @@ export async function createDbClient( const dbType = detectDbType(connectionString); try { + if (isLibsqlUrl(connectionString)) { + const client = libsqlClient(connectionString); + await client.query("SELECT 1"); + return client; + } + if (dbType === "sqlite") { const client = sqliteClient(connectionString); // bun:sqlite opens lazily, so a missing file would not surface until the @@ -166,6 +295,13 @@ export function describeDbError(error: unknown, platform?: DbPlatform): string { return "Could not reach the database. Check the host and port, and that the server accepts connections from here."; } + if (/\b401\b|unauthorized|not authorized/i.test(message)) { + return ( + "The libsql server rejected that token.\n" + + "Append ?authToken=… to the URL, or set TURSO_AUTH_TOKEN (`turso db tokens create `)." + ); + } + if (/password authentication failed|access denied/i.test(message)) { return "The database rejected those credentials. Check the user and password in the connection string."; } From 94b15e82b46b89fed2c8820d9fd741c9efc048f4 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 14 Sep 2026 11:28:13 -0400 Subject: [PATCH 032/141] feat(migrate): ask again when a database connection fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A connection string is long, pasted by hand, masked as it is typed, and wrong in ways nothing can check until something connects: a typo'd host, an expired token, the pooler URL where the direct one was needed, the right server but the wrong database. Any of those ended the command, charging the operator a full re-run — platform, log directory, output path and all — for one mistyped line they could not see. `supabase`, `authjs` and `betterauth` now run the database work through `withDbRetry`, which explains the failure and puts the prompt back up. Only the database read is inside the loop, so an export that has already written its file cannot run twice. `-y`, agent mode and a non-TTY fail as before: there is nobody to ask, and a loop that cannot prompt is a loop that cannot end. A libsql 404 is explained rather than sent to the generic advice. Turso resolves every `*.turso.io` name, so a typo'd database answers 404 instead of failing to connect, and "check the host" points at the half that is right. --- .../cli-core/src/commands/migrate/README.md | 8 + .../src/commands/migrate/export/authjs.ts | 27 ++-- .../src/commands/migrate/export/betterauth.ts | 35 +++-- .../src/commands/migrate/export/db-options.ts | 53 ++++++- .../commands/migrate/export/db-retry.test.ts | 139 ++++++++++++++++++ .../src/commands/migrate/export/supabase.ts | 27 ++-- .../cli-core/src/commands/migrate/lib/db.ts | 10 ++ 7 files changed, 266 insertions(+), 33 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/export/db-retry.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 9a4d001e0..ca9e41364 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -252,6 +252,14 @@ clerk migrate export betterauth --db-url "./db.sqlite" clerk migrate export betterauth --db-url "libsql://app-org.turso.io?authToken=..." # or set TURSO_AUTH_TOKEN ``` +**A connection that fails is asked for again.** The string is long, pasted by +hand, masked as it is typed, and wrong in ways nothing can check until +something connects — a typo'd host, an expired token, the pooler URL where the +direct one was needed, the right server but the wrong database. The failure is +explained and the prompt comes back, so a mistyped line costs one line rather +than a re-run of the platform, log directory and output path already answered. +`-y`, agent mode and a non-TTY still fail outright: there is nobody to ask. + Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; `libsql://` (Turso) over the server's HTTP pipeline endpoint, since `bun:sqlite` only opens local files and `@libsql/client` ships native optional dependencies. diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index d9b3f2c72..08c36b965 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -16,7 +16,12 @@ import { log } from "../../../lib/log.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; -import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; +import { + resolveDbUrl, + withDbRetry, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; /** Table names to try, in order. Prisma capitalizes; Drizzle does not. */ const TABLE_CANDIDATES = ["User", "user", "users"] as const; @@ -105,21 +110,25 @@ export function buildAuthJsExport(rows: AuthJsRow[], dateTime: string) { }; } +const AUTHJS_DB = { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Auth.js adapter uses.", +} as const satisfies ResolveConfig; + export async function exportAuthJs(options: DbExportOptions): Promise { - const dbUrl = await resolveDbUrl(options, { - platform: "authjs", - envVar: "AUTHJS_DB_URL", - prompt: "Auth.js database connection string", - hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Auth.js adapter uses.", - }); + const dbUrl = await resolveDbUrl(options, AUTHJS_DB); const destination = await resolveOutputPath("authjs", options.output); await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const { rows, table } = await withSpinner("Reading the user table...", () => - withDbClient(dbUrl, "authjs", fetchAuthJsUsers), + const { rows, table } = await withDbRetry(dbUrl, AUTHJS_DB, async (connectionString) => + withSpinner("Reading the user table...", () => + withDbClient(connectionString, "authjs", fetchAuthJsUsers), + ), ); log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 348271a6d..6eea6262c 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -20,7 +20,12 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; -import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; +import { + resolveDbUrl, + withDbRetry, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; /** Columns a Better Auth plugin adds to the user table. */ export const PLUGIN_COLUMNS = [ @@ -152,25 +157,29 @@ export function buildBetterAuthExport(rows: BetterAuthRow[], dateTime: string) { }; } +const BETTERAUTH_DB = { + platform: "betterauth", + envVar: "BETTERAUTH_DB_URL", + prompt: "Better Auth database connection string", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Better Auth install uses.", +} as const satisfies ResolveConfig; + export async function exportBetterAuth(options: DbExportOptions): Promise { - const dbUrl = await resolveDbUrl(options, { - platform: "betterauth", - envVar: "BETTERAUTH_DB_URL", - prompt: "Better Auth database connection string", - hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Better Auth install uses.", - }); + const dbUrl = await resolveDbUrl(options, BETTERAUTH_DB); const destination = await resolveOutputPath("betterauth", options.output); await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const { rows, plugins } = await withSpinner("Reading the user table...", () => - withDbClient(dbUrl, "betterauth", async (client) => { - const plugins = await detectPluginColumns(client); - const rows = await client.query(buildBetterAuthQuery(client, plugins)); - return { rows, plugins }; - }), + const { rows, plugins } = await withDbRetry(dbUrl, BETTERAUTH_DB, async (connectionString) => + withSpinner("Reading the user table...", () => + withDbClient(connectionString, "betterauth", async (client) => { + const plugins = await detectPluginColumns(client); + const rows = await client.query(buildBetterAuthQuery(client, plugins)); + return { rows, plugins }; + }), + ), ); log.info( diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 3ef77efbe..253fc8e08 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -5,7 +5,7 @@ * string, from a flag, an environment variable, or a prompt. */ -import { throwUsageError } from "../../../lib/errors.ts"; +import { CliError, throwUsageError } from "../../../lib/errors.ts"; import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; @@ -18,7 +18,7 @@ export type DbExportOptions = { output?: string; }; -type ResolveConfig = { +export type ResolveConfig = { platform: DbPlatform; /** Environment variable checked when `--db-url` is absent. */ envVar: string; @@ -135,6 +135,17 @@ export async function resolveDbUrl( if (config.hint) log.info(dim(config.hint)); + return promptDbUrl(config); +} + +/** + * Asks for a connection string, masked. + * + * Masked because a connection string carries the database password inline. The + * validator runs on the normalized value, so a password that needed encoding is + * judged as the driver will see it, not as it was typed. + */ +async function promptDbUrl(config: ResolveConfig): Promise { const answer = await passwordPrompt({ message: config.prompt, validate: (value) => @@ -146,6 +157,44 @@ export async function resolveDbUrl( return normalizeConnectionString(answer); } +/** + * Runs `work` against the database, asking for another connection string each + * time it fails. + * + * A connection string is long, pasted by hand, and wrong in ways nothing can + * check until something connects: a typo'd host, an expired token, the pooler + * URL where the direct one was needed, the right server but the wrong database. + * Ending the command there charges the operator a full re-run — platform, log + * directory, output path and all — for a single mistyped line, and the string + * is masked as they type it, so they cannot even see what to correct. + * + * Only the database work belongs in `work`: everything retried here is retried + * whole, and an export that has already written its file must not run twice. + * + * `-y`, agent mode and a non-TTY get the failure as before — there is nobody to + * ask, and a loop that cannot prompt is a loop that cannot end. + */ +export async function withDbRetry( + dbUrl: string, + config: ResolveConfig, + work: (connectionString: string) => Promise, +): Promise { + let connectionString = dbUrl; + + for (;;) { + try { + return await work(connectionString); + } catch (error) { + // Everything the database layer raises is a CliError carrying its own + // explanation; anything else (an interrupt, a bug) is not ours to retry. + if (!(error instanceof CliError) || !isHuman() || isAgent()) throw error; + + log.error(error.message); + connectionString = await promptDbUrl(config); + } + } +} + /** Describes the target for the run's opening line, credentials removed. */ export function describeTarget(connectionString: string): string { const label = isLibsqlUrl(connectionString) ? "libsql" : detectDbType(connectionString); diff --git a/packages/cli-core/src/commands/migrate/export/db-retry.test.ts b/packages/cli-core/src/commands/migrate/export/db-retry.test.ts new file mode 100644 index 000000000..b0027dbc5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-retry.test.ts @@ -0,0 +1,139 @@ +/** + * `withDbRetry` — the loop that puts the connection-string prompt back up when + * the database work fails. + * + * Its own file because `mock.module` registrations last for the process, and + * `bun test --parallel` puts several files in each worker — a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +let answers: string[] = []; + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../../lib/prompts.ts", () => ({ + password: async () => answers.shift() ?? "", + text: async () => "", + confirm: async () => true, + multiselect: async () => [], + select: async () => "", + editor: async () => "{}", + note: () => {}, +})); + +const { withDbRetry } = await import("./db-options.ts"); + +const captured = useCaptureLog(); + +const CONFIG = { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", +} as const; + +const FIRST = "libsql://typo.turso.io?authToken=t"; +const SECOND = "libsql://right.turso.io?authToken=t"; + +let originalMode: Mode; + +beforeAll(() => { + originalMode = getMode(); +}); + +afterAll(() => { + setMode(originalMode); +}); + +beforeEach(() => { + setMode("human"); + answers = []; +}); + +describe("withDbRetry", () => { + test("returns the first result without prompting when the work succeeds", async () => { + const seen: string[] = []; + + const result = await withDbRetry(FIRST, CONFIG, (url) => { + seen.push(url); + return Promise.resolve("rows"); + }); + + expect(result).toBe("rows"); + expect(seen).toEqual([FIRST]); + }); + + test("asks again after a failure and runs with the new connection string", async () => { + answers = [SECOND]; + const seen: string[] = []; + + const result = await withDbRetry(FIRST, CONFIG, (url) => { + seen.push(url); + if (url === FIRST) { + throw new CliError("Could not reach libsql://***@typo.turso.io", { + code: ERROR_CODE.USAGE_ERROR, + }); + } + return Promise.resolve("rows"); + }); + + expect(result).toBe("rows"); + expect(seen).toEqual([FIRST, SECOND]); + // The operator has to be told what was wrong with the string they cannot see. + expect(captured.err).toContain("Could not reach"); + }); + + test("keeps asking until a connection string works", async () => { + answers = [FIRST, FIRST, SECOND]; + let attempts = 0; + + await withDbRetry(FIRST, CONFIG, (url) => { + attempts++; + if (url !== SECOND) throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); + return Promise.resolve("rows"); + }); + + expect(attempts).toBe(4); + }); + + // Cancelling the prompt is an answer: it ends the command rather than + // looping on a question the operator has already declined. + test("lets a cancelled prompt out of the loop", async () => { + mock.module("../../../lib/prompts.ts", () => ({ + password: async () => { + throw new UserAbortError(); + }, + text: async () => "", + confirm: async () => true, + multiselect: async () => [], + select: async () => "", + editor: async () => "{}", + note: () => {}, + })); + + await expect( + withDbRetry(FIRST, CONFIG, () => { + throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); + }), + ).rejects.toThrow(UserAbortError); + }); + + test("throws without prompting when there is nobody to ask", async () => { + setMode("agent"); + let attempts = 0; + + await expect( + withDbRetry(FIRST, CONFIG, () => { + attempts++; + throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); + }), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index cf257f35f..10d3ee18f 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -15,7 +15,12 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; -import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; +import { + resolveDbUrl, + withDbRetry, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; /** * `display_name` is coalesced into `first_name` here rather than in the @@ -103,21 +108,25 @@ export function buildSupabaseExport(rows: SupabaseRow[], dateTime: string) { }; } +const SUPABASE_DB = { + platform: "supabase", + envVar: "SUPABASE_DB_URL", + prompt: "Supabase Postgres connection string", + hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", +} as const satisfies ResolveConfig; + export async function exportSupabase(options: DbExportOptions): Promise { - const dbUrl = await resolveDbUrl(options, { - platform: "supabase", - envVar: "SUPABASE_DB_URL", - prompt: "Supabase Postgres connection string", - hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", - }); + const dbUrl = await resolveDbUrl(options, SUPABASE_DB); const destination = await resolveOutputPath("supabase", options.output); await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const rows = await withSpinner("Reading auth.users...", () => - withDbClient(dbUrl, "supabase", fetchSupabaseUsers), + const rows = await withDbRetry(dbUrl, SUPABASE_DB, async (connectionString) => + withSpinner("Reading auth.users...", () => + withDbClient(connectionString, "supabase", fetchSupabaseUsers), + ), ); const { users, coverage } = buildSupabaseExport(rows, dateTime); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index 64191c125..9b1681568 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -295,6 +295,16 @@ export function describeDbError(error: unknown, platform?: DbPlatform): string { return "Could not reach the database. Check the host and port, and that the server accepts connections from here."; } + // Turso resolves every `*.turso.io` name, so a typo'd database does not fail + // to connect — it answers 404. "Check the host" would send the reader after + // the half that is right. + if (/\b404\b/.test(message)) { + return ( + "No database at that libsql host. Check the database name in the URL —\n" + + "`turso db show ` prints the URL to use." + ); + } + if (/\b401\b|unauthorized|not authorized/i.test(message)) { return ( "The libsql server rejected that token.\n" + From 563ff35019ab4d22d7cdca24cbe389e66b474ed6 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 14 Sep 2026 12:20:30 -0400 Subject: [PATCH 033/141] refactor(migrate): retry any rejected credential, not just a connection string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `withDbRetry` only ever knew how to re-ask for a connection string, but the shape it handled is not specific to databases: every credential a migration takes is long, pasted by hand, masked as it is typed, and wrong in ways nothing local can check. A Firebase key revoked in the console and an Auth0 application missing `read:users` both read as valid input right up until the far end says otherwise — and both ended the command there, after the operator had already answered every other question it asked. It becomes `withInputRetry` in `migrate/lib/`, taking the input, a way to ask for another, and the step that proves it. `export firebase` and `export auth0` now run their token exchange through it, alongside the three database exports. The helper hands back the input that finally worked, so the rest of the export runs against that one — a Firebase export reads its project id off the key Google accepted, not the key first offered. Only the proving step goes inside the loop: a fetch already under way or a file already written must not run twice. `-y`, agent mode and a non-TTY fail as before, and a cancelled prompt leaves the loop, since declining the question is an answer. --- .../cli-core/src/commands/migrate/README.md | 19 +- .../src/commands/migrate/export/auth0.ts | 35 +++- .../src/commands/migrate/export/authjs.ts | 16 +- .../src/commands/migrate/export/betterauth.ts | 24 ++- .../src/commands/migrate/export/db-options.ts | 42 +--- .../commands/migrate/export/db-retry.test.ts | 139 ------------- .../src/commands/migrate/export/firebase.ts | 28 ++- .../src/commands/migrate/export/supabase.ts | 14 +- .../commands/migrate/lib/input-retry.test.ts | 189 ++++++++++++++++++ .../src/commands/migrate/lib/input-retry.ts | 62 ++++++ 10 files changed, 351 insertions(+), 217 deletions(-) delete mode 100644 packages/cli-core/src/commands/migrate/export/db-retry.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/input-retry.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/input-retry.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index ca9e41364..3363a72c3 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -152,6 +152,17 @@ the registry; given, it runs directly. Each platform resolves its own flags — what Auth0 needs (a tenant domain and M2M credentials) has nothing in common with what a database export needs. +**A credential the far end rejects is asked for again.** Connection strings, +Firebase service account keys and Auth0 client secrets are all long, pasted by +hand, masked as they are typed, and wrong in ways nothing local can check: a +typo'd host, a revoked key, an expired token, the right server but the wrong +database. Only the connection or the token exchange can say, and by then the +operator has answered every other question the command asked. So that step — +and only that step, never a fetch already under way or a file already written — +runs inside a retry: the failure is explained, the prompt comes back, and the +rest of the export continues against whichever credential worked. `-y`, agent +mode and a non-TTY fail outright instead, having nobody to ask. + | Platform | Source | Feeds | | ------------ | -------------------------------- | -------------------------- | | `clerk` | Clerk Backend API | `--transformer clerk` | @@ -252,14 +263,6 @@ clerk migrate export betterauth --db-url "./db.sqlite" clerk migrate export betterauth --db-url "libsql://app-org.turso.io?authToken=..." # or set TURSO_AUTH_TOKEN ``` -**A connection that fails is asked for again.** The string is long, pasted by -hand, masked as it is typed, and wrong in ways nothing can check until -something connects — a typo'd host, an expired token, the pooler URL where the -direct one was needed, the right server but the wrong database. The failure is -explained and the prompt comes back, so a mistyped line costs one line rather -than a re-run of the platform, log directory and output path already answered. -`-y`, agent mode and a non-TTY still fail outright: there is nobody to ask. - Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; `libsql://` (Turso) over the server's HTTP pipeline endpoint, since `bun:sqlite` only opens local files and `@libsql/client` ships native optional dependencies. diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index b8a1a6308..d45e74a73 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -23,6 +23,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spin import { isAgent, isHuman } from "../../../mode.ts"; import { findMigrateEnvValue } from "../lib/env-file.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; const PAGE_SIZE = 100; @@ -113,20 +114,33 @@ export async function resolveAuth0Credentials( "Auth0 needs a machine-to-machine application with the `read:users` scope. Create one under Applications → APIs → Auth0 Management API → Machine to Machine Applications.", ); + return promptAuth0Credentials(resolved); +} + +/** + * Asks for whichever of the three are still missing. + * + * Called with nothing known after Auth0 has rejected a set: its error names no + * field, and the operator may have mistyped any of them — so all three are + * asked again rather than guessing which one to keep. + */ +export async function promptAuth0Credentials( + known: Partial = {}, +): Promise { const domain = - resolved.domain ?? + known.domain ?? (await text({ message: "Auth0 tenant domain (e.g. my-tenant.us.auth0.com)", validate: (value) => (value?.trim() ? undefined : "A domain is required"), })); const clientId = - resolved.clientId ?? + known.clientId ?? (await text({ message: "Machine-to-machine client ID", validate: (value) => (value?.trim() ? undefined : "A client ID is required"), })); const clientSecret = - resolved.clientSecret ?? + known.clientSecret ?? (await passwordPrompt({ message: "Machine-to-machine client secret", validate: (value) => (value?.trim() ? undefined : "A client secret is required"), @@ -314,16 +328,23 @@ export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0Exp } export async function exportAuth0(options: ExportAuth0Options): Promise { - const credentials = await resolveAuth0Credentials(options); + const resolved = await resolveAuth0Credentials(options); const destination = await resolveOutputPath("auth0", options.output); await withGutter("Exporting users from Auth0", async ({ setNextSteps }) => { const dateTime = await startLogging(); - log.info(`Exporting from ${credentials.domain}.`); - const token = await withSpinner("Authenticating with Auth0...", () => - fetchAuth0Token(credentials), + // Only Auth0 can say whether these three go together, and whether the + // application carries the `read:users` scope, so a rejected set is asked + // for again here. + const { value: token, input: credentials } = await withInputRetry( + resolved, + () => promptAuth0Credentials(), + async (candidate) => { + log.info(`Exporting from ${candidate.domain}.`); + return withSpinner("Authenticating with Auth0...", () => fetchAuth0Token(candidate)); + }, ); const users = await withSpinner("Fetching users from Auth0...", (spinner) => diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 08c36b965..2f80c1f9e 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -17,11 +17,12 @@ import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { + promptDbUrl, resolveDbUrl, - withDbRetry, type DbExportOptions, type ResolveConfig, } from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; /** Table names to try, in order. Prisma capitalizes; Drizzle does not. */ const TABLE_CANDIDATES = ["User", "user", "users"] as const; @@ -125,10 +126,15 @@ export async function exportAuthJs(options: DbExportOptions): Promise { await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const { rows, table } = await withDbRetry(dbUrl, AUTHJS_DB, async (connectionString) => - withSpinner("Reading the user table...", () => - withDbClient(connectionString, "authjs", fetchAuthJsUsers), - ), + const { + value: { rows, table }, + } = await withInputRetry( + dbUrl, + () => promptDbUrl(AUTHJS_DB), + async (connectionString) => + withSpinner("Reading the user table...", () => + withDbClient(connectionString, "authjs", fetchAuthJsUsers), + ), ); log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 6eea6262c..e9b901063 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -21,11 +21,12 @@ import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { + promptDbUrl, resolveDbUrl, - withDbRetry, type DbExportOptions, type ResolveConfig, } from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; /** Columns a Better Auth plugin adds to the user table. */ export const PLUGIN_COLUMNS = [ @@ -172,14 +173,19 @@ export async function exportBetterAuth(options: DbExportOptions): Promise await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const { rows, plugins } = await withDbRetry(dbUrl, BETTERAUTH_DB, async (connectionString) => - withSpinner("Reading the user table...", () => - withDbClient(connectionString, "betterauth", async (client) => { - const plugins = await detectPluginColumns(client); - const rows = await client.query(buildBetterAuthQuery(client, plugins)); - return { rows, plugins }; - }), - ), + const { + value: { rows, plugins }, + } = await withInputRetry( + dbUrl, + () => promptDbUrl(BETTERAUTH_DB), + async (connectionString) => + withSpinner("Reading the user table...", () => + withDbClient(connectionString, "betterauth", async (client) => { + const plugins = await detectPluginColumns(client); + const rows = await client.query(buildBetterAuthQuery(client, plugins)); + return { rows, plugins }; + }), + ), ); log.info( diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 253fc8e08..c0f92a92f 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -5,7 +5,7 @@ * string, from a flag, an environment variable, or a prompt. */ -import { CliError, throwUsageError } from "../../../lib/errors.ts"; +import { throwUsageError } from "../../../lib/errors.ts"; import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; @@ -145,7 +145,7 @@ export async function resolveDbUrl( * validator runs on the normalized value, so a password that needed encoding is * judged as the driver will see it, not as it was typed. */ -async function promptDbUrl(config: ResolveConfig): Promise { +export async function promptDbUrl(config: ResolveConfig): Promise { const answer = await passwordPrompt({ message: config.prompt, validate: (value) => @@ -157,44 +157,6 @@ async function promptDbUrl(config: ResolveConfig): Promise { return normalizeConnectionString(answer); } -/** - * Runs `work` against the database, asking for another connection string each - * time it fails. - * - * A connection string is long, pasted by hand, and wrong in ways nothing can - * check until something connects: a typo'd host, an expired token, the pooler - * URL where the direct one was needed, the right server but the wrong database. - * Ending the command there charges the operator a full re-run — platform, log - * directory, output path and all — for a single mistyped line, and the string - * is masked as they type it, so they cannot even see what to correct. - * - * Only the database work belongs in `work`: everything retried here is retried - * whole, and an export that has already written its file must not run twice. - * - * `-y`, agent mode and a non-TTY get the failure as before — there is nobody to - * ask, and a loop that cannot prompt is a loop that cannot end. - */ -export async function withDbRetry( - dbUrl: string, - config: ResolveConfig, - work: (connectionString: string) => Promise, -): Promise { - let connectionString = dbUrl; - - for (;;) { - try { - return await work(connectionString); - } catch (error) { - // Everything the database layer raises is a CliError carrying its own - // explanation; anything else (an interrupt, a bug) is not ours to retry. - if (!(error instanceof CliError) || !isHuman() || isAgent()) throw error; - - log.error(error.message); - connectionString = await promptDbUrl(config); - } - } -} - /** Describes the target for the run's opening line, credentials removed. */ export function describeTarget(connectionString: string): string { const label = isLibsqlUrl(connectionString) ? "libsql" : detectDbType(connectionString); diff --git a/packages/cli-core/src/commands/migrate/export/db-retry.test.ts b/packages/cli-core/src/commands/migrate/export/db-retry.test.ts deleted file mode 100644 index b0027dbc5..000000000 --- a/packages/cli-core/src/commands/migrate/export/db-retry.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -/** - * `withDbRetry` — the loop that puts the connection-string prompt back up when - * the database work fails. - * - * Its own file because `mock.module` registrations last for the process, and - * `bun test --parallel` puts several files in each worker — a mocked - * `prompts.ts` would leak into any file that later lands in the same worker and - * imports the real one. - */ - -import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; -import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; -import { getMode, setMode, type Mode } from "../../../mode.ts"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; - -let answers: string[] = []; - -// Every export of the real module must appear here — a missing one is a link -// error at import time, which takes down the whole file rather than one prompt. -mock.module("../../../lib/prompts.ts", () => ({ - password: async () => answers.shift() ?? "", - text: async () => "", - confirm: async () => true, - multiselect: async () => [], - select: async () => "", - editor: async () => "{}", - note: () => {}, -})); - -const { withDbRetry } = await import("./db-options.ts"); - -const captured = useCaptureLog(); - -const CONFIG = { - platform: "authjs", - envVar: "AUTHJS_DB_URL", - prompt: "Auth.js database connection string", -} as const; - -const FIRST = "libsql://typo.turso.io?authToken=t"; -const SECOND = "libsql://right.turso.io?authToken=t"; - -let originalMode: Mode; - -beforeAll(() => { - originalMode = getMode(); -}); - -afterAll(() => { - setMode(originalMode); -}); - -beforeEach(() => { - setMode("human"); - answers = []; -}); - -describe("withDbRetry", () => { - test("returns the first result without prompting when the work succeeds", async () => { - const seen: string[] = []; - - const result = await withDbRetry(FIRST, CONFIG, (url) => { - seen.push(url); - return Promise.resolve("rows"); - }); - - expect(result).toBe("rows"); - expect(seen).toEqual([FIRST]); - }); - - test("asks again after a failure and runs with the new connection string", async () => { - answers = [SECOND]; - const seen: string[] = []; - - const result = await withDbRetry(FIRST, CONFIG, (url) => { - seen.push(url); - if (url === FIRST) { - throw new CliError("Could not reach libsql://***@typo.turso.io", { - code: ERROR_CODE.USAGE_ERROR, - }); - } - return Promise.resolve("rows"); - }); - - expect(result).toBe("rows"); - expect(seen).toEqual([FIRST, SECOND]); - // The operator has to be told what was wrong with the string they cannot see. - expect(captured.err).toContain("Could not reach"); - }); - - test("keeps asking until a connection string works", async () => { - answers = [FIRST, FIRST, SECOND]; - let attempts = 0; - - await withDbRetry(FIRST, CONFIG, (url) => { - attempts++; - if (url !== SECOND) throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); - return Promise.resolve("rows"); - }); - - expect(attempts).toBe(4); - }); - - // Cancelling the prompt is an answer: it ends the command rather than - // looping on a question the operator has already declined. - test("lets a cancelled prompt out of the loop", async () => { - mock.module("../../../lib/prompts.ts", () => ({ - password: async () => { - throw new UserAbortError(); - }, - text: async () => "", - confirm: async () => true, - multiselect: async () => [], - select: async () => "", - editor: async () => "{}", - note: () => {}, - })); - - await expect( - withDbRetry(FIRST, CONFIG, () => { - throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); - }), - ).rejects.toThrow(UserAbortError); - }); - - test("throws without prompting when there is nobody to ask", async () => { - setMode("agent"); - let attempts = 0; - - await expect( - withDbRetry(FIRST, CONFIG, () => { - attempts++; - throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); - }), - ).rejects.toThrow(CliError); - - expect(attempts).toBe(1); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 93a6f53af..f016d7b08 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -32,6 +32,7 @@ import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; /** Identity Toolkit's maximum for `accounts:batchGet`. */ @@ -166,6 +167,19 @@ async function resolveServiceAccount(options: ExportFirebaseOptions): Promise { const answer = await passwordPrompt({ message: "Path to the service account key file, or paste the key JSON", validate: (value) => { @@ -482,16 +496,22 @@ export function formatHashConfigGuidance( export async function exportFirebase(options: ExportFirebaseOptions): Promise { // Read and validate before anything reaches the network, so a wrong file // fails in a second rather than after an auth round-trip. - const account = await resolveServiceAccount(options); + const resolved = await resolveServiceAccount(options); const destination = await resolveOutputPath("firebase", options.output); await withGutter("Exporting users from Firebase", async ({ setNextSteps }) => { const dateTime = await startLogging(); - log.info(`Exporting from the ${account.project_id} project.`); - const token = await withSpinner("Authenticating with Google...", () => - fetchAccessToken(account), + // Only Google can say whether a well-formed key is still a valid one, so a + // revoked or deleted key fails here and is asked for again. + const { value: token, input: account } = await withInputRetry( + resolved, + promptServiceAccount, + async (candidate) => { + log.info(`Exporting from the ${candidate.project_id} project.`); + return withSpinner("Authenticating with Google...", () => fetchAccessToken(candidate)); + }, ); const users = await withSpinner("Fetching users from Firebase...", (spinner) => diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 10d3ee18f..275a2bbd6 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -16,11 +16,12 @@ import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { + promptDbUrl, resolveDbUrl, - withDbRetry, type DbExportOptions, type ResolveConfig, } from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; /** * `display_name` is coalesced into `first_name` here rather than in the @@ -123,10 +124,13 @@ export async function exportSupabase(options: DbExportOptions): Promise { await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const rows = await withDbRetry(dbUrl, SUPABASE_DB, async (connectionString) => - withSpinner("Reading auth.users...", () => - withDbClient(connectionString, "supabase", fetchSupabaseUsers), - ), + const { value: rows } = await withInputRetry( + dbUrl, + () => promptDbUrl(SUPABASE_DB), + async (connectionString) => + withSpinner("Reading auth.users...", () => + withDbClient(connectionString, "supabase", fetchSupabaseUsers), + ), ); const { users, coverage } = buildSupabaseExport(rows, dateTime); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts new file mode 100644 index 000000000..8e7e6e26b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -0,0 +1,189 @@ +/** + * `withInputRetry` — the loop that puts a credential prompt back up when the + * far end rejects what it was given. + * + * Its own file because `mock.module` registrations last for the process, and + * `bun test --parallel` puts several files in each worker — a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +let answers: string[] = []; +let cancelPrompt = false; + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../../lib/prompts.ts", () => ({ + password: async () => { + if (cancelPrompt) throw new UserAbortError(); + return answers.shift() ?? ""; + }, + text: async () => answers.shift() ?? "", + confirm: async () => true, + multiselect: async () => [], + select: async () => "", + editor: async () => "{}", + note: () => {}, +})); + +const { withInputRetry } = await import("./input-retry.ts"); +const { promptDbUrl } = await import("../export/db-options.ts"); + +const captured = useCaptureLog(); + +const CONFIG = { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", +} as const; + +const FIRST = "libsql://typo.turso.io?authToken=t"; +const SECOND = "libsql://right.turso.io?authToken=t"; + +const rejected = () => new CliError("Could not reach it", { code: ERROR_CODE.USAGE_ERROR }); + +let originalMode: Mode; + +beforeAll(() => { + originalMode = getMode(); +}); + +afterAll(() => { + setMode(originalMode); +}); + +beforeEach(() => { + setMode("human"); + answers = []; + cancelPrompt = false; +}); + +describe("withInputRetry", () => { + test("returns the first result without prompting when the work succeeds", async () => { + const seen: string[] = []; + + const { value, input } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + seen.push(url); + return Promise.resolve("rows"); + }, + ); + + expect(value).toBe("rows"); + expect(input).toBe(FIRST); + expect(seen).toEqual([FIRST]); + }); + + test("asks again after a failure and runs with the new input", async () => { + answers = [SECOND]; + const seen: string[] = []; + + const { value } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + seen.push(url); + if (url === FIRST) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(value).toBe("rows"); + expect(seen).toEqual([FIRST, SECOND]); + // The operator has to be told what was wrong with a string they cannot see. + expect(captured.err).toContain("Could not reach it"); + }); + + // Later steps run against the credential that worked, not the one first tried + // — a Firebase export reads its project id off the key that Google accepted. + test("reports the input that finally worked", async () => { + answers = [SECOND]; + + const { input } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + if (url === FIRST) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(input).toBe(SECOND); + }); + + test("keeps asking until an input works", async () => { + answers = [FIRST, FIRST, SECOND]; + let attempts = 0; + + await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + attempts++; + if (url !== SECOND) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(attempts).toBe(4); + }); + + // Cancelling the prompt is an answer: it ends the command rather than looping + // on a question the operator has already declined. + test("lets a cancelled prompt out of the loop", async () => { + cancelPrompt = true; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + throw rejected(); + }, + ), + ).rejects.toThrow(UserAbortError); + }); + + test("throws without prompting when there is nobody to ask", async () => { + setMode("agent"); + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + + // A bug inside the work, or an interrupt, is not a wrong answer to a prompt. + test("does not retry an error the database layer did not raise", async () => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw new TypeError("undefined is not a function"); + }, + ), + ).rejects.toThrow(TypeError); + + expect(attempts).toBe(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts new file mode 100644 index 000000000..36805de7d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -0,0 +1,62 @@ +/** + * Retrying the *answer*, not the request. + * + * Distinct from `retry.ts`, which re-sends an identical request after a 429: + * there the request was right and the server was busy. Here the request was + * fine and the input was wrong, so nothing changes until the operator supplies + * something better. + * + * Every credential a migration takes — a connection string, a Firebase service + * account key, an Auth0 client secret — is long, pasted by hand, masked as it + * is typed, and wrong in ways nothing local can check: a typo'd host, an + * expired token, a key that was revoked, the right server but the wrong + * database. Only the remote end can say, and by then the operator has already + * answered every other question the command asked. Ending there charges them a + * full re-run for one line they could not see. + */ + +import { CliError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; + +/** + * Runs `work`, and on failure asks for the input again and runs it once more. + * + * Keep `work` to the step that *proves* the input — the connection, the token + * exchange. Everything inside runs again on each attempt, so work that has + * already written a file, or a long fetch the credential has already been + * accepted for, does not belong in here. + * + * `-y`, agent mode and a non-TTY get the failure unchanged: there is nobody to + * ask, and a loop that cannot prompt is a loop that cannot end. A cancelled + * prompt throws {@link UserAbortError}, which is not a `CliError` and so leaves + * the loop — declining the question is an answer. + * + * @param input - What to try first: a flag, an environment value, or the + * answer to the prompt the caller has already put up. + * @param reprompt - Asks for a replacement. Called once per failure. + * @param work - The step the input has to survive. + * @returns The result, and the input that produced it — which is not `input` + * when it took a retry, and later steps need the one that worked. + */ +export async function withInputRetry( + input: I, + reprompt: () => Promise, + work: (input: I) => Promise, +): Promise<{ value: T; input: I }> { + let candidate = input; + + for (;;) { + try { + return { value: await work(candidate), input: candidate }; + } catch (error) { + // Everything these steps raise for a bad credential is a CliError + // carrying its own explanation; anything else (an interrupt, a bug) is + // not ours to retry. + if (!(error instanceof CliError) || !isHuman() || isAgent()) throw error; + + log.error(error.message); + candidate = await reprompt(); + } + } +} From 534894f223c533a42dd40c0e3206597f3b7ecd69 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 14 Sep 2026 12:40:53 -0400 Subject: [PATCH 034/141] fix(migrate): keep the Firebase import command on one line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hash parameters are printed inside the gutter, which prefixes every line it is given with `│`. The command was split over four lines with backslash continuations, so copying it took three of those bars along with it and the shell read them as arguments: error: too many arguments for 'import'. Expected 0 arguments but got 3: │, │, │. The command a user is told to run has to survive being copied, so it is one line however long it gets. A line that wraps on screen carries no bar and pastes back as what was printed. Reported against a real Firebase export. --- packages/cli-core/src/commands/migrate/README.md | 11 ++++++++--- .../src/commands/migrate/export/firebase.test.ts | 10 ++++++++++ .../src/commands/migrate/export/firebase.ts | 13 +++++++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 3363a72c3..d2cf4b051 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -332,11 +332,16 @@ prints the exact import command: ``` Password hash parameters Read from the project. Import with: - clerk migrate import -y --transformer firebase --file exports/firebase-export.json \ - --firebase-signer-key "…" --firebase-salt-separator "…" \ - --firebase-rounds 8 --firebase-mem-cost 14 + clerk migrate import -y --transformer firebase --file exports/firebase-export.json --firebase-signer-key "…" --firebase-salt-separator "…" --firebase-rounds 8 --firebase-mem-cost 14 ``` +On one line however long it gets: this prints inside the gutter, which prefixes +every line given to it with `│`. Split over lines with backslash continuations, +that character lands in the middle of the command and is copied along with it — +the shell then reads each one as another argument and rejects the import. A line +that wraps on screen carries no such character and pastes back as what was +printed. + Reading the config needs a broader role than listing users, so if it is denied the export still succeeds and points at **Authentication → Users → (⋮) → Password hash parameters** instead. An export with no password hashes says so diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index d4a440b7e..4d432e0a0 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -410,6 +410,16 @@ describe("formatHashConfigGuidance", () => { expect(text).toContain("--firebase-rounds 8 --firebase-mem-cost 14"); }); + // The command is printed inside the gutter, which prefixes every line it is + // given with `│`. Split over lines, that character lands mid-command and is + // copied with it — the shell then reads each one as another argument and + // rejects the import. + test("keeps the command on one line, so it can be copied out of the gutter", () => { + const [command] = formatHashConfigGuidance(config, "out.json", 3).slice(-1); + expect(command).not.toContain("\n"); + expect(command).not.toContain("\\"); + }); + test("says where to find them when the project would not say", () => { const text = formatHashConfigGuidance(null, "out.json", 3).join("\n"); expect(text).toContain("Password hash parameters"); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index f016d7b08..3f983c5e9 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -484,11 +484,16 @@ export function formatHashConfigGuidance( return [ bold("Password hash parameters"), "Read from the project. Import with:", + // One line, however long. Inside the gutter every line printed here is + // prefixed with `│`, and backslash continuations put that character in the + // middle of the command — copied along with it, and rejected by the shell + // as three extra arguments. A line that wraps on screen has no such + // character in it and pastes back as what was printed. dim( - ` clerk migrate import -y --transformer firebase --file ${outputPath} \\\n` + - ` --firebase-signer-key "${config.signerKey}" \\\n` + - ` --firebase-salt-separator "${config.saltSeparator}" \\\n` + - ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, + ` clerk migrate import -y --transformer firebase --file ${outputPath}` + + ` --firebase-signer-key "${config.signerKey}"` + + ` --firebase-salt-separator "${config.saltSeparator}"` + + ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, ), ]; } From 37ec7ec876ca4d50c4fcd4f8769c3543dbd234be Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 16 Sep 2026 17:54:06 -0400 Subject: [PATCH 035/141] fix(migrate): mark promise-returning callbacks async MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `promise-function-async` was enabled on main in #436, after this branch's migrate work was written, and arrived here through a later merge of main. That turned 41 call sites across the command into lint errors without any of them changing. Every one is the same shape: a callback handed to Commander, the API scheduler, or a spinner/retry wrapper that returns a promise without being declared `async`. Adding `async` is the sanctioned fix rather than a suppression — `require-await` is deliberately left off (see .claude/rules/promises.md) so exactly these callback shapes can satisfy the rule. Also drops two eslint-disable directives whose violations no longer fire, which `--report-unused-disable-directives-severity=error` counts as failures. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X4wXePuWJHWiocgoBaAX5n --- .../cli-core/src/commands/migrate/delete.ts | 10 ++--- .../src/commands/migrate/export/auth0.ts | 6 +-- .../src/commands/migrate/export/authjs.ts | 4 +- .../src/commands/migrate/export/betterauth.ts | 4 +- .../src/commands/migrate/export/clerk.ts | 4 +- .../src/commands/migrate/export/firebase.ts | 6 ++- .../src/commands/migrate/export/supabase.ts | 4 +- .../src/commands/migrate/import-users.ts | 39 ++++++++++--------- .../cli-core/src/commands/migrate/index.ts | 6 +-- .../cli-core/src/commands/migrate/lib/db.ts | 6 +-- .../src/commands/migrate/lib/scheduler.ts | 2 +- .../src/commands/migrate/logs/index.ts | 10 +++-- .../src/commands/migrate/readme.test.ts | 1 - packages/cli-core/src/commands/migrate/run.ts | 11 +++--- .../src/commands/migrate/settings/index.ts | 6 +-- .../migrate/transformers/list.test.ts | 1 - 16 files changed, 64 insertions(+), 56 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index f80133f76..5b5a78318 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -136,7 +136,7 @@ export async function findMigratedUsers(options: { params.set("limit", String(EXTERNAL_ID_BATCH)); for (const id of ids) params.append("external_id", id); - const response = await retryOn429(() => + const response = await retryOn429(async () => bapiRequest({ method: "GET", path: `/v1/users?${params.toString()}`, @@ -201,8 +201,8 @@ export async function deleteMigratedUsers(options: { const deleteOne = async (user: MigratedUser): Promise => { try { await retryOn429( - () => - schedule(() => + async () => + schedule(async () => bapiRequest({ method: "DELETE", path: `/v1/users/${user.id}`, secretKey }), ), { @@ -272,7 +272,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise + const users = await withSpinner("Finding migrated users...", async (spinner) => findMigratedUsers({ externalIds, secretKey, spinner }), ); @@ -317,7 +317,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise + const summary = await withSpinner(`Deleting users: [0/${users.length}]...`, async (spinner) => deleteMigratedUsers({ users, secretKey, limits, dateTime, spinner }), ); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index d45e74a73..9ae0f4a32 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -340,14 +340,14 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { // for again here. const { value: token, input: credentials } = await withInputRetry( resolved, - () => promptAuth0Credentials(), + async () => promptAuth0Credentials(), async (candidate) => { log.info(`Exporting from ${candidate.domain}.`); - return withSpinner("Authenticating with Auth0...", () => fetchAuth0Token(candidate)); + return withSpinner("Authenticating with Auth0...", async () => fetchAuth0Token(candidate)); }, ); - const users = await withSpinner("Fetching users from Auth0...", (spinner) => + const users = await withSpinner("Fetching users from Auth0...", async (spinner) => fetchAllAuth0Users({ credentials, token, spinner }), ); diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 2f80c1f9e..c5b5b319d 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -130,9 +130,9 @@ export async function exportAuthJs(options: DbExportOptions): Promise { value: { rows, table }, } = await withInputRetry( dbUrl, - () => promptDbUrl(AUTHJS_DB), + async () => promptDbUrl(AUTHJS_DB), async (connectionString) => - withSpinner("Reading the user table...", () => + withSpinner("Reading the user table...", async () => withDbClient(connectionString, "authjs", fetchAuthJsUsers), ), ); diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index e9b901063..d637845a7 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -177,9 +177,9 @@ export async function exportBetterAuth(options: DbExportOptions): Promise value: { rows, plugins }, } = await withInputRetry( dbUrl, - () => promptDbUrl(BETTERAUTH_DB), + async () => promptDbUrl(BETTERAUTH_DB), async (connectionString) => - withSpinner("Reading the user table...", () => + withSpinner("Reading the user table...", async () => withDbClient(connectionString, "betterauth", async (client) => { const plugins = await detectPluginColumns(client); const rows = await client.query(buildBetterAuthQuery(client, plugins)); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 28d5a1e4d..57af5cc88 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -161,7 +161,7 @@ export async function fetchAllClerkUsers(options: { const all: BapiUser[] = []; for (let offset = 0; ; offset += PAGE_SIZE) { - const response = await retryOn429(() => + const response = await retryOn429(async () => bapiRequest({ method: "GET", path: `/v1/users?limit=${PAGE_SIZE}&offset=${offset}`, @@ -239,7 +239,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); - const users = await withSpinner("Fetching users from Clerk...", (spinner) => + const users = await withSpinner("Fetching users from Clerk...", async (spinner) => fetchAllClerkUsers({ secretKey: source.secretKey, spinner }), ); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 3f983c5e9..162c8e2be 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -515,11 +515,13 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { log.info(`Exporting from the ${candidate.project_id} project.`); - return withSpinner("Authenticating with Google...", () => fetchAccessToken(candidate)); + return withSpinner("Authenticating with Google...", async () => + fetchAccessToken(candidate), + ); }, ); - const users = await withSpinner("Fetching users from Firebase...", (spinner) => + const users = await withSpinner("Fetching users from Firebase...", async (spinner) => fetchAllFirebaseUsers({ account, token, spinner }), ); diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 275a2bbd6..3ab72ff78 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -126,9 +126,9 @@ export async function exportSupabase(options: DbExportOptions): Promise { const { value: rows } = await withInputRetry( dbUrl, - () => promptDbUrl(SUPABASE_DB), + async () => promptDbUrl(SUPABASE_DB), async (connectionString) => - withSpinner("Reading auth.users...", () => + withSpinner("Reading auth.users...", async () => withDbClient(connectionString, "supabase", fetchSupabaseUsers), ), ); diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 80402b6d9..4676a7565 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -190,7 +190,7 @@ async function attachIdentifier( : { user_id: clerkUserId, phone_number: value, primary: false, verified }; try { - await ctx.schedule(() => + await ctx.schedule(async () => bapiRequest({ method: "POST", path, @@ -225,7 +225,7 @@ async function createUser( ): Promise { const identifiers = splitIdentifiers(user); - const response = await ctx.schedule(() => + const response = await ctx.schedule(async () => bapiRequest({ method: "POST", path: "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/v1/users", @@ -239,16 +239,16 @@ async function createUser( // Extra identifiers are best-effort: a duplicate secondary email should not // undo a user who was otherwise imported successfully. await Promise.all([ - ...identifiers.additionalEmails.map((email) => + ...identifiers.additionalEmails.map(async (email) => attachIdentifier(ctx, user.userId, clerkUserId, "email", email, true), ), - ...identifiers.unverifiedEmails.map((email) => + ...identifiers.unverifiedEmails.map(async (email) => attachIdentifier(ctx, user.userId, clerkUserId, "email", email, false), ), - ...identifiers.additionalPhones.map((phone) => + ...identifiers.additionalPhones.map(async (phone) => attachIdentifier(ctx, user.userId, clerkUserId, "phone", phone, true), ), - ...identifiers.unverifiedPhones.map((phone) => + ...identifiers.unverifiedPhones.map(async (phone) => attachIdentifier(ctx, user.userId, clerkUserId, "phone", phone, false), ), ]); @@ -313,17 +313,20 @@ export async function importUsers(options: ImportUsersOptions): Promise => { try { - const clerkUserId = await retryOn429(() => createUser(ctx, user, skipPasswordRequirement), { - onRetry: ({ message }) => - errorLogger( - { - userId: user.userId, - status: "429_retry", - errors: [{ code: "rate_limit_retry", message, longMessage: message }], - }, - dateTime, - ), - }); + const clerkUserId = await retryOn429( + async () => createUser(ctx, user, skipPasswordRequirement), + { + onRetry: ({ message }) => + errorLogger( + { + userId: user.userId, + status: "429_retry", + errors: [{ code: "rate_limit_retry", message, longMessage: message }], + }, + dateTime, + ), + }, + ); successful++; processed++; importLogger({ userId: user.userId, status: "success", clerkUserId }, dateTime); @@ -341,7 +344,7 @@ export async function importUsers(options: ImportUsersOptions): Promise processUser(user))); + await Promise.all(users.map(async (user) => processUser(user))); return { totalProcessed: total, successful, failed, validationFailed, errorBreakdown }; } diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index e006cb6dd..f5a1553a2 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -97,7 +97,7 @@ export function registerMigrate(program: Program): void { description: "Skip Supabase users whose only provider is not enabled in Clerk", }, ]) - .action((_opts, cmd) => + .action(async (_opts, cmd) => migrate.run(cmd.optsWithGlobals() as Parameters[0]), ); @@ -117,7 +117,7 @@ export function registerMigrate(program: Program): void { }, { command: "clerk migrate delete -y", description: "Undo without prompting" }, ]) - .action((_opts, cmd) => + .action(async (_opts, cmd) => migrate.delete(cmd.optsWithGlobals() as Parameters[0]), ); @@ -152,7 +152,7 @@ export function registerMigrate(program: Program): void { description: "Include one you wrote", }, ]) - .action((_opts, cmd) => + .action(async (_opts, cmd) => migrate.transformersList( cmd.optsWithGlobals() as Parameters[0], ), diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index 9b1681568..0f2dd4e8b 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -210,7 +210,7 @@ function libsqlClient( }, placeholder: () => "?", quote: QUOTING.sqlite, - close() { + async close() { return Promise.resolve(); }, }; @@ -221,13 +221,13 @@ function sqliteClient(connectionString: string): DbClient { return { dbType: "sqlite", - query>(query: string, params: unknown[] = []) { + async query>(query: string, params: unknown[] = []) { // bun:sqlite is synchronous; the Promise keeps one interface for callers. return Promise.resolve(database.query(query).all(...(params as never[])) as T[]); }, placeholder: () => "?", quote: QUOTING.sqlite, - close() { + async close() { database.close(); return Promise.resolve(); }, diff --git a/packages/cli-core/src/commands/migrate/lib/scheduler.ts b/packages/cli-core/src/commands/migrate/lib/scheduler.ts index 3f6704be9..1a7e65992 100644 --- a/packages/cli-core/src/commands/migrate/lib/scheduler.ts +++ b/packages/cli-core/src/commands/migrate/lib/scheduler.ts @@ -18,7 +18,7 @@ export function createApiScheduler(concurrencyLimit: number, rateLimit: number): let active = 0; let nextRequestAt = 0; - function acquire(): Promise { + async function acquire(): Promise { if (active < maxConcurrent) { active++; return Promise.resolve(); diff --git a/packages/cli-core/src/commands/migrate/logs/index.ts b/packages/cli-core/src/commands/migrate/logs/index.ts index d12a5d29b..abb352d34 100644 --- a/packages/cli-core/src/commands/migrate/logs/index.ts +++ b/packages/cli-core/src/commands/migrate/logs/index.ts @@ -38,7 +38,9 @@ export function registerMigrateLogs(migrateCommand: Command<[], Record logs.list(cmd.optsWithGlobals() as Parameters[0])); + .action(async (_opts, cmd) => + logs.list(cmd.optsWithGlobals() as Parameters[0]), + ); logsCommand .command("clean") @@ -48,7 +50,9 @@ export function registerMigrateLogs(migrateCommand: Command<[], Record logs.clean(cmd.optsWithGlobals() as Parameters[0])); + .action(async (_opts, cmd) => + logs.clean(cmd.optsWithGlobals() as Parameters[0]), + ); logsCommand .command("convert") @@ -63,7 +67,7 @@ export function registerMigrateLogs(migrateCommand: Command<[], Record + .action(async (files, _opts, cmd) => logs.convert({ ...(cmd.optsWithGlobals() as Parameters[0]), files, diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index 4887b0b64..150bf3713 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -82,7 +82,6 @@ function flagsOf(command: Command): string[] { const defaultChild = command.commands.find( // Commander records the default subcommand on the parent, not the child. - // eslint-disable-next-line @typescript-eslint/no-explicit-any (child) => child.name() === (command as any)._defaultCommandName, ); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 8ef97230b..905add1a6 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -332,7 +332,7 @@ async function skipDisabledProviderUsers( return users; } - const settings = await withSpinner("Checking enabled providers...", () => + const settings = await withSpinner("Checking enabled providers...", async () => fetchInstanceSettings(secretKey), ); const enabled = settings ? enabledSocialProviders(settings) : null; @@ -463,7 +463,7 @@ async function offerSettingChanges( return []; } - await withSpinner(`Updating settings on ${target.label}...`, () => + await withSpinner(`Updating settings on ${target.label}...`, async () => writeInstanceConfig(target, buildChangePayload(applied), { method: "PATCH", failureContext: "Failed to update instance settings", @@ -491,7 +491,7 @@ async function showReadinessReport( ): Promise { if (input.skipReport) return; - let settings = await withSpinner("Checking instance settings...", () => + let settings = await withSpinner("Checking instance settings...", async () => fetchInstanceSettings(input.secretKey), ); const fileSide = { ...(await readFileSide(input)), users: input.users }; @@ -662,7 +662,8 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const { users: loaded, validationFailed } = await withSpinner( `Loading users from ${file}...`, - () => loadUsersFromFile(file, transformer, dateTime, { context: { firebaseHashConfig } }), + async () => + loadUsersFromFile(file, transformer, dateTime, { context: { firebaseHashConfig } }), ); let users = applyResumeAfter(loaded, options.resumeAfter); @@ -742,7 +743,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ...(options.skipUnsupportedProviders ? { skipUnsupportedProviders: true } : {}), }); - const summary = await withSpinner(`Importing users: [0/${users.length}]...`, (spinner) => + const summary = await withSpinner(`Importing users: [0/${users.length}]...`, async (spinner) => importUsers({ users, secretKey, diff --git a/packages/cli-core/src/commands/migrate/settings/index.ts b/packages/cli-core/src/commands/migrate/settings/index.ts index 2a9aa0fdf..833e0335c 100644 --- a/packages/cli-core/src/commands/migrate/settings/index.ts +++ b/packages/cli-core/src/commands/migrate/settings/index.ts @@ -80,7 +80,7 @@ export function registerMigrateSettings( { command: "clerk migrate settings list", description: "Credentials shown redacted" }, { command: "clerk migrate settings list --json", description: "Machine-readable listing" }, ]) - .action((_opts, cmd) => + .action(async (_opts, cmd) => settings.list(cmd.optsWithGlobals() as Parameters[0]), ); @@ -99,7 +99,7 @@ export function registerMigrateSettings( description: "Write a credential to .env.clerk-migrate", }, ]) - .action((name, value) => settings.set(name, value)); + .action(async (name, value) => settings.set(name, value)); settingsCommand .command("clear") @@ -114,7 +114,7 @@ export function registerMigrateSettings( }, { command: "clerk migrate settings clear -y", description: "Clear without prompting" }, ]) - .action((name, _opts, cmd) => + .action(async (name, _opts, cmd) => settings.clear(cmd.optsWithGlobals() as Parameters[0], name), ); } diff --git a/packages/cli-core/src/commands/migrate/transformers/list.test.ts b/packages/cli-core/src/commands/migrate/transformers/list.test.ts index 51c57eca2..a297ec919 100644 --- a/packages/cli-core/src/commands/migrate/transformers/list.test.ts +++ b/packages/cli-core/src/commands/migrate/transformers/list.test.ts @@ -10,7 +10,6 @@ import { transformers } from "./registry.ts"; const captured = useCaptureLog(); -// eslint-disable-next-line no-control-regex const stripAnsi = (value: string) => value.replace(/\[[0-9;]*m/g, ""); let workDir: string; From 8893e60d0696c2f37cb8ba592ca4003d20c1de3c Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 16 Sep 2026 17:54:25 -0400 Subject: [PATCH 036/141] feat(migrate): add WorkOS as an export platform and transformer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clerk migrate export workos` pulls users out of a WorkOS tenant through the User Management API, and `--transformer workos` reads the file back. WorkOS is API-only: there is no bring-your-own-database option, so this has no `--db-url` sibling. Apps commonly mirror WorkOS users into their own store through webhooks, but that mirror is a derived copy holding no credentials. Pagination is cursor-based, so unlike Auth0 there is no record ceiling. No credential survives the move, and the run says so rather than leaving it to be found when nobody can sign in. WorkOS accepts password hashes on import and never returns them, and `totp.secret` comes back on enrol only. The transformer therefore names no `passwordHasher` — every other one names its platform's hasher, but there is no digest here to verify, and naming one would imply a column that cannot exist. Users are created with `skip_password_requirement` and no password credential at all, which is safer than a synthetic one: a placeholder digest is a real, working credential on every migrated account. The coverage table carries a permanent `0/N have a password` row so the gap is visible before the import, not after. `--with-identities` is off by default because WorkOS has no bulk endpoint for OAuth identities — it is one request per user, turning ten requests into 1,010 for a thousand users, and Clerk's import has no external-accounts field to put the result in. The interactive path asks once, after the user count is known, so the question names the real cost. The breakdown prints as its own OAuth providers block rather than as coverage rows: a coverage row means "N of the M users have this field", while one user can hold two providers and `not readable` is not a property of the user at all. A failed lookup is counted on its own row rather than folded into `no OAuth provider`, since flattening the two would understate social sign-in. Non-interactive runs get progress on stderr during both fetches. `withSpinner` hands a no-op to anything that is not a TTY, so an agent exporting a large tenant would otherwise see nothing until the run finished. Covered by the existing `clerk migrate` changeset. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X4wXePuWJHWiocgoBaAX5n --- packages/cli-core/src/commands/init/scan.ts | 5 + .../cli-core/src/commands/migrate/README.md | 89 +++- .../src/commands/migrate/export/index.ts | 41 +- .../commands/migrate/export/registry.test.ts | 1 + .../src/commands/migrate/export/registry.ts | 20 +- .../src/commands/migrate/export/shared.ts | 18 + .../commands/migrate/export/workos.test.ts | 456 ++++++++++++++++ .../src/commands/migrate/export/workos.ts | 488 ++++++++++++++++++ .../commands/migrate/transformers/registry.ts | 2 + .../migrate/transformers/transformers.test.ts | 52 +- .../commands/migrate/transformers/workos.ts | 39 ++ .../src/commands/migrate/wizard.test.ts | 1 + 12 files changed, 1188 insertions(+), 24 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/export/workos.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/workos.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/workos.ts diff --git a/packages/cli-core/src/commands/init/scan.ts b/packages/cli-core/src/commands/init/scan.ts index 7dbc599a3..6f8458906 100644 --- a/packages/cli-core/src/commands/init/scan.ts +++ b/packages/cli-core/src/commands/init/scan.ts @@ -54,6 +54,11 @@ const AUTH_LIBRARY_SCANS: AuthLibraryScan[] = [ name: "Better Auth", docsUrl: "https://clerk.com/docs/migrations/overview", }, + { + packages: ["@workos-inc/authkit-nextjs", "@workos-inc/node"], + name: "WorkOS", + docsUrl: "https://clerk.com/docs/migrations/overview", + }, { packages: ["@kinde-oss/kinde-auth-nextjs"], name: "Kinde", diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index d2cf4b051..688bb55fa 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -145,6 +145,7 @@ clerk migrate export # pick a platform clerk migrate export clerk --output users.json clerk migrate export auth0 --domain my-tenant.us.auth0.com \ --client-id … --client-secret … +clerk migrate export workos --api-key sk_… ``` The platform is an optional positional. Omitted, you get a picker built from @@ -171,6 +172,7 @@ mode and a non-TTY fail outright instead, having nobody to ask. | `authjs` | Auth.js database | `--transformer authjs` | | `betterauth` | Better Auth database | `--transformer betterauth` | | `firebase` | Firebase Identity Toolkit | `--transformer firebase` | +| `workos` | WorkOS User Management API | `--transformer workos` | Every export asks where to save the file before it starts, proposing `./exports/-export-.json`. Press enter to take it, @@ -196,6 +198,8 @@ unattended rather than stalling on a prompt with everything held in memory. | `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | | `--client-id ` | `auth0` | Machine-to-machine application client ID | | `--client-secret ` | `auth0` | Machine-to-machine application client secret | +| `--api-key ` | `workos` | WorkOS secret API key, the one starting `sk_` | +| `--with-identities` | `workos` | Also record each user's OAuth providers | `export clerk` also takes the targeting flags — it reads from a Clerk instance, so it resolves a key the same way `clerk migrate import` does, with one extra @@ -240,7 +244,7 @@ Exported 3 users to /project/exports/clerk-export-20260817-1432.json Every export also writes `logs/export-.log`, so `migrate logs list` sees it alongside imports and deletions. -#### Neither platform exports passwords +#### Three platforms export no passwords - **Clerk** never returns password digests, TOTP secrets or backup codes over the API — only the `*_enabled` booleans. Migrated users must reset their @@ -248,9 +252,13 @@ sees it alongside imports and deletions. - **Auth0**'s Management API does not return password hashes either; they come only from a support request. Add a `passwordHash` field to each user before importing, or migrate without passwords. +- **WorkOS** returns neither password hashes nor TOTP secrets, and has no + support-request escape hatch: hashes go in on import and never come back, and + `totp.secret` is returned on enrol only. There is nothing to add to the file. -Both say so on every run. The coverage row counts users who _have_ a password, -so the size of the gap is visible up front. +All three say so on every run. The coverage row counts users who _have_ a +password, so the size of the gap is visible up front — `workos` prints that row +at zero unconditionally, because zero is the only value it can take. #### Database-backed exports (`supabase`, `authjs`, `betterauth`) @@ -373,6 +381,59 @@ Auth0 pages this endpoint only through the first **1000** users. Past that the export stops and says so, pointing at Auth0's bulk export job — silently returning the first thousand would read as "that is everyone". +#### WorkOS credentials + +Needs a secret API key — the one starting `sk_`, from the WorkOS dashboard +under API Keys. Resolved from `--api-key`, then `WORKOS_API_KEY`, then a +prompt; agent mode exits naming both instead. + +There is no `--db-url` sibling because there is no database to point it at. +WorkOS is API-only: apps commonly mirror users into their own store through +webhooks, but that mirror is a derived copy holding no credentials, so the +User Management API is the only source. Pagination is cursor-based, so unlike +Auth0 there is no record ceiling — `after` runs to the end of the tenant. + +**`--with-identities` is off by default, and it is not free.** WorkOS has no +bulk endpoint for OAuth identities, so it is one request per user: ten requests +becomes 1,010 for a thousand users. Nothing it returns can be imported — +`POST /v1/users` has no external-accounts field — so it buys a provider +breakdown in the coverage report, and an `identities` array kept in the export +file for whoever runs the migration. The interactive path asks once, after the +user count is known, defaulting to no; agent mode takes the flag's answer and +asks nothing. + +The breakdown prints as its own **OAuth providers** block under the coverage +table, not as extra coverage rows: + +``` +Field coverage + ✓ 6/6 have an email address + ✗ 0/6 have a password (WorkOS returns none) + +OAuth providers + GoogleOAuth 2 users + MicrosoftOAuth 1 user + no OAuth provider 2 users + not readable 1 user + Those users have no `identities` field in the export, rather than an empty one. +``` + +Separate because the two kinds of row do not mean the same thing. A coverage +row is "N of the M users have this field"; a provider row has no such +denominator — one user holding two providers is counted under both, so the +counts can sum past the user count, and `not readable` is not a property of the +user at all. + +A lookup that fails is counted on its own row rather than folded into +`no OAuth provider`. "Lookup failed" and "has no providers" are different +facts, and flattening the first into the second would understate social +sign-in. + +Non-interactive runs get progress on stderr every 500 users during the fan-out, +and every 10 pages during the user fetch. `withSpinner` hands a no-op to +anything that is not a TTY, so without this an agent exporting a large tenant +would see nothing at all until the run finished. + ### `clerk migrate delete` The undo for a bad migration. Deletes the users a previous @@ -553,6 +614,7 @@ platform is one file in `transformers/` plus one line in `transformers/registry. | `betterauth` | Better Auth export | `bcrypt` | Reads the credential account's `password_hash` | | `firebase` | `firebase auth:export` | `scrypt_firebase` | CSV or JSON; needs the four hash parameters below | | `supabase` | Supabase `auth.users` export | `bcrypt` | Supports `--skip-unsupported-providers` | +| `workos` | WorkOS User Management API | none | No hasher default: WorkOS returns no digest to name one for | ### `clerk migrate transformers list` @@ -1142,18 +1204,21 @@ instances), and its settings-change offer writes through the Platform API: | ------- | ----------------------------------------------------------------- | ------------------------------------------------------ | | `PATCH` | `/v1/platform/applications/{appID}/instances/{instanceID}/config` | Applying the settings changes selected from the report | -Two exports talk to their own platform rather than to Clerk: +Three exports talk to their own platform rather than to Clerk: -| Method | Path | Used by | -| ------ | ---------------------------------------------- | ---------------------------------------------------- | -| `POST` | `https:///oauth/token` | `export auth0` — Management API access token | -| `GET` | `https:///api/v2/users` | `export auth0` — 100 per page, 1000 users maximum | -| `POST` | `https://oauth2.googleapis.com/token` | `export firebase` — RS256 assertion → access token | -| `GET` | `…/v1/projects/{project_id}/accounts:batchGet` | `export firebase` — pages users, 1000 at a time | -| `GET` | `…/admin/v2/projects/{project_id}/config` | `export firebase` — reads the scrypt hash parameters | +| Method | Path | Used by | +| ------ | ---------------------------------------------- | -------------------------------------------------------- | +| `POST` | `https:///oauth/token` | `export auth0` — Management API access token | +| `GET` | `https:///api/v2/users` | `export auth0` — 100 per page, 1000 users maximum | +| `POST` | `https://oauth2.googleapis.com/token` | `export firebase` — RS256 assertion → access token | +| `GET` | `…/v1/projects/{project_id}/accounts:batchGet` | `export firebase` — pages users, 1000 at a time | +| `GET` | `…/admin/v2/projects/{project_id}/config` | `export firebase` — reads the scrypt hash parameters | +| `GET` | `…/user_management/users` | `export workos` — 100 per page, cursor-paginated | +| `GET` | `…/user_management/users/{id}/identities` | `export workos` — `--with-identities` only, one per user | The two Identity Toolkit paths are on `identitytoolkit.googleapis.com`, or on -`FIREBASE_AUTH_EMULATOR_HOST` when that is set. +`FIREBASE_AUTH_EMULATOR_HOST` when that is set. The two WorkOS paths are on +`api.workos.com`. The three database exports (`supabase`, `authjs`, `betterauth`) make no HTTP calls at all — they connect over `--db-url`. diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index e72066676..f8585bfa2 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -8,6 +8,7 @@ import { exportBetterAuth } from "./betterauth.ts"; import { exportClerk } from "./clerk.ts"; import { exportFirebase } from "./firebase.ts"; import { exportSupabase } from "./supabase.ts"; +import { exportWorkOs } from "./workos.ts"; import type { DbExportOptions } from "./db-options.ts"; import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; @@ -56,6 +57,7 @@ const handlers = { authjs: exportAuthJs, betterauth: exportBetterAuth, firebase: exportFirebase, + workos: exportWorkOs, }; /** The three platforms that read a database, which share `--db-url`. */ @@ -97,7 +99,9 @@ export function registerMigrateExport(migrateCommand: Command<[], Record handlers.picker(cmd.optsWithGlobals() as Record)); + .action(async (_opts, cmd) => + handlers.picker(cmd.optsWithGlobals() as Record), + ); exportCommand .command("clerk") @@ -118,7 +122,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record + .action(async (_opts, cmd) => handlers.clerk(cmd.optsWithGlobals() as Parameters[0]), ); @@ -142,7 +146,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record + .action(async (_opts, cmd) => handlers.auth0(cmd.optsWithGlobals() as Parameters[0]), ); @@ -159,10 +163,35 @@ export function registerMigrateExport(migrateCommand: Command<[], Record + .action(async (_opts, cmd) => handlers.firebase(cmd.optsWithGlobals() as Parameters[0]), ); + exportCommand + .command("workos") + .description( + "Export users from a WorkOS tenant (default: ./exports/workos-export-.json)", + ) + .option("--api-key ", "WorkOS secret API key, the one starting `sk_`") + .option( + "--with-identities", + "Also record each user's OAuth providers — one extra request per user", + ) + .option("-o, --output ", "Where to write the export, relative to the current directory") + .setExamples([ + { + command: "clerk migrate export workos --api-key sk_…", + description: "Export with an explicit API key", + }, + { + command: "clerk migrate export workos", + description: "Read WORKOS_API_KEY, or prompt", + }, + ]) + .action(async (_opts, cmd) => + handlers.workos(cmd.optsWithGlobals() as Parameters[0]), + ); + // All three take exactly one connection string, so they are registered from // a table rather than three near-identical blocks. for (const platform of DB_PLATFORMS) { @@ -183,6 +212,8 @@ export function registerMigrateExport(migrateCommand: Command<[], Record handlers[platform.key](cmd.optsWithGlobals() as DbExportOptions)); + .action(async (_opts, cmd) => + handlers[platform.key](cmd.optsWithGlobals() as DbExportOptions), + ); } } diff --git a/packages/cli-core/src/commands/migrate/export/registry.test.ts b/packages/cli-core/src/commands/migrate/export/registry.test.ts index 1acded24f..f8471aa8b 100644 --- a/packages/cli-core/src/commands/migrate/export/registry.test.ts +++ b/packages/cli-core/src/commands/migrate/export/registry.test.ts @@ -11,6 +11,7 @@ describe("export registry", () => { "authjs", "firebase", "betterauth", + "workos", ]); }); diff --git a/packages/cli-core/src/commands/migrate/export/registry.ts b/packages/cli-core/src/commands/migrate/export/registry.ts index 93c4f2605..a5d5a6c15 100644 --- a/packages/cli-core/src/commands/migrate/export/registry.ts +++ b/packages/cli-core/src/commands/migrate/export/registry.ts @@ -16,6 +16,7 @@ import { exportBetterAuth } from "./betterauth.ts"; import { exportClerk } from "./clerk.ts"; import { exportFirebase } from "./firebase.ts"; import { exportSupabase } from "./supabase.ts"; +import { exportWorkOs } from "./workos.ts"; export type ExportRegistryEntry = { key: string; @@ -32,42 +33,49 @@ export const exportPlatforms: ExportRegistryEntry[] = [ label: "Clerk", description: "Another Clerk instance, e.g. development → production", transformerKey: "clerk", - run: (options) => exportClerk(options), + run: async (options) => exportClerk(options), }, { key: "auth0", label: "Auth0", description: "An Auth0 tenant, via the Management API", transformerKey: "auth0", - run: (options) => exportAuth0(options), + run: async (options) => exportAuth0(options), }, { key: "supabase", label: "Supabase", description: "A Supabase Postgres database — includes password hashes", transformerKey: "supabase", - run: (options) => exportSupabase(options), + run: async (options) => exportSupabase(options), }, { key: "authjs", label: "Auth.js (NextAuth)", description: "An Auth.js database — Postgres, MySQL or SQLite", transformerKey: "authjs", - run: (options) => exportAuthJs(options), + run: async (options) => exportAuthJs(options), }, { key: "firebase", label: "Firebase", description: "A Firebase project, via Identity Toolkit", transformerKey: "firebase", - run: (options) => exportFirebase(options), + run: async (options) => exportFirebase(options), }, { key: "betterauth", label: "Better Auth", description: "A Better Auth database — plugin columns detected automatically", transformerKey: "betterauth", - run: (options) => exportBetterAuth(options), + run: async (options) => exportBetterAuth(options), + }, + { + key: "workos", + label: "WorkOS", + description: "A WorkOS tenant, via the User Management API — no password hashes", + transformerKey: "workos", + run: async (options) => exportWorkOs(options), }, ]; diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index 7f968d845..e443b803e 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -96,11 +96,23 @@ export function formatFieldCoverage(fields: CoverageField[], total: number): str }); } +/** + * An extra block printed under the coverage table. + * + * For a breakdown that is not "how many users have this field" — WorkOS's + * OAuth providers, where one user can appear in two rows and the denominator + * is not the user count. Folding that into the coverage table would put rows + * of two different kinds under one heading. + */ +export type ExportSection = { title: string; rows: string[] }; + export type ExportSummary = { platform: string; userCount: number; outputPath: string; coverage: CoverageField[]; + /** Extra blocks, printed under the coverage table in order. */ + sections?: ExportSection[]; /** The transformer that reads this file, for the "what next" line. */ transformerKey: string; }; @@ -124,6 +136,12 @@ export function reportExport(summary: ExportSummary): readonly string[] { log.info(line); } + for (const section of summary.sections ?? []) { + log.blank(); + log.info(section.title); + for (const row of section.rows) log.info(row); + } + log.blank(); log.success( `Exported ${summary.userCount} user${summary.userCount === 1 ? "" : "s"} to ${summary.outputPath}`, diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts new file mode 100644 index 000000000..e61103bf3 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -0,0 +1,456 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildIdentityReport, + buildWorkOsExport, + exportWorkOs, + fetchAllWorkOsIdentities, + fetchAllWorkOsUsers, + fetchWorkOsIdentities, + fetchWorkOsPage, + mapWorkOsUserToExport, + resolveWithIdentities, + resolveWorkOsApiKey, + type WorkOsIdentity, +} from "./workos.ts"; + +/** A cwd with no `.env` files, so these tests exercise only the injected env. */ +const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); + +const captured = useCaptureLog(); +useMigrateLogDir(); + +const API_KEY = "sk_test"; + +/** Colour is on or off depending on the runner, so rows are compared bare. */ +const stripAnsi = (value: string): string => value.replace(/\u001b\[[0-9;]*m/g, ""); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: string[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expworkos-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a leaked + // "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +const workosUser = (i: number, overrides: Record = {}) => ({ + id: `user_0${i}`, + email: `a${i}@x.dev`, + email_verified: true, + first_name: `Given${i}`, + last_name: `Family${i}`, + ...overrides, +}); + +/** + * Stubs one users page per entry in `pages`, chaining the cursor, plus an + * identities response per user id in `identities`. + */ +function stubWorkOs( + pages: Record[][], + identities: Record = {}, +) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = input.toString(); + requests.push(url); + + const identityMatch = /\/users\/([^/]+)\/identities/.exec(url); + if (identityMatch) { + const entry = identities[identityMatch[1] as string]; + if (entry === "fail") return new Response("nope", { status: 500 }); + return Response.json(entry ?? []); + } + + const after = new URL(url).searchParams.get("after"); + const index = after ? Number(after.replace("cursor", "")) : 0; + const isLast = index >= pages.length - 1; + return Response.json({ + data: pages[index] ?? [], + list_metadata: { after: isLast ? null : `cursor${index + 1}` }, + }); + }) as unknown as typeof fetch; +} + +describe("resolveWorkOsApiKey", () => { + test("prefers the flag", async () => { + expect(await resolveWorkOsApiKey({ apiKey: "sk_flag" }, NO_ENV_FILES, {})).toBe("sk_flag"); + }); + + test("falls back to the environment", async () => { + expect(await resolveWorkOsApiKey({}, NO_ENV_FILES, { WORKOS_API_KEY: "sk_env" })).toBe( + "sk_env", + ); + }); + + // Tests run non-TTY, the same signal an agent gives. + test("names the flag and the variable when neither supplied one", async () => { + await expect(resolveWorkOsApiKey({}, NO_ENV_FILES, {})).rejects.toThrow( + /Missing: --api-key \(or WORKOS_API_KEY\)\./, + ); + }); +}); + +describe("fetchWorkOsPage", () => { + test("asks for the documented page size", async () => { + stubWorkOs([[]]); + await fetchWorkOsPage(API_KEY); + expect(requests[0]).toContain("limit=100"); + }); + + test("explains a rejection instead of surfacing a raw status", async () => { + globalThis.fetch = (async () => + Response.json({ message: "Unauthorized" }, { status: 401 })) as unknown as typeof fetch; + + await expect(fetchWorkOsPage(API_KEY)).rejects.toThrow( + /WorkOS returned 401 listing users: Unauthorized/, + ); + }); + + // The usual cause is a publishable key, or a key from the other environment. + test("points at the key itself", async () => { + globalThis.fetch = (async () => new Response("{}", { status: 401 })) as unknown as typeof fetch; + await expect(fetchWorkOsPage(API_KEY)).rejects.toThrow(/secret key/); + }); +}); + +describe("fetchAllWorkOsUsers", () => { + test("follows the cursor until it comes back null", async () => { + stubWorkOs([ + Array.from({ length: 100 }, (_, i) => workosUser(i)), + Array.from({ length: 4 }, (_, i) => workosUser(100 + i)), + ]); + + const all = await fetchAllWorkOsUsers({ apiKey: API_KEY }); + + expect(all).toHaveLength(104); + expect(requests).toHaveLength(2); + expect(requests[1]).toContain("after=cursor1"); + }); + + // Cursor pagination has no record ceiling, so a full page that happens to be + // the last one must not read as "there is more". + test("stops on a full final page", async () => { + stubWorkOs([Array.from({ length: 100 }, (_, i) => workosUser(i))]); + expect(await fetchAllWorkOsUsers({ apiKey: API_KEY })).toHaveLength(100); + expect(requests).toHaveLength(1); + }); + + test("reuses a page already fetched rather than asking twice", async () => { + stubWorkOs([[workosUser(0)]]); + const firstPage = await fetchWorkOsPage(API_KEY); + requests = []; + + expect(await fetchAllWorkOsUsers({ apiKey: API_KEY, firstPage })).toHaveLength(1); + expect(requests).toHaveLength(0); + }); +}); + +describe("resolveWithIdentities", () => { + test("is on when the flag asked for it", async () => { + expect(await resolveWithIdentities({ withIdentities: true }, 10)).toBe(true); + }); + + // The fan-out is one request per user and nothing it returns can be + // imported, so it is never the default. + test("is off without the flag when there is nobody to ask", async () => { + expect(await resolveWithIdentities({}, 10)).toBe(false); + }); + + test("does not ask when there are no users to ask about", async () => { + const originalMode = getMode(); + setMode("human"); + try { + expect(await resolveWithIdentities({}, 0)).toBe(false); + } finally { + setMode(originalMode); + } + }); +}); + +describe("fetchAllWorkOsIdentities", () => { + test("collects each user's providers", async () => { + stubWorkOs([[]], { + user_00: [{ provider: "GoogleOAuth", idp_id: "g1", type: "OAuth" }], + user_01: [], + }); + + const { identities, failed } = await fetchAllWorkOsIdentities({ + apiKey: API_KEY, + users: [workosUser(0), workosUser(1)], + }); + + expect(identities.get("user_00")).toEqual([ + { provider: "GoogleOAuth", idp_id: "g1", type: "OAuth" }, + ]); + expect(identities.get("user_01")).toEqual([]); + expect(failed).toBe(0); + }); + + // "Lookup failed" and "has no providers" are different facts, and flattening + // the first into the second would put a wrong number in the report. + test("leaves a failed lookup absent rather than empty, and counts it", async () => { + stubWorkOs([[]], { user_00: "fail", user_01: [] }); + + const { identities, failed } = await fetchAllWorkOsIdentities({ + apiKey: API_KEY, + users: [workosUser(0), workosUser(1)], + }); + + expect(identities.has("user_00")).toBe(false); + expect(identities.get("user_01")).toEqual([]); + expect(failed).toBe(1); + }); +}); + +describe("buildIdentityReport", () => { + const rowsOf = (section: { rows: string[] }) => + section.rows.map((row) => stripAnsi(row).trimEnd()); + + test("ranks providers by use, and counts users with none", () => { + const section = buildIdentityReport( + [workosUser(0), workosUser(1), workosUser(2), workosUser(3)], + new Map([ + ["user_00", [{ provider: "GoogleOAuth" }]], + ["user_01", [{ provider: "GoogleOAuth" }, { provider: "MicrosoftOAuth" }]], + ["user_02", []], + ["user_03", []], + ]), + 0, + ); + + expect(section.title).toBe("OAuth providers"); + expect(rowsOf(section)).toEqual([ + " GoogleOAuth 2 users", + " MicrosoftOAuth 1 user", + " no OAuth provider 2 users", + ]); + }); + + // Counting a failed lookup as "no provider" would understate social sign-in. + test("reports unreadable lookups on their own row, with the caveat", () => { + const section = buildIdentityReport( + [workosUser(0), workosUser(1)], + new Map([["user_01", []]]), + 1, + ); + + const rows = rowsOf(section); + expect(rows).toContain(" not readable 1 user"); + expect(rows).toContain(" no OAuth provider 1 user"); + expect(rows.at(-1)).toContain("no `identities` field in the export, rather than an empty one"); + }); + + test("says nothing about unreadable lookups when there were none", () => { + const section = buildIdentityReport([workosUser(0)], new Map([["user_00", []]]), 0); + expect(rowsOf(section)).toEqual([" no OAuth provider 1 user"]); + }); +}); + +describe("mapWorkOsUserToExport", () => { + test("keeps the fields the workos transformer maps from", () => { + expect(mapWorkOsUserToExport(workosUser(0, { created_at: "2025-01-01" }))).toEqual({ + id: "user_00", + email: "a0@x.dev", + first_name: "Given0", + last_name: "Family0", + created_at: "2025-01-01", + email_verified: true, + }); + }); + + // Dropping a false flag would import an unconfirmed address as verified. + test("keeps email_verified when it is false", () => { + expect(mapWorkOsUserToExport(workosUser(0, { email_verified: false })).email_verified).toBe( + false, + ); + }); + + test("drops tenant fields the import has no use for", () => { + const mapped = mapWorkOsUserToExport( + workosUser(0, { + locale: "en-GB", + profile_picture_url: "https://x.dev/a.png", + last_sign_in_at: "2026-01-01", + updated_at: "2026-01-01", + external_id: "cust_1", + }), + ); + for (const noise of [ + "locale", + "profile_picture_url", + "last_sign_in_at", + "updated_at", + "external_id", + ]) { + expect(noise in mapped).toBe(false); + } + }); + + test("omits empty metadata", () => { + expect("metadata" in mapWorkOsUserToExport(workosUser(0, { metadata: {} }))).toBe(false); + expect(mapWorkOsUserToExport(workosUser(0, { metadata: { plan: "pro" } })).metadata).toEqual({ + plan: "pro", + }); + }); + + test("carries identities only when they were fetched", () => { + expect("identities" in mapWorkOsUserToExport(workosUser(0))).toBe(false); + expect(mapWorkOsUserToExport(workosUser(0), [{ provider: "GoogleOAuth" }]).identities).toEqual([ + { provider: "GoogleOAuth" }, + ]); + }); +}); + +describe("buildWorkOsExport", () => { + test("counts coverage and logs each user", () => { + const { users, coverage } = buildWorkOsExport( + [workosUser(0), workosUser(1, { first_name: undefined })], + "2026-01-01T00:00:00", + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + + const logged = fs.readdirSync(getLogDir()); + expect(logged[0]).toMatch(/^export-/); + }); + + // Always shown, always zero: seeing it before the import is the point. + test("reports the password row even though it can only ever be zero", () => { + const { coverage } = buildWorkOsExport([workosUser(0)], "2026-01-01T00:00:00"); + expect(coverage.at(-1)).toEqual({ + label: "have a password (WorkOS returns none)", + count: 0, + }); + }); + + // Providers get their own block: a coverage row means "N of M users have + // this field", and a provider count can exceed M. + test("keeps providers out of the coverage table", () => { + const { coverage } = buildWorkOsExport( + [workosUser(0)], + "2026-01-01T00:00:00", + new Map([["user_00", [{ provider: "GoogleOAuth" }]]]), + ); + expect(coverage.some((row) => row.label.toLowerCase().includes("oauth"))).toBe(false); + }); +}); + +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + +describe("exportWorkOs", () => { + test("writes the default path and reports coverage", async () => { + stubWorkOs([[workosUser(0)]]); + + await exportWorkOs({ apiKey: API_KEY }); + + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^workos-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; + expect(written[0]?.id).toBe("user_00"); + expect(captured.err).toContain("Field coverage"); + }); + + test("names the command that consumes the file", async () => { + stubWorkOs([[workosUser(0)]]); + // The suggestion rides the gutter's Next steps block, which only renders + // in human mode. + const originalMode = getMode(); + setMode("human"); + try { + // --output answers the destination prompt, and --with-identities answers + // the providers question, so human mode stops on neither. + await exportWorkOs({ apiKey: API_KEY, output: "exports/mine.json", withIdentities: true }); + } finally { + setMode(originalMode); + } + expect(captured.err).toContain("migrate import --transformer workos --file exports/mine.json"); + }); + + test("--output controls the destination", async () => { + stubWorkOs([[workosUser(0)]]); + + await exportWorkOs({ apiKey: API_KEY, output: "tenant.json" }); + + expect(fs.existsSync(path.join(workDir, "tenant.json"))).toBe(true); + }); + + test("skips the per-user identity fan-out unless asked", async () => { + stubWorkOs([[workosUser(0), workosUser(1)]]); + + await exportWorkOs({ apiKey: API_KEY, output: "plain.json" }); + + expect(requests.filter((url) => url.includes("/identities"))).toHaveLength(0); + }); + + test("--with-identities records each user's providers in the file", async () => { + stubWorkOs([[workosUser(0)]], { user_00: [{ provider: "GoogleOAuth", idp_id: "g1" }] }); + + await exportWorkOs({ apiKey: API_KEY, output: "rich.json", withIdentities: true }); + + const written = JSON.parse(fs.readFileSync(path.join(workDir, "rich.json"), "utf-8")) as Record< + string, + unknown + >[]; + expect(written[0]?.identities).toEqual([{ provider: "GoogleOAuth", idp_id: "g1" }]); + }); + + // Finding this out after the import means nobody can sign in. + test("says plainly that no credentials are in the file", async () => { + stubWorkOs([[workosUser(0)]]); + await exportWorkOs({ apiKey: API_KEY, output: "warned.json" }); + expect(captured.err).toContain("does not return password hashes or TOTP secrets"); + }); +}); + +describe("fetchWorkOsIdentities", () => { + test("accepts the bare array the endpoint returns", async () => { + stubWorkOs([[]], { user_00: [{ provider: "GoogleOAuth" }] }); + expect(await fetchWorkOsIdentities(API_KEY, "user_00")).toEqual([{ provider: "GoogleOAuth" }]); + }); + + // A move to WorkOS's usual envelope must not read as "no providers". + test("accepts a { data } envelope too", async () => { + globalThis.fetch = (async () => + Response.json({ data: [{ provider: "AppleOAuth" }] })) as unknown as typeof fetch; + expect(await fetchWorkOsIdentities(API_KEY, "user_00")).toEqual([{ provider: "AppleOAuth" }]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts new file mode 100644 index 000000000..f8ccf5df6 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -0,0 +1,488 @@ +/** + * `clerk migrate export workos` — pull users out of a WorkOS tenant. + * + * Structurally the Auth0 case, and written against the same shape: two REST + * calls through `loggedFetch` rather than the `@workos-inc/node` SDK, so + * everything the command sends shows up under `--verbose`. See the header of + * `auth0.ts` and `.claude/rules/debug-logging.md`. + * + * **WorkOS is API-only, and no credential leaves it.** There is no + * bring-your-own-database option — apps mirror WorkOS users into their own + * store through webhooks, but that mirror is a derived copy holding no secrets, + * which is why this has no `--db-url` sibling. Password hashes are accepted on + * import and never returned; TOTP secrets come back on enrol only, never on + * list or get. So a WorkOS migration moves identities, and every password user + * signs in again by reset. The run says so rather than leaving it to be + * discovered when nobody can sign in. + */ + +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { confirm, password as passwordPrompt } from "../../../lib/prompts.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; +import { createApiScheduler } from "../lib/scheduler.ts"; +import { + reportExport, + resolveOutputPath, + writeExportOutput, + type ExportSection, +} from "./shared.ts"; + +const API_BASE = "https://api.workos.com/user_management"; + +/** WorkOS caps `limit` at 100. */ +const PAGE_SIZE = 100; + +/** + * Pacing for the per-user identity fan-out. + * + * WorkOS allows 6,000 requests a minute, so this is nowhere near the ceiling — + * it is here so a 50,000-user tenant does not open 50,000 sockets at once. + */ +const IDENTITY_CONCURRENCY = 10; +const IDENTITY_RATE_PER_SECOND = 20; + +/** + * How often a non-interactive run says where it has got to. + * + * `withSpinner` hands a no-op `update` to anything that is not a TTY, so an + * agent exporting 50,000 users with `--with-identities` would otherwise see + * nothing at all for the ten minutes the fan-out takes. These two print + * through `log.info` instead, which a non-TTY does get. + */ +const IDENTITY_PROGRESS_EVERY = 500; +const USER_PROGRESS_EVERY_PAGES = 10; + +const NO_PROVIDER_LABEL = "no OAuth provider"; +const NOT_READABLE_LABEL = "not readable"; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; + +export type ExportWorkOsOptions = { + apiKey?: string; + withIdentities?: boolean; + output?: string; +}; + +export type WorkOsUser = Record & { id?: string }; + +export type WorkOsIdentity = { idp_id?: string; type?: string; provider?: string }; + +/** + * Resolves the API key: flag, then environment, then a prompt. + * + * One value rather than Auth0's three, so there is no "name everything that is + * missing" pass — there is only ever the one thing. + * + * @throws CliError in agent mode when nothing supplied it. + */ +export async function resolveWorkOsApiKey( + options: ExportWorkOsOptions, + cwd: string = process.cwd(), + env: Record = process.env, +): Promise { + const resolved = + options.apiKey ?? (await findMigrateEnvValue(["WORKOS_API_KEY"], cwd, env))?.value; + + if (resolved) return resolved.trim(); + + if (isAgent() || !isHuman()) { + throwUsageError( + "`clerk migrate export workos` needs a WorkOS API key and cannot prompt here.\n" + + "Missing: --api-key (or WORKOS_API_KEY).", + DOCS_URL, + undefined, + [ + { + command: "clerk migrate export workos --api-key sk_…", + description: "Export with an explicit API key", + }, + ], + ); + } + + log.info( + "WorkOS needs a secret API key, the one starting `sk_`. Find it in the WorkOS dashboard under API Keys.", + ); + + return promptWorkOsApiKey(); +} + +export async function promptWorkOsApiKey(): Promise { + const key = await passwordPrompt({ + message: "WorkOS secret API key (sk_…)", + validate: (value) => (value?.trim() ? undefined : "An API key is required"), + }); + return key.trim(); +} + +/** + * Whatever WorkOS put in an error body, in one string. + * + * Read as text and parsed from that, rather than `response.json()` with a text + * fallback: the failed parse disturbs the stream, so the fallback could never + * actually run. A non-JSON body — a proxy's HTML error page — is the case worth + * surfacing verbatim. + */ +async function describeFailure(response: Response): Promise { + const raw = (await response.text().catch(() => "")).trim(); + if (!raw) return "no detail returned"; + + try { + const body = JSON.parse(raw) as { + message?: string; + error_description?: string; + error?: string; + }; + return body.message ?? body.error_description ?? body.error ?? raw; + } catch { + return raw; + } +} + +export type WorkOsPage = { users: WorkOsUser[]; after?: string }; + +/** + * Fetches one page of users. + * + * Cursor pagination, so unlike Auth0's offset endpoint there is no record + * ceiling to warn about — `after` runs to the end of the tenant. + */ +export async function fetchWorkOsPage(apiKey: string, after?: string): Promise { + const url = new URL(`${API_BASE}/users`); + url.searchParams.set("limit", String(PAGE_SIZE)); + url.searchParams.set("order", "asc"); + if (after) url.searchParams.set("after", after); + + const response = await loggedFetch(url, { + tag: "workos", + method: "GET", + headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, + }); + + if (!response.ok) { + throw new CliError( + `WorkOS returned ${response.status} listing users: ${await describeFailure(response)}\n` + + "Check that the key is a secret key (`sk_…`) for the right environment, and that it has not been revoked.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + const body = (await response.json()) as { + data?: WorkOsUser[]; + list_metadata?: { after?: string | null }; + }; + + return { users: body.data ?? [], after: body.list_metadata?.after ?? undefined }; +} + +/** + * Pages through the tenant's users. + * + * @param firstPage - A page already fetched, so the request that proved the API + * key is not sent twice. + */ +export async function fetchAllWorkOsUsers(options: { + apiKey: string; + firstPage?: WorkOsPage; + spinner?: SpinnerControls; +}): Promise { + let page = options.firstPage ?? (await fetchWorkOsPage(options.apiKey)); + const all = [...page.users]; + options.spinner?.update(`Fetching users from WorkOS: ${all.length} so far...`); + + // Counted in pages rather than users: a short page would knock a + // `users % N` check off its multiple and silence every later one. + for (let pages = 1; page.after; pages++) { + page = await fetchWorkOsPage(options.apiKey, page.after); + all.push(...page.users); + options.spinner?.update(`Fetching users from WorkOS: ${all.length} so far...`); + if (!isHuman() && pages % USER_PROGRESS_EVERY_PAGES === 0) { + log.info(`Fetched ${all.length} users from WorkOS so far...`); + } + } + + return all; +} + +/** + * Whether to spend one request per user on OAuth providers. + * + * Off unless asked for, both times. WorkOS has no bulk identities endpoint, so + * this is the difference between ten requests and one per user — and nothing it + * returns can be imported, because `POST /v1/users` has no external-accounts + * field. It is a line in the coverage report, and a record kept in the file. + * + * Agent mode gets the flag's answer and no question: there is nobody to ask. + */ +export async function resolveWithIdentities( + options: ExportWorkOsOptions, + userCount: number, +): Promise { + if (options.withIdentities) return true; + if (userCount === 0 || isAgent() || !isHuman()) return false; + + return confirm({ + message: `Also fetch each user's OAuth providers? That is ${userCount} extra request${userCount === 1 ? "" : "s"}, and the result is report-only — Clerk's import cannot take external accounts.`, + default: false, + }); +} + +/** Fetches one user's OAuth identities. */ +export async function fetchWorkOsIdentities( + apiKey: string, + userId: string, +): Promise { + const response = await loggedFetch(new URL(`${API_BASE}/users/${userId}/identities`), { + tag: "workos", + method: "GET", + headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, + }); + + if (!response.ok) { + throw new CliError( + `WorkOS returned ${response.status} listing identities for ${userId}: ${await describeFailure(response)}`, + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + // The endpoint returns a bare array; the envelope is handled too so a future + // move to WorkOS's usual `{ data }` shape does not read as "no providers". + const body = (await response.json()) as WorkOsIdentity[] | { data?: WorkOsIdentity[] }; + return Array.isArray(body) ? body : (body.data ?? []); +} + +/** + * Fetches identities for every user. + * + * A user missing from the returned map is one whose lookup **failed**, which is + * not the same as one with no providers — so failures are counted and returned + * separately rather than flattened into an empty list. + */ +export async function fetchAllWorkOsIdentities(options: { + apiKey: string; + users: WorkOsUser[]; + spinner?: SpinnerControls; +}): Promise<{ identities: Map; failed: number }> { + const schedule = createApiScheduler(IDENTITY_CONCURRENCY, IDENTITY_RATE_PER_SECOND); + const total = options.users.length; + const identities = new Map(); + let failed = 0; + let done = 0; + + await Promise.all( + options.users.map(async (user) => + schedule(async () => { + const userId = String(user.id ?? ""); + try { + if (userId) identities.set(userId, await fetchWorkOsIdentities(options.apiKey, userId)); + } catch { + failed++; + } + done++; + options.spinner?.update(`Fetching OAuth providers: ${done}/${total}...`); + if (!isHuman() && done % IDENTITY_PROGRESS_EVERY === 0) { + log.info(`Fetched OAuth providers for ${done}/${total} users...`); + } + }), + ), + ); + + return { identities, failed }; +} + +/** + * The OAuth provider breakdown, as its own block under the coverage table. + * + * Kept out of coverage on purpose: a coverage row means "N of the M users have + * this field", and these rows do not. One user can hold two providers, so the + * counts can sum past the user count, and "not readable" is not a property of + * the user at all. Two kinds of row under one heading would make both harder + * to read. + */ +export function buildIdentityReport( + users: WorkOsUser[], + identities: Map, + failed: number, +): ExportSection { + const byProvider = new Map(); + let none = 0; + + for (const user of users) { + const found = identities.get(String(user.id ?? "")); + // Absent means the lookup failed; `failed` already counts it. + if (!found) continue; + if (found.length === 0) { + none++; + continue; + } + for (const identity of found) { + const provider = identity.provider ?? "unknown"; + byProvider.set(provider, (byProvider.get(provider) ?? 0) + 1); + } + } + + // Busiest provider first; alphabetical within a tie so two runs of the same + // tenant print the same order. + const entries = [...byProvider].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); + + const labels = [ + ...entries.map(([provider]) => provider), + NO_PROVIDER_LABEL, + ...(failed > 0 ? [NOT_READABLE_LABEL] : []), + ]; + const width = Math.max(...labels.map((label) => label.length)); + const row = (label: string, count: number) => + ` ${label.padEnd(width)} ${dim(`${count} user${count === 1 ? "" : "s"}`)}`; + + const rows = [ + ...entries.map(([provider, count]) => row(provider, count)), + row(NO_PROVIDER_LABEL, none), + ]; + + if (failed > 0) { + rows.push(row(NOT_READABLE_LABEL, failed)); + rows.push( + dim(" Those users have no `identities` field in the export, rather than an empty one."), + ); + } + + return { title: "OAuth providers", rows }; +} + +/** + * Keeps the fields the `workos` transformer maps from, plus `identities` when + * they were fetched. + * + * A copy rather than the raw record: WorkOS also returns `locale`, + * `profile_picture_url`, `last_sign_in_at` and `updated_at`, none of which + * `POST /v1/users` accepts. `identities` has no target field either and is + * dropped at validation, so it rides along purely as a record for whoever runs + * the migration. + */ +export function mapWorkOsUserToExport( + user: WorkOsUser, + identities?: WorkOsIdentity[], +): Record { + const exported: Record = {}; + + for (const field of ["id", "email", "first_name", "last_name", "created_at"] as const) { + if (user[field]) exported[field] = user[field]; + } + + // Meaningful when false: dropping it would import an address WorkOS never + // confirmed as a verified one. + if (user.email_verified !== undefined) exported.email_verified = user.email_verified; + + const metadata = user.metadata; + if (metadata && typeof metadata === "object" && Object.keys(metadata).length > 0) { + exported.metadata = metadata; + } + + if (identities && identities.length > 0) exported.identities = identities; + + return exported; +} + +export type WorkOsExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +export function buildWorkOsExport( + users: WorkOsUser[], + dateTime: string, + identities?: Map, +): WorkOsExportResult { + const exported: Record[] = []; + const counts = { email: 0, firstName: 0, lastName: 0, metadata: 0 }; + + for (const user of users) { + const userId = String(user.id ?? ""); + try { + const mapped = mapWorkOsUserToExport(user, identities?.get(userId)); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.first_name) counts.firstName++; + if (mapped.last_name) counts.lastName++; + if (mapped.metadata) counts.metadata++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + { label: "have metadata", count: counts.metadata }, + // Always present, always zero. WorkOS returns no digest for anyone, and + // seeing that before the import is the whole reason the row is here. + { label: "have a password (WorkOS returns none)", count: 0 }, + ], + }; +} + +export async function exportWorkOs(options: ExportWorkOsOptions): Promise { + const resolved = await resolveWorkOsApiKey(options); + + const destination = await resolveOutputPath("workos", options.output); + + await withGutter("Exporting users from WorkOS", async ({ setNextSteps }) => { + const dateTime = await startLogging(); + + // Only WorkOS can say whether the key is live, for the right environment, + // and not revoked — so a rejected key is asked for again here. The page it + // fetches is kept and reused, so proving the key costs no extra request. + const { value: firstPage, input: apiKey } = await withInputRetry( + resolved, + async () => promptWorkOsApiKey(), + async (candidate) => + withSpinner("Authenticating with WorkOS...", async () => fetchWorkOsPage(candidate)), + ); + + const users = await withSpinner("Fetching users from WorkOS...", async (spinner) => + fetchAllWorkOsUsers({ apiKey, firstPage, spinner }), + ); + + const providers = (await resolveWithIdentities(options, users.length)) + ? await withSpinner("Fetching OAuth providers...", async (spinner) => + fetchAllWorkOsIdentities({ apiKey, users, spinner }), + ) + : undefined; + + const { users: exported, coverage } = buildWorkOsExport(users, dateTime, providers?.identities); + const outputPath = writeExportOutput(exported, destination); + + setNextSteps( + reportExport({ + platform: "workos", + userCount: exported.length, + outputPath, + coverage, + sections: providers + ? [buildIdentityReport(users, providers.identities, providers.failed)] + : [], + transformerKey: "workos", + }), + ); + + if (exported.length > 0) { + log.warn( + "WorkOS does not return password hashes or TOTP secrets, and there is no export that does. Imported users who signed in with a password must reset it on their first Clerk sign-in, and anyone using an authenticator app has to re-enrol. Users on SSO or social sign-in are unaffected.", + ); + log.info(dim(`See ${DOCS_URL}`)); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/registry.ts b/packages/cli-core/src/commands/migrate/transformers/registry.ts index df9448595..ae2f133b8 100644 --- a/packages/cli-core/src/commands/migrate/transformers/registry.ts +++ b/packages/cli-core/src/commands/migrate/transformers/registry.ts @@ -15,6 +15,7 @@ import betterAuthTransformer from "./betterauth.ts"; import clerkTransformer from "./clerk.ts"; import firebaseTransformer from "./firebase.ts"; import supabaseTransformer from "./supabase.ts"; +import workosTransformer from "./workos.ts"; export const transformers: TransformerRegistryEntry[] = [ clerkTransformer, @@ -23,6 +24,7 @@ export const transformers: TransformerRegistryEntry[] = [ betterAuthTransformer, firebaseTransformer, supabaseTransformer, + workosTransformer, ]; /** diff --git a/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts b/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts index ae492f01d..b0005a471 100644 --- a/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts +++ b/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts @@ -48,7 +48,7 @@ const one = (key: string, record: Record, context = {}) => | undefined; describe("registry", () => { - test("registers all six platforms", () => { + test("registers all seven platforms", () => { expect(transformerKeys()).toEqual([ "clerk", "auth0", @@ -56,6 +56,7 @@ describe("registry", () => { "betterauth", "firebase", "supabase", + "workos", ]); }); @@ -163,6 +164,55 @@ describe("auth0", () => { }); }); +describe("workos", () => { + const base = { id: "user_01ABC", email: "a@x.dev" }; + + test("maps identity, name and metadata onto the Clerk schema", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, first_name: "Ada", last_name: "Lovelace" }, + ]); + expect(users[0]).toMatchObject({ + userId: "user_01ABC", + email: "a@x.dev", + firstName: "Ada", + lastName: "Lovelace", + }); + }); + + test.each([ + [true, "email", undefined], + [false, undefined, "a@x.dev"], + [undefined, undefined, "a@x.dev"], + ])("email_verified=%p routes the address correctly", (verified, kept, unverified) => { + const user = one("workos", { ...base, email_verified: verified }); + expect(user?.email).toBe(kept ? "a@x.dev" : undefined); + expect(user?.unverifiedEmailAddresses).toBe(unverified); + }); + + test("keeps metadata public", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, metadata: { plan: "pro" } }, + ]); + expect(users[0]?.publicMetadata).toEqual({ plan: "pro" }); + }); + + // No other transformer omits it. WorkOS never returns a digest, so naming a + // hasher would imply a password column that cannot exist. + test("names no password hasher, because WorkOS returns no hashes", () => { + expect(getTransformer("workos").defaults).toBeUndefined(); + }); + + // The export carries these so whoever runs the migration can see who used + // social sign-in; Clerk's import has no field for them. + test("drops OAuth identities carried through from the export", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, identities: [{ provider: "GoogleOAuth", idp_id: "1" }] }, + ]); + expect(users[0]?.userId).toBe("user_01ABC"); + expect("identities" in (users[0] ?? {})).toBe(false); + }); +}); + describe("authjs", () => { const base = { id: "cuid1", email: "a@x.dev" }; diff --git a/packages/cli-core/src/commands/migrate/transformers/workos.ts b/packages/cli-core/src/commands/migrate/transformers/workos.ts new file mode 100644 index 000000000..9e5b8e349 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/workos.ts @@ -0,0 +1,39 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification } from "./shared.ts"; + +/** + * WorkOS → Clerk transformer. + * + * Works with WorkOS's User Management API. `id` is a `user_…` string and is + * carried through as the Clerk user's `external_id`. + * + * **There is no `passwordHasher` default here, and that is deliberate.** Every + * other transformer names the hasher its platform ships so a digest can be + * verified; WorkOS returns no digest to verify. It accepts password hashes on + * import and never gives them back, and its TOTP secrets are returned on enrol + * only — so a WorkOS migration moves identities, not credentials. Naming a + * hasher here would imply a password column that cannot exist. + * + * WorkOS has no phone number and no username, which is why the map is short: + * those fields have nothing to come from. + */ +const workosTransformer = { + key: "workos", + label: "WorkOS", + description: + "Works with WorkOS's User Management API. WorkOS returns no password hashes, so imported users sign in by reset or SSO.", + transformer: { + id: "userId", + email: "email", + email_verified: "emailVerified", + first_name: "firstName", + last_name: "lastName", + metadata: "publicMetadata", + created_at: "createdAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "boolean"); + }, +} satisfies TransformerRegistryEntry; + +export default workosTransformer; diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts index e913a0064..cee841da4 100644 --- a/packages/cli-core/src/commands/migrate/wizard.test.ts +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -75,6 +75,7 @@ describe("transformer picker", () => { "betterauth", "firebase", "supabase", + "workos", ]); }); From 4e5c4bd52ffcc50dcce69fa9a3f4c193be3e8260 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 16 Sep 2026 17:54:33 -0400 Subject: [PATCH 037/141] docs: sync the migrate description in the root README `9a56d9f docs(migrate): mention migration command` added the row with a truncated description, so `readme.test.ts` failed comparing the committed help block against `clerk --help`. The command's own description is the longer form. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X4wXePuWJHWiocgoBaAX5n --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 484873e52..56d40fd14 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Commands: init [options] Initialize Clerk in your project link [options] Link this project to a Clerk application mcp Manage the Clerk remote MCP server connection for AI editors and CLIs - migrate Migrate users into Clerk from another auth provider + migrate Migrate users into Clerk from another auth provider or another Clerk instance open Open Clerk resources in your browser telemetry Control CLI usage telemetry (status, disable, enable) unlink [options] Unlink this project from its Clerk application From fe3beec29878811817dd629f9af9c05ca078a61c Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 17 Sep 2026 02:31:10 -0400 Subject: [PATCH 038/141] docs(migrate): describe the export source picker and counts as they are MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four places where the documentation had drifted from the code. `export clerk`'s own help example was `--instance prod --output prod-users.json`, described as "Export a specific instance to a chosen path" — which reads as a run that does not prompt. It does prompt: `resolveSource` treats only `--secret-key` as a choice the user typed, so `--app` and `--instance` decide whose instances lead the picker and nothing more. The example now shows `--secret-key`, which is the flag that actually skips it, and the README says so in the bullet rather than leaving it to be discovered. The README also claimed the flat instance picker opens when there is nothing to resolve at all — no link, no key, no flags. That tier falls through to `resolveUsersInstanceContext({})`, the application picker `users list` uses, which offers "create a new application" as well. `clerk-source.ts`'s own header comment said the same thing, so both are corrected; leaving one would re-seed the other. And two counts: the transformers listing example still showed six built-ins and no WorkOS row, and the settings example showed four of seven settings when there are eight, omitting three rows the command always prints. Covered by the existing `clerk migrate` changeset. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X4wXePuWJHWiocgoBaAX5n --- .../cli-core/src/commands/migrate/README.md | 27 ++++++++++++++----- .../commands/migrate/export/clerk-source.ts | 8 +++--- .../src/commands/migrate/export/index.ts | 4 +-- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 688bb55fa..acbdb392d 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -208,6 +208,8 @@ as the source without asking is how a run exports an instance and imports it back into itself. Instead: - `--secret-key ` names the source instance outright and runs unquestioned. + It is the only flag that does: `--app` and `--instance` decide whose instances + lead the picker, but the picker still opens. - Anything resolved on your behalf — the linked project, a keyless app — is never taken silently. A picker of every **instance** on your account opens instead — one flat row each, `my-app - Production instance (ins_…)`, not an @@ -215,12 +217,16 @@ back into itself. Instead: application's instances listed **first** so taking one is still a single Enter. Only when there are no instances to offer does it stop and list `--secret-key`, `--app`/`--instance` and `clerk link` instead. -- With nothing to resolve at all (no link, no key, no flags), that same picker - opens directly, rather than an error about an unlinked directory. +- With nothing to resolve at all (no link, no key, no flags), you get the + application picker `clerk users` uses — `Select a Clerk application to use:`, + followed by an instance picker when the application has more than one — + rather than an error about an unlinked directory. That is `clerk link`'s + picker, so it does offer `+ Create a new application`; a brand-new + application has no users to export, so it is never the answer here. -The picker has no "create a new application" choice, unlike `clerk link`'s — a -new application has no users to export. Rows are searchable by what they show, -so typing an application name, `production`, or an instance id all narrow it. +The instance picker (the second tier) has no "create a new application" choice. +Its rows are searchable by what they show, so typing an application name, +`production`, or an instance id all narrow it. In agent mode the resolved instance is used without a prompt; pass `--secret-key` or `--app`/`--instance` to be explicit. @@ -651,7 +657,11 @@ Transformers: … -6 built-in transformers + workos WorkOS + Works with WorkOS's User Management API. WorkOS returns no password hashes, + so imported users sign in by reset or SSO. + +7 built-in transformers Migrating from something else? Write a transformer and pass --transformer-file. ``` @@ -728,11 +738,14 @@ Each setting is named after the `clerk migrate import` flag it stands in for. SETTING VALUE SOURCE DESCRIPTION transformer firebase clerk config Source platform the export came from file users.json clerk config Export file to import users from +skip-unsupported-providers not set Skip users with no provider enabled in Clerk (Supabase) +log-dir ./logs clerk config Directory migration logs are written to firebase-signer-key [REDACTED] .env.clerk-migrate Firebase base64 signer key +firebase-salt-separator not set Firebase base64 salt separator firebase-rounds 8 .env.local (ROUNDS) Firebase scrypt rounds firebase-mem-cost 14 MEM_COST env var Firebase scrypt memory cost -4 of 7 settings set. Credentials are shown redacted. +6 of 8 settings set. Credentials are shown redacted. → Run `clerk migrate settings set ` to change one → Run `clerk migrate settings clear ` to forget one diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts index 0754114d5..3c701ae09 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -14,9 +14,11 @@ * keyless app) is not taken silently: every instance on the account is * offered, with the resolved application's instances first so "yes, that * one" is still a single Enter. - * 3. Nothing to resolve at all — no link, no key, no flags — offers those same - * instances, the trade `users list` makes, rather than failing on an - * unlinked directory. + * 3. Nothing to resolve at all — no link, no key, no flags — falls back to the + * application picker `users list` uses, rather than failing on an unlinked + * directory. That one is `clerk link`'s, so it also offers "create a new + * application"; an empty application is never the right source, but the + * alternative here is an error, not a better list. */ import { fetchAppsTolerantly } from "../../../lib/app-picker.ts"; diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index f8585bfa2..0241004ed 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -118,8 +118,8 @@ export function registerMigrateExport(migrateCommand: Command<[], Record From 58f98c7d7f8796f0d853b2d4f9bda739a8c90716 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 17 Sep 2026 02:31:57 -0400 Subject: [PATCH 039/141] feat(migrate): make `-y` mean "do not prompt" on every export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `-y` did not exist on any `migrate export` subcommand, and `ensureLogDir` never consulted it — yet the README and both functions' own doc comments claimed it suppressed the credential-retry loop and took the default log directory. Three prompts could still stop a run that had asked not to be asked. `-y` is not the same question as agent mode. Agent mode says the CLI *cannot* prompt; `-y` says the operator does not want it to. A confirm is skipped by either, but the two places that take a default instead of asking need to know a human chose it, so they cannot be collapsed. It is held per-run in `lib/assume-yes.ts` and set by a `preAction` hook on the `migrate` group, mirroring how `mode.ts` resolves `--mode` once and is read everywhere. The readers sit three layers below the parse: the log-directory question runs inside the gutter of seven commands and the retry loop under every credential prompt, so threading a `yes` parameter down would have put one on every export handler signature on the way. Hooks are inherited, so a subcommand that declares no `-y` resolves to false rather than to nothing. The three prompts do not all answer it the same way: - **A rejected credential** fails on the first attempt instead of re-asking. - **The log directory** takes `./logs` without asking and without saving. Landing on a default is not a choice, and recording one would retire the question for a human who never saw it. - **The export path** fails, naming `--output` and handing back the whole command with the proposed path already in it. It is the one prompt whose default cannot be undone by running the command again: a file written where nobody chose it has to be found and moved, and the second run writes a second copy. Guessing is the expensive answer here, so it does not guess. Agent mode keeps defaulting on that last one even when it also passes `-y` — there was never a prompt on that path to suppress, and agents pass `-y` reflexively, so erroring would turn working automation into a usage failure for nothing. The hook is the only link between the flag and its readers, and every unit test around them sets the flag directly, so a hook that stopped firing would revert all three behaviours with the suite still green. `index.test.ts` therefore parses real argv through the real program and asserts the flag both sets and clears. Covered by the existing `clerk migrate` changeset. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X4wXePuWJHWiocgoBaAX5n --- .../cli-core/src/commands/migrate/README.md | 34 +++++++----- .../src/commands/migrate/export/index.ts | 5 ++ .../commands/migrate/export/shared.test.ts | 54 +++++++++++++++++++ .../src/commands/migrate/export/shared.ts | 30 ++++++++++- .../src/commands/migrate/index.test.ts | 41 ++++++++++++++ .../cli-core/src/commands/migrate/index.ts | 9 ++++ .../src/commands/migrate/lib/assume-yes.ts | 32 +++++++++++ .../commands/migrate/lib/input-retry.test.ts | 23 ++++++++ .../src/commands/migrate/lib/input-retry.ts | 3 +- .../migrate/lib/log-dir-prompt.test.ts | 28 ++++++++++ .../src/commands/migrate/lib/logger.ts | 3 +- 11 files changed, 247 insertions(+), 15 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/assume-yes.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index acbdb392d..f83c076fa 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -161,8 +161,9 @@ database. Only the connection or the token exchange can say, and by then the operator has answered every other question the command asked. So that step — and only that step, never a fetch already under way or a file already written — runs inside a retry: the failure is explained, the prompt comes back, and the -rest of the export continues against whichever credential worked. `-y`, agent -mode and a non-TTY fail outright instead, having nobody to ask. +rest of the export continues against whichever credential worked. Agent mode +and a non-TTY fail outright instead, having nobody to ask, and `-y` fails too, +having been told not to. | Platform | Source | Feeds | | ------------ | -------------------------------- | -------------------------- | @@ -187,19 +188,28 @@ never silently overwrites the first. takes the proposed path. `--output` resolves against the **current directory**, like every other path flag here. +`-y` does neither: it **fails**, naming `--output` and handing back the whole +command with the proposed path already in it, to run again. This is the one +prompt whose default cannot be undone by re-running — a file written where +nobody chose it has to be found and moved, and the second run writes a second +copy. Every other question `-y` silences has a default that costs nothing to +land on. Agent mode keeps defaulting even when it also passes `-y`, since there +was no prompt on that path to suppress. + The question comes before any users are fetched, so a long export can be left unattended rather than stalling on a prompt with everything held in memory. -| Flag | Platforms | Description | -| -------------------------- | ---------------------------------- | --------------------------------------------------------- | -| `-o, --output ` | all | Where to write the export | -| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL, libsql/Turso or SQLite connection string | -| `--service-account ` | `firebase` | Path to a service account key JSON file | -| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | -| `--client-id ` | `auth0` | Machine-to-machine application client ID | -| `--client-secret ` | `auth0` | Machine-to-machine application client secret | -| `--api-key ` | `workos` | WorkOS secret API key, the one starting `sk_` | -| `--with-identities` | `workos` | Also record each user's OAuth providers | +| Flag | Platforms | Description | +| -------------------------- | ---------------------------------- | ----------------------------------------------------------- | +| `-o, --output ` | all | Where to write the export | +| `-y, --yes` | all | Do not prompt: require `--output`, fail on a bad credential | +| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL, libsql/Turso or SQLite connection string | +| `--service-account ` | `firebase` | Path to a service account key JSON file | +| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | +| `--client-id ` | `auth0` | Machine-to-machine application client ID | +| `--client-secret ` | `auth0` | Machine-to-machine application client secret | +| `--api-key ` | `workos` | WorkOS secret API key, the one starting `sk_` | +| `--with-identities` | `workos` | Also record each user's OAuth providers | `export clerk` also takes the targeting flags — it reads from a Clerk instance, so it resolves a key the same way `clerk migrate import` does, with one extra diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 0241004ed..07eae878d 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -109,6 +109,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)", ) .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") .option("--secret-key ", "Backend API secret key to use") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") @@ -135,6 +136,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Machine-to-machine application client ID") .option("--client-secret ", "Machine-to-machine application client secret") .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") .setExamples([ { command: @@ -157,6 +159,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Path to a service account key JSON file") .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") .setExamples([ { command: "clerk migrate export firebase --service-account ./service-account.json", @@ -178,6 +181,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Where to write the export, relative to the current directory") + .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") .setExamples([ { command: "clerk migrate export workos --api-key sk_…", @@ -202,6 +206,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Postgres, MySQL, libsql/Turso or SQLite connection string") .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") .setExamples([ { command: `clerk migrate export ${platform.key} --db-url "${platform.example}"`, diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts index 23b387db1..1c505ef81 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.test.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, mock, test } from "bun:test"; +import { type CliError, ERROR_CODE, EXIT_CODE } from "../../../lib/errors.ts"; const mockText = mock(); mock.module("../../../lib/prompts.ts", () => ({ @@ -14,9 +15,11 @@ mock.module("../../../mode.ts", () => ({ })); const { defaultOutputPath, outputStamp, resolveOutputPath } = await import("./shared.ts"); +const { setAssumeYes } = await import("../lib/assume-yes.ts"); beforeEach(() => { human = true; + setAssumeYes(false); mockText.mockReset(); }); @@ -79,4 +82,55 @@ describe("resolveOutputPath", () => { ); expect(mockText).not.toHaveBeenCalled(); }); + + // The one prompt whose default cannot be undone by running the command + // again: a file at a path nobody chose has to be found and moved, and a + // second run writes a second copy. So `-y` fails here rather than guessing. + describe("with -y", () => { + beforeEach(() => setAssumeYes(true)); + + test("fails rather than prompting or defaulting", async () => { + await expect(resolveOutputPath("supabase")).rejects.toThrow( + /needs an export location and will not prompt for one with -y/, + ); + expect(mockText).not.toHaveBeenCalled(); + }); + + test("is a usage error, so the exit code says what to fix", async () => { + const error = (await resolveOutputPath("supabase").catch((e: unknown) => e)) as CliError; + + expect(error.code).toBe(ERROR_CODE.USAGE_ERROR); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + }); + + // The whole point of failing instead of defaulting: the error has to hand + // back a line that runs, or it has cost the operator the run for nothing. + test("hands back the command to re-run, proposed path and all", async () => { + const error = (await resolveOutputPath("supabase").catch((e: unknown) => e)) as CliError; + + expect(error.examples?.[0]?.command).toMatch( + /^clerk migrate export supabase -y --output exports\/supabase-export-\d{8}-\d{4}\.json$/, + ); + }); + + test("names the platform that was actually run", async () => { + const error = (await resolveOutputPath("firebase").catch((e: unknown) => e)) as CliError; + + expect(error.message).toContain("`clerk migrate export firebase`"); + }); + + test("stays quiet when --output already answered it", async () => { + expect(await resolveOutputPath("clerk", "somewhere/mine.json")).toBe("somewhere/mine.json"); + }); + + // An agent passes `-y` reflexively and has no prompt to suppress, so the + // flag must not turn a working export into a usage error there. + test("still defaults in agent mode", async () => { + human = false; + + expect(await resolveOutputPath("supabase")).toMatch( + /^exports\/supabase-export-\d{8}-\d{4}\.json$/, + ); + }); + }); }); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index e443b803e..ed8370475 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -12,10 +12,12 @@ import fs from "node:fs"; import path from "node:path"; import { dim, green, yellow } from "../../../lib/color.ts"; +import { throwUsageError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { NEXT_STEPS } from "../../../lib/next-steps.ts"; import { text } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; /** * `YYYYMMDD-HHmm`, local time — ISO 8601 basic format, minus seconds. @@ -52,14 +54,40 @@ export function defaultOutputPath(platform: string, now?: Date): string { * One prompt, not a confirm followed by a path prompt: the proposed path is * prefilled, so Enter accepts it and typing replaces it. * - * `--output` is an answer already given, and agent mode has nobody to ask. + * `--output` is an answer already given, and agent mode has nobody to ask, so + * it takes the proposal. + * + * `-y` is neither: somebody is there, and they said not to ask. It fails + * instead of defaulting, because this is the one prompt whose default cannot + * be undone by running the command again — a file written to a path nobody + * chose has to be found and moved, and a second run writes a second copy. + * Silencing that question is what `--output` is for, so the error hands over + * the exact line, proposed path and all. (The log-directory question does take + * its default under `-y`: `./logs` is where the reader would look anyway, and + * nothing is saved.) */ export async function resolveOutputPath(platform: string, output?: string): Promise { if (output) return output; const proposed = defaultOutputPath(platform); + // Ordered so agent mode keeps defaulting even when it also passes `-y`: + // there was never a prompt on that path to suppress. if (!isHuman()) return proposed; + if (isAssumeYes()) { + throwUsageError( + `\`clerk migrate export ${platform}\` needs an export location and will not prompt for one with -y.\nPass --output, then run it again.`, + undefined, + undefined, + [ + { + command: `clerk migrate export ${platform} -y --output ${proposed}`, + description: "Re-run with the proposed path", + }, + ], + ); + } + const chosen = await text({ message: "Save the export to:", default: proposed, diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index cb24f0384..81c636f72 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { createProgram } from "../../cli-program.ts"; import { exportPlatformKeys } from "./export/registry.ts"; +import { isAssumeYes, setAssumeYes } from "./lib/assume-yes.ts"; import { transformerKeys } from "./transformers/registry.ts"; function findCommand(names: string[]) { @@ -197,4 +198,44 @@ describe("registerMigrate", () => { const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === long); expect(option?.short).toBe(short); }); + + // Every export takes `-y`: it is what turns the credential-retry loop off, + // and the loop is on every one of them. + test.each(exportPlatformKeys())("migrate export %s accepts --yes", (platform) => { + const flags = findCommand(["migrate", "export", platform])?.options.map((o) => o.long); + expect(flags).toContain("--yes"); + }); +}); + +/** + * The hook is the only link between the parsed flag and the two places that + * read it, three layers down. If it stopped firing — a Commander upgrade that + * dropped hook inheritance, an action registered outside the group — both + * behaviours would silently revert and every unit test around them would still + * pass, because they set the flag directly. + */ +describe("the migrate group's -y hook", () => { + async function parse(argv: string[]) { + const program = createProgram(); + // `exitOverride` so a usage error inside the action throws here instead of + // taking the test runner down with it; the hook has already run by then. + program.exitOverride(); + try { + await program.parseAsync(["node", "clerk", ...argv]); + } catch { + // The action is allowed to fail — only the hook's effect is under test. + } + return isAssumeYes(); + } + + test("records -y on an export", async () => { + expect(await parse(["migrate", "export", "supabase", "-y", "--db-url", "./none.sqlite"])).toBe( + true, + ); + }); + + test("records its absence, so a previous run cannot leak into this one", async () => { + setAssumeYes(true); + expect(await parse(["migrate", "export", "supabase", "--db-url", "./none.sqlite"])).toBe(false); + }); }); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index f5a1553a2..e49dcd3ec 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -2,6 +2,7 @@ import { createOption } from "@commander-js/extra-typings"; import type { Program } from "../../cli-program.ts"; import { parseIntegerOption } from "../../lib/option-parsers.ts"; import { deleteMigration } from "./delete.ts"; +import { setAssumeYes } from "./lib/assume-yes.ts"; import { registerMigrateExport } from "./export/index.ts"; import { registerMigrateLogs } from "./logs/index.ts"; import { registerMigrateSettings } from "./settings/index.ts"; @@ -39,6 +40,14 @@ export function registerMigrate(program: Program): void { { command: "clerk migrate delete", description: "Undo the last migration" }, ]); + // `-y` is read three layers down — by the log-directory question and by the + // credential-retry loop — so it is resolved once here rather than threaded + // through every export handler. Hooks are inherited, so this fires for every + // subcommand under `migrate`; one that declares no `-y` resolves to false. + migrateCommand.hook("preAction", (_thisCommand, actionCommand) => { + setAssumeYes(Boolean(actionCommand.opts().yes)); + }); + // Named, not `isDefault`. `import` and `export` are the two directions this // group moves users in, and neither is implied by the bare group name — a // default would make `clerk migrate --file users.json` mean "import" while diff --git a/packages/cli-core/src/commands/migrate/lib/assume-yes.ts b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts new file mode 100644 index 000000000..6315520f2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts @@ -0,0 +1,32 @@ +/** + * Whether this run was given `-y`. + * + * `-y` is not the same question as {@link isAgent}. Agent mode says the CLI + * *cannot* prompt; `-y` says the operator does not want it to. Most prompts + * only care about the first — a confirm is skipped by either — but the two + * places that take a default instead of asking need to know a human chose it, + * so the two cannot be collapsed into one flag. + * + * Held per-run rather than threaded through, because the readers are three + * layers below the command that parses it: `ensureLogDir` runs inside the + * gutter of seven different commands, and `withInputRetry` sits under every + * credential prompt. Passing it down would put a `yes` parameter on every + * export handler signature on the way. This mirrors `mode.ts`, which resolves + * `--mode` once in a `preAction` hook and is read the same way. + * + * Set by the `migrate` group's `preAction` hook, so every subcommand under it + * is covered whether or not it declares the flag — one that does not simply + * resolves to `false`. + */ + +let assumeYes = false; + +/** Records this run's `-y`. Called once per invocation, before the action. */ +export function setAssumeYes(value: boolean): void { + assumeYes = value; +} + +/** Whether `-y` was passed to the command now running. */ +export function isAssumeYes(): boolean { + return assumeYes; +} diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts index 8e7e6e26b..c414d6d1b 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -33,6 +33,7 @@ mock.module("../../../lib/prompts.ts", () => ({ const { withInputRetry } = await import("./input-retry.ts"); const { promptDbUrl } = await import("../export/db-options.ts"); +const { setAssumeYes } = await import("./assume-yes.ts"); const captured = useCaptureLog(); @@ -59,6 +60,7 @@ afterAll(() => { beforeEach(() => { setMode("human"); + setAssumeYes(false); answers = []; cancelPrompt = false; }); @@ -169,6 +171,27 @@ describe("withInputRetry", () => { expect(attempts).toBe(1); }); + // `-y` is a human on a TTY who could be asked and said not to. Agent mode + // cannot reach the prompt at all; this one can and declines to, so it needs + // its own check rather than riding on the mode assertion above. + test("throws without prompting when `-y` said not to ask", async () => { + setAssumeYes(true); + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + // A bug inside the work, or an interrupt, is not a wrong answer to a prompt. test("does not retry an error the database layer did not raise", async () => { let attempts = 0; diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts index 36805de7d..13a3604b7 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -18,6 +18,7 @@ import { CliError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { isAgent, isHuman } from "../../../mode.ts"; +import { isAssumeYes } from "./assume-yes.ts"; /** * Runs `work`, and on failure asks for the input again and runs it once more. @@ -53,7 +54,7 @@ export async function withInputRetry( // Everything these steps raise for a bad credential is a CliError // carrying its own explanation; anything else (an interrupt, a bug) is // not ours to retry. - if (!(error instanceof CliError) || !isHuman() || isAgent()) throw error; + if (!(error instanceof CliError) || !isHuman() || isAgent() || isAssumeYes()) throw error; log.error(error.message); candidate = await reprompt(); diff --git a/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts b/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts index 766145d24..7aba346fe 100644 --- a/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts @@ -32,6 +32,7 @@ mock.module("../../../lib/prompts.ts", () => ({ const { _resetLogDir, ensureLogDir } = await import("./logger.ts"); const { loadSettings, saveSettings } = await import("./settings.ts"); +const { setAssumeYes } = await import("./assume-yes.ts"); useCaptureLog(); @@ -68,6 +69,7 @@ beforeEach(() => { mockText.mockClear(); answer = ""; setMode("human"); + setAssumeYes(false); }); afterEach(() => _resetLogDir()); @@ -111,3 +113,29 @@ describe("ensureLogDir asks once", () => { }); }); }); + +describe("ensureLogDir with `-y`", () => { + // Agent mode cannot reach this prompt; `-y` is a human on a TTY who could be + // asked and said not to be, so it needs its own check. + test("takes ./logs without asking", async () => { + setAssumeYes(true); + + expect(await ensureLogDir()).toBe(path.join(workDir, "logs")); + expect(mockText).not.toHaveBeenCalled(); + }); + + // Landing on a default is not a choice, and recording one would retire the + // question for a human who never saw it. + test("saves nothing, so the next interactive run still asks", async () => { + setAssumeYes(true); + await ensureLogDir(); + expect((await loadSettings()).logDir).toBeUndefined(); + + _resetLogDir(); + setAssumeYes(false); + answer = "./audit"; + + expect(await ensureLogDir()).toBe(path.join(workDir, "audit")); + expect(mockText).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.ts b/packages/cli-core/src/commands/migrate/lib/logger.ts index 814fe4914..3eca2cdce 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.ts @@ -17,6 +17,7 @@ import { log } from "../../../lib/log.ts"; import { text } from "../../../lib/prompts.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { envNames, findSetting } from "../settings/registry.ts"; +import { isAssumeYes } from "./assume-yes.ts"; import { findMigrateEnvValue } from "./env-file.ts"; import { loadSettings, saveSettings } from "./settings.ts"; import type { @@ -105,7 +106,7 @@ export async function resolveLogDir(): Promise { export async function ensureLogDir(): Promise { const chosen = await chosenLogDir(); if (chosen) return remember(chosen); - if (!isHuman() || isAgent()) return remember(DEFAULT_LOG_DIR); + if (!isHuman() || isAgent() || isAssumeYes()) return remember(DEFAULT_LOG_DIR); const answer = await text({ message: "Where should migration logs be saved?", From 0541bb33e5b7ffa1089655d0ca4c1bb68291728a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 17 Sep 2026 02:53:06 -0400 Subject: [PATCH 040/141] fix(migrate): let a named instance skip the export source picker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `resolveSource` treated only `--secret-key` as a choice the user had made, so `--app`, `--instance` and an exported `CLERK_SECRET_KEY` all opened the instance picker anyway — and then overrode whatever they had resolved with the picker's answer. Two consequences, both wrong. `clerk migrate export clerk` could not be scripted outside agent mode. Every run stopped at a prompt no flag could pre-answer, which is not a reasonable ask of an export that may be the slow half of a migration runbook. And it made this the one command in the family where exporting `CLERK_SECRET_KEY` did less than not exporting it. `resolveBapiSecretKey` puts an exported key above the linked profile everywhere else, with a comment saying so; here the key was resolved and then discarded. The distinction the picker exists for is not "did the CLI look anything up" but "did the user say which instance". A flag or an exported key is a sentence typed for this run; the linked profile is a choice made for some other purpose, and normally names the migration's destination rather than its source. Only the second is worth asking about, and the catch branch below already drew the line in exactly that place — `named` is that predicate, hoisted so both branches read it. `--instance` counts on its own: with the linked application it names one instance, and with `--app` it addresses any instance on the account. Covered by the existing `clerk migrate` changeset. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X4wXePuWJHWiocgoBaAX5n --- .../cli-core/src/commands/migrate/README.md | 16 +++++++-- .../migrate/export/clerk-source.test.ts | 30 ++++++++++++++++ .../commands/migrate/export/clerk-source.ts | 35 +++++++++++++------ 3 files changed, 67 insertions(+), 14 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index f83c076fa..15d1e18ab 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -217,9 +217,12 @@ step. The linked project is usually the migration's _destination_, so taking it as the source without asking is how a run exports an instance and imports it back into itself. Instead: -- `--secret-key ` names the source instance outright and runs unquestioned. - It is the only flag that does: `--app` and `--instance` decide whose instances - lead the picker, but the picker still opens. +- **Naming the instance runs unquestioned.** `--secret-key `, `--app`, + `--instance`, or an exported `CLERK_SECRET_KEY` — any of them is a sentence + you typed for this run, so none of them opens a picker. That is what makes + the export scriptable outside agent mode, and it keeps an exported key + outranking the linked profile here the way it does everywhere else in the + CLI. - Anything resolved on your behalf — the linked project, a keyless app — is never taken silently. A picker of every **instance** on your account opens instead — one flat row each, `my-app - Production instance (ins_…)`, not an @@ -707,6 +710,13 @@ It only ever edits `.env.clerk-migrate`; a value coming from the app's own env file or the shell is named in the listing's source column and has to be removed there. +**A bare `settings clear` needs `-y` where it cannot ask.** It forgets every +setting and every credential in `.env.clerk-migrate`, so a non-interactive or +agent run refuses rather than assuming, the way `migrate logs clean` and +`migrate delete` already do. `settings clear ` does not: naming the one +setting to forget is itself the confirmation, the same way `settings set` needs +none. + A misspelled name gets the closest match back, not just the list: ``` diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts index 0b7fa6e46..1e5d95566 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -69,6 +69,36 @@ describe("resolveClerkSource", () => { expect(mockSearch).not.toHaveBeenCalled(); }); + // An export has to be scriptable outside agent mode. Before this, `chosen` + // keyed off `--secret-key` alone, so every other way of naming an instance + // still stopped at a picker no flag could answer. + test.each([ + ["--app", { app: "app_1" }], + ["--instance", { instance: "prod" }], + ["--app and --instance", { app: "app_1", instance: "prod" }], + ])("%s names the instance, so nothing is asked", async (_label, options) => { + stubResolved("my-app (production)"); + + const source = await resolveClerkSource(options); + + expect(source).toEqual({ secretKey: "sk_test_resolved", target: "my-app (production)" }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // `resolveBapiSecretKey` puts an exported key above the linked profile for + // every other command in this family. Opening a picker here — and then + // overriding the key with whatever it returned — made this the one command + // where exporting CLERK_SECRET_KEY did less than not exporting it. + test("an exported CLERK_SECRET_KEY is not second-guessed in a linked directory", async () => { + process.env.CLERK_SECRET_KEY = "sk_test_from_env"; + stubResolved("my-app (development)", "sk_test_from_env"); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_test_from_env", target: "my-app (development)" }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + // Exporting the instance that is about to be imported *into* is the failure // this whole module exists to prevent, so a resolved instance is offered as // one choice among the account's applications rather than taken silently. diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts index 3c701ae09..78dfa201c 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -9,7 +9,11 @@ * * So the source is resolved in three tiers: * - * 1. `--secret-key` names an instance outright — it runs unquestioned. + * 1. The user named the instance — `--secret-key`, `--app`, `--instance`, or + * an exported `CLERK_SECRET_KEY` — and it runs unquestioned. An exported + * key outranks the linked profile everywhere else in this CLI + * (`resolveBapiSecretKey`), and this command is not the one place that + * should differ. * 2. Anything the CLI resolved on the user's behalf (the linked project, a * keyless app) is not taken silently: every instance on the account is * offered, with the resolved application's instances first so "yes, that @@ -59,24 +63,33 @@ export type ClerkExportSource = { type ResolvedSource = ClerkExportSource & { chosen: boolean }; async function resolveSource(options: ResolveClerkSourceOptions): Promise { + // What separates the two tiers is not "did the CLI have to look anything up" + // but "did the user say which instance". A flag or an exported key is a + // sentence they typed for this run; the linked project is a choice they made + // for some other purpose, possibly months ago, and normally names the + // migration's destination rather than its source. Only the second is worth + // asking about. + // + // `--instance` counts on its own: paired with the linked application it + // names one instance, and pairing it with `--app` addresses any instance on + // the account. Without this an export could not be scripted at all outside + // agent mode — every run would stop at a picker no flag could answer. + const named = + Boolean(options.secretKey) || + Boolean(options.app) || + Boolean(options.instance) || + Boolean(process.env.CLERK_SECRET_KEY); + try { return { target: await describeBapiTarget(options), secretKey: await resolveBapiSecretKey(options), - // Flags and env keys are a choice the user typed; the linked project is - // one they made for some other purpose, possibly months ago. - chosen: Boolean(options.secretKey), + chosen: named, }; } catch (error) { - const hasExplicitTarget = - Boolean(options.secretKey) || - Boolean(options.app) || - Boolean(options.instance) || - Boolean(process.env.CLERK_SECRET_KEY); - if ( !isHuman() || - hasExplicitTarget || + named || !(error instanceof CliError) || error.code !== ERROR_CODE.NO_SECRET_KEY ) { From 24f2cb9dc6366ac46076f4ca75c54cf158dbedfd Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 17 Sep 2026 02:53:06 -0400 Subject: [PATCH 041/141] fix(migrate): require -y to clear every setting where nothing can be asked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clerk migrate settings clear` forgets the whole project entry and every credential in `.env.clerk-migrate`. In agent mode or off a TTY it did both with no prompt and no flag, while `migrate logs clean` and `migrate delete` — the other two destructive commands in this tree — refuse outright without `-y`. Of the three it is the one that destroys secrets, and the only one whose loss cannot be recovered from a log. Reading silence as consent is the one interpretation that cannot be walked back, so it now refuses the same way, naming the file it would have emptied. `settings clear ` is left as it was: naming the single setting to forget is itself the confirmation, the same way `settings set` needs none. Covered by the existing `clerk migrate` changeset. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X4wXePuWJHWiocgoBaAX5n --- .../src/commands/migrate/settings/clear.ts | 17 ++++++++++++++++- .../commands/migrate/settings/settings.test.ts | 14 ++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/settings/clear.ts b/packages/cli-core/src/commands/migrate/settings/clear.ts index 23871d90e..a7d8a981d 100644 --- a/packages/cli-core/src/commands/migrate/settings/clear.ts +++ b/packages/cli-core/src/commands/migrate/settings/clear.ts @@ -113,7 +113,22 @@ export async function clear(options: SettingsClearOptions = {}, name?: string): const saved = await loadSettings(); const hadConfig = Object.keys(saved).length > 0; - if (!options.yes && isHuman() && !isAgent()) { + if (!options.yes) { + if (isAgent() || !isHuman()) { + throwUsageError( + "`clerk migrate settings clear` forgets this project's settings and every credential in " + + `${MIGRATE_ENV_FILE}, and cannot prompt here. Pass -y to confirm.`, + undefined, + undefined, + [ + { + command: "clerk migrate settings clear -y", + description: "Forget them all without prompting", + }, + ], + ); + } + if (hadConfig) warnAboutUndo(saved); const proceed = await confirm({ message: "Clear this project's migration settings?", diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index 220a6cf35..90838c5f3 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -249,6 +249,20 @@ describe("clear", () => { expect(envFileContent()).toBe("OTHER=keep\n"); }); + + // It destroys credentials, like `logs clean` destroys logs and `migrate + // delete` destroys users — and those two both refuse rather than assume. + // Proceeding here because nobody could be asked is the one reading of + // silence that cannot be undone. + test("refuses rather than assuming when it cannot prompt", async () => { + await set("transformer", "firebase"); + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + + await expect(clear({})).rejects.toThrow(/cannot prompt here\. Pass -y to confirm/); + + expect(await loadSettings()).toMatchObject({ transformer: "firebase" }); + expect(envFileContent()).toContain("CLERK_FIREBASE_SIGNER_KEY"); + }); }); describe("clear ", () => { From e447998bd0aaeba8b38759294c373b930821183e Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 18 Sep 2026 15:10:56 -0400 Subject: [PATCH 042/141] build: drop the unused `build` script and compile in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The root `build` script only forwarded to cli-core's `build`, which emitted a plain `dist/cli.js` bundle nothing consumed — the published artifact is the compiled binary from `build:compile`. CI now runs `build:compile` directly, so the bundle step no longer runs at all. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 2 +- package.json | 1 - packages/cli-core/package.json | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d1f1fc71..4d5fa8f5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }} restore-keys: bun-${{ runner.os }}- - run: bun install --frozen-lockfile - - run: bun run build + - run: bun run build:compile lint: name: Lint diff --git a/package.json b/package.json index 2a9f82875..be6b45f89 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,6 @@ "packages/*" ], "scripts": { - "build": "bun run --filter @clerk/cli-core build", "dev": "bun run --cwd packages/cli-core dev", "test": "bun run scripts/check-bun-version.ts && bun test 'packages/cli-core/src/' 'packages/extras/src/' 'scripts/' --parallel --only-failures", "test:e2e": "bun run scripts/check-bun-version.ts && bun test 'test/e2e/' --retry 1 --parallel --only-failures", diff --git a/packages/cli-core/package.json b/packages/cli-core/package.json index 8bf0bf0fe..2ef1869c3 100644 --- a/packages/cli-core/package.json +++ b/packages/cli-core/package.json @@ -7,7 +7,6 @@ }, "type": "module", "scripts": { - "build": "bun build ./src/cli.ts --outfile ./dist/cli.js --target node --external @napi-rs/keyring --minify", "build:compile": "bun build --compile --minify --no-compile-autoload-dotenv --no-compile-autoload-bunfig ./src/cli.ts --outfile ./dist/clerk", "dev": "bun run ./src/cli.ts", "typecheck": "tsc --noEmit -p tsconfig.json", From 3030f7920b1d98724afb4d24cb122ca3b1efc3ab Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 18 Sep 2026 15:19:47 -0400 Subject: [PATCH 043/141] chore: set bun.lock configVersion to 0 Co-Authored-By: Claude Opus 5 (1M context) --- bun.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bun.lock b/bun.lock index 67c0f8e0b..3c2fbecb6 100644 --- a/bun.lock +++ b/bun.lock @@ -1,6 +1,6 @@ { "lockfileVersion": 1, - "configVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "@clerk/cli-workspace", From f46c0fc408f3b550689e4ba81fa3938100b3316f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 21 Sep 2026 18:07:21 -0400 Subject: [PATCH 044/141] feat(migrate): accept --no-with-identities and assume it under -y `-y` says the operator does not want to be asked, but `export workos` asked about the OAuth provider fan-out anyway, so an unattended run stalled on a prompt. `-y` now answers it the way a `yes` would. That leaves no way to decline without being asked, so the flag gains Commander's negation: `--no-with-identities` sets it to false, and an explicit flag in either direction wins over `-y`. Matches `clerk init --no-skills`. Co-Authored-By: Claude Opus 5 (1M context) --- .../cli-core/src/commands/migrate/README.md | 4 +++- .../src/commands/migrate/export/index.ts | 6 ++++- .../commands/migrate/export/workos.test.ts | 23 +++++++++++++++++++ .../src/commands/migrate/export/workos.ts | 10 ++++++-- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 15d1e18ab..b5ee4bb37 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -210,6 +210,7 @@ unattended rather than stalling on a prompt with everything held in memory. | `--client-secret ` | `auth0` | Machine-to-machine application client secret | | `--api-key ` | `workos` | WorkOS secret API key, the one starting `sk_` | | `--with-identities` | `workos` | Also record each user's OAuth providers | +| `--no-with-identities` | `workos` | Skip the OAuth provider fan-out without being asked | `export clerk` also takes the targeting flags — it reads from a Clerk instance, so it resolves a key the same way `clerk migrate import` does, with one extra @@ -419,7 +420,8 @@ becomes 1,010 for a thousand users. Nothing it returns can be imported — breakdown in the coverage report, and an `identities` array kept in the export file for whoever runs the migration. The interactive path asks once, after the user count is known, defaulting to no; agent mode takes the flag's answer and -asks nothing. +asks nothing. `-y` answers the question `yes`, so pass +`--no-with-identities` to skip the fan-out without being asked. The breakdown prints as its own **OAuth providers** block under the coverage table, not as extra coverage rows: diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 07eae878d..506654063 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -180,8 +180,12 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Where to write the export, relative to the current directory") - .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .option( + "-y, --yes", + "Do not prompt: require --output, fail on a rejected credential, and assume --with-identities", + ) .setExamples([ { command: "clerk migrate export workos --api-key sk_…", diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts index e61103bf3..7cf60563f 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.test.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -5,6 +5,7 @@ import os from "node:os"; import path from "node:path"; import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; +import { setAssumeYes } from "../lib/assume-yes.ts"; import { buildIdentityReport, buildWorkOsExport, @@ -184,6 +185,28 @@ describe("resolveWithIdentities", () => { expect(await resolveWithIdentities({}, 10)).toBe(false); }); + test("is off when `--no-with-identities` said so, even under -y", async () => { + setAssumeYes(true); + try { + expect(await resolveWithIdentities({ withIdentities: false }, 10)).toBe(false); + } finally { + setAssumeYes(false); + } + }); + + // `-y` is "answer the prompts yes", and the prompt is "also fetch providers?". + test("is on under -y, which answers the question rather than asking it", async () => { + const originalMode = getMode(); + setMode("human"); + setAssumeYes(true); + try { + expect(await resolveWithIdentities({}, 10)).toBe(true); + } finally { + setAssumeYes(false); + setMode(originalMode); + } + }); + test("does not ask when there are no users to ask about", async () => { const originalMode = getMode(); setMode("human"); diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index f8ccf5df6..c61270a93 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -25,6 +25,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spin import { isAgent, isHuman } from "../../../mode.ts"; import { findMigrateEnvValue } from "../lib/env-file.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { createApiScheduler } from "../lib/scheduler.ts"; import { @@ -66,6 +67,7 @@ const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; export type ExportWorkOsOptions = { apiKey?: string; + /** Unset means "ask"; `--no-with-identities` sets it to false. */ withIdentities?: boolean; output?: string; }; @@ -220,13 +222,17 @@ export async function fetchAllWorkOsUsers(options: { * field. It is a line in the coverage report, and a record kept in the file. * * Agent mode gets the flag's answer and no question: there is nobody to ask. + * `-y` answers the question the way a `yes` would, so `--no-with-identities` + * is the way to say no without being asked. */ export async function resolveWithIdentities( options: ExportWorkOsOptions, userCount: number, ): Promise { - if (options.withIdentities) return true; - if (userCount === 0 || isAgent() || !isHuman()) return false; + if (options.withIdentities !== undefined) return options.withIdentities; + if (userCount === 0) return false; + if (isAssumeYes()) return true; + if (isAgent() || !isHuman()) return false; return confirm({ message: `Also fetch each user's OAuth providers? That is ${userCount} extra request${userCount === 1 ? "" : "s"}, and the result is report-only — Clerk's import cannot take external accounts.`, From a2a5165f25f11d0214720b3d050ce1aba6acae3c Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 21 Sep 2026 18:07:33 -0400 Subject: [PATCH 045/141] fix(migrate): keep the log directory an import saved at its start `startLogging` settles `logDir` and saves it before the first user is processed, but the write that records the transformer and file replaced the whole entry with a fresh object, dropping it again mid-run. A run that was interrupted after that point left `log-dir` unset in `clerk migrate settings`, so `clerk migrate logs` had no directory to read the log the run had just written. Spread the saved entry rather than overwriting it. Every other `saveSettings` caller already did. Co-Authored-By: Claude Opus 5 (1M context) --- packages/cli-core/src/commands/migrate/run.test.ts | 12 +++++++++++- packages/cli-core/src/commands/migrate/run.ts | 8 +++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index f7e5364c2..b1a655e38 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -11,7 +11,7 @@ import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); import { getLogDir } from "./lib/logger.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; -import { loadSettings } from "./lib/settings.ts"; +import { loadSettings, saveSettings } from "./lib/settings.ts"; import { applyResumeAfter, explainErrors, run, validateRunOptions } from "./run.ts"; import type { User } from "./types.ts"; @@ -176,6 +176,16 @@ describe("run", () => { expect(await loadSettings()).toEqual({ transformer: "clerk", file: "export.json" }); }); + // `startLogging` settles and saves `logDir` before the first user is + // processed. Writing the transformer and file as a fresh object dropped it + // again mid-run, so `clerk migrate logs` had nowhere to read the log the run + // had just written. + test("keeps the log directory it saved at the start of the run", async () => { + await saveSettings({ logDir: "./custom-logs" }); + await run(baseOptions); + expect((await loadSettings()).logDir).toBe("./custom-logs"); + }); + test("--require-password imports only the users that have one", async () => { await run({ ...baseOptions, requirePassword: true }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 905add1a6..afa6ab288 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -58,7 +58,7 @@ import { } from "./lib/modify-settings.ts"; import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; import { startLogging, getLogFilePath } from "./lib/logger.ts"; -import { saveSettings } from "./lib/settings.ts"; +import { loadSettings, saveSettings } from "./lib/settings.ts"; import { countSocialProviders, findDisabledProviders, @@ -737,7 +737,13 @@ export async function run(rawOptions: MigrateRunOptions): Promise { // The Firebase hash parameters are deliberately not among these: the signer // key is a secret, and remembering it would write it to disk in plaintext. + // + // Spread over what is already saved rather than written fresh: `logDir` was + // settled and saved by `startLogging` at the top of this run, and a bare + // object here would drop it — leaving `clerk migrate logs` with no + // directory to read the run it just wrote. await saveSettings({ + ...(await loadSettings()), transformer, file, ...(options.skipUnsupportedProviders ? { skipUnsupportedProviders: true } : {}), From e4db5785d48daa832c66e03f44df7bc05a0cf424 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 21 Sep 2026 18:07:48 -0400 Subject: [PATCH 046/141] feat(migrate): print the import command where an agent can see it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suggested `migrate import` command was handed to `setNextSteps`, which renders only in the gutter outro — and `withGutter` and `printNextSteps` both return early for an agent or a non-TTY. So an agent was told what had been exported and never how to import it. It now prints through `log.info`, alongside the coverage table that already reaches every mode, and carries two things it did not say before: - `-y`, when the export was given it. On import `-y` also waves through the development-instance user-limit warning, so it is carried across rather than always printed, and replaced by a hint line when the export went without it. Firebase's hash-parameter command mirrors the same choice, so the two commands a Firebase operator sees agree. - How to reach production. One command rather than a development and a production variant: no flag's absence means "development", since the resolved key decides, so the note names `--instance prod` instead of labelling a bare command that `CLERK_SECRET_KEY=sk_live_…` would send to production. `migrate import` and `migrate delete` get the same treatment through `printAgentNextSteps`: their steps name the log file holding the per-user record of what landed and the command that undoes the run, which an agent that lost users mid-import has no other way to find. Co-Authored-By: Claude Opus 5 (1M context) --- .../cli-core/src/commands/migrate/README.md | 22 ++++++++- .../cli-core/src/commands/migrate/delete.ts | 3 +- .../src/commands/migrate/export/auth0.ts | 18 +++---- .../src/commands/migrate/export/authjs.ts | 18 +++---- .../src/commands/migrate/export/betterauth.ts | 18 +++---- .../src/commands/migrate/export/clerk.ts | 18 +++---- .../commands/migrate/export/firebase.test.ts | 20 ++++++++ .../src/commands/migrate/export/firebase.ts | 21 ++++---- .../commands/migrate/export/shared.test.ts | 34 ++++++++++++- .../src/commands/migrate/export/shared.ts | 49 ++++++++++++++++--- .../src/commands/migrate/export/supabase.ts | 18 +++---- .../src/commands/migrate/export/workos.ts | 24 +++++---- packages/cli-core/src/commands/migrate/run.ts | 9 ++-- packages/cli-core/src/lib/next-steps.ts | 30 +++++++++--- 14 files changed, 206 insertions(+), 96 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index b5ee4bb37..95dadc83f 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -257,10 +257,28 @@ Field coverage ! 2/3 have a password (not exportable — see below) Exported 3 users to /project/exports/clerk-export-20260817-1432.json -└ Next steps - → Run `clerk migrate import --transformer clerk --file exports/clerk-export-20260817-1432.json` to import them + +Import them with: + clerk migrate import --transformer clerk --file exports/clerk-export-20260817-1432.json + + Imports into whichever instance the resolved secret key belongs to. + For production, add `--instance prod` or use a production secret key. + Add `-y` to skip the import confirmation prompt. ``` +The import command prints through the same channel as the coverage table +rather than the gutter's **Next steps** outro, which is human-only — an agent +would otherwise be told what was exported and never how to import it. `-y` is +carried across from the export that was given it, and replaced by the hint line +above when it was not: on import `-y` also waves through the +development-instance user-limit warning, so it is not a flag to suggest to +someone who never asked for it. + +There is one command, not a development and a production variant, because no +flag's absence means "development" — the resolved key decides, through +`--secret-key`, `--app`, `CLERK_SECRET_KEY`, the keyless project and the linked +profile in that order. + Every export also writes `logs/export-.log`, so `migrate logs list` sees it alongside imports and deletions. diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 5b5a78318..b48605e8d 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -31,7 +31,7 @@ import { } from "../../lib/errors.ts"; import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; import { log } from "../../lib/log.ts"; -import { NEXT_STEPS } from "../../lib/next-steps.ts"; +import { NEXT_STEPS, printAgentNextSteps } from "../../lib/next-steps.ts"; import { confirm } from "../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; @@ -324,6 +324,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise 0) process.exitCode = 1; }); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 9ae0f4a32..43ce12f0a 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -332,7 +332,7 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { const destination = await resolveOutputPath("auth0", options.output); - await withGutter("Exporting users from Auth0", async ({ setNextSteps }) => { + await withGutter("Exporting users from Auth0", async () => { const dateTime = await startLogging(); // Only Auth0 can say whether these three go together, and whether the @@ -354,15 +354,13 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { const { users: exported, coverage } = buildAuth0Export(users, dateTime); const outputPath = writeExportOutput(exported, destination); - setNextSteps( - reportExport({ - platform: "auth0", - userCount: exported.length, - outputPath, - coverage, - transformerKey: "auth0", - }), - ); + reportExport({ + platform: "auth0", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "auth0", + }); if (exported.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index c5b5b319d..e407b8948 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -123,7 +123,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { const destination = await resolveOutputPath("authjs", options.output); - await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { + await withGutter("Exporting users from Auth.js", async () => { const dateTime = await startLogging(); const { @@ -141,15 +141,13 @@ export async function exportAuthJs(options: DbExportOptions): Promise { const { users, coverage } = buildAuthJsExport(rows, dateTime); const outputPath = writeExportOutput(users, destination); - setNextSteps( - reportExport({ - platform: "authjs", - userCount: users.length, - outputPath, - coverage, - transformerKey: "authjs", - }), - ); + reportExport({ + platform: "authjs", + userCount: users.length, + outputPath, + coverage, + transformerKey: "authjs", + }); if (users.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index d637845a7..21aee4787 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -170,7 +170,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const destination = await resolveOutputPath("betterauth", options.output); - await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { + await withGutter("Exporting users from Better Auth", async () => { const dateTime = await startLogging(); const { @@ -197,14 +197,12 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const { users, coverage } = buildBetterAuthExport(rows, dateTime); const outputPath = writeExportOutput(users, destination); - setNextSteps( - reportExport({ - platform: "betterauth", - userCount: users.length, - outputPath, - coverage, - transformerKey: "betterauth", - }), - ); + reportExport({ + platform: "betterauth", + userCount: users.length, + outputPath, + coverage, + transformerKey: "betterauth", + }); }); } diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 57af5cc88..638c06bec 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -234,7 +234,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { const destination = await resolveOutputPath("clerk", options.output); - await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { + await withGutter("Exporting users from Clerk", async () => { const dateTime = await startLogging(); log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); @@ -246,15 +246,13 @@ export async function exportClerk(options: ExportClerkOptions): Promise { const { users: exported, coverage } = buildClerkExport(users, dateTime); const outputPath = writeExportOutput(exported, destination); - setNextSteps( - reportExport({ - platform: "clerk", - userCount: exported.length, - outputPath, - coverage, - transformerKey: "clerk", - }), - ); + reportExport({ + platform: "clerk", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "clerk", + }); if (exported.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 4d432e0a0..0bae0c8b3 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -4,6 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; +import { setAssumeYes } from "../lib/assume-yes.ts"; import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { @@ -420,6 +421,25 @@ describe("formatHashConfigGuidance", () => { expect(command).not.toContain("\\"); }); + // The shared import block carries `-y` across from the export; this command + // is the one a Firebase operator actually copies, so it has to agree. + test("carries -y across from the export that was given it", () => { + setAssumeYes(true); + try { + expect(formatHashConfigGuidance(config, "out.json", 3).join("\n")).toContain( + "clerk migrate import -y --transformer firebase", + ); + } finally { + setAssumeYes(false); + } + }); + + test("leaves -y out when the export was not given it", () => { + expect(formatHashConfigGuidance(config, "out.json", 3).join("\n")).toContain( + "clerk migrate import --transformer firebase", + ); + }); + test("says where to find them when the project would not say", () => { const text = formatHashConfigGuidance(null, "out.json", 3).join("\n"); expect(text).toContain("Password hash parameters"); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 162c8e2be..671bd7cb7 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -31,6 +31,7 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; @@ -490,7 +491,7 @@ export function formatHashConfigGuidance( // as three extra arguments. A line that wraps on screen has no such // character in it and pastes back as what was printed. dim( - ` clerk migrate import -y --transformer firebase --file ${outputPath}` + + ` clerk migrate import ${isAssumeYes() ? "-y " : ""}--transformer firebase --file ${outputPath}` + ` --firebase-signer-key "${config.signerKey}"` + ` --firebase-salt-separator "${config.saltSeparator}"` + ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, @@ -505,7 +506,7 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { + await withGutter("Exporting users from Firebase", async () => { const dateTime = await startLogging(); // Only Google can say whether a well-formed key is still a valid one, so a @@ -528,15 +529,13 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise entry.label.includes("password"))?.count ?? 0; const hashConfig = passwordCount > 0 ? await fetchHashConfig(account, token) : null; diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts index 1c505ef81..43cf7281f 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.test.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -14,7 +14,8 @@ mock.module("../../../mode.ts", () => ({ setMode: () => {}, })); -const { defaultOutputPath, outputStamp, resolveOutputPath } = await import("./shared.ts"); +const { defaultOutputPath, formatImportCommand, outputStamp, resolveOutputPath } = + await import("./shared.ts"); const { setAssumeYes } = await import("../lib/assume-yes.ts"); beforeEach(() => { @@ -134,3 +135,34 @@ describe("resolveOutputPath", () => { }); }); }); + +describe("formatImportCommand", () => { + const stripAnsi = (value: string): string => value.replace(/\u001b\[[0-9;]*m/g, ""); + const render = () => stripAnsi(formatImportCommand("supabase", "exports/mine.json").join("\n")); + + test("names the transformer and the file just written", () => { + expect(render()).toContain( + "clerk migrate import --transformer supabase --file exports/mine.json", + ); + }); + + // The instance comes from the resolved key, so there is no flag whose + // absence means development — the note says what actually decides. + test("says how to reach production", () => { + expect(render()).toContain("--instance prod"); + }); + + test("offers -y when the export was not given it", () => { + expect(render()).toContain("Add `-y` to skip the import confirmation prompt."); + }); + + // Carried across rather than always printed: on import `-y` also waves + // through the development-instance user-limit warning. + test("carries -y across from the export that was given it", () => { + setAssumeYes(true); + + const text = render(); + expect(text).toContain("clerk migrate import -y --transformer supabase"); + expect(text).not.toContain("Add `-y`"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index ed8370475..7c4dc1614 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -14,7 +14,6 @@ import path from "node:path"; import { dim, green, yellow } from "../../../lib/color.ts"; import { throwUsageError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; -import { NEXT_STEPS } from "../../../lib/next-steps.ts"; import { text } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; @@ -146,17 +145,19 @@ export type ExportSummary = { }; /** - * Reports the coverage table. + * Reports the coverage table and the import command that reads the file. * - * @returns The next steps for the caller to hand to `setNextSteps`, so the - * suggested import command closes the gutter like every other command's. - * Empty when nothing was exported — there is nothing to import. + * The command prints through `log.info`, alongside the coverage table, rather + * than being handed back for `setNextSteps`. The gutter's next-steps outro is + * human-only — `withGutter` and `printNextSteps` both return early for an agent + * or a non-TTY — and this is the one line that says what to do with the file + * just written. An agent that cannot see it has to guess the invocation. */ -export function reportExport(summary: ExportSummary): readonly string[] { +export function reportExport(summary: ExportSummary): void { log.blank(); if (summary.userCount === 0) { log.warn(`No users found to export. Wrote an empty file to ${summary.outputPath}.`); - return []; + return; } log.info("Field coverage"); @@ -175,7 +176,39 @@ export function reportExport(summary: ExportSummary): readonly string[] { `Exported ${summary.userCount} user${summary.userCount === 1 ? "" : "s"} to ${summary.outputPath}`, ); - return NEXT_STEPS.MIGRATE_EXPORT(summary.transformerKey, relativeIfInside(summary.outputPath)); + log.blank(); + for (const line of formatImportCommand( + summary.transformerKey, + relativeIfInside(summary.outputPath), + )) { + log.info(line); + } +} + +/** + * The import command for the file just written, and what it will target. + * + * One command rather than a development and a production variant, because + * there is no flag whose absence means "development": the key decides, through + * `--secret-key`, `--app`, `CLERK_SECRET_KEY`, the keyless project and the + * linked profile in that order. A line labelled "development" would be wrong + * for anyone holding `CLERK_SECRET_KEY=sk_live_…`, which is the reader who can + * least afford it. So the note names what picks the instance instead. + * + * `-y` is carried across from this export rather than always printed: on + * import it also waves through the development-instance user-limit warning, so + * it is not a flag to suggest to someone who never asked for it. + */ +export function formatImportCommand(transformerKey: string, file: string): string[] { + const yes = isAssumeYes() ? "-y " : ""; + return [ + "Import them with:", + dim(` clerk migrate import ${yes}--transformer ${transformerKey} --file ${file}`), + "", + dim(" Imports into whichever instance the resolved secret key belongs to."), + dim(" For production, add `--instance prod` or use a production secret key."), + ...(isAssumeYes() ? [] : [dim(" Add `-y` to skip the import confirmation prompt.")]), + ]; } /** Shortens a path for display when it sits under the working directory. */ diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 3ab72ff78..d911891f6 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -121,7 +121,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { const destination = await resolveOutputPath("supabase", options.output); - await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { + await withGutter("Exporting users from Supabase", async () => { const dateTime = await startLogging(); const { value: rows } = await withInputRetry( @@ -136,15 +136,13 @@ export async function exportSupabase(options: DbExportOptions): Promise { const { users, coverage } = buildSupabaseExport(rows, dateTime); const outputPath = writeExportOutput(users, destination); - setNextSteps( - reportExport({ - platform: "supabase", - userCount: users.length, - outputPath, - coverage, - transformerKey: "supabase", - }), - ); + reportExport({ + platform: "supabase", + userCount: users.length, + outputPath, + coverage, + transformerKey: "supabase", + }); if (users.length > 0) { log.info( diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index c61270a93..1865accd2 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -445,7 +445,7 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise const destination = await resolveOutputPath("workos", options.output); - await withGutter("Exporting users from WorkOS", async ({ setNextSteps }) => { + await withGutter("Exporting users from WorkOS", async () => { const dateTime = await startLogging(); // Only WorkOS can say whether the key is live, for the right environment, @@ -471,18 +471,16 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise const { users: exported, coverage } = buildWorkOsExport(users, dateTime, providers?.identities); const outputPath = writeExportOutput(exported, destination); - setNextSteps( - reportExport({ - platform: "workos", - userCount: exported.length, - outputPath, - coverage, - sections: providers - ? [buildIdentityReport(users, providers.identities, providers.failed)] - : [], - transformerKey: "workos", - }), - ); + reportExport({ + platform: "workos", + userCount: exported.length, + outputPath, + coverage, + sections: providers + ? [buildIdentityReport(users, providers.identities, providers.failed)] + : [], + transformerKey: "workos", + }); if (exported.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index afa6ab288..23fe660ef 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -30,7 +30,7 @@ import { type InstanceTarget, } from "../../lib/keyless-target.ts"; import { log } from "../../lib/log.ts"; -import { NEXT_STEPS } from "../../lib/next-steps.ts"; +import { NEXT_STEPS, printAgentNextSteps } from "../../lib/next-steps.ts"; import { confirm, multiselect } from "../../lib/prompts.ts"; import { withGutter, withSpinner } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; @@ -767,9 +767,10 @@ export async function run(rawOptions: MigrateRunOptions): Promise { // reading the log and knowing how to undo it matters most. When users did // fail, the per-user record of *why* leads, since the breakdown above only // counts each error and never names who hit it. - setNextSteps( - summary.failed > 0 ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(logFile) : NEXT_STEPS.MIGRATE_DONE, - ); + const steps = + summary.failed > 0 ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(logFile) : NEXT_STEPS.MIGRATE_DONE; + setNextSteps(steps); + printAgentNextSteps(steps); if (summary.failed > 0) process.exitCode = 1; }); diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index 35c34dfdb..642a0dd7c 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -87,11 +87,6 @@ export const NEXT_STEPS = { "Run `clerk migrate settings clear ` to forget one", "Run `clerk migrate settings clear` to forget them all, credentials included", ], - // A suggested import is worthless unless it names the transformer that reads - // this export and the file just written. - MIGRATE_EXPORT: (transformerKey: string, file: string) => [ - `Run \`clerk migrate import --transformer ${transformerKey} --file ${file}\` to import them`, - ], } as const; /** @@ -99,7 +94,30 @@ export const NEXT_STEPS = { * Only shown in human/interactive mode — agents get AGENT_PROMPT instead. */ export function printNextSteps(steps: readonly string[]): void { - if (!isHuman() || steps.length === 0) return; + if (!isHuman()) return; + renderNextSteps(steps); +} + +/** + * The same suggestions, on the paths a human never takes: agent mode and a + * non-TTY, where `printNextSteps` and `withGutter`'s outro both print nothing. + * + * A no-op for a human, who gets them from the outro — so a caller pairs this + * with `setNextSteps` rather than choosing between the two. + * + * Opt-in rather than folded into `printNextSteps`, because most steps are a + * nudge towards a command someone might like to run next. The migrate ones are + * not: they name the log file holding the per-user record of what landed, and + * the command that undoes the run. An agent that imported 10,000 users and lost + * 300 of them needs both, and has no gutter to read them from. + */ +export function printAgentNextSteps(steps: readonly string[]): void { + if (isHuman()) return; + renderNextSteps(steps); +} + +function renderNextSteps(steps: readonly string[]): void { + if (steps.length === 0) return; for (const step of steps) { log.info(` ${cyan("\u2192")} ${step}`); } From 95a0ca7d0fa440ace899abd9aac34e199bf8b2b6 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 21 Sep 2026 18:24:24 -0400 Subject: [PATCH 047/141] build: track the playwright-core patch against 1.63.0 playwright-core moved to 1.63.0 while `patchedDependencies` still keyed the patch to 1.60.0, so `check:patches` failed and, more quietly, Bun installed playwright unpatched. A rename was not enough: 1.63.0 renamed the local `url2` to `url3` in both hunks, so the patch no longer applied. Recreated it with `bun patch` against 1.63.0, carrying the header explaining why it exists. Co-Authored-By: Claude Opus 5 (1M context) --- bun.lock | 3 ++ package.json | 2 +- ...0.0.patch => playwright-core@1.63.0.patch} | 29 ++++++++++--------- 3 files changed, 19 insertions(+), 15 deletions(-) rename patches/{playwright-core@1.60.0.patch => playwright-core@1.63.0.patch} (54%) diff --git a/bun.lock b/bun.lock index 3c2fbecb6..05c5db0e6 100644 --- a/bun.lock +++ b/bun.lock @@ -60,6 +60,9 @@ }, }, }, + "patchedDependencies": { + "playwright-core@1.63.0": "patches/playwright-core@1.63.0.patch", + }, "overrides": { "tmp": "^0.2.6", }, diff --git a/package.json b/package.json index be6b45f89..cd531217c 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,6 @@ "bun": ">=1.3.13" }, "patchedDependencies": { - "playwright-core@1.60.0": "patches/playwright-core@1.60.0.patch" + "playwright-core@1.63.0": "patches/playwright-core@1.63.0.patch" } } diff --git a/patches/playwright-core@1.60.0.patch b/patches/playwright-core@1.63.0.patch similarity index 54% rename from patches/playwright-core@1.60.0.patch rename to patches/playwright-core@1.63.0.patch index a92f2fcc1..fdebad832 100644 --- a/patches/playwright-core@1.60.0.patch +++ b/patches/playwright-core@1.63.0.patch @@ -7,24 +7,25 @@ # Once Bun stops leaking a non-empty `url` onto client IncomingMessages, this # patch can be removed. diff --git a/lib/coreBundle.js b/lib/coreBundle.js +index 8ed8bb5de1b12c7c050191123b22a119298826bb..9c23e184bf1e6dfd58a317bb84385e8344fb1c8e 100644 --- a/lib/coreBundle.js +++ b/lib/coreBundle.js -@@ -25987,7 +25987,8 @@ - progress2.log(`\u2190 ${response2.statusCode} ${response2.statusMessage}`); +@@ -26590,7 +26590,8 @@ ${text2}`; + fetchLog(`\u2190 ${response2.statusCode} ${response2.statusMessage}`); for (const [name, value2] of Object.entries(response2.headers)) - progress2.log(` ${name}: ${value2}`); -- const cookies = this._parseSetCookieHeader(response2.url || url2.toString(), response2.headers["set-cookie"]); -+ const resolvedResponseUrl = response2.url && response2.url.startsWith("http") ? response2.url : url2.toString(); + fetchLog(` ${name}: ${value2}`); +- const cookies = this._parseSetCookieHeader(response2.url || url3.toString(), response2.headers["set-cookie"]); ++ const resolvedResponseUrl = response2.url && response2.url.startsWith("http") ? response2.url : url3.toString(); + const cookies = this._parseSetCookieHeader(resolvedResponseUrl, response2.headers["set-cookie"]); if (cookies.length) { try { await this.addCookies(cookies); -@@ -26065,7 +26066,7 @@ - const body2 = Buffer.concat(chunks); - notifyRequestFinished(body2); - fulfill({ -- url: response2.url || url2.toString(), -+ url: response2.url && response2.url.startsWith("http") ? response2.url : url2.toString(), - status: response2.statusCode || 0, - statusText: response2.statusMessage || "", - headers: toHeadersArray(response2.rawHeaders), +@@ -26669,7 +26670,7 @@ ${text2}`; + body: body2, + log: log2, + response: { +- url: response2.url || url3.toString(), ++ url: response2.url && response2.url.startsWith("http") ? response2.url : url3.toString(), + status: response2.statusCode || 0, + statusText: response2.statusMessage || "", + headers: toHeadersArray(response2.rawHeaders), From c33e9f8b3da6e65133684b6371a115471d2d12a0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 21 Sep 2026 18:24:25 -0400 Subject: [PATCH 048/141] test(migrate): compare the satellite host by hostname, not substring CodeQL flagged the substring check as incomplete URL sanitization, and it is: `satellite.example.com` can sit in a path or a query parameter of a URL that never addressed that host, so the assertion could pass a request it should catch and fail one it should not. Parse the URL and compare the hostname. Co-Authored-By: Claude Opus 5 (1M context) --- .../cli-core/src/commands/migrate/lib/clerk-config.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts index a94479cd9..efceabe1d 100644 --- a/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts @@ -58,8 +58,12 @@ describe("fetchInstanceSettings", () => { await fetchInstanceSettings("sk_test_abc"); + // Compared as a parsed hostname rather than a substring: the satellite's + // domain can appear anywhere in a URL — in a path or a query parameter — + // so `includes` would pass a request that never went near that host, and + // fail one that did. const urls = mockFetch.mock.calls.map(([input]) => String(input)); - expect(urls.every((url) => !url.includes("satellite.example.com"))).toBe(true); + expect(urls.every((url) => new URL(url).hostname !== "satellite.example.com")).toBe(true); }); test("skips the dev browser bootstrap for a production key", async () => { From dbd907fa6cf9449760a56324f73b9619ccd4eb86 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 22 Sep 2026 00:20:26 -0400 Subject: [PATCH 049/141] ci: match the Playwright image to the version installed `playwright` was caret-ranged at `^1.60.0`, so installs resolved to 1.63.0 while the E2E job still ran inside `mcr.microsoft.com/playwright:v1.60.0-noble`. That image ships only the browsers 1.60.0 can launch, so every browser test failed on a missing `chrome-headless-shell`, and the same drift left `patchedDependencies` keyed to a version no longer installed. Bump the image to v1.63.0-noble and pin the dependency exactly, so the npm version, the image tag and the patch key can only move together, on purpose. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 8 +++++++- bun.lock | 2 +- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d5fa8f5b..8fddfaba9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,13 @@ jobs: (github.event_name != 'pull_request' && inputs.run-e2e) runs-on: blacksmith-8vcpu-ubuntu-2404 container: - image: mcr.microsoft.com/playwright:v1.60.0-noble + # Ships the browser build that only this Playwright version can launch, + # so the tag must match the `playwright` version pinned in package.json + # (and the `patches/playwright-core@.patch` key alongside it). + # The dependency is pinned exactly rather than caret-ranged for that + # reason: a minor bump that moved on its own left the image behind and + # every browser test failed on a missing chrome-headless-shell. + image: mcr.microsoft.com/playwright:v1.63.0-noble timeout-minutes: 30 steps: - name: Install unzip (required by setup-bun) diff --git a/bun.lock b/bun.lock index 05c5db0e6..9d7121358 100644 --- a/bun.lock +++ b/bun.lock @@ -13,7 +13,7 @@ "oxfmt": "^0.64.0", "oxlint": "^1.79.0", "oxlint-tsgolint": "^7.0.2001", - "playwright": "^1.60.0", + "playwright": "1.63.0", "semver": "^7.8.5", "typescript": "^7", }, diff --git a/package.json b/package.json index cd531217c..97e9756e4 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "oxfmt": "^0.64.0", "oxlint": "^1.79.0", "oxlint-tsgolint": "^7.0.2001", - "playwright": "^1.60.0", + "playwright": "1.63.0", "semver": "^7.8.5", "typescript": "^7" }, From 9edb83230f7d628e4b0c8db32bf056c75f91790b Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 22 Sep 2026 12:01:25 -0400 Subject: [PATCH 050/141] build: restore a build alias for main's CI --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 97e9756e4..c8b414712 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "packages/*" ], "scripts": { + "build": "bun run build:compile", "dev": "bun run --cwd packages/cli-core dev", "test": "bun run scripts/check-bun-version.ts && bun test 'packages/cli-core/src/' 'packages/extras/src/' 'scripts/' --parallel --only-failures", "test:e2e": "bun run scripts/check-bun-version.ts && bun test 'test/e2e/' --retry 1 --parallel --only-failures", From c7e54524ea175300bbb2133607882addb0ec80c1 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 22 Sep 2026 15:18:44 -0400 Subject: [PATCH 051/141] build: hold Playwright at 1.60.0 until main's CI image moves Reverts the 1.63.0 bump and the matching patch rename. `!snapshot` runs release.yml from the default branch, so the E2E job's container comes from main's ci.yml (`v1.60.0-noble`) no matter what this branch's ci.yml says. Installing 1.63.0 into that image fails every browser test on a missing chrome-headless-shell. Pinned exactly rather than restored to the caret range: `^1.60.0` is what let the install drift to 1.63.0 in the first place, and the pin costs nothing while we are holding a version on purpose. Redo the bump (npm version, patch key, ci.yml image tag together) once this branch has merged and main's ci.yml owns the newer image. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 8 +---- bun.lock | 10 ++++--- package.json | 4 +-- ...3.0.patch => playwright-core@1.60.0.patch} | 29 +++++++++---------- 4 files changed, 23 insertions(+), 28 deletions(-) rename patches/{playwright-core@1.63.0.patch => playwright-core@1.60.0.patch} (54%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fddfaba9..4d5fa8f5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,13 +101,7 @@ jobs: (github.event_name != 'pull_request' && inputs.run-e2e) runs-on: blacksmith-8vcpu-ubuntu-2404 container: - # Ships the browser build that only this Playwright version can launch, - # so the tag must match the `playwright` version pinned in package.json - # (and the `patches/playwright-core@.patch` key alongside it). - # The dependency is pinned exactly rather than caret-ranged for that - # reason: a minor bump that moved on its own left the image behind and - # every browser test failed on a missing chrome-headless-shell. - image: mcr.microsoft.com/playwright:v1.63.0-noble + image: mcr.microsoft.com/playwright:v1.60.0-noble timeout-minutes: 30 steps: - name: Install unzip (required by setup-bun) diff --git a/bun.lock b/bun.lock index 9d7121358..f9614f5f0 100644 --- a/bun.lock +++ b/bun.lock @@ -13,7 +13,7 @@ "oxfmt": "^0.64.0", "oxlint": "^1.79.0", "oxlint-tsgolint": "^7.0.2001", - "playwright": "1.63.0", + "playwright": "1.60.0", "semver": "^7.8.5", "typescript": "^7", }, @@ -61,7 +61,7 @@ }, }, "patchedDependencies": { - "playwright-core@1.63.0": "patches/playwright-core@1.63.0.patch", + "playwright-core@1.60.0": "patches/playwright-core@1.60.0.patch", }, "overrides": { "tmp": "^0.2.6", @@ -429,6 +429,8 @@ "fs-extra": ["fs-extra@7.0.1", "", { "dependencies": { "graceful-fs": "^4.1.2", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw=="], + "fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], + "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], @@ -567,9 +569,9 @@ "pkce-challenge": ["pkce-challenge@5.0.1", "", {}, "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ=="], - "playwright": ["playwright@1.63.0", "", { "dependencies": { "playwright-core": "1.63.0" }, "bin": { "playwright": "cli.js" } }, "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg=="], + "playwright": ["playwright@1.60.0", "", { "dependencies": { "playwright-core": "1.60.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA=="], - "playwright-core": ["playwright-core@1.63.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg=="], + "playwright-core": ["playwright-core@1.60.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA=="], "prettier": ["prettier@2.8.8", "", { "bin": { "prettier": "bin-prettier.js" } }, "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q=="], diff --git a/package.json b/package.json index c8b414712..cedddf21e 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "oxfmt": "^0.64.0", "oxlint": "^1.79.0", "oxlint-tsgolint": "^7.0.2001", - "playwright": "1.63.0", + "playwright": "1.60.0", "semver": "^7.8.5", "typescript": "^7" }, @@ -58,6 +58,6 @@ "bun": ">=1.3.13" }, "patchedDependencies": { - "playwright-core@1.63.0": "patches/playwright-core@1.63.0.patch" + "playwright-core@1.60.0": "patches/playwright-core@1.60.0.patch" } } diff --git a/patches/playwright-core@1.63.0.patch b/patches/playwright-core@1.60.0.patch similarity index 54% rename from patches/playwright-core@1.63.0.patch rename to patches/playwright-core@1.60.0.patch index fdebad832..a92f2fcc1 100644 --- a/patches/playwright-core@1.63.0.patch +++ b/patches/playwright-core@1.60.0.patch @@ -7,25 +7,24 @@ # Once Bun stops leaking a non-empty `url` onto client IncomingMessages, this # patch can be removed. diff --git a/lib/coreBundle.js b/lib/coreBundle.js -index 8ed8bb5de1b12c7c050191123b22a119298826bb..9c23e184bf1e6dfd58a317bb84385e8344fb1c8e 100644 --- a/lib/coreBundle.js +++ b/lib/coreBundle.js -@@ -26590,7 +26590,8 @@ ${text2}`; - fetchLog(`\u2190 ${response2.statusCode} ${response2.statusMessage}`); +@@ -25987,7 +25987,8 @@ + progress2.log(`\u2190 ${response2.statusCode} ${response2.statusMessage}`); for (const [name, value2] of Object.entries(response2.headers)) - fetchLog(` ${name}: ${value2}`); -- const cookies = this._parseSetCookieHeader(response2.url || url3.toString(), response2.headers["set-cookie"]); -+ const resolvedResponseUrl = response2.url && response2.url.startsWith("http") ? response2.url : url3.toString(); + progress2.log(` ${name}: ${value2}`); +- const cookies = this._parseSetCookieHeader(response2.url || url2.toString(), response2.headers["set-cookie"]); ++ const resolvedResponseUrl = response2.url && response2.url.startsWith("http") ? response2.url : url2.toString(); + const cookies = this._parseSetCookieHeader(resolvedResponseUrl, response2.headers["set-cookie"]); if (cookies.length) { try { await this.addCookies(cookies); -@@ -26669,7 +26670,7 @@ ${text2}`; - body: body2, - log: log2, - response: { -- url: response2.url || url3.toString(), -+ url: response2.url && response2.url.startsWith("http") ? response2.url : url3.toString(), - status: response2.statusCode || 0, - statusText: response2.statusMessage || "", - headers: toHeadersArray(response2.rawHeaders), +@@ -26065,7 +26066,7 @@ + const body2 = Buffer.concat(chunks); + notifyRequestFinished(body2); + fulfill({ +- url: response2.url || url2.toString(), ++ url: response2.url && response2.url.startsWith("http") ? response2.url : url2.toString(), + status: response2.statusCode || 0, + statusText: response2.statusMessage || "", + headers: toHeadersArray(response2.rawHeaders), From d76b4553fac463feec8345cd9b1042804b3bbe11 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 23 Sep 2026 17:57:56 -0400 Subject: [PATCH 052/141] feat(migrate): remember what an export produced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An export computed the transformer, the file and — for Firebase — the four scrypt parameters, printed them as a command line, and threw them away. The settings table listed all six as `not set` immediately after the run that produced them, and nothing but `settings set` could ever fill them: no export path called `saveSettings` at all. `reportExport` is the one place all seven exporters pass through and already holds the transformer and the output path, so the save goes there rather than in each exporter. Firebase's hash parameters are credentials, so they go to `.env.clerk-migrate` under the variables the `firebase-*` settings read, and the file is named in the output rather than written silently. `-y` saves nothing, matching `ensureLogDir`: it means "do not stop to ask me", and a remembered value is one a later run picks up silently. Co-Authored-By: Claude Opus 5 (1M context) --- .../cli-core/src/commands/migrate/README.md | 24 +++++++++ .../src/commands/migrate/export/auth0.ts | 2 +- .../src/commands/migrate/export/authjs.ts | 2 +- .../src/commands/migrate/export/betterauth.ts | 2 +- .../src/commands/migrate/export/clerk.ts | 2 +- .../src/commands/migrate/export/firebase.ts | 35 ++++++++++++- .../commands/migrate/export/shared.test.ts | 50 ++++++++++++++++++- .../src/commands/migrate/export/shared.ts | 33 ++++++++++-- .../src/commands/migrate/export/supabase.ts | 2 +- .../src/commands/migrate/export/workos.ts | 2 +- 10 files changed, 141 insertions(+), 13 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 95dadc83f..f4066fd0d 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -282,6 +282,22 @@ profile in that order. Every export also writes `logs/export-.log`, so `migrate logs list` sees it alongside imports and deletions. +#### What an export remembers + +The printed command is one half of the handoff; the other is that an export +**saves what it just produced** as this project's `transformer` and `file` +settings, so a bare `clerk migrate import` picks up where the export left off. +`clerk migrate settings` shows both afterwards, sourced from the CLI config. + +The path is remembered the way it was printed — relative while it sits under +the project — so the remembered value and the copyable command never disagree +about which file they mean. + +**`-y` saves nothing.** It means "do not stop to ask me", and a remembered +value is one a later run picks up silently; a non-interactive export leaves the +settings untouched and the printed command as the only handoff. This is the +rule [`log-dir`](#where-logs-go) already follows. + #### Three platforms export no passwords - **Clerk** never returns password digests, TOTP secrets or backup codes over @@ -388,6 +404,14 @@ the shell then reads each one as another argument and rejects the import. A line that wraps on screen carries no such character and pastes back as what was printed. +The four parameters are also **saved to `.env.clerk-migrate`** (created +gitignored), under the variables the `firebase-*` settings read, so the import +can be run without pasting them back. They are credentials, which is why they +go to that file rather than the CLI config — the same split +`settings set firebase-signer-key` uses. The path is named in the output: +writing a credential file is not something to do silently. As everywhere else, +`-y` saves nothing. + Reading the config needs a broader role than listing users, so if it is denied the export still succeeds and points at **Authentication → Users → (⋮) → Password hash parameters** instead. An export with no password hashes says so diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 43ce12f0a..5cd62f7b9 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -354,7 +354,7 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { const { users: exported, coverage } = buildAuth0Export(users, dateTime); const outputPath = writeExportOutput(exported, destination); - reportExport({ + await reportExport({ platform: "auth0", userCount: exported.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index e407b8948..963238911 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -141,7 +141,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { const { users, coverage } = buildAuthJsExport(rows, dateTime); const outputPath = writeExportOutput(users, destination); - reportExport({ + await reportExport({ platform: "authjs", userCount: users.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 21aee4787..963b223a6 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -197,7 +197,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const { users, coverage } = buildBetterAuthExport(rows, dateTime); const outputPath = writeExportOutput(users, destination); - reportExport({ + await reportExport({ platform: "betterauth", userCount: users.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 638c06bec..193262287 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -246,7 +246,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { const { users: exported, coverage } = buildClerkExport(users, dateTime); const outputPath = writeExportOutput(exported, destination); - reportExport({ + await reportExport({ platform: "clerk", userCount: exported.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 671bd7cb7..471c22b8f 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -32,7 +32,9 @@ import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; +import { writeMigrateEnvValues } from "../lib/env-file.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; +import { findSetting } from "../settings/registry.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; @@ -499,6 +501,35 @@ export function formatHashConfigGuidance( ]; } +/** + * Saves the scrypt parameters where `clerk migrate import` will read them. + * + * They are credentials, so they land in `.env.clerk-migrate` — created + * gitignored — rather than the CLI config, which is the split the settings + * registry owns. The variable names come from that registry instead of being + * spelled again here: a setting and the variable behind it must not drift into + * two names the user has to know separately. + * + * `-y` opts out, for the reason {@link reportExport} gives. Writing a + * credential file is also not something to do silently, so the path is named. + */ +async function rememberHashConfig(config: HashConfig | null): Promise { + if (!config || isAssumeYes()) return; + + const variable = (name: string) => + (findSetting(name) as NonNullable>).envVar as string; + + const file = await writeMigrateEnvValues({ + [variable("firebase-signer-key")]: config.signerKey, + [variable("firebase-salt-separator")]: config.saltSeparator, + [variable("firebase-rounds")]: String(config.rounds), + [variable("firebase-mem-cost")]: String(config.memoryCost), + }); + + log.blank(); + log.info(dim(`Saved to ${file} (gitignored), so the import can be run without them.`)); +} + export async function exportFirebase(options: ExportFirebaseOptions): Promise { // Read and validate before anything reaches the network, so a wrong file // fails in a second rather than after an auth round-trip. @@ -529,7 +560,7 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise ({ text: (...args: unknown[]) => mockText(...args), })); +const mockSaveSettings = mock(async () => {}); +mock.module("../lib/settings.ts", () => ({ + loadSettings: async () => ({}), + saveSettings: (...args: unknown[]) => mockSaveSettings(...(args as [])), +})); + let human = true; mock.module("../../../mode.ts", () => ({ isHuman: () => human, @@ -14,7 +22,7 @@ mock.module("../../../mode.ts", () => ({ setMode: () => {}, })); -const { defaultOutputPath, formatImportCommand, outputStamp, resolveOutputPath } = +const { defaultOutputPath, formatImportCommand, outputStamp, reportExport, resolveOutputPath } = await import("./shared.ts"); const { setAssumeYes } = await import("../lib/assume-yes.ts"); @@ -22,6 +30,7 @@ beforeEach(() => { human = true; setAssumeYes(false); mockText.mockReset(); + mockSaveSettings.mockReset(); }); describe("outputStamp", () => { @@ -166,3 +175,42 @@ describe("formatImportCommand", () => { expect(text).not.toContain("Add `-y`"); }); }); + +describe("reportExport", () => { + useCaptureLog(); + + const summary = { + platform: "supabase", + userCount: 2, + outputPath: path.resolve(process.cwd(), "exports/mine.json"), + coverage: [{ label: "have an email address", count: 2 }], + transformerKey: "supabase", + }; + + // Relative, matching the command printed alongside it: the remembered value + // and the copyable one must not disagree about the same file. + test("remembers the transformer and the file for the next import", async () => { + await reportExport(summary); + + expect(mockSaveSettings).toHaveBeenCalledWith({ + transformer: "supabase", + file: path.join("exports", "mine.json"), + }); + }); + + // `-y` means "do not stop to ask me", and a remembered value is one a later + // run picks up silently. + test("remembers nothing under -y", async () => { + setAssumeYes(true); + + await reportExport(summary); + + expect(mockSaveSettings).not.toHaveBeenCalled(); + }); + + test("remembers nothing when there was nothing to export", async () => { + await reportExport({ ...summary, userCount: 0 }); + + expect(mockSaveSettings).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index 7c4dc1614..7dc54461a 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -17,6 +17,7 @@ import { log } from "../../../lib/log.ts"; import { text } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; +import { loadSettings, saveSettings } from "../lib/settings.ts"; /** * `YYYYMMDD-HHmm`, local time — ISO 8601 basic format, minus seconds. @@ -152,8 +153,11 @@ export type ExportSummary = { * human-only — `withGutter` and `printNextSteps` both return early for an agent * or a non-TTY — and this is the one line that says what to do with the file * just written. An agent that cannot see it has to guess the invocation. + * + * Also remembers the transformer and the file, so the import can be run + * without retyping what was just printed. */ -export function reportExport(summary: ExportSummary): void { +export async function reportExport(summary: ExportSummary): Promise { log.blank(); if (summary.userCount === 0) { log.warn(`No users found to export. Wrote an empty file to ${summary.outputPath}.`); @@ -176,13 +180,32 @@ export function reportExport(summary: ExportSummary): void { `Exported ${summary.userCount} user${summary.userCount === 1 ? "" : "s"} to ${summary.outputPath}`, ); + const file = relativeIfInside(summary.outputPath); + log.blank(); - for (const line of formatImportCommand( - summary.transformerKey, - relativeIfInside(summary.outputPath), - )) { + for (const line of formatImportCommand(summary.transformerKey, file)) { log.info(line); } + + await rememberExport(summary.transformerKey, file); +} + +/** + * Remembers what this export produced, so `clerk migrate import` can be run + * without repeating the flags just printed. + * + * `-y` opts out. It means "do not stop to ask me", and a remembered value is + * something a later run picks up silently — the same reason `ensureLogDir` + * saves nothing under `-y`. Both settings stay unset until somebody who was + * watching the output produced them. + * + * Stores the path the way it was printed: relative while it sits under the + * project, so the remembered value and the copyable command say the same + * thing. + */ +async function rememberExport(transformer: string, file: string): Promise { + if (isAssumeYes()) return; + await saveSettings({ ...(await loadSettings()), transformer, file }); } /** diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index d911891f6..e4453aa02 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -136,7 +136,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { const { users, coverage } = buildSupabaseExport(rows, dateTime); const outputPath = writeExportOutput(users, destination); - reportExport({ + await reportExport({ platform: "supabase", userCount: users.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index 1865accd2..fee4fa397 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -471,7 +471,7 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise const { users: exported, coverage } = buildWorkOsExport(users, dateTime, providers?.identities); const outputPath = writeExportOutput(exported, destination); - reportExport({ + await reportExport({ platform: "workos", userCount: exported.length, outputPath, From 447432d9a62d6c79e7cf1d7c4a1f1960a4a22105 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 29 Sep 2026 17:29:26 -0400 Subject: [PATCH 053/141] refactor(migrate)!: remove `clerk migrate settings` and the state it kept `settings` stored migration state in three places: the CLI config (`migrations`, keyed by project), `.env.clerk-migrate`, and a remembered log directory. A run could pick up any of them without saying so. The run store that replaces this keeps state in one place, so everything `settings` added comes out first, as if it had never existed: - `commands/migrate/settings/`, `lib/settings.ts`, `lib/env-file.ts` and the log-directory prompt are deleted. - The CLI config loses `migrations`, `getMigrationEntry`, `setMigrationEntry` and `getProjectKey`. - `findEnvValue` goes back to the private `findKeyInProject` in `keyless-target.ts`, and `REDACTED` goes back into `users.ts`. `ensureGitignoreEntry` stays in `lib/git.ts`, because the run store needs it. - Imports and exports no longer save anything. Exports and Firebase hash settings are no longer remembered, and the wizard no longer pre-fills. - Export credentials and `CLERK_MIGRATE_LOG_DIR` are read from `process.env` only. The Firebase hash parameters come from the flags only. - `migrate delete` is removed. It depended on the saved record, and `migrate undo ` replaces it. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 285 +----------- .../src/commands/migrate/delete.test.ts | 428 ------------------ .../cli-core/src/commands/migrate/delete.ts | 331 -------------- .../src/commands/migrate/export/auth0.test.ts | 25 +- .../src/commands/migrate/export/auth0.ts | 17 +- .../src/commands/migrate/export/authjs.ts | 6 +- .../src/commands/migrate/export/betterauth.ts | 6 +- .../src/commands/migrate/export/clerk.test.ts | 3 +- .../src/commands/migrate/export/clerk.ts | 6 +- .../migrate/export/db-exports.test.ts | 41 +- .../src/commands/migrate/export/db-options.ts | 6 +- .../commands/migrate/export/firebase.test.ts | 3 +- .../src/commands/migrate/export/firebase.ts | 39 +- .../commands/migrate/export/shared.test.ts | 50 +- .../src/commands/migrate/export/shared.ts | 33 +- .../src/commands/migrate/export/supabase.ts | 6 +- .../commands/migrate/export/workos.test.ts | 14 +- .../src/commands/migrate/export/workos.ts | 11 +- .../src/commands/migrate/index.test.ts | 14 - .../cli-core/src/commands/migrate/index.ts | 35 +- .../src/commands/migrate/lib/assume-yes.ts | 8 +- .../src/commands/migrate/lib/env-file.test.ts | 171 ------- .../src/commands/migrate/lib/env-file.ts | 162 ------- .../migrate/lib/firebase-hash.test.ts | 184 +------- .../src/commands/migrate/lib/firebase-hash.ts | 97 +--- .../migrate/lib/log-dir-prompt.test.ts | 141 ------ .../src/commands/migrate/lib/logger.test.ts | 74 +-- .../src/commands/migrate/lib/logger.ts | 110 +---- .../src/commands/migrate/lib/settings.test.ts | 74 --- .../src/commands/migrate/lib/settings.ts | 39 -- .../src/commands/migrate/logs/clean.ts | 3 +- .../src/commands/migrate/logs/convert.ts | 3 +- .../src/commands/migrate/logs/index.ts | 4 +- .../src/commands/migrate/logs/list.ts | 5 +- .../commands/migrate/run-interactive.test.ts | 76 +--- .../cli-core/src/commands/migrate/run.test.ts | 24 - packages/cli-core/src/commands/migrate/run.ts | 23 +- .../src/commands/migrate/settings/clear.ts | 154 ------- .../src/commands/migrate/settings/index.ts | 120 ----- .../src/commands/migrate/settings/list.ts | 155 ------- .../src/commands/migrate/settings/registry.ts | 211 --------- .../src/commands/migrate/settings/set.ts | 48 -- .../migrate/settings/settings.test.ts | 341 -------------- .../src/commands/migrate/wizard.test.ts | 36 -- .../cli-core/src/commands/migrate/wizard.ts | 32 +- packages/cli-core/src/lib/config.test.ts | 39 -- packages/cli-core/src/lib/config.ts | 45 +- packages/cli-core/src/lib/constants.ts | 12 - packages/cli-core/src/lib/dotenv.ts | 70 --- packages/cli-core/src/lib/git.ts | 6 +- packages/cli-core/src/lib/keyless-target.ts | 47 +- packages/cli-core/src/lib/next-steps.ts | 12 +- packages/cli-core/src/lib/users.ts | 2 +- packages/cli-core/src/test/lib/stubs.ts | 35 +- 54 files changed, 193 insertions(+), 3729 deletions(-) delete mode 100644 packages/cli-core/src/commands/migrate/delete.test.ts delete mode 100644 packages/cli-core/src/commands/migrate/delete.ts delete mode 100644 packages/cli-core/src/commands/migrate/lib/env-file.test.ts delete mode 100644 packages/cli-core/src/commands/migrate/lib/env-file.ts delete mode 100644 packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts delete mode 100644 packages/cli-core/src/commands/migrate/lib/settings.test.ts delete mode 100644 packages/cli-core/src/commands/migrate/lib/settings.ts delete mode 100644 packages/cli-core/src/commands/migrate/settings/clear.ts delete mode 100644 packages/cli-core/src/commands/migrate/settings/index.ts delete mode 100644 packages/cli-core/src/commands/migrate/settings/list.ts delete mode 100644 packages/cli-core/src/commands/migrate/settings/registry.ts delete mode 100644 packages/cli-core/src/commands/migrate/settings/set.ts delete mode 100644 packages/cli-core/src/commands/migrate/settings/settings.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index f4066fd0d..216744f2f 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -39,10 +39,8 @@ clerk migrate import ``` It picks the transformer from a list built off the registry, asks for the file, -collects Firebase's hash parameters when they are needed, and pre-fills the -platform and file from the last run so a repeat migration is mostly pressing -enter. Anything already passed as a flag is not asked for. Firebase's hash -parameters are never pre-filled — see [below](#--firebase--firebase). +and collects Firebase's hash parameters when they are needed. Anything already +passed as a flag is not asked for. Then it prints the [Migration Readiness report](#migration-readiness-report), offers to [change whatever it flagged](#changing-the-flagged-settings), and @@ -282,22 +280,6 @@ profile in that order. Every export also writes `logs/export-.log`, so `migrate logs list` sees it alongside imports and deletions. -#### What an export remembers - -The printed command is one half of the handoff; the other is that an export -**saves what it just produced** as this project's `transformer` and `file` -settings, so a bare `clerk migrate import` picks up where the export left off. -`clerk migrate settings` shows both afterwards, sourced from the CLI config. - -The path is remembered the way it was printed — relative while it sits under -the project — so the remembered value and the copyable command never disagree -about which file they mean. - -**`-y` saves nothing.** It means "do not stop to ask me", and a remembered -value is one a later run picks up silently; a non-interactive export leaves the -settings untouched and the printed command as the only handoff. This is the -rule [`log-dir`](#where-logs-go) already follows. - #### Three platforms export no passwords - **Clerk** never returns password digests, TOTP secrets or backup codes over @@ -404,14 +386,6 @@ the shell then reads each one as another argument and rejects the import. A line that wraps on screen carries no such character and pastes back as what was printed. -The four parameters are also **saved to `.env.clerk-migrate`** (created -gitignored), under the variables the `firebase-*` settings read, so the import -can be run without pasting them back. They are credentials, which is why they -go to that file rather than the CLI config — the same split -`settings set firebase-signer-key` uses. The path is named in the output: -writing a credential file is not something to do silently. As everywhere else, -`-y` saves nothing. - Reading the config needs a broader role than listing users, so if it is denied the export still succeeds and points at **Authentication → Users → (⋮) → Password hash parameters** instead. An export with no password hashes says so @@ -497,61 +471,13 @@ and every 10 pages during the user fetch. `withSpinner` hands a no-op to anything that is not a TTY, so without this an agent exporting a large tenant would see nothing at all until the run finished. -### `clerk migrate delete` - -The undo for a bad migration. Deletes the users a previous -`clerk migrate import` created in this directory, matched by the `external_id` -the import stamped on each one. - -```sh -clerk migrate delete # confirms first -clerk migrate delete -y # non-interactive -``` - -Takes the same targeting flags as `clerk migrate import` (`--secret-key`, `--app`, -`--instance`). - -Flat rather than under a noun group: it is the one command in this tree that -destroys data **in Clerk**, and is worth keeping short and prominent. (Contrast -`migrate logs clean`, which only removes local files.) - -#### What it will and will not touch - -The saved migration record is the only account of what a run created, so that -is what identifies the migration being undone. Without it the command fails and -explains — deleting nothing silently would look like a successful undo. - -Users are found with `GET /v1/users?external_id=…`, 100 IDs per request. Only a -user Clerk itself reports as carrying one of _this_ migration's external IDs is -ever deleted; anything else in the instance is out of scope. IDs with no -matching user are skipped and reported, which is the normal case for a partial -migration or one already partly undone. - -It confirms before acting — defaulting to **no** — and requires `-y` in -non-interactive or agent mode. - -#### Failures - -Rate limiting and 429 retries are literally the same code path as the import -(`lib/retry.ts`), not a second implementation that drifts. - -A failure on one user is logged and the rest continue: a half-undone migration -with no record of which half is far worse than a reported failure. Every -attempt lands in a timestamped `logs/delete-.log`, carrying -both the source ID and the Clerk ID. The command exits non-zero if any deletion -failed. - ### `clerk migrate logs` -Everything that touches the local log directory — `./logs` unless the project -says otherwise; see [Where logs go](#where-logs-go). Noun-verb like every +Everything that touches the local log directory — `./logs` unless +`CLERK_MIGRATE_LOG_DIR` says otherwise. Noun-verb like every other group in the CLI (`config pull`, `users list`), rather than the standalone tool's `clean-logs`/`convert-logs`, which were npm script names. -Grouping also disambiguates the two deletes in this tree: `migrate logs clean` -removes **local files**, `migrate delete` removes **users from a Clerk -instance**. - ```sh clerk migrate logs # defaults to list clerk migrate logs list --json @@ -567,32 +493,7 @@ clerk migrate logs convert import-2026-01-01T12-00-00.log | `logs convert` | `[file…]`, `--all` | NDJSON → a JSON array, written as `.json` | All three read the directory through one shared enumerator, which is what makes -`logs list` nearly free. All three **resolve** the directory without ever asking -for one: they are read-only, and "where should logs go?" is not a question to -put in front of someone who asked to see the logs they already have. - -#### Where logs go - -`./logs`, relative to the current directory, until the project says otherwise. -Resolution order, highest first: - -| Source | Set by | -| --------------------------- | ----------------------------------------------------- | -| `CLERK_MIGRATE_LOG_DIR` | The shell, `.env`, `.env.local`, `.env.clerk-migrate` | -| `log-dir` in the CLI config | The first-run prompt, or `settings set` | -| `./logs` | The fallback | - -The first time `migrate import`, `migrate export` or `migrate delete` runs -interactively in a project with none of those set, it asks where logs should be -saved and offers `./logs`. The answer is saved under `log-dir`, so it is asked -once per project and never again. `-y`, agent mode and a non-TTY take `./logs` -without asking **and without saving it** — landing on a default is not a choice, -and recording one would retire the question for a human who never saw it. - -Logs are the only record of which users landed and which failed, and -`migrate delete` reads them to undo a run, so where they go is worth the one -question. Change it later with `clerk migrate settings set log-dir `, or -clear it with `clerk migrate settings clear log-dir` to be asked again. +`logs list` nearly free. #### `logs list` @@ -725,124 +626,6 @@ Migrating from something else? Write a transformer and pass --transformer-file. `--json` gives an agent the same data, including which source field each transformer maps to `userId`. -### `clerk migrate settings` - -What a run in this directory would pick up, and where each value comes from. -Listing is the default, because it is the read-only one: a bare `clerk migrate -settings` shows, never changes. - -```sh -clerk migrate settings # list -clerk migrate settings list --json -clerk migrate settings set transformer firebase -clerk migrate settings set firebase-signer-key abc123 -clerk migrate settings clear firebase-signer-key # forget one -clerk migrate settings clear -y # forget them all -``` - -| Subcommand | Takes | Description | -| ----------------------------- | --------------------- | -------------------------------------------------------- | -| `settings list` | `--json` | Every setting, its value and the source it resolved from | -| `settings set ` | ` ` | Change one setting | -| `settings clear [name]` | `[name]`, `-y, --yes` | Forget one setting, or every setting and its credentials | - -`settings clear ` leaves the rest of the project's settings alone. For a -credential it drops every variable the setting answers to, aliases included — -clearing `firebase-rounds` while a bare `ROUNDS` stayed behind in the same file -would report the setting cleared and leave the next run reading the old value. -It only ever edits `.env.clerk-migrate`; a value coming from the app's own env -file or the shell is named in the listing's source column and has to be removed -there. - -**A bare `settings clear` needs `-y` where it cannot ask.** It forgets every -setting and every credential in `.env.clerk-migrate`, so a non-interactive or -agent run refuses rather than assuming, the way `migrate logs clean` and -`migrate delete` already do. `settings clear ` does not: naming the one -setting to forget is itself the confirmation, the same way `settings set` needs -none. - -A misspelled name gets the closest match back, not just the list: - -``` -$ clerk migrate settings clear logs-dir -error: command-argument value 'logs-dir' is invalid for argument 'name'. - Did you mean "log-dir"? Allowed choices are transformer, file, … -``` - -Setting names are kebab-case and identical to the `clerk migrate import` flag -each one backs, so `firebase-signer-key` here is `--firebase-signer-key` there -rather than a second spelling to learn. The description column carries the -prose. - -The source column is the point. A migration reads from flags, the environment, -two of the app's env files and the CLI's config, so when a run picks up a stale -value the question is never "what is it" but "which of those won". A value that -arrived under one of the accepted aliases names the variable alongside the file. - -It names a **file** wherever there is one to name. Bun loads `.env`/`.env.local` -into the environment before the CLI runs, so a value a developer typed into -`.env.local` would otherwise be reported as "`ROUNDS` env var" — true, and no -help to someone asking which file to edit. Attribution is by value: a file -holding the same key with a _different_ value lost to something exported in the -shell, and that row keeps saying `ROUNDS env var`, because that is exactly the -case this column exists to catch. - -A setting with no value leaves the column empty rather than filling it with a -placeholder — the source column already reads `not set` on that row, and the -blank is what makes the settings that do have a value stand out. - -It closes on next steps naming the two commands that change what it just -showed — the same block `clerk mcp list` and `clerk whoami` end on, and human -only. The full command surface stays in `--help`. - -``` -A migration run in this directory picks these up unless a flag overrides them. -Each setting is named after the `clerk migrate import` flag it stands in for. - -SETTING VALUE SOURCE DESCRIPTION -transformer firebase clerk config Source platform the export came from -file users.json clerk config Export file to import users from -skip-unsupported-providers not set Skip users with no provider enabled in Clerk (Supabase) -log-dir ./logs clerk config Directory migration logs are written to -firebase-signer-key [REDACTED] .env.clerk-migrate Firebase base64 signer key -firebase-salt-separator not set Firebase base64 salt separator -firebase-rounds 8 .env.local (ROUNDS) Firebase scrypt rounds -firebase-mem-cost 14 MEM_COST env var Firebase scrypt memory cost - -6 of 8 settings set. Credentials are shown redacted. - - → Run `clerk migrate settings set ` to change one - → Run `clerk migrate settings clear ` to forget one - → Run `clerk migrate settings clear` to forget them all, credentials included -``` - -#### Where each setting is kept - -Two stores, split by what the value **is** rather than by which command wrote it: - -| Store | Holds | Why | -| -------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------- | -| CLI config | `transformer`, `file`, `skip-unsupported-providers`, `log-dir` | Project state, not secret, useless outside the CLI | -| `.env.clerk-migrate` | `firebase-*` | Credentials: gitignored on write, and hand-editable for rotation | - -`log-dir` is the one setting that answers to both: it is remembered in the CLI -config, and `CLERK_MIGRATE_LOG_DIR` outranks what is remembered, so a directory -can be pinned for one shell without disturbing the project. The listing's source -column says which is winning, and `settings clear log-dir` clears both — half a -clear would report the setting gone while the next run still read it. - -`.env.clerk-migrate` is the migration's own file rather than the app's -`.env.local`, because a Firebase signer key is of no use to the application -being migrated and does not belong in the file its developers read daily. The -CLI adds it to `.gitignore` the first time it writes it, and deletes it when -`settings clear` removes the last value. - -Credentials are withheld wherever they are displayed, including under `--json`, -so the output is safe to paste into an issue. They display as `[REDACTED]` — -the same thing `clerk users create --dry-run` prints for a password — rather -than a truncation like `aVer…3456`: the source column already says which value -is in play, and a partial secret is one the reader has to recognise as partial. - ### Custom transformers (`--transformer-file`) Migrating from a platform with no built-in, without recompiling the CLI: @@ -898,7 +681,7 @@ rejected with the specific problem rather than crashing mid-pipeline: The `userId` check is the load-bearing one: without it the import would run to completion and create every user with no `external_id`, which is what makes a -migration re-runnable and what `migrate delete` matches on. +migration re-runnable. ### Verified vs unverified identifiers @@ -930,32 +713,8 @@ naming what is missing. A partial set produces a well-formed digest that verifies against nothing, so users would import successfully and then be unable to sign in. -They never go into the CLI's config: the signer key is a Firebase secret, and -that file is not a secret store. To avoid re-passing all four on every run, set -them once with [`clerk migrate settings`](#clerk-migrate-settings), or export -them yourself: - -| Flag | Variable | Also accepted | -| --------------------------- | ------------------------------- | --------------------------------------------------------- | -| `--firebase-signer-key` | `CLERK_FIREBASE_SIGNER_KEY` | `FIREBASE_BASE64_SIGNER_KEY`, `BASE64_SIGNER_KEY` | -| `--firebase-salt-separator` | `CLERK_FIREBASE_SALT_SEPARATOR` | `FIREBASE_BASE64_SALT_SEPARATOR`, `BASE64_SALT_SEPARATOR` | -| `--firebase-rounds` | `CLERK_FIREBASE_ROUNDS` | `FIREBASE_ROUNDS`, `ROUNDS` | -| `--firebase-mem-cost` | `CLERK_FIREBASE_MEM_COST` | `FIREBASE_MEM_COST`, `MEM_COST` | - -The unprefixed names are what Firebase itself calls these (`base64_signer_key`, -`rounds`) and what every guide, Clerk's own standalone migration script -included, tells you to paste into `.env`. Someone who followed one has the -values the import needs, spelled the way the source platform spells them, so -they are read rather than reported as missing. - -They are a fallback, not a synonym: a `CLERK_FIREBASE_*` variable wins wherever -both exist, and `clerk migrate settings` names the variable it read alongside -the file — `ROUNDS` is generic enough to mean something else in an app that was -never a Firebase project, and that should be visible rather than silent. - -Resolution order is flag, then exported variable, then `.env.clerk-migrate`, -then the app's `.env.local`/`.env`. The sources can be mixed as long as all four -end up supplied. Run with `--verbose` to see which one each came from. +They are read from the flags only. The signer key is a Firebase secret, and the +CLI stores none of them. An export with no password hashes needs no parameters at all. @@ -983,7 +742,7 @@ The schema lives in `validator.ts`; adding a source platform means adding a transformer, not editing it. **Required:** `userId` (`string`). It becomes the Clerk user's `external_id`, -which is what makes a migration re-runnable and what `migrate delete` matches on. +which is what makes a migration re-runnable. **Identifiers.** At least one of these must be present, or the user is logged as a validation failure and skipped. Each accepts a single value or an array. @@ -1222,15 +981,7 @@ rather than "which project is linked here". | ------------------------------------------ | --------------------------------------------------------------------- | | `./logs/export-.log` | NDJSON: one line per exported user | | `./logs/import-.log` | NDJSON: one line per user, plus validation failures and retry notices | -| `./logs/delete-.log` | NDJSON: one line per `migrate delete` attempt | | `./exports/-export-.json` | The export itself, unless `--output` says otherwise | -| `./.env.clerk-migrate` | Migration credentials, written by `settings set` and gitignored | - -The transformer and file of the last run are **not** written here. They go to -the `migrations` section of the CLI's own config file, keyed by project the -same way a linked profile is. That is what `migrate delete` reads to know which -migration to undo, so it is load-bearing rather than a convenience — and it has -no business being written into the repository being migrated. Log writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. Use the last @@ -1262,16 +1013,14 @@ NDJSON is. The original `.log` stays put. ## API Endpoints -| Method | Path | Used by | -| -------- | -------------------------- | ------------------------------------------------------------------------------------ | -| `POST` | `/v1/users` | `migrate import` — creates each user | -| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | -| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | -| `GET` | `/v1/users?external_id=…` | `migrate delete` — finds this migration's users, 100 IDs a call | -| `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | -| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | -| `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | -| `GET` | `/v1/domains` | Readiness report and `--skip-unsupported-providers` — resolves the Frontend API host | +| Method | Path | Used by | +| ------ | -------------------------- | ------------------------------------------------------------------------------------ | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | +| `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | +| `GET` | `/v1/domains` | Readiness report and `--skip-unsupported-providers` — resolves the Frontend API host | The readiness report also reads the instance's Frontend API `GET /v1/environment` (bootstrapping a dev browser first on development diff --git a/packages/cli-core/src/commands/migrate/delete.test.ts b/packages/cli-core/src/commands/migrate/delete.test.ts deleted file mode 100644 index 3ae3131ac..000000000 --- a/packages/cli-core/src/commands/migrate/delete.test.ts +++ /dev/null @@ -1,428 +0,0 @@ -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { _setConfigDir } from "../../lib/config.ts"; -import { CliError } from "../../lib/errors.ts"; -import { useCaptureLog } from "../../test/lib/stubs.ts"; -import { - batch, - deleteMigration, - deleteMigratedUsers, - findMigratedUsers, - readMigratedExternalIds, - resolveMigrationToUndo, -} from "./delete.ts"; -import type { ResolvedLimits } from "./lib/instance.ts"; -import { getLogDir } from "./lib/logger.ts"; -import { saveSettings } from "./lib/settings.ts"; - -const captured = useCaptureLog(); - -const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurrencyLimit: 8 }; -const DATE_TIME = "2026-01-01T00:00:00"; - -let workDir: string; -let configDir: string; -let originalCwd: string; -let originalFetch: typeof globalThis.fetch; -let requests: { method: string; url: string }[]; - -const EXPORT = [ - { id: "legacy_a", primary_email_address: "a@x.dev" }, - { id: "legacy_b", primary_email_address: "b@x.dev" }, -]; - -beforeAll(() => { - originalCwd = process.cwd(); - originalFetch = globalThis.fetch; - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-delete-"))); - configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-delete-config-")); - _setConfigDir(configDir); - process.chdir(workDir); -}); - -afterAll(() => { - globalThis.fetch = originalFetch; - _setConfigDir(undefined); - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); - fs.rmSync(configDir, { recursive: true, force: true }); -}); - -beforeEach(() => { - requests = []; - fs.rmSync(getLogDir(), { recursive: true, force: true }); - fs.rmSync(path.join(configDir, "config.json"), { force: true }); - fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); -}); - -afterEach(() => { - globalThis.fetch = originalFetch; - process.exitCode = 0; -}); - -/** - * Stubs BAPI: `GET /v1/users` answers with whichever of `present` the request - * asked for, mirroring how Clerk ignores external IDs it does not find. - */ -function stubBapi(present: Record, onDelete?: (id: string) => Response) { - globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { - const url = input.toString(); - requests.push({ method: init?.method ?? "GET", url }); - - if (url.includes("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/v1/users?")) { - const asked = new URL(url).searchParams.getAll("external_id"); - return Response.json( - asked - .filter((externalId) => externalId in present) - .map((externalId) => ({ id: present[externalId], external_id: externalId })), - ); - } - - const match = /\/v1\/users\/([^/?]+)/.exec(url); - if (init?.method === "DELETE" && match) { - return onDelete ? onDelete(match[1] as string) : Response.json({ deleted: true }); - } - return Response.json({}); - }) as unknown as typeof fetch; -} - -const logEntries = () => - fs - .readdirSync(getLogDir()) - .flatMap((name) => fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n")) - .map((line) => JSON.parse(line) as Record); - -const deleteCalls = () => requests.filter((r) => r.method === "DELETE").map((r) => r.url); - -describe("resolveMigrationToUndo", () => { - test("reads the file and transformer from the saved migration", async () => { - await saveSettings({ transformer: "clerk", file: "export.json" }); - expect(await resolveMigrationToUndo()).toEqual({ file: "export.json", key: "clerk" }); - }); - - // Deleting nothing silently would look like a successful undo. - test("explains when there is no saved migration at all", async () => { - await expect(resolveMigrationToUndo()).rejects.toThrow(/no record of a previous/); - }); - - test.each([ - ["no file", { transformer: "clerk" }], - ["no transformer", { file: "export.json" }], - ])("explains when the saved migration has %s", async (_label, settings) => { - await saveSettings(settings); - await expect(resolveMigrationToUndo()).rejects.toThrow(CliError); - }); - - test("explains when the migration file has since been removed", async () => { - await saveSettings({ transformer: "clerk", file: "gone.json" }); - await expect(resolveMigrationToUndo()).rejects.toThrow(/no longer there/); - }); -}); - -describe("readMigratedExternalIds", () => { - test("returns the source IDs the import stamped as external_id", async () => { - expect(await readMigratedExternalIds("export.json", "clerk")).toEqual(["legacy_a", "legacy_b"]); - }); - - test("uses each transformer's own id field", async () => { - fs.writeFileSync( - path.join(workDir, "auth0.json"), - JSON.stringify([{ user_id: "auth0|1", email: "a@x.dev" }]), - ); - expect(await readMigratedExternalIds("auth0.json", "auth0")).toEqual(["auth0|1"]); - }); - - // Firebase's postTransform demands the project's hash parameters; deleting - // must not require them, so only the field mapping runs. - test("reads a firebase export without needing its password hash parameters", async () => { - fs.writeFileSync( - path.join(workDir, "firebase.json"), - JSON.stringify({ - users: [{ localId: "fb1", email: "a@x.dev", passwordHash: "H", salt: "S" }], - }), - ); - expect(await readMigratedExternalIds("firebase.json", "firebase")).toEqual(["fb1"]); - }); - - test("dedupes repeated IDs", async () => { - fs.writeFileSync( - path.join(workDir, "dupes.json"), - JSON.stringify([{ id: "legacy_a" }, { id: "legacy_a" }]), - ); - expect(await readMigratedExternalIds("dupes.json", "clerk")).toEqual(["legacy_a"]); - }); - - test("skips rows with no ID rather than matching on an empty string", async () => { - fs.writeFileSync( - path.join(workDir, "partial.json"), - JSON.stringify([{ id: "legacy_a" }, { primary_email_address: "b@x.dev" }, { id: "" }]), - ); - expect(await readMigratedExternalIds("partial.json", "clerk")).toEqual(["legacy_a"]); - }); -}); - -describe("batch", () => { - test.each([ - [0, 0], - [1, 1], - [100, 1], - [101, 2], - [250, 3], - ])("%i ids become %i request(s)", (count, expected) => { - const ids = Array.from({ length: count }, (_, i) => `u${i}`); - expect(batch(ids, 100)).toHaveLength(expected); - }); - - test("keeps every item, in order", () => { - expect(batch([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]); - }); -}); - -describe("findMigratedUsers", () => { - test("queries by external_id instead of listing the instance", async () => { - stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); - - const found = await findMigratedUsers({ - externalIds: ["legacy_a", "legacy_b"], - secretKey: "sk_test_x", - }); - - expect(found).toEqual([ - { id: "user_1", externalId: "legacy_a" }, - { id: "user_2", externalId: "legacy_b" }, - ]); - expect(requests).toHaveLength(1); - expect(requests[0]?.url).toContain("external_id=legacy_a"); - }); - - test("omits IDs the instance does not have", async () => { - stubBapi({ legacy_a: "user_1" }); - - const found = await findMigratedUsers({ - externalIds: ["legacy_a", "legacy_b"], - secretKey: "sk_test_x", - }); - - expect(found).toEqual([{ id: "user_1", externalId: "legacy_a" }]); - }); - - test("pages in batches of 100, the BAPI limit", async () => { - const ids = Array.from({ length: 150 }, (_, i) => `legacy_${i}`); - stubBapi(Object.fromEntries(ids.map((id, i) => [id, `user_${i}`]))); - - const found = await findMigratedUsers({ externalIds: ids, secretKey: "sk_test_x" }); - - expect(found).toHaveLength(150); - expect(requests).toHaveLength(2); - }); - - test("asks for a page large enough to hold the whole batch", async () => { - stubBapi({ legacy_a: "user_1" }); - await findMigratedUsers({ externalIds: ["legacy_a"], secretKey: "sk_test_x" }); - expect(requests[0]?.url).toContain("limit=100"); - }); - - // The guard that keeps this command from touching anything it did not create. - test("ignores a user whose external_id was not asked for", async () => { - globalThis.fetch = (async (input: string | URL | Request) => { - requests.push({ method: "GET", url: input.toString() }); - return Response.json([ - { id: "user_1", external_id: "legacy_a" }, - { id: "user_999", external_id: "somebody_else" }, - { id: "user_888" }, - ]); - }) as unknown as typeof fetch; - - const found = await findMigratedUsers({ externalIds: ["legacy_a"], secretKey: "sk_test_x" }); - - expect(found).toEqual([{ id: "user_1", externalId: "legacy_a" }]); - }); -}); - -describe("deleteMigratedUsers", () => { - const users = [ - { id: "user_1", externalId: "legacy_a" }, - { id: "user_2", externalId: "legacy_b" }, - ]; - - test("deletes each user and logs the outcome", async () => { - stubBapi({}); - - const summary = await deleteMigratedUsers({ - users, - secretKey: "sk_test_x", - limits: LIMITS, - dateTime: DATE_TIME, - }); - - expect(summary).toMatchObject({ deleted: 2, failed: 0 }); - expect(deleteCalls()).toHaveLength(2); - expect(logEntries().filter((e) => e.status === "success")).toHaveLength(2); - }); - - test("records the source ID alongside the Clerk ID in the log", async () => { - stubBapi({}); - - await deleteMigratedUsers({ - users: [users[0] as (typeof users)[0]], - secretKey: "sk_test_x", - limits: LIMITS, - dateTime: DATE_TIME, - }); - - expect(logEntries()[0]).toMatchObject({ - userId: "legacy_a", - clerkUserId: "user_1", - status: "success", - }); - }); - - // A half-undone migration with no record of which half is worse than a - // reported failure. - test("keeps going after one user fails", async () => { - stubBapi({}, (id) => - id === "user_1" - ? new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { - status: 422, - }) - : Response.json({ deleted: true }), - ); - - const summary = await deleteMigratedUsers({ - users, - secretKey: "sk_test_x", - limits: LIMITS, - dateTime: DATE_TIME, - }); - - expect(summary).toMatchObject({ deleted: 1, failed: 1 }); - expect(deleteCalls()).toHaveLength(2); - expect(logEntries().some((e) => e.status === "error" && e.code === "422")).toBe(true); - }); - - test("retries a 429 and logs the attempt", async () => { - const attempts = new Map(); - stubBapi({}, (id) => { - const attempt = (attempts.get(id) ?? 0) + 1; - attempts.set(id, attempt); - return attempt === 1 - ? new Response(JSON.stringify({ errors: [{ code: "e", message: "slow down" }] }), { - status: 429, - headers: { "retry-after": "1" }, - }) - : Response.json({ deleted: true }); - }); - - const summary = await deleteMigratedUsers({ - users: [users[0] as (typeof users)[0]], - secretKey: "sk_test_x", - limits: LIMITS, - dateTime: DATE_TIME, - }); - - expect(summary).toMatchObject({ deleted: 1, failed: 0 }); - expect(deleteCalls()).toHaveLength(2); - expect(logEntries().some((e) => e.status === "429_retry")).toBe(true); - }); - - test("groups identical failures in the breakdown", async () => { - stubBapi( - {}, - () => - new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { - status: 422, - }), - ); - - const summary = await deleteMigratedUsers({ - users, - secretKey: "sk_test_x", - limits: LIMITS, - dateTime: DATE_TIME, - }); - - expect([...summary.errorBreakdown.values()]).toEqual([2]); - }); -}); - -describe("deleteMigration", () => { - const baseOptions = { yes: true, secretKey: "sk_test_x" }; - - beforeEach(async () => { - await saveSettings({ transformer: "clerk", file: "export.json" }); - }); - - test("deletes the users the last run created", async () => { - stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); - - await deleteMigration(baseOptions); - - expect(deleteCalls()).toEqual([ - expect.stringContaining("/v1/users/user_1"), - expect.stringContaining("/v1/users/user_2"), - ]); - expect(captured.err).toContain("Deleted:"); - }); - - test("writes a timestamped NDJSON deletion log", async () => { - stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); - - await deleteMigration(baseOptions); - - const logs = fs.readdirSync(getLogDir()); - expect(logs).toHaveLength(1); - expect(logs[0]).toMatch(/^delete-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); - }); - - test("leaves users the migration did not create alone", async () => { - stubBapi({ legacy_a: "user_1" }); - - await deleteMigration(baseOptions); - - expect(deleteCalls()).toEqual([expect.stringContaining("/v1/users/user_1")]); - expect(captured.err).toContain("1 of the file's users is not in this instance"); - }); - - test("does nothing when none of the migration's users are present", async () => { - stubBapi({}); - - await deleteMigration(baseOptions); - - expect(deleteCalls()).toHaveLength(0); - expect(captured.err).toContain("Nothing to delete"); - }); - - // Tests run non-TTY, which is the same signal an agent gives. - test("refuses without -y when it cannot prompt, and says how many are at stake", async () => { - stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); - - await expect(deleteMigration({ secretKey: "sk_test_x" })).rejects.toThrow( - /permanently deletes 2 users and cannot prompt here/, - ); - expect(deleteCalls()).toHaveLength(0); - }); - - test("fails before any API call when there is no saved migration", async () => { - fs.rmSync(path.join(configDir, "config.json"), { force: true }); - stubBapi({ legacy_a: "user_1" }); - - await expect(deleteMigration(baseOptions)).rejects.toThrow(CliError); - expect(requests).toHaveLength(0); - }); - - test("exits non-zero when a deletion failed", async () => { - stubBapi( - { legacy_a: "user_1" }, - () => - new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { - status: 422, - }), - ); - - await deleteMigration(baseOptions); - - expect(process.exitCode).toBe(1); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts deleted file mode 100644 index b48605e8d..000000000 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ /dev/null @@ -1,331 +0,0 @@ -/** - * `clerk migrate delete` — undo a migration. - * - * Ported from the standalone migration-tool's `src/delete/index.ts`, with two - * substantive changes: - * - * - **Users are looked up by `external_id`, not by downloading the instance.** - * The original paged through every user in the instance 500 at a time and - * intersected client-side, which on a large instance means fetching hundreds - * of thousands of users to delete a few hundred. `GET /v1/users` filters on - * up to 100 `external_id`s per call and ignores IDs it does not find, so the - * work is proportional to the migration rather than to the instance. - * - **IDs come from the existing transform pipeline.** The original - * re-implemented per-format ID extraction with its own Firebase CSV header - * list and a chain of `userId`/`user_id`/`localId`/`id` fallbacks. The - * transformer already declares which source field becomes `userId`. - * - * This is the one command in the `migrate` tree that destroys data in Clerk, so - * it stays flat and prominent rather than buried under a noun group, and it - * confirms before acting. - */ - -import { bapiRequest } from "../../lib/bapi.ts"; -import { bold, dim, green, red } from "../../lib/color.ts"; -import { - BapiError, - CliError, - ERROR_CODE, - throwUsageError, - throwUserAbort, -} from "../../lib/errors.ts"; -import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; -import { log } from "../../lib/log.ts"; -import { NEXT_STEPS, printAgentNextSteps } from "../../lib/next-steps.ts"; -import { confirm } from "../../lib/prompts.ts"; -import { withGutter, withSpinner, type SpinnerControls } from "../../lib/spinner.ts"; -import { isAgent, isHuman } from "../../mode.ts"; -import { normalizeErrorMessage } from "./import-users.ts"; -import { resolveLimits, type ResolvedLimits } from "./lib/instance.ts"; -import { deleteErrorLogger, deleteLogger, startLogging, getLogFilePath } from "./lib/logger.ts"; -import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; -import { createApiScheduler } from "./lib/scheduler.ts"; -import { loadSettings } from "./lib/settings.ts"; -import { fileExists, readRawUsers, transformKeys } from "./lib/transform.ts"; -import { getTransformer } from "./transformers/registry.ts"; - -/** BAPI accepts at most 100 `external_id` values per `GET /v1/users` call. */ -const EXTERNAL_ID_BATCH = 100; - -export type MigrateDeleteOptions = { - yes?: boolean; - secretKey?: string; - app?: string; - instance?: string; -}; - -export type MigratedUser = { - /** The Clerk user ID to delete. */ - id: string; - /** The source platform's ID, stamped on the user as `external_id`. */ - externalId: string; -}; - -/** - * Resolves which migration is being undone. - * - * The saved migration record is the only account of that — this command has no - * independent way to know what a previous run created, which is why it is - * coupled to `run`. - */ -export async function resolveMigrationToUndo(): Promise<{ file: string; key: string }> { - const settings = await loadSettings(); - - if (!settings.file || !settings.transformer) { - throw new CliError( - "No migration to undo: this project has no record of a previous `clerk migrate import`.\n" + - "Run `clerk migrate delete` from the project you migrated from.", - { code: ERROR_CODE.FILE_NOT_FOUND }, - ); - } - - if (!fileExists(settings.file)) { - throw new CliError( - `The migration file ${settings.file} is no longer there, so the users it created cannot be identified.`, - { code: ERROR_CODE.FILE_NOT_FOUND }, - ); - } - - return { file: settings.file, key: settings.transformer }; -} - -/** - * The source IDs a migration stamped onto Clerk users as `external_id`. - * - * Runs the transformer's field mapping but not its `postTransform`: only the - * ID matters here, and Firebase's post-transform would demand the project's - * password hash parameters to rebuild digests nobody is importing. - */ -export async function readMigratedExternalIds(file: string, key: string): Promise { - const transformer = getTransformer(key); - const rows = await readRawUsers(file, key); - - const ids = new Set(); - for (const row of rows) { - const userId = transformKeys(row, transformer).userId; - if (typeof userId === "string" && userId.length > 0) ids.add(userId); - } - return [...ids]; -} - -/** Splits `items` into chunks of at most `size`. */ -export function batch(items: T[], size: number): T[][] { - const batches: T[][] = []; - for (let i = 0; i < items.length; i += size) batches.push(items.slice(i, i + size)); - return batches; -} - -/** - * Finds the Clerk users a migration created, by `external_id`. - * - * IDs with no matching user are simply absent from the result — a partial - * migration, or one already partly undone, is the normal case. - */ -export async function findMigratedUsers(options: { - externalIds: string[]; - secretKey: string; - spinner?: SpinnerControls; -}): Promise { - const found: MigratedUser[] = []; - const batches = batch(options.externalIds, EXTERNAL_ID_BATCH); - - for (const [index, ids] of batches.entries()) { - options.spinner?.update(`Finding migrated users: batch ${index + 1}/${batches.length}...`); - - const params = new URLSearchParams(); - params.set("limit", String(EXTERNAL_ID_BATCH)); - for (const id of ids) params.append("external_id", id); - - const response = await retryOn429(async () => - bapiRequest({ - method: "GET", - path: `/v1/users?${params.toString()}`, - secretKey: options.secretKey, - }), - ); - - const users = (response.body ?? []) as { id?: string; external_id?: string }[]; - for (const user of Array.isArray(users) ? users : []) { - // Never delete on a partial match: only a user Clerk itself reports as - // carrying one of this migration's external IDs is in scope. - if (user.id && user.external_id && ids.includes(user.external_id)) { - found.push({ id: user.id, externalId: user.external_id }); - } - } - } - - return found; -} - -export type DeleteSummary = { - deleted: number; - failed: number; - errorBreakdown: Map; -}; - -/** Deletes each user, rate-limited and 429-retried exactly as the import is. */ -export async function deleteMigratedUsers(options: { - users: MigratedUser[]; - secretKey: string; - limits: ResolvedLimits; - dateTime: string; - spinner?: SpinnerControls; -}): Promise { - const { users, secretKey, limits, dateTime, spinner } = options; - const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); - const errorBreakdown = new Map(); - - let processed = 0; - let deleted = 0; - let failed = 0; - - const progress = () => - spinner?.update( - `Deleting users: [${processed}/${users.length}] (${deleted} deleted, ${failed} failed)...`, - ); - - // A failure on one user must not abort the rest: a half-undone migration - // with no record of which half is far worse than a reported failure. - const recordFailure = (user: MigratedUser, message: string, code: string) => { - failed++; - processed++; - const normalized = normalizeErrorMessage(message); - errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); - deleteLogger( - { userId: user.externalId, clerkUserId: user.id, status: "error", error: message, code }, - dateTime, - ); - progress(); - }; - - const deleteOne = async (user: MigratedUser): Promise => { - try { - await retryOn429( - async () => - schedule(async () => - bapiRequest({ method: "DELETE", path: `/v1/users/${user.id}`, secretKey }), - ), - { - onRetry: ({ message }) => - deleteErrorLogger( - { - userId: user.externalId, - status: "429_retry", - errors: [{ code: "rate_limit_retry", message, longMessage: message }], - }, - dateTime, - ), - }, - ); - - deleted++; - processed++; - deleteLogger({ userId: user.externalId, clerkUserId: user.id, status: "success" }, dateTime); - progress(); - } catch (error) { - if (error instanceof RateLimitExceededError) { - recordFailure(user, error.message, "429"); - return; - } - const apiError = error as BapiError; - const message = apiError.longMessage ?? apiError.message ?? "Unknown error"; - recordFailure(user, message, String(apiError.status ?? "unknown")); - } - }; - - progress(); - await Promise.all(users.map(deleteOne)); - - return { deleted, failed, errorBreakdown }; -} - -function formatSummary(summary: DeleteSummary, logFile: string): string { - const lines = [ - `${bold("Deleted:")} ${green(String(summary.deleted))}`, - `${bold("Failed:")} ${red(String(summary.failed))}`, - ]; - - if (summary.errorBreakdown.size > 0) { - lines.push("", bold("Error breakdown:")); - for (const [error, count] of summary.errorBreakdown) { - lines.push(` ${count} user${count === 1 ? "" : "s"}: ${error}`); - } - } - lines.push("", dim(`Log: ${logFile}`)); - - return lines.join("\n"); -} - -export async function deleteMigration(options: MigrateDeleteOptions): Promise { - const { file, key } = await resolveMigrationToUndo(); - - await withGutter("Undoing a migration", async ({ setNextSteps }) => { - const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); - const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); - const limits = resolveLimits(secretKey); - const dateTime = await startLogging(); - const logFile = getLogFilePath("delete", dateTime); - - const externalIds = await readMigratedExternalIds(file, key); - if (externalIds.length === 0) { - log.warn(`No user IDs found in ${file}; nothing to undo.`); - return; - } - - const users = await withSpinner("Finding migrated users...", async (spinner) => - findMigratedUsers({ externalIds, secretKey, spinner }), - ); - - if (users.length === 0) { - log.info( - `None of the ${externalIds.length} user${externalIds.length === 1 ? "" : "s"} in ${file} are in ${target ?? "this instance"}. Nothing to delete.`, - ); - return; - } - - log.warn( - `About to delete ${users.length} user${users.length === 1 ? "" : "s"} from ` + - `${target ?? "the resolved instance"}, matched to ${file} by external ID.`, - ); - if (users.length < externalIds.length) { - log.info( - dim( - `${externalIds.length - users.length} of the file's users ${externalIds.length - users.length === 1 ? "is" : "are"} not in this instance and will be left alone.`, - ), - ); - } - - if (!options.yes) { - if (isAgent() || !isHuman()) { - throwUsageError( - `\`clerk migrate delete\` permanently deletes ${users.length} user${users.length === 1 ? "" : "s"} and cannot prompt here. Pass -y to confirm.`, - undefined, - undefined, - [ - { - command: "clerk migrate delete -y", - description: "Delete the migrated users without prompting", - }, - ], - ); - } - - const proceed = await confirm({ - message: `Permanently delete ${users.length} user${users.length === 1 ? "" : "s"}?`, - default: false, - }); - if (!proceed) throwUserAbort(); - } - - const summary = await withSpinner(`Deleting users: [0/${users.length}]...`, async (spinner) => - deleteMigratedUsers({ users, secretKey, limits, dateTime, spinner }), - ); - - log.info(formatSummary(summary, logFile)); - - setNextSteps(NEXT_STEPS.MIGRATE_DELETE); - printAgentNextSteps(NEXT_STEPS.MIGRATE_DELETE); - - if (summary.failed > 0) process.exitCode = 1; - }); -} diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index 14eaf9585..8355a23f7 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; -import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { buildAuth0Export, @@ -16,11 +16,7 @@ import { resolveAuth0Credentials, } from "./auth0.ts"; -/** A cwd with no `.env` files, so these tests exercise only the injected env. */ -const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); - const captured = useCaptureLog(); -useMigrateLogDir(); const CREDENTIALS = { domain: "t.auth0.com", clientId: "cid", clientSecret: "csec" }; @@ -96,25 +92,26 @@ describe("resolveAuth0Credentials", () => { test("prefers flags", async () => { const resolved = await resolveAuth0Credentials( { domain: "flag.auth0.com", clientId: "f", clientSecret: "s" }, - NO_ENV_FILES, { AUTH0_DOMAIN: "env.auth0.com" }, ); expect(resolved.domain).toBe("flag.auth0.com"); }); test("falls back to the environment", async () => { - const resolved = await resolveAuth0Credentials({}, NO_ENV_FILES, { - AUTH0_DOMAIN: "env.auth0.com", - AUTH0_CLIENT_ID: "e", - AUTH0_CLIENT_SECRET: "s", - }); + const resolved = await resolveAuth0Credentials( + {}, + { + AUTH0_DOMAIN: "env.auth0.com", + AUTH0_CLIENT_ID: "e", + AUTH0_CLIENT_SECRET: "s", + }, + ); expect(resolved).toEqual({ domain: "env.auth0.com", clientId: "e", clientSecret: "s" }); }); test("normalizes a domain that came with a scheme", async () => { const resolved = await resolveAuth0Credentials( { domain: "https://t.auth0.com/", clientId: "c", clientSecret: "s" }, - NO_ENV_FILES, {}, ); expect(resolved.domain).toBe("t.auth0.com"); @@ -122,14 +119,14 @@ describe("resolveAuth0Credentials", () => { // Tests run non-TTY, the same signal an agent gives. test("names every missing credential at once rather than one at a time", async () => { - await expect(resolveAuth0Credentials({}, NO_ENV_FILES, {})).rejects.toThrow( + await expect(resolveAuth0Credentials({}, {})).rejects.toThrow( /--domain \(or AUTH0_DOMAIN\), --client-id \(or AUTH0_CLIENT_ID\), --client-secret \(or AUTH0_CLIENT_SECRET\)/, ); }); test("names only what is actually missing", async () => { await expect( - resolveAuth0Credentials({ domain: "t.auth0.com", clientId: "c" }, NO_ENV_FILES, {}), + resolveAuth0Credentials({ domain: "t.auth0.com", clientId: "c" }, {}), ).rejects.toThrow(/Missing: --client-secret \(or AUTH0_CLIENT_SECRET\)\./); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 5cd62f7b9..260abe1eb 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -21,8 +21,7 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt, text } from "../../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; -import { findMigrateEnvValue } from "../lib/env-file.ts"; -import { exportLogger, startLogging } from "../lib/logger.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; @@ -66,16 +65,12 @@ export function normalizeAuth0Domain(domain: string): string { */ export async function resolveAuth0Credentials( options: ExportAuth0Options, - cwd: string = process.cwd(), env: Record = process.env, ): Promise { - const fromEnv = async (name: string): Promise => - (await findMigrateEnvValue([name], cwd, env))?.value; - const resolved = { - domain: options.domain ?? (await fromEnv("AUTH0_DOMAIN")), - clientId: options.clientId ?? (await fromEnv("AUTH0_CLIENT_ID")), - clientSecret: options.clientSecret ?? (await fromEnv("AUTH0_CLIENT_SECRET")), + domain: options.domain ?? env.AUTH0_DOMAIN, + clientId: options.clientId ?? env.AUTH0_CLIENT_ID, + clientSecret: options.clientSecret ?? env.AUTH0_CLIENT_SECRET, }; const missing = ( @@ -333,7 +328,7 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { const destination = await resolveOutputPath("auth0", options.output); await withGutter("Exporting users from Auth0", async () => { - const dateTime = await startLogging(); + const dateTime = getDateTimeStamp(); // Only Auth0 can say whether these three go together, and whether the // application carries the `read:users` scope, so a rejected set is asked @@ -354,7 +349,7 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { const { users: exported, coverage } = buildAuth0Export(users, dateTime); const outputPath = writeExportOutput(exported, destination); - await reportExport({ + reportExport({ platform: "auth0", userCount: exported.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 963238911..51628be7f 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -13,7 +13,7 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { log } from "../../../lib/log.ts"; -import { exportLogger, startLogging } from "../lib/logger.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { @@ -124,7 +124,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { const destination = await resolveOutputPath("authjs", options.output); await withGutter("Exporting users from Auth.js", async () => { - const dateTime = await startLogging(); + const dateTime = getDateTimeStamp(); const { value: { rows, table }, @@ -141,7 +141,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { const { users, coverage } = buildAuthJsExport(rows, dateTime); const outputPath = writeExportOutput(users, destination); - await reportExport({ + reportExport({ platform: "authjs", userCount: users.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 963b223a6..4e8adf089 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -17,7 +17,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; -import { exportLogger, startLogging } from "../lib/logger.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { @@ -171,7 +171,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const destination = await resolveOutputPath("betterauth", options.output); await withGutter("Exporting users from Better Auth", async () => { - const dateTime = await startLogging(); + const dateTime = getDateTimeStamp(); const { value: { rows, plugins }, @@ -197,7 +197,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const { users, coverage } = buildBetterAuthExport(rows, dateTime); const outputPath = writeExportOutput(users, destination); - await reportExport({ + reportExport({ platform: "betterauth", userCount: users.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 67f825172..75e1f0565 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -3,7 +3,7 @@ import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { buildClerkExport, @@ -13,7 +13,6 @@ import { } from "./clerk.ts"; const captured = useCaptureLog(); -useMigrateLogDir(); let workDir: string; let originalCwd: string; diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 193262287..22305feed 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -17,7 +17,7 @@ import { bapiRequest } from "../../../lib/bapi.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; -import { exportLogger, startLogging } from "../lib/logger.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { retryOn429 } from "../lib/retry.ts"; import { resolveClerkSource } from "./clerk-source.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; @@ -235,7 +235,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { const destination = await resolveOutputPath("clerk", options.output); await withGutter("Exporting users from Clerk", async () => { - const dateTime = await startLogging(); + const dateTime = getDateTimeStamp(); log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); @@ -246,7 +246,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { const { users: exported, coverage } = buildClerkExport(users, dateTime); const outputPath = writeExportOutput(exported, destination); - await reportExport({ + reportExport({ platform: "clerk", userCount: exported.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 62097f2b2..7cfc62a9a 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -31,9 +31,6 @@ import { resolveDbUrl, } from "./db-options.ts"; -/** A cwd with no `.env` files, so these tests exercise only the injected env. */ -const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); - const captured = useCaptureLog(); let workDir: string; @@ -139,55 +136,37 @@ describe("resolveDbUrl", () => { const config = { platform: "authjs" as const, envVar: "AUTHJS_DB_URL", prompt: "url" }; test("prefers the flag", async () => { - const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, NO_ENV_FILES, { + const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, { AUTHJS_DB_URL: "mysql://u:p@h/db", }); expect(url).toBe("postgres://u:p@h/db"); }); test("falls back to the environment variable", async () => { - expect( - await resolveDbUrl({}, config, NO_ENV_FILES, { AUTHJS_DB_URL: "mysql://u:p@h/db" }), - ).toBe("mysql://u:p@h/db"); + expect(await resolveDbUrl({}, config, { AUTHJS_DB_URL: "mysql://u:p@h/db" })).toBe( + "mysql://u:p@h/db", + ); }); test("encodes a raw password passed to the flag", async () => { - const url = await resolveDbUrl( - { dbUrl: "postgres://u:p#ss@host:5432/db" }, - config, - NO_ENV_FILES, - {}, - ); + const url = await resolveDbUrl({ dbUrl: "postgres://u:p#ss@host:5432/db" }, config, {}); expect(decodeURIComponent(new URL(url).password)).toBe("p#ss"); }); test("rejects a flag that is not a connection string, naming the encoding trap", async () => { - await expect(resolveDbUrl({ dbUrl: "not a url" }, config, NO_ENV_FILES, {})).rejects.toThrow( - /URL-encode it/, - ); + await expect(resolveDbUrl({ dbUrl: "not a url" }, config, {})).rejects.toThrow(/URL-encode it/); }); test("warns and moves on when the environment variable is unusable", async () => { // Tests run non-TTY, so it then hits the agent-mode branch. - await expect( - resolveDbUrl({}, config, NO_ENV_FILES, { AUTHJS_DB_URL: "garbage" }), - ).rejects.toThrow(/cannot prompt here/); + await expect(resolveDbUrl({}, config, { AUTHJS_DB_URL: "garbage" })).rejects.toThrow( + /cannot prompt here/, + ); expect(captured.err).toContain("AUTHJS_DB_URL is not a valid connection string"); }); - // The env var reaching process.env is the runtime's job; this is the fallback - // for when it did not, and is the rung the secret key has always had. - test("falls back to a .env file when the variable is not in the environment", async () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-dburl-env-")); - fs.writeFileSync(path.join(dir, ".env.local"), "AUTHJS_DB_URL=postgres://u:p@h/db\n"); - - expect(await resolveDbUrl({}, config, dir, {})).toBe("postgres://u:p@h/db"); - }); - test("names both the flag and the variable when it cannot prompt", async () => { - await expect(resolveDbUrl({}, config, NO_ENV_FILES, {})).rejects.toThrow( - /--db-url.*AUTHJS_DB_URL/s, - ); + await expect(resolveDbUrl({}, config, {})).rejects.toThrow(/--db-url.*AUTHJS_DB_URL/s); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index c0f92a92f..3baeee59a 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -11,7 +11,6 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { detectDbType, isLibsqlUrl, redactConnectionString, type DbPlatform } from "../lib/db.ts"; -import { findMigrateEnvValue } from "../lib/env-file.ts"; export type DbExportOptions = { dbUrl?: string; @@ -96,7 +95,6 @@ export function looksLikeConnectionString(value: string): boolean { export async function resolveDbUrl( options: DbExportOptions, config: ResolveConfig, - cwd: string = process.cwd(), env: Record = process.env, ): Promise { const fromFlag = options.dbUrl ? normalizeConnectionString(options.dbUrl) : undefined; @@ -110,8 +108,8 @@ export async function resolveDbUrl( return fromFlag; } - const located = await findMigrateEnvValue([config.envVar], cwd, env); - const fromEnv = located ? normalizeConnectionString(located.value) : undefined; + const raw = env[config.envVar]; + const fromEnv = raw ? normalizeConnectionString(raw) : undefined; if (fromEnv) { if (looksLikeConnectionString(fromEnv)) return fromEnv; // Falling through silently would make the prompt look unexplained. diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 0bae0c8b3..de8234467 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -5,7 +5,7 @@ import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import { setAssumeYes } from "../lib/assume-yes.ts"; -import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { buildFirebaseExport, @@ -22,7 +22,6 @@ import { } from "./firebase.ts"; const captured = useCaptureLog(); -useMigrateLogDir(); let workDir: string; let originalCwd: string; diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 471c22b8f..be65f9ea4 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -32,9 +32,7 @@ import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; -import { writeMigrateEnvValues } from "../lib/env-file.ts"; -import { exportLogger, startLogging } from "../lib/logger.ts"; -import { findSetting } from "../settings/registry.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; @@ -501,35 +499,6 @@ export function formatHashConfigGuidance( ]; } -/** - * Saves the scrypt parameters where `clerk migrate import` will read them. - * - * They are credentials, so they land in `.env.clerk-migrate` — created - * gitignored — rather than the CLI config, which is the split the settings - * registry owns. The variable names come from that registry instead of being - * spelled again here: a setting and the variable behind it must not drift into - * two names the user has to know separately. - * - * `-y` opts out, for the reason {@link reportExport} gives. Writing a - * credential file is also not something to do silently, so the path is named. - */ -async function rememberHashConfig(config: HashConfig | null): Promise { - if (!config || isAssumeYes()) return; - - const variable = (name: string) => - (findSetting(name) as NonNullable>).envVar as string; - - const file = await writeMigrateEnvValues({ - [variable("firebase-signer-key")]: config.signerKey, - [variable("firebase-salt-separator")]: config.saltSeparator, - [variable("firebase-rounds")]: String(config.rounds), - [variable("firebase-mem-cost")]: String(config.memoryCost), - }); - - log.blank(); - log.info(dim(`Saved to ${file} (gitignored), so the import can be run without them.`)); -} - export async function exportFirebase(options: ExportFirebaseOptions): Promise { // Read and validate before anything reaches the network, so a wrong file // fails in a second rather than after an auth round-trip. @@ -538,7 +507,7 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { - const dateTime = await startLogging(); + const dateTime = getDateTimeStamp(); // Only Google can say whether a well-formed key is still a valid one, so a // revoked or deleted key fails here and is asked for again. @@ -560,7 +529,7 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise ({ text: (...args: unknown[]) => mockText(...args), })); -const mockSaveSettings = mock(async () => {}); -mock.module("../lib/settings.ts", () => ({ - loadSettings: async () => ({}), - saveSettings: (...args: unknown[]) => mockSaveSettings(...(args as [])), -})); - let human = true; mock.module("../../../mode.ts", () => ({ isHuman: () => human, @@ -22,7 +14,7 @@ mock.module("../../../mode.ts", () => ({ setMode: () => {}, })); -const { defaultOutputPath, formatImportCommand, outputStamp, reportExport, resolveOutputPath } = +const { defaultOutputPath, formatImportCommand, outputStamp, resolveOutputPath } = await import("./shared.ts"); const { setAssumeYes } = await import("../lib/assume-yes.ts"); @@ -30,7 +22,6 @@ beforeEach(() => { human = true; setAssumeYes(false); mockText.mockReset(); - mockSaveSettings.mockReset(); }); describe("outputStamp", () => { @@ -175,42 +166,3 @@ describe("formatImportCommand", () => { expect(text).not.toContain("Add `-y`"); }); }); - -describe("reportExport", () => { - useCaptureLog(); - - const summary = { - platform: "supabase", - userCount: 2, - outputPath: path.resolve(process.cwd(), "exports/mine.json"), - coverage: [{ label: "have an email address", count: 2 }], - transformerKey: "supabase", - }; - - // Relative, matching the command printed alongside it: the remembered value - // and the copyable one must not disagree about the same file. - test("remembers the transformer and the file for the next import", async () => { - await reportExport(summary); - - expect(mockSaveSettings).toHaveBeenCalledWith({ - transformer: "supabase", - file: path.join("exports", "mine.json"), - }); - }); - - // `-y` means "do not stop to ask me", and a remembered value is one a later - // run picks up silently. - test("remembers nothing under -y", async () => { - setAssumeYes(true); - - await reportExport(summary); - - expect(mockSaveSettings).not.toHaveBeenCalled(); - }); - - test("remembers nothing when there was nothing to export", async () => { - await reportExport({ ...summary, userCount: 0 }); - - expect(mockSaveSettings).not.toHaveBeenCalled(); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index 7dc54461a..7c4dc1614 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -17,7 +17,6 @@ import { log } from "../../../lib/log.ts"; import { text } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; -import { loadSettings, saveSettings } from "../lib/settings.ts"; /** * `YYYYMMDD-HHmm`, local time — ISO 8601 basic format, minus seconds. @@ -153,11 +152,8 @@ export type ExportSummary = { * human-only — `withGutter` and `printNextSteps` both return early for an agent * or a non-TTY — and this is the one line that says what to do with the file * just written. An agent that cannot see it has to guess the invocation. - * - * Also remembers the transformer and the file, so the import can be run - * without retyping what was just printed. */ -export async function reportExport(summary: ExportSummary): Promise { +export function reportExport(summary: ExportSummary): void { log.blank(); if (summary.userCount === 0) { log.warn(`No users found to export. Wrote an empty file to ${summary.outputPath}.`); @@ -180,32 +176,13 @@ export async function reportExport(summary: ExportSummary): Promise { `Exported ${summary.userCount} user${summary.userCount === 1 ? "" : "s"} to ${summary.outputPath}`, ); - const file = relativeIfInside(summary.outputPath); - log.blank(); - for (const line of formatImportCommand(summary.transformerKey, file)) { + for (const line of formatImportCommand( + summary.transformerKey, + relativeIfInside(summary.outputPath), + )) { log.info(line); } - - await rememberExport(summary.transformerKey, file); -} - -/** - * Remembers what this export produced, so `clerk migrate import` can be run - * without repeating the flags just printed. - * - * `-y` opts out. It means "do not stop to ask me", and a remembered value is - * something a later run picks up silently — the same reason `ensureLogDir` - * saves nothing under `-y`. Both settings stay unset until somebody who was - * watching the output produced them. - * - * Stores the path the way it was printed: relative while it sits under the - * project, so the remembered value and the copyable command say the same - * thing. - */ -async function rememberExport(transformer: string, file: string): Promise { - if (isAssumeYes()) return; - await saveSettings({ ...(await loadSettings()), transformer, file }); } /** diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index e4453aa02..20972154e 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -12,7 +12,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; -import { exportLogger, startLogging } from "../lib/logger.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { @@ -122,7 +122,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { const destination = await resolveOutputPath("supabase", options.output); await withGutter("Exporting users from Supabase", async () => { - const dateTime = await startLogging(); + const dateTime = getDateTimeStamp(); const { value: rows } = await withInputRetry( dbUrl, @@ -136,7 +136,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { const { users, coverage } = buildSupabaseExport(rows, dateTime); const outputPath = writeExportOutput(users, destination); - await reportExport({ + reportExport({ platform: "supabase", userCount: users.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts index 7cf60563f..9dbf2edbd 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.test.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -3,7 +3,7 @@ import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { setAssumeYes } from "../lib/assume-yes.ts"; import { @@ -20,11 +20,7 @@ import { type WorkOsIdentity, } from "./workos.ts"; -/** A cwd with no `.env` files, so these tests exercise only the injected env. */ -const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); - const captured = useCaptureLog(); -useMigrateLogDir(); const API_KEY = "sk_test"; @@ -102,18 +98,16 @@ function stubWorkOs( describe("resolveWorkOsApiKey", () => { test("prefers the flag", async () => { - expect(await resolveWorkOsApiKey({ apiKey: "sk_flag" }, NO_ENV_FILES, {})).toBe("sk_flag"); + expect(await resolveWorkOsApiKey({ apiKey: "sk_flag" }, {})).toBe("sk_flag"); }); test("falls back to the environment", async () => { - expect(await resolveWorkOsApiKey({}, NO_ENV_FILES, { WORKOS_API_KEY: "sk_env" })).toBe( - "sk_env", - ); + expect(await resolveWorkOsApiKey({}, { WORKOS_API_KEY: "sk_env" })).toBe("sk_env"); }); // Tests run non-TTY, the same signal an agent gives. test("names the flag and the variable when neither supplied one", async () => { - await expect(resolveWorkOsApiKey({}, NO_ENV_FILES, {})).rejects.toThrow( + await expect(resolveWorkOsApiKey({}, {})).rejects.toThrow( /Missing: --api-key \(or WORKOS_API_KEY\)\./, ); }); diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index fee4fa397..1e96682f9 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -23,8 +23,7 @@ import { log } from "../../../lib/log.ts"; import { confirm, password as passwordPrompt } from "../../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; -import { findMigrateEnvValue } from "../lib/env-file.ts"; -import { exportLogger, startLogging } from "../lib/logger.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { createApiScheduler } from "../lib/scheduler.ts"; @@ -86,11 +85,9 @@ export type WorkOsIdentity = { idp_id?: string; type?: string; provider?: string */ export async function resolveWorkOsApiKey( options: ExportWorkOsOptions, - cwd: string = process.cwd(), env: Record = process.env, ): Promise { - const resolved = - options.apiKey ?? (await findMigrateEnvValue(["WORKOS_API_KEY"], cwd, env))?.value; + const resolved = options.apiKey ?? env.WORKOS_API_KEY; if (resolved) return resolved.trim(); @@ -446,7 +443,7 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise const destination = await resolveOutputPath("workos", options.output); await withGutter("Exporting users from WorkOS", async () => { - const dateTime = await startLogging(); + const dateTime = getDateTimeStamp(); // Only WorkOS can say whether the key is live, for the right environment, // and not revoked — so a rejected key is asked for again here. The page it @@ -471,7 +468,7 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise const { users: exported, coverage } = buildWorkOsExport(users, dateTime, providers?.identities); const outputPath = writeExportOutput(exported, destination); - await reportExport({ + reportExport({ platform: "workos", userCount: exported.length, outputPath, diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index 81c636f72..2aea5a480 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -137,20 +137,6 @@ describe("registerMigrate", () => { ); }); - // Flat rather than under a noun group: it is the one command in this tree - // that destroys data in Clerk. - test("registers delete as a direct subcommand of migrate", () => { - expect(findCommand(["migrate", "delete"])).toBeDefined(); - expect(findCommand(["migrate", "delete"])?.description()).toContain("last migration"); - }); - - test.each(["--yes", "--secret-key", "--app", "--instance"])( - "migrate delete accepts %s", - (flag) => { - expect(findCommand(["migrate", "delete"])?.options.map((o) => o.long)).toContain(flag); - }, - ); - test.each([[["logs"]], [["logs", "list"]], [["logs", "clean"]], [["logs", "convert"]]])( "registers migrate %p", (names) => { diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index e49dcd3ec..63b14c130 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -1,16 +1,14 @@ import { createOption } from "@commander-js/extra-typings"; import type { Program } from "../../cli-program.ts"; import { parseIntegerOption } from "../../lib/option-parsers.ts"; -import { deleteMigration } from "./delete.ts"; import { setAssumeYes } from "./lib/assume-yes.ts"; import { registerMigrateExport } from "./export/index.ts"; import { registerMigrateLogs } from "./logs/index.ts"; -import { registerMigrateSettings } from "./settings/index.ts"; import { run } from "./run.ts"; import { list as transformersList } from "./transformers/list.ts"; import { transformerKeys } from "./transformers/registry.ts"; -const migrate = { run, delete: deleteMigration, transformersList }; +const migrate = { run, transformersList }; export function registerMigrate(program: Program): void { const migrateCommand = program @@ -30,18 +28,12 @@ export function registerMigrate(program: Program): void { command: "clerk migrate export supabase", description: "Export users from Supabase, ready to import", }, - { command: "clerk migrate settings", description: "Show what a run here would pick up" }, - { - command: "clerk migrate settings set firebase-signer-key abc123", - description: "Save a credential to .env.clerk-migrate", - }, { command: "clerk migrate logs", description: "List the local migration logs" }, { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, - { command: "clerk migrate delete", description: "Undo the last migration" }, ]); - // `-y` is read three layers down — by the log-directory question and by the - // credential-retry loop — so it is resolved once here rather than threaded + // `-y` is read several layers down — by the credential-retry loop and the + // export commands — so it is resolved once here rather than threaded // through every export handler. Hooks are inherited, so this fires for every // subcommand under `migrate`; one that declares no `-y` resolves to false. migrateCommand.hook("preAction", (_thisCommand, actionCommand) => { @@ -110,26 +102,6 @@ export function registerMigrate(program: Program): void { migrate.run(cmd.optsWithGlobals() as Parameters[0]), ); - // Flat, not under a noun group: this is the one command in the tree that - // destroys data in Clerk, and it is worth keeping short and prominent. - migrateCommand - .command("delete") - .description("Delete the users created by the last migration for this project") - .option("-y, --yes", "Skip the confirmation prompt") - .option("--secret-key ", "Backend API secret key to use") - .option("--app ", "Application ID to target (works from any directory)") - .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") - .setExamples([ - { - command: "clerk migrate delete", - description: "Undo the last migration after confirming", - }, - { command: "clerk migrate delete -y", description: "Undo without prompting" }, - ]) - .action(async (_opts, cmd) => - migrate.delete(cmd.optsWithGlobals() as Parameters[0]), - ); - registerMigrateExport(migrateCommand); // A compiled binary has no source tree to grep, so the available mappings @@ -168,5 +140,4 @@ export function registerMigrate(program: Program): void { ); registerMigrateLogs(migrateCommand); - registerMigrateSettings(migrateCommand); } diff --git a/packages/cli-core/src/commands/migrate/lib/assume-yes.ts b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts index 6315520f2..dd277dad0 100644 --- a/packages/cli-core/src/commands/migrate/lib/assume-yes.ts +++ b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts @@ -8,10 +8,10 @@ * so the two cannot be collapsed into one flag. * * Held per-run rather than threaded through, because the readers are three - * layers below the command that parses it: `ensureLogDir` runs inside the - * gutter of seven different commands, and `withInputRetry` sits under every - * credential prompt. Passing it down would put a `yes` parameter on every - * export handler signature on the way. This mirrors `mode.ts`, which resolves + * layers below the command that parses it: `withInputRetry` sits under every + * credential prompt, and the export commands read it to decide what to print. + * Passing it down would put a `yes` parameter on every export handler + * signature on the way. This mirrors `mode.ts`, which resolves * `--mode` once in a `preAction` hook and is read the same way. * * Set by the `migrate` group's `preAction` hook, so every subcommand under it diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.test.ts b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts deleted file mode 100644 index 855cc9274..000000000 --- a/packages/cli-core/src/commands/migrate/lib/env-file.test.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { - clearMigrateEnvValues, - findMigrateEnvValue, - MIGRATE_ENV_FILE, - writeMigrateEnvValues, -} from "./env-file.ts"; - -let workDir: string; - -const envFile = () => path.join(workDir, MIGRATE_ENV_FILE); -const read = (file: string) => fs.readFileSync(path.join(workDir, file), "utf-8"); - -beforeEach(() => { - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-envfile-"))); -}); - -afterEach(() => { - fs.rmSync(workDir, { recursive: true, force: true }); -}); - -describe("writeMigrateEnvValues", () => { - test("creates the file and gitignores it", async () => { - await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); - - expect(read(MIGRATE_ENV_FILE)).toBe("CLERK_FIREBASE_ROUNDS=8\n"); - expect(read(".gitignore")).toContain(MIGRATE_ENV_FILE); - }); - - test("appends to an existing .gitignore without duplicating the entry", async () => { - fs.writeFileSync(path.join(workDir, ".gitignore"), "node_modules\n"); - - await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); - await writeMigrateEnvValues({ CLERK_FIREBASE_MEM_COST: "14" }, workDir); - - expect(read(".gitignore")).toBe(`node_modules\n${MIGRATE_ENV_FILE}\n`); - }); - - // The header `mergeEnvVars` adds is right for an app's shared .env and wrong - // here — one `settings set` per key would stack one header per call. - test("adds no section header, however many times it is called", async () => { - await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); - await writeMigrateEnvValues({ CLERK_FIREBASE_MEM_COST: "14" }, workDir); - await writeMigrateEnvValues({ CLERK_FIREBASE_SIGNER_KEY: "k" }, workDir); - - expect(read(MIGRATE_ENV_FILE)).not.toContain("#"); - }); - - test("updates a key in place rather than appending a second copy", async () => { - await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); - await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "10" }, workDir); - - expect(read(MIGRATE_ENV_FILE)).toBe("CLERK_FIREBASE_ROUNDS=10\n"); - }); - - // The file is meant to be hand-editable, so a write must not flatten it. - test("preserves hand-written comments and unrelated keys", async () => { - fs.writeFileSync(envFile(), "# my note\nOTHER=keep\n"); - - await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); - - expect(read(MIGRATE_ENV_FILE)).toBe("# my note\nOTHER=keep\nCLERK_FIREBASE_ROUNDS=8\n"); - }); -}); - -describe("findMigrateEnvValue", () => { - test("reads a value out of the file", async () => { - await writeMigrateEnvValues({ CLERK_FIREBASE_SIGNER_KEY: "from-file" }, workDir); - - const located = await findMigrateEnvValue(["CLERK_FIREBASE_SIGNER_KEY"], workDir, {}); - expect(located).toEqual({ - value: "from-file", - name: "CLERK_FIREBASE_SIGNER_KEY", - source: MIGRATE_ENV_FILE, - }); - }); - - test("beats the app's own .env.local", async () => { - fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=1\n"); - await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); - - const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, {}); - expect(located?.value).toBe("8"); - }); - - // An exported variable is the one thing an operator can change per-invocation. - test("loses to an exported environment variable", async () => { - await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); - - const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { - CLERK_FIREBASE_ROUNDS: "99", - }); - expect(located).toEqual({ - value: "99", - name: "CLERK_FIREBASE_ROUNDS", - source: "CLERK_FIREBASE_ROUNDS env var", - }); - }); - - test("returns nothing when the setting is absent everywhere", async () => { - expect(await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, {})).toBeUndefined(); - }); - - // Bun loads `.env.local` into process.env before the CLI runs, so a value a - // developer put in a file arrives looking like an exported variable. Naming - // the variable answers nothing — the question is which file to edit. - describe("attributing an environment value to the file it came from", () => { - test("names the file when it holds the same value", async () => { - fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); - - const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { - CLERK_FIREBASE_ROUNDS: "8", - }); - expect(located?.source).toBe(".env.local"); - }); - - // The one case the source column exists for: the file lost, so naming it - // would point at the value that is not being used. - test("keeps the variable when the file holds a different value", async () => { - fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); - - const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { - CLERK_FIREBASE_ROUNDS: "99", - }); - expect(located?.source).toBe("CLERK_FIREBASE_ROUNDS env var"); - }); - - test("prefers the file the runtime would have loaded last", async () => { - fs.writeFileSync(path.join(workDir, ".env"), "CLERK_FIREBASE_ROUNDS=8\n"); - fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); - - const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { - CLERK_FIREBASE_ROUNDS: "8", - }); - expect(located?.source).toBe(".env.local"); - }); - - test("keeps the variable when no file holds it at all", async () => { - const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { - CLERK_FIREBASE_ROUNDS: "8", - }); - expect(located?.source).toBe("CLERK_FIREBASE_ROUNDS env var"); - }); - }); -}); - -describe("clearMigrateEnvValues", () => { - test("removes only the named settings", async () => { - fs.writeFileSync(envFile(), "OTHER=keep\nCLERK_FIREBASE_ROUNDS=8\n"); - - expect(await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir)).toEqual([ - "CLERK_FIREBASE_ROUNDS", - ]); - expect(read(MIGRATE_ENV_FILE)).toBe("OTHER=keep\n"); - }); - - // Left behind, it reads as "there is config here" when there is not. - test("deletes the file when nothing but comments would remain", async () => { - fs.writeFileSync(envFile(), "# a note\nCLERK_FIREBASE_ROUNDS=8\n"); - - await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir); - expect(fs.existsSync(envFile())).toBe(false); - }); - - test("reports nothing dropped when there is no file", async () => { - expect(await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir)).toEqual([]); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.ts b/packages/cli-core/src/commands/migrate/lib/env-file.ts deleted file mode 100644 index 2c15c9163..000000000 --- a/packages/cli-core/src/commands/migrate/lib/env-file.ts +++ /dev/null @@ -1,162 +0,0 @@ -/** - * `.env.clerk-migrate` — the migration's own env file. - * - * Migration credentials are a Firebase signer key, an Auth0 client secret, a - * database URL: things the app being migrated has no use for. Writing them into - * the app's `.env.local` mixes two unrelated sets of config in the file a - * developer reads every day, so they get their own. - * - * Read ahead of `.env`/`.env.local`, so a value set here wins over a stale one - * left in the app's file. An exported shell variable still beats both — that is - * {@link findEnvValue}'s contract for every value the CLI resolves. - * - * Always added to `.gitignore` on write. The CLI creating a credential-bearing - * file in someone's repository without that is how one ends up committed. - */ - -import { unlink } from "node:fs/promises"; -import { join } from "node:path"; -import { - findEnvValue, - parseEnvFile, - serializeEnvFile, - type EnvLine, - type LocatedEnvValue, -} from "../../../lib/dotenv.ts"; -import { ensureGitignoreEntry } from "../../../lib/git.ts"; -import { log } from "../../../lib/log.ts"; - -export const MIGRATE_ENV_FILE = ".env.clerk-migrate"; - -/** Lowest priority first: the migration's own file overrides the app's. */ -const MIGRATE_ENV_FILES = [".env", ".env.local", MIGRATE_ENV_FILE] as const; - -/** - * The project env file a value in the environment actually came from, if any. - * - * Bun loads `.env`, `.env.local` and friends into `process.env` before the CLI - * runs, so a variable a developer wrote into `.env.local` reaches - * {@link findEnvValue} as an environment variable and gets reported as one. - * That is true but useless: "`ROUNDS` env var" does not tell anyone which of - * their files to edit. - * - * Attribution is by value, not by presence. A file that holds the same key with - * a *different* value lost to something exported in the shell, and saying - * `.env.local` there would name the file that is not winning — the one case - * this column exists to catch. Highest-priority file first, matching the order - * the runtime loaded them in. - */ -async function fileHolding( - cwd: string, - { name, value }: LocatedEnvValue, -): Promise { - for (const envFile of [...MIGRATE_ENV_FILES].reverse()) { - const file = Bun.file(join(cwd, envFile)); - if (!(await file.exists())) continue; - - for (const line of parseEnvFile(await file.text())) { - if (line.type === "entry" && line.key === name && line.value === value) return envFile; - } - } - return undefined; -} - -/** Resolves a migration setting: environment first, then the project's env files. */ -export async function findMigrateEnvValue( - names: string[], - cwd: string = process.cwd(), - env: Record = process.env, -): Promise { - const located = await findEnvValue(cwd, names, { env, files: MIGRATE_ENV_FILES }); - if (!located) return undefined; - - // `findEnvValue` reports the environment before it reads a file, so a value - // the runtime loaded out of `.env.local` is credited to the variable rather - // than to the file the user would edit. Put the file back. - const source = located.source.endsWith(" env var") - ? ((await fileHolding(cwd, located)) ?? located.source) - : located.source; - - log.debug(`migrate: ${names[0]} from ${source}`); - return { ...located, source }; -} - -/** - * Merges `values` into the parsed file: existing keys update in place, new ones - * append. - * - * Deliberately not `mergeEnvVars` from `lib/dotenv.ts`. That one prepends a - * `# Clerk` section header when the file holds none of the keys being written, - * which is right for `env pull` dropping Clerk keys into an app's shared `.env` - * — and wrong here twice over: every key in this file is already Clerk's, and - * writing one setting at a time means the check fires again on every call, - * stacking a fresh header per `settings set`. - */ -function mergeMigrateEnv(lines: EnvLine[], values: Record): EnvLine[] { - const remaining = { ...values }; - - const merged = lines.map((line): EnvLine => { - if (line.type !== "entry" || !(line.key in remaining)) return line; - const value = remaining[line.key]!; - delete remaining[line.key]; - return { type: "entry", key: line.key, value, raw: `${line.key}=${value}` }; - }); - - for (const [key, value] of Object.entries(remaining)) { - merged.push({ type: "entry", key, value, raw: `${key}=${value}` }); - } - return merged; -} - -/** - * Writes settings into `.env.clerk-migrate`, creating and gitignoring it first. - * - * Existing comments, blank lines and key order survive — the file is meant to - * be hand-edited, so rewriting it wholesale would discard the user's notes. - */ -export async function writeMigrateEnvValues( - values: Record, - cwd: string = process.cwd(), -): Promise { - const target = join(cwd, MIGRATE_ENV_FILE); - const existing = await Bun.file(target) - .text() - .catch(() => ""); - - await Bun.write(target, serializeEnvFile(mergeMigrateEnv(parseEnvFile(existing), values))); - await ensureGitignoreEntry(cwd, MIGRATE_ENV_FILE); - - return MIGRATE_ENV_FILE; -} - -/** Removes the named settings from `.env.clerk-migrate`, leaving the rest. */ -export async function clearMigrateEnvValues( - names: string[], - cwd: string = process.cwd(), -): Promise { - const target = join(cwd, MIGRATE_ENV_FILE); - const existing = await Bun.file(target) - .text() - .catch(() => ""); - if (!existing) return []; - - const dropped: string[] = []; - const kept = parseEnvFile(existing).filter((line) => { - if (line.type !== "entry" || !names.includes(line.key)) return true; - dropped.push(line.key); - return false; - }); - - if (dropped.length === 0) return dropped; - - // A file holding nothing but the comments that described the settings it no - // longer has is worse than no file: it reads as "there is config here". - if (kept.some((line) => line.type === "entry")) { - await Bun.write(target, serializeEnvFile(kept)); - } else { - await unlink(target).catch(() => {}); - log.debug(`migrate: removed empty ${MIGRATE_ENV_FILE}`); - } - - return dropped; -} diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts index ec5d89fb2..b9d536f9d 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts @@ -1,12 +1,6 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { describe, expect, test } from "bun:test"; import { resolveFirebaseHashConfig } from "./firebase-hash.ts"; -const captured = useCaptureLog(); - const ALL_FLAGS = { firebaseSignerKey: "SIGNER", firebaseSaltSeparator: "Bw==", @@ -14,108 +8,22 @@ const ALL_FLAGS = { firebaseMemCost: 14, }; -const ENV = { - CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER", - CLERK_FIREBASE_SALT_SEPARATOR: "Bw==", - CLERK_FIREBASE_ROUNDS: "8", - CLERK_FIREBASE_MEM_COST: "14", -}; - -let workDir: string; -let originalCwd: string; - -const setEnv = (vars: Partial) => Object.assign(process.env, vars); - -beforeEach(() => { - originalCwd = process.cwd(); - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-fbhash-"))); - process.chdir(workDir); -}); - -afterEach(() => { - for (const name of Object.keys(ENV)) delete process.env[name]; - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); -}); - describe("gating on the transformer", () => { - // `migrate import` is one command for every platform, so a signer key left in - // .env.clerk-migrate after a Firebase migration is in scope for whatever runs - // next unless the transformer says otherwise. test.each([["clerk"], ["supabase"], ["auth0"], ["authjs"], ["betterauth"]])( - "reads nothing for the %s transformer", - async (transformer) => { - setEnv(ENV); - expect(await resolveFirebaseHashConfig({}, transformer)).toBeUndefined(); + "ignores even explicit flags for the %s transformer", + (transformer) => { + expect(resolveFirebaseHashConfig(ALL_FLAGS, transformer)).toBeUndefined(); }, ); - test("stays silent about a complete set on another platform's run", async () => { - setEnv(ENV); - await resolveFirebaseHashConfig({}, "supabase"); - expect(captured.err).toBe(""); - }); - - // The case that regressed: half a set used to fail every later run. - test("stays silent about a partial set on another platform's run", async () => { - fs.writeFileSync(path.join(workDir, ".env.clerk-migrate"), "CLERK_FIREBASE_SIGNER_KEY=left\n"); - - expect(await resolveFirebaseHashConfig({}, "supabase")).toBeUndefined(); - expect(captured.err).toBe(""); - }); - - test("ignores even explicit flags when the platform is not firebase", async () => { - expect(await resolveFirebaseHashConfig(ALL_FLAGS, "supabase")).toBeUndefined(); - }); - - test("resolves nothing before the platform is known", async () => { - setEnv(ENV); - expect(await resolveFirebaseHashConfig({}, undefined)).toBeUndefined(); + test("resolves nothing before the platform is known", () => { + expect(resolveFirebaseHashConfig(ALL_FLAGS, undefined)).toBeUndefined(); }); }); describe("on a firebase run", () => { - test("builds the config from flags", async () => { - expect(await resolveFirebaseHashConfig(ALL_FLAGS, "firebase")).toEqual({ - base64_signer_key: "SIGNER", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }); - }); - - test("falls back to the environment", async () => { - setEnv(ENV); - expect((await resolveFirebaseHashConfig({}, "firebase"))?.base64_signer_key).toBe("ENV_SIGNER"); - }); - - test("reads .env.clerk-migrate when the variable is not exported", async () => { - fs.writeFileSync( - path.join(workDir, ".env.clerk-migrate"), - Object.entries(ENV) - .map(([key, value]) => `${key}=${value}`) - .join("\n"), - ); - - expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); - }); - - test("prefers a flag over the environment", async () => { - setEnv(ENV); - expect((await resolveFirebaseHashConfig(ALL_FLAGS, "firebase"))?.base64_signer_key).toBe( - "SIGNER", - ); - }); - - test("fills only the gaps the flags left", async () => { - setEnv({ CLERK_FIREBASE_ROUNDS: "8", CLERK_FIREBASE_MEM_COST: "14" }); - - expect( - await resolveFirebaseHashConfig( - { firebaseSignerKey: "SIGNER", firebaseSaltSeparator: "Bw==" }, - "firebase", - ), - ).toEqual({ + test("builds the config from flags", () => { + expect(resolveFirebaseHashConfig(ALL_FLAGS, "firebase")).toEqual({ base64_signer_key: "SIGNER", base64_salt_separator: "Bw==", rounds: 8, @@ -130,82 +38,20 @@ describe("on a firebase run", () => { ["firebaseSaltSeparator", "--firebase-salt-separator"], ["firebaseRounds", "--firebase-rounds"], ["firebaseMemCost", "--firebase-mem-cost"], - ] as const)("rejects a flag set missing %s, naming it", async (omit, flag) => { + ] as const)("rejects a flag set missing %s, naming it", (omit, flag) => { const partial = { ...ALL_FLAGS }; delete (partial as Record)[omit]; - await expect(resolveFirebaseHashConfig(partial, "firebase")).rejects.toThrow(new RegExp(flag)); - }); - - test("names every missing flag at once", async () => { - await expect( - resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" }, "firebase"), - ).rejects.toThrow(/--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/); - }); - - // Saved config is a leftover, not an instruction — but on a Firebase import - // it is the reason the passwords will not come across, so it is said aloud. - test("warns and continues when only saved config is partial", async () => { - setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); - - expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); - expect(captured.err).toContain("Ignoring an incomplete Firebase hash configuration"); + expect(() => resolveFirebaseHashConfig(partial, "firebase")).toThrow(new RegExp(flag)); }); - test("still fails when a flag supplied part of the set", async () => { - setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); - - await expect(resolveFirebaseHashConfig({ firebaseRounds: 8 }, "firebase")).rejects.toThrow( - /--firebase-salt-separator/, + test("names every missing flag at once", () => { + expect(() => resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" }, "firebase")).toThrow( + /--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/, ); }); - // An empty variable is how a shell spells "unset". - test("ignores an empty variable", async () => { - setEnv({ CLERK_FIREBASE_SIGNER_KEY: "" }); - expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); - }); - - test("returns nothing when neither flags nor the environment supply a config", async () => { - expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); - }); -}); - -// Firebase names these `base64_signer_key`, `rounds` and friends, and that is -// how every guide — Clerk's own standalone script included — tells you to write -// them into `.env`. A project that followed one has the values already. -describe("the names Firebase itself uses", () => { - const writeEnvLocal = (contents: string) => - fs.writeFileSync(path.join(workDir, ".env.local"), contents); - - test("reads a set written under the unprefixed names", async () => { - writeEnvLocal("BASE64_SIGNER_KEY=SIGNER\nBASE64_SALT_SEPARATOR=Bw==\nROUNDS=8\nMEM_COST=14\n"); - - expect(await resolveFirebaseHashConfig({}, "firebase")).toEqual({ - base64_signer_key: "SIGNER", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }); - }); - - test("reads a set written under the FIREBASE_ prefix", async () => { - writeEnvLocal( - "FIREBASE_BASE64_SIGNER_KEY=SIGNER\nFIREBASE_BASE64_SALT_SEPARATOR=Bw==\n" + - "FIREBASE_ROUNDS=8\nFIREBASE_MEM_COST=14\n", - ); - - expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); - }); - - // The alias is a fallback, not a synonym: `ROUNDS` in an app's own env file - // is not necessarily about Firebase at all. - test("prefers the prefixed variable in the same file", async () => { - writeEnvLocal( - "ROUNDS=99\nCLERK_FIREBASE_ROUNDS=8\nBASE64_SIGNER_KEY=SIGNER\n" + - "BASE64_SALT_SEPARATOR=Bw==\nMEM_COST=14\n", - ); - - expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); + test("returns nothing when no flag supplies a config", () => { + expect(resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts index d32fc3c58..bab30d1a7 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts @@ -1,37 +1,22 @@ /** - * Firebase's four scrypt parameters: where they come from, and when they are - * looked for at all. + * Firebase's four scrypt parameters, read from the `--firebase-*` flags. * * **Only read when the transformer is `firebase`.** `migrate import` is one - * command serving every platform, so a `CLERK_FIREBASE_SIGNER_KEY` left in - * `.env.clerk-migrate` after a Firebase migration is in scope for the Supabase - * run that follows it unless something says otherwise. Nothing downstream would - * misuse it — only the Firebase transformer reads the config off - * {@link TransformContext} — but resolving it means a stale or partial set can - * warn, or fail, a run that never mentioned Firebase. So the gate is here, - * before the lookup, rather than a filter after it. - * - * The per-platform export commands need no such gate: `migrate export auth0` - * reads `AUTH0_*` and nothing else, because the command itself is the platform. - * This is the only place one command spans them all. + * command serving every platform, and nothing but the Firebase transformer + * reads the config off {@link TransformContext}. */ import { throwUsageError } from "../../../lib/errors.ts"; -import { log } from "../../../lib/log.ts"; -import { envNames, findSetting } from "../settings/registry.ts"; -import { findMigrateEnvValue } from "./env-file.ts"; import type { FirebaseHashConfig } from "../types.ts"; -/** The `--firebase-*` flags, and the setting each falls back to. */ +/** The `--firebase-*` flags, keyed by their option name. */ export const FIREBASE_FLAGS = [ - ["firebaseSignerKey", "--firebase-signer-key", "firebase-signer-key"], - ["firebaseSaltSeparator", "--firebase-salt-separator", "firebase-salt-separator"], - ["firebaseRounds", "--firebase-rounds", "firebase-rounds"], - ["firebaseMemCost", "--firebase-mem-cost", "firebase-mem-cost"], + ["firebaseSignerKey", "--firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost"], ] as const; -const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); - export type FirebaseHashFlags = { firebaseSignerKey?: string; firebaseSaltSeparator?: string; @@ -40,74 +25,30 @@ export type FirebaseHashFlags = { }; /** - * Overlays the saved environment values onto whichever flags were not passed. - * - * The variables come from the settings registry — `CLERK_FIREBASE_*` and the - * unprefixed names Firebase itself uses — so `clerk migrate settings` and the - * import read exactly the same set. Resolved through - * {@link findMigrateEnvValue}: the environment first, then - * `.env.clerk-migrate`, then the app's own `.env` files. The signer key is a - * Firebase secret, so it is never written to the CLI's config — - * `.env.clerk-migrate` is gitignored on creation. - */ -async function withFirebaseEnv(flags: FirebaseHashFlags): Promise { - const merged: FirebaseHashFlags = { ...flags }; - for (const [key, , settingName] of FIREBASE_FLAGS) { - if (merged[key] !== undefined) continue; - const setting = findSetting(settingName); - const located = setting && (await findMigrateEnvValue(envNames(setting))); - if (!located || located.value.trim() === "") continue; - // A non-numeric round count is left to fail the flag's own validation - // rather than silently becoming NaN. - (merged as Record)[key] = FIREBASE_NUMERIC.has(key) - ? Number(located.value) - : located.value; - } - return merged; -} - -/** - * Resolves the four parameters from flags, then the `CLERK_FIREBASE_*` - * variables, then the project's env files. + * Resolves the four parameters from the flags. * * The four are required as a set: a digest built from a partial set is * well-formed but verifies against nothing, so every migrated user would fail - * to sign in with no error at import time. How a partial set is treated depends - * on where it came from — flags are an instruction, saved config is not. + * to sign in with no error at import time. * * @param transformer - The platform being migrated. Anything but `firebase` - * returns immediately, without reading the environment. + * returns immediately. * @returns The config, or `undefined` when none was supplied — which is fine * for an export that carries no password hashes. */ -export async function resolveFirebaseHashConfig( +export function resolveFirebaseHashConfig( flags: FirebaseHashFlags, transformer: string | undefined, -): Promise { +): FirebaseHashConfig | undefined { if (transformer !== "firebase") return undefined; - const fromFlags = FIREBASE_FLAGS.filter(([key]) => flags[key] !== undefined); - const resolved = await withFirebaseEnv(flags); - const provided = FIREBASE_FLAGS.filter(([key]) => resolved[key] !== undefined); - + const provided = FIREBASE_FLAGS.filter(([key]) => flags[key] !== undefined); if (provided.length === 0) return undefined; if (provided.length < FIREBASE_FLAGS.length) { - const missing = FIREBASE_FLAGS.filter(([key]) => resolved[key] === undefined).map( + const missing = FIREBASE_FLAGS.filter(([key]) => flags[key] === undefined).map( ([, flag]) => flag, ); - - // Saved config is a leftover, not an instruction: half a set in - // `.env.clerk-migrate` should not fail the run, but on a Firebase import it - // is the reason the passwords will not come across, so it is said out loud. - if (fromFlags.length === 0) { - log.warn( - `Ignoring an incomplete Firebase hash configuration (no ${missing.join(", ")}). ` + - "Run `clerk migrate settings` to see what is set.", - ); - return undefined; - } - throwUsageError( `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", @@ -116,9 +57,9 @@ export async function resolveFirebaseHashConfig( } return { - base64_signer_key: resolved.firebaseSignerKey as string, - base64_salt_separator: resolved.firebaseSaltSeparator as string, - rounds: resolved.firebaseRounds as number, - mem_cost: resolved.firebaseMemCost as number, + base64_signer_key: flags.firebaseSignerKey as string, + base64_salt_separator: flags.firebaseSaltSeparator as string, + rounds: flags.firebaseRounds as number, + mem_cost: flags.firebaseMemCost as number, }; } diff --git a/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts b/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts deleted file mode 100644 index 7aba346fe..000000000 --- a/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -/** - * The one path `logger.test.ts` cannot cover: `ensureLogDir` actually asking. - * - * Its own file because `mock.module` registrations last for the process, and - * `bun test --parallel` puts several files in each worker — a mocked - * `prompts.ts` would leak into any file that later lands in the same worker and - * imports the real one. - */ - -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { _setConfigDir } from "../../../lib/config.ts"; -import { getMode, setMode, type Mode } from "../../../mode.ts"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; - -type TextConfig = { message: string; default?: string; placeholder?: string }; - -let answer = ""; -const mockText = mock(async (_config: TextConfig) => answer); - -// Every export of the real module must appear here — a missing one is a link -// error at import time, which takes down the whole file rather than one prompt. -mock.module("../../../lib/prompts.ts", () => ({ - text: (...args: unknown[]) => mockText(...(args as [TextConfig])), - confirm: async () => true, - multiselect: async () => [], - password: async () => "", - editor: async () => "{}", -})); - -const { _resetLogDir, ensureLogDir } = await import("./logger.ts"); -const { loadSettings, saveSettings } = await import("./settings.ts"); -const { setAssumeYes } = await import("./assume-yes.ts"); - -useCaptureLog(); - -let workDir: string; -let configDir: string; -let originalCwd: string; -let originalMode: Mode; -let originalEnv: string | undefined; - -beforeAll(() => { - originalCwd = process.cwd(); - originalMode = getMode(); - originalEnv = process.env.CLERK_MIGRATE_LOG_DIR; - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-"))); - configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-cfg-")); - _setConfigDir(configDir); - process.chdir(workDir); -}); - -afterAll(() => { - setMode(originalMode); - if (originalEnv === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; - else process.env.CLERK_MIGRATE_LOG_DIR = originalEnv; - _setConfigDir(undefined); - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); - fs.rmSync(configDir, { recursive: true, force: true }); -}); - -beforeEach(() => { - _resetLogDir(); - delete process.env.CLERK_MIGRATE_LOG_DIR; - fs.rmSync(path.join(configDir, "config.json"), { force: true }); - mockText.mockClear(); - answer = ""; - setMode("human"); - setAssumeYes(false); -}); - -afterEach(() => _resetLogDir()); - -describe("ensureLogDir asks once", () => { - test("saves the answer, so the next run does not ask", async () => { - answer = "./migration-logs"; - - expect(await ensureLogDir()).toBe(path.join(workDir, "migration-logs")); - expect(await loadSettings()).toMatchObject({ logDir: "./migration-logs" }); - - _resetLogDir(); - expect(await ensureLogDir()).toBe(path.join(workDir, "migration-logs")); - expect(mockText).toHaveBeenCalledTimes(1); - }); - - test("offers ./logs as the default", async () => { - await ensureLogDir(); - expect(mockText.mock.calls[0]?.[0]).toMatchObject({ default: "./logs" }); - }); - - // Enter on the prompt is an answer, not a skip: it settles the question so - // the next run goes straight to importing. - test("treats an empty answer as ./logs and remembers it", async () => { - answer = " "; - - expect(await ensureLogDir()).toBe(path.join(workDir, "logs")); - expect(await loadSettings()).toMatchObject({ logDir: "./logs" }); - }); - - test("leaves the project's other settings alone", async () => { - await saveSettings({ transformer: "firebase", file: "users.json" }); - answer = "./audit"; - - await ensureLogDir(); - - expect(await loadSettings()).toEqual({ - transformer: "firebase", - file: "users.json", - logDir: "./audit", - }); - }); -}); - -describe("ensureLogDir with `-y`", () => { - // Agent mode cannot reach this prompt; `-y` is a human on a TTY who could be - // asked and said not to be, so it needs its own check. - test("takes ./logs without asking", async () => { - setAssumeYes(true); - - expect(await ensureLogDir()).toBe(path.join(workDir, "logs")); - expect(mockText).not.toHaveBeenCalled(); - }); - - // Landing on a default is not a choice, and recording one would retire the - // question for a human who never saw it. - test("saves nothing, so the next interactive run still asks", async () => { - setAssumeYes(true); - await ensureLogDir(); - expect((await loadSettings()).logDir).toBeUndefined(); - - _resetLogDir(); - setAssumeYes(false); - answer = "./audit"; - - expect(await ensureLogDir()).toBe(path.join(workDir, "audit")); - expect(mockText).toHaveBeenCalledTimes(1); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.test.ts b/packages/cli-core/src/commands/migrate/lib/logger.test.ts index e9176df9a..65000aaa9 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.test.ts @@ -3,21 +3,13 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { - _resetLogDir, - DEFAULT_LOG_DIR, - ensureLogDir, errorLogger, getDateTimeStamp, getLogDir, getLogFilePath, importLogger, - resolveLogDir, validationLogger, } from "./logger.ts"; -import { _setConfigDir } from "../../../lib/config.ts"; -import { getMode, setMode, type Mode } from "../../../mode.ts"; -import { MIGRATE_ENV_FILE } from "./env-file.ts"; -import { loadSettings, saveSettings } from "./settings.ts"; const DATE_TIME = "2026-01-01T12:00:00"; @@ -38,7 +30,6 @@ afterAll(() => { }); beforeEach(() => { - _resetLogDir(); fs.rmSync(getLogDir(), { recursive: true, force: true }); }); @@ -119,87 +110,30 @@ describe("log writers", () => { }); describe("resolving the log directory", () => { - let configDir: string; - let originalMode: Mode; let originalEnv: string | undefined; beforeAll(() => { - originalMode = getMode(); originalEnv = process.env.CLERK_MIGRATE_LOG_DIR; - configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-cfg-")); - _setConfigDir(configDir); }); afterAll(() => { - setMode(originalMode); if (originalEnv === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; else process.env.CLERK_MIGRATE_LOG_DIR = originalEnv; - _setConfigDir(undefined); - fs.rmSync(configDir, { recursive: true, force: true }); }); beforeEach(() => { delete process.env.CLERK_MIGRATE_LOG_DIR; - fs.rmSync(path.join(configDir, "config.json"), { force: true }); - fs.rmSync(path.join(workDir, MIGRATE_ENV_FILE), { force: true }); - setMode("agent"); }); - test("falls back to ./logs when nothing has chosen one", async () => { - expect(await resolveLogDir()).toBe(path.join(workDir, "logs")); - }); - - test("prefers the saved setting over the default", async () => { - await saveSettings({ logDir: "./audit" }); - expect(await resolveLogDir()).toBe(path.join(workDir, "audit")); - }); - - // A variable exported for one shell is the narrower statement of the two. - test("prefers the environment over the saved setting", async () => { - await saveSettings({ logDir: "./audit" }); - process.env.CLERK_MIGRATE_LOG_DIR = "./from-env"; - - expect(await resolveLogDir()).toBe(path.join(workDir, "from-env")); - }); - - test("reads the migration's own env file", async () => { - fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "CLERK_MIGRATE_LOG_DIR=./from-file\n"); - expect(await resolveLogDir()).toBe(path.join(workDir, "from-file")); + test("falls back to ./logs", () => { + expect(getLogDir()).toBe(path.join(workDir, "logs")); }); - // Every synchronous write reads the settled value, so resolving is what makes - // the log files land anywhere but ./logs. - test("settles the directory the log writers use", async () => { - await saveSettings({ logDir: "./audit" }); - await resolveLogDir(); + test("reads CLERK_MIGRATE_LOG_DIR", () => { + process.env.CLERK_MIGRATE_LOG_DIR = "./audit"; expect(getLogFilePath("import", DATE_TIME)).toBe( path.join(workDir, "audit", `import-${DATE_TIME.replace(/:/g, "-")}.log`), ); }); - - describe("ensureLogDir", () => { - test("takes the default without saving it when nobody can be asked", async () => { - expect(await ensureLogDir()).toBe(path.join(workDir, DEFAULT_LOG_DIR)); - // Nothing saved: the question stays open for the first interactive run. - expect(await loadSettings()).toEqual({}); - }); - - test("does not ask once the setting is saved", async () => { - await saveSettings({ logDir: "./audit" }); - setMode("human"); - - // Reaching the prompt in a test without a TTY throws, so returning is the - // assertion. - expect(await ensureLogDir()).toBe(path.join(workDir, "audit")); - }); - - test("does not ask when the environment already answers", async () => { - process.env.CLERK_MIGRATE_LOG_DIR = "./from-env"; - setMode("human"); - - expect(await ensureLogDir()).toBe(path.join(workDir, "from-env")); - expect(await loadSettings()).toEqual({}); - }); - }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.ts b/packages/cli-core/src/commands/migrate/lib/logger.ts index 3eca2cdce..3d7ef5133 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.ts @@ -14,12 +14,6 @@ import fs from "node:fs"; import path from "node:path"; import { log } from "../../../lib/log.ts"; -import { text } from "../../../lib/prompts.ts"; -import { isAgent, isHuman } from "../../../mode.ts"; -import { envNames, findSetting } from "../settings/registry.ts"; -import { isAssumeYes } from "./assume-yes.ts"; -import { findMigrateEnvValue } from "./env-file.ts"; -import { loadSettings, saveSettings } from "./settings.ts"; import type { DeleteLogEntry, ErrorLog, @@ -29,110 +23,12 @@ import type { ValidationErrorPayload, } from "../types.ts"; -/** Where logs go when nobody has said otherwise. */ +/** Where logs go when `CLERK_MIGRATE_LOG_DIR` is not set. */ export const DEFAULT_LOG_DIR = "./logs"; -/** - * The directory settled for this process, once something has settled it. - * - * The log writers are synchronous — a run interrupted with Ctrl-C has to leave - * a complete record of what it already processed — but resolving the directory - * reads the config, the env files and possibly the operator. So resolution - * happens once, up front, and every synchronous write reads the answer from - * here. {@link resolveLogDir} and {@link ensureLogDir} are the only writers. - */ -let settled: string | undefined; - -function remember(dir: string): string { - settled = path.resolve(process.cwd(), dir); - return settled; -} - -/** Forgets the settled directory. Tests only — each one resolves its own. */ -export function _resetLogDir(): void { - settled = undefined; -} - -/** - * Absolute path of the log directory. - * - * Falls back to `./logs` when nothing has resolved yet, so a caller that - * forgets to is wrong about *where*, never broken. - */ +/** Absolute path of the log directory: `CLERK_MIGRATE_LOG_DIR`, else `./logs`. */ export function getLogDir(): string { - return settled ?? path.resolve(process.cwd(), DEFAULT_LOG_DIR); -} - -/** The `log-dir` setting, which owns both the env var and the config key. */ -const LOG_DIR = findSetting("log-dir") as NonNullable>; - -/** - * The directory the operator has already chosen, by either route. - * - * The environment wins over the remembered value, matching every other setting - * the CLI resolves: a variable exported for one shell is the narrower, more - * deliberate statement of the two. - */ -async function chosenLogDir(): Promise { - const located = await findMigrateEnvValue(envNames(LOG_DIR)); - if (located?.value) return located.value; - return (await loadSettings()).logDir; -} - -/** - * Settles the log directory without asking: environment, then the saved - * setting, then `./logs`. - * - * For the read-only log commands. Landing on the default here does not save it - * — an operator who has only ever *listed* logs has still made no choice, and - * recording one on their behalf would skip the question forever. - */ -export async function resolveLogDir(): Promise { - return remember((await chosenLogDir()) ?? DEFAULT_LOG_DIR); -} - -/** - * Settles the log directory, asking a human who has not chosen yet. - * - * Migration logs are the only record of which users landed and which failed, - * and `migrate delete` reads them to undo a run — so where they go is worth one - * question, once per project, before the first thing is written. The answer is - * saved, so it is asked once and never again. - * - * `-y`, agent mode and a non-TTY take the default rather than a prompt they - * cannot answer, and save nothing: the question stays open for the first - * interactive run. - */ -export async function ensureLogDir(): Promise { - const chosen = await chosenLogDir(); - if (chosen) return remember(chosen); - if (!isHuman() || isAgent() || isAssumeYes()) return remember(DEFAULT_LOG_DIR); - - const answer = await text({ - message: "Where should migration logs be saved?", - default: DEFAULT_LOG_DIR, - placeholder: DEFAULT_LOG_DIR, - }); - const dir = answer.trim() || DEFAULT_LOG_DIR; - - await saveSettings({ ...(await loadSettings()), logDir: dir }); - log.info( - `Saving migration logs to ${dir}. Change it with \`clerk migrate settings set log-dir \`.`, - ); - - return remember(dir); -} - -/** - * Settles where this run's logs go, and stamps it. - * - * Every command that writes a log starts here rather than calling - * {@link getDateTimeStamp} directly, so there is no path on which a log file is - * named before its directory has been resolved. - */ -export async function startLogging(): Promise { - await ensureLogDir(); - return getDateTimeStamp(); + return path.resolve(process.cwd(), process.env.CLERK_MIGRATE_LOG_DIR || DEFAULT_LOG_DIR); } /** diff --git a/packages/cli-core/src/commands/migrate/lib/settings.test.ts b/packages/cli-core/src/commands/migrate/lib/settings.test.ts deleted file mode 100644 index 7ad3cf9ed..000000000 --- a/packages/cli-core/src/commands/migrate/lib/settings.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { afterAll, afterEach, beforeAll, beforeEach, expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { _setConfigDir, getMigrationEntry, getProjectKey } from "../../../lib/config.ts"; -import { loadSettings, saveSettings } from "./settings.ts"; - -let workDir: string; -let configDir: string; -let originalCwd: string; - -beforeAll(() => { - originalCwd = process.cwd(); - // Realpath'd because the project key is derived from `process.cwd()`, which - // resolves the /var → /private/var symlink macOS puts in front of tmpdir. - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-"))); - process.chdir(workDir); -}); - -afterAll(() => { - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); -}); - -beforeEach(() => { - configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-config-")); - _setConfigDir(configDir); -}); - -afterEach(() => { - _setConfigDir(undefined); - fs.rmSync(configDir, { recursive: true, force: true }); -}); - -test("returns empty settings when nothing was saved", async () => { - expect(await loadSettings()).toEqual({}); -}); - -test("round-trips the transformer key and file path", async () => { - await saveSettings({ transformer: "clerk", file: "users.json" }); - expect(await loadSettings()).toEqual({ transformer: "clerk", file: "users.json" }); -}); - -test("writes to the CLI config file, not the working directory", async () => { - await saveSettings({ transformer: "clerk" }); - - expect(fs.existsSync(path.join(workDir, ".settings"))).toBe(false); - const config = JSON.parse(fs.readFileSync(path.join(configDir, "config.json"), "utf-8")); - expect(config.migrations).toEqual({ [await getProjectKey(workDir)]: { transformer: "clerk" } }); -}); - -test("keys the record by project, so another directory does not see it", async () => { - await saveSettings({ transformer: "clerk", file: "users.json" }); - - const elsewhere = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-other-"))); - try { - expect(await getMigrationEntry(await getProjectKey(elsewhere))).toBeUndefined(); - } finally { - fs.rmSync(elsewhere, { recursive: true, force: true }); - } -}); - -test("treats a corrupt config file as empty rather than failing the run", async () => { - fs.writeFileSync(path.join(configDir, "config.json"), "{not json"); - expect(await loadSettings()).toEqual({}); -}); - -test("leaves the run standing when the config cannot be written", async () => { - fs.rmSync(configDir, { recursive: true, force: true }); - fs.writeFileSync(configDir, "not a directory"); - - await saveSettings({ transformer: "clerk" }); - expect(await loadSettings()).toEqual({}); -}); diff --git a/packages/cli-core/src/commands/migrate/lib/settings.ts b/packages/cli-core/src/commands/migrate/lib/settings.ts deleted file mode 100644 index 958bf1cd2..000000000 --- a/packages/cli-core/src/commands/migrate/lib/settings.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * What this project last migrated, and with which transformer. - * - * Kept in the CLI's own config file under `migrations`, keyed by project — the - * same shape `clerk webhooks listen` files its relay token under. An earlier - * version wrote a `.settings` file into the user's cwd instead, which the CLI - * cannot gitignore on the user's behalf and which put migration state inside - * the repository being migrated. - * - * Both halves fail silently: an unreadable or unwritable config only costs the - * user a remembered default, so it must not take the run down with it. - */ - -import { - getMigrationEntry, - getProjectKey, - setMigrationEntry, - type MigrationEntry, -} from "../../../lib/config.ts"; -import { log } from "../../../lib/log.ts"; - -/** Reads saved settings, or `{}` when absent or unreadable. */ -export async function loadSettings(): Promise { - try { - return (await getMigrationEntry(await getProjectKey(process.cwd()))) ?? {}; - } catch (error) { - log.debug(`config: could not read migration settings — ${error}`); - return {}; - } -} - -/** Persists settings for the next run in this project. */ -export async function saveSettings(settings: MigrationEntry): Promise { - try { - await setMigrationEntry(await getProjectKey(process.cwd()), settings); - } catch (error) { - log.debug(`config: could not save migration settings — ${error}`); - } -} diff --git a/packages/cli-core/src/commands/migrate/logs/clean.ts b/packages/cli-core/src/commands/migrate/logs/clean.ts index 4d17d684d..3c609f5ad 100644 --- a/packages/cli-core/src/commands/migrate/logs/clean.ts +++ b/packages/cli-core/src/commands/migrate/logs/clean.ts @@ -16,7 +16,7 @@ import { confirm } from "../../../lib/prompts.ts"; import { withGutter } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { listLogFiles } from "../lib/log-files.ts"; -import { getLogDir, resolveLogDir } from "../lib/logger.ts"; +import { getLogDir } from "../lib/logger.ts"; export type LogsCleanOptions = { yes?: boolean; @@ -24,7 +24,6 @@ export type LogsCleanOptions = { export async function clean(options: LogsCleanOptions = {}): Promise { await withGutter("Cleaning migration logs", async () => { - await resolveLogDir(); const files = listLogFiles(); if (files.length === 0) { diff --git a/packages/cli-core/src/commands/migrate/logs/convert.ts b/packages/cli-core/src/commands/migrate/logs/convert.ts index 4536e70a5..33adf2bae 100644 --- a/packages/cli-core/src/commands/migrate/logs/convert.ts +++ b/packages/cli-core/src/commands/migrate/logs/convert.ts @@ -15,7 +15,7 @@ import { multiselect } from "../../../lib/prompts.ts"; import { withGutter } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { findLogFile, listLogFiles, readNdjson, type LogFile } from "../lib/log-files.ts"; -import { getLogDir, resolveLogDir } from "../lib/logger.ts"; +import { getLogDir } from "../lib/logger.ts"; export type LogsConvertOptions = { all?: boolean; @@ -85,7 +85,6 @@ export async function convert(options: LogsConvertOptions = {}): Promise { // The multiselect lives inside the gutter so cancelling it closes with // `└ Paused` rather than leaving a half-drawn frame. await withGutter("Converting migration logs", async () => { - await resolveLogDir(); const targets = await resolveTargets(options); if (targets.length === 0) return; diff --git a/packages/cli-core/src/commands/migrate/logs/index.ts b/packages/cli-core/src/commands/migrate/logs/index.ts index abb352d34..640a88a36 100644 --- a/packages/cli-core/src/commands/migrate/logs/index.ts +++ b/packages/cli-core/src/commands/migrate/logs/index.ts @@ -11,9 +11,7 @@ const logs = { clean, convert, list }; * * Noun-verb, matching every other group in the CLI (`config pull`, `users * list`) rather than the standalone tool's `clean-logs`/`convert-logs`, which - * were npm script names. Grouping also disambiguates the two deletes in this - * tree: `migrate logs clean` removes local files, `migrate delete` removes - * users from a Clerk instance. + * were npm script names. */ export function registerMigrateLogs(migrateCommand: Command<[], Record>): void { const logsCommand = migrateCommand diff --git a/packages/cli-core/src/commands/migrate/logs/list.ts b/packages/cli-core/src/commands/migrate/logs/list.ts index 81806166e..ae0f53928 100644 --- a/packages/cli-core/src/commands/migrate/logs/list.ts +++ b/packages/cli-core/src/commands/migrate/logs/list.ts @@ -10,7 +10,7 @@ import { bold, cyan, dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { withGutter } from "../../../lib/spinner.ts"; import { formatSize, listLogFiles, type LogFile, type LogKind } from "../lib/log-files.ts"; -import { displayLogDir, resolveLogDir } from "../lib/logger.ts"; +import { displayLogDir } from "../lib/logger.ts"; /** Every kind a log file can be, and what one entry in it records. */ const KIND_LEGEND: Record, string> = { @@ -53,9 +53,6 @@ export function formatTimestamp(stamp: string): string { } export async function list(options: LogsListOptions = {}): Promise { - // Resolve, never ask: listing is read-only, and "where should logs go?" is - // not a question to answer before showing someone the ones they have. - await resolveLogDir(); const files = listLogFiles(); if (options.json) { diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 1c7728c1d..c8b0308e9 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -14,12 +14,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { getMode, setMode, type Mode } from "../../mode.ts"; -import { - keylessTargetStubs, - listageStubs, - useCaptureLog, - useMigrateLogDir, -} from "../../test/lib/stubs.ts"; +import { keylessTargetStubs, listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; import type { InstanceTarget } from "../../lib/keyless-target.ts"; const mockSelect = mock(async () => "clerk" as unknown); @@ -70,13 +65,10 @@ mock.module("../../lib/prompts.ts", () => ({ })); const { run } = await import("./run.ts"); -const { deleteMigration } = await import("./delete.ts"); const { UserAbortError } = await import("../../lib/errors.ts"); -const { loadSettings, saveSettings } = await import("./lib/settings.ts"); const { _setConfigDir } = await import("../../lib/config.ts"); const captured = useCaptureLog(); -useMigrateLogDir(); let workDir: string; let configDir: string; @@ -197,12 +189,6 @@ describe("the wizard fills in missing flags", () => { expect(mockSelect).not.toHaveBeenCalled(); expect(mockText).toHaveBeenCalledTimes(1); }); - - test("records the wizard's answers for the next run", async () => { - await run({ secretKey: "sk_test_x" }); - - expect(await loadSettings()).toMatchObject({ transformer: "clerk", file: "export.json" }); - }); }); describe("the readiness report", () => { @@ -540,63 +526,3 @@ describe("guards that still apply interactively", () => { expect(created()).toHaveLength(0); }); }); - -describe("migrate delete confirmation", () => { - /** Answers the external-id lookup, then the deletes. */ - function stubDeleteTargets(present: Record) { - globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { - const url = input.toString(); - requests.push({ method: init?.method ?? "GET", url, body: null }); - - if (url.includes("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/v1/users?")) { - const asked = new URL(url).searchParams.getAll("external_id"); - return Response.json( - asked - .filter((externalId) => externalId in present) - .map((externalId) => ({ id: present[externalId], external_id: externalId })), - ); - } - return Response.json({ deleted: true }); - }) as unknown as typeof fetch; - } - - const deleted = () => requests.filter((r) => r.method === "DELETE"); - - beforeEach(async () => { - await saveSettings({ transformer: "clerk", file: "export.json" }); - stubDeleteTargets({ legacy_a: "user_1", legacy_b: "user_2" }); - fs.writeFileSync( - path.join(workDir, "export.json"), - JSON.stringify([ - { id: "legacy_a", primary_email_address: "a@x.dev" }, - { id: "legacy_b", primary_email_address: "b@x.dev" }, - ]), - ); - }); - - test("reports the count and confirms before deleting", async () => { - confirmAnswer = true; - - await deleteMigration({ secretKey: "sk_test_x" }); - - expect(captured.err).toContain("About to delete 2 users"); - expect(deleted()).toHaveLength(2); - }); - - // The undo for a bad undo does not exist, so declining must cost nothing. - test("declining deletes nobody", async () => { - confirmAnswer = false; - - await expect(deleteMigration({ secretKey: "sk_test_x" })).rejects.toThrow(UserAbortError); - - expect(deleted()).toHaveLength(0); - }); - - test("-y skips the prompt", async () => { - confirmAnswer = false; - - await deleteMigration({ yes: true, secretKey: "sk_test_x" }); - - expect(deleted()).toHaveLength(2); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index b1a655e38..7e531b1e7 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -11,7 +11,6 @@ import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); import { getLogDir } from "./lib/logger.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; -import { loadSettings, saveSettings } from "./lib/settings.ts"; import { applyResumeAfter, explainErrors, run, validateRunOptions } from "./run.ts"; import type { User } from "./types.ts"; @@ -171,21 +170,6 @@ describe("run", () => { expect(entries.filter((e) => e.status === "success")).toHaveLength(2); }); - test("records the run's transformer and file for the next run", async () => { - await run(baseOptions); - expect(await loadSettings()).toEqual({ transformer: "clerk", file: "export.json" }); - }); - - // `startLogging` settles and saves `logDir` before the first user is - // processed. Writing the transformer and file as a fresh object dropped it - // again mid-run, so `clerk migrate logs` had nowhere to read the log the run - // had just written. - test("keeps the log directory it saved at the start of the run", async () => { - await saveSettings({ logDir: "./custom-logs" }); - await run(baseOptions); - expect((await loadSettings()).logDir).toBe("./custom-logs"); - }); - test("--require-password imports only the users that have one", async () => { await run({ ...baseOptions, requirePassword: true }); @@ -718,14 +702,6 @@ describe("run", () => { expect(created()).toHaveLength(2); expect(captured.err).toContain("only applies to supabase"); }); - - test("records the flag for the next run", async () => { - stubInstance({ oauth_discord: { enabled: true } }); - - await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); - - expect((await loadSettings()).skipUnsupportedProviders).toBe(true); - }); }); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 23fe660ef..e3b6386a5 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -57,8 +57,7 @@ import { type SettingChange, } from "./lib/modify-settings.ts"; import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; -import { startLogging, getLogFilePath } from "./lib/logger.ts"; -import { loadSettings, saveSettings } from "./lib/settings.ts"; +import { getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; import { countSocialProviders, findDisabledProviders, @@ -539,7 +538,7 @@ async function resolveMissingOptions(options: MigrateRunOptions): Promise { const options = await resolveMissingOptions(rawOptions); const { transformer, file } = validateRunOptions(options); - const firebaseHashConfig = await resolveFirebaseHashConfig(options, transformer); + const firebaseHashConfig = resolveFirebaseHashConfig(options, transformer); await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const limits = resolveLimits(secretKey); - const dateTime = await startLogging(); + const dateTime = getDateTimeStamp(); const logFile = getLogFilePath("import", dateTime); const { users: loaded, validationFailed } = await withSpinner( @@ -735,20 +734,6 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (!proceed) throwUserAbort(); } - // The Firebase hash parameters are deliberately not among these: the signer - // key is a secret, and remembering it would write it to disk in plaintext. - // - // Spread over what is already saved rather than written fresh: `logDir` was - // settled and saved by `startLogging` at the top of this run, and a bare - // object here would drop it — leaving `clerk migrate logs` with no - // directory to read the run it just wrote. - await saveSettings({ - ...(await loadSettings()), - transformer, - file, - ...(options.skipUnsupportedProviders ? { skipUnsupportedProviders: true } : {}), - }); - const summary = await withSpinner(`Importing users: [0/${users.length}]...`, async (spinner) => importUsers({ users, diff --git a/packages/cli-core/src/commands/migrate/settings/clear.ts b/packages/cli-core/src/commands/migrate/settings/clear.ts deleted file mode 100644 index a7d8a981d..000000000 --- a/packages/cli-core/src/commands/migrate/settings/clear.ts +++ /dev/null @@ -1,154 +0,0 @@ -/** - * `clerk migrate settings clear [name]` — forget this project's migration - * settings, or just one of them. - * - * Clears both stores when given no name. The credentials half is the reason - * this command exists: after a migration finishes, a Firebase signer key - * sitting in the repo has no further use, and "delete the file yourself" is a - * step people skip. - * - * `migrate delete` reads the saved transformer and file to know what to undo, - * so clearing is confirmed unless `-y` — an operator who clears and then wants - * to undo has no record left to undo from. - */ - -import { throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; -import { log } from "../../../lib/log.ts"; -import { confirm } from "../../../lib/prompts.ts"; -import { isAgent, isHuman } from "../../../mode.ts"; -import { clearMigrateEnvValues, MIGRATE_ENV_FILE } from "../lib/env-file.ts"; -import { loadSettings, saveSettings } from "../lib/settings.ts"; -import type { MigrationEntry } from "../../../lib/config.ts"; -import { envNames, findSetting, SETTING_NAMES, SETTINGS } from "./registry.ts"; - -export type SettingsClearOptions = { - yes?: boolean; -}; - -/** - * Every variable the migration settings own, `log-dir`'s included. - * - * Keyed on declaring an `envVar` rather than on `store === "env"`: `log-dir` is - * remembered in the config but still answers to a variable, and a full clear - * that left that variable behind would not have cleared the setting. - */ -const ENV_VARS = SETTINGS.filter((s) => s.envVar).map((s) => s.envVar as string); - -/** - * Warns that clearing this is what `migrate delete` reads to find the users the - * last run created. - */ -function warnAboutUndo(saved: MigrationEntry): void { - if (!saved.file) return; - log.warn( - `\`clerk migrate delete\` uses the saved file (${saved.file}) to identify the users the last run created. ` + - "Clearing it leaves nothing to undo from.", - ); -} - -/** - * Clears one named setting, leaving the rest of the project's settings alone. - * - * Both stores are cleared, because a setting can sit in either and `log-dir` - * can sit in both. Clearing half of one is worse than clearing none: the - * command reports the setting gone while the next run still reads it. - * - * An `env` value goes under every spelling the setting answers to, not just the - * prefixed one — dropping `CLERK_FIREBASE_ROUNDS` while `ROUNDS` stayed in the - * same file would leave the old value winning. - */ -async function clearOne(name: string, options: SettingsClearOptions): Promise { - const setting = findSetting(name); - if (!setting) { - throwUsageError( - `Unknown setting "${name}". Valid names: ${SETTING_NAMES.join(", ")}.`, - undefined, - undefined, - [ - { - command: "clerk migrate settings", - description: "List the settings and their current values", - }, - ], - ); - } - - const saved = await loadSettings(); - - if (!options.yes && isHuman() && !isAgent()) { - // Only the file itself is what `migrate delete` cannot do without; the - // transformer it can be told again. - if (setting.configKey === "file") warnAboutUndo(saved); - const proceed = await confirm({ message: `Clear \`${name}\`?`, default: false }); - if (!proceed) throwUserAbort(); - } - - const cleared: string[] = []; - - if (setting.envVar && (await clearMigrateEnvValues(envNames(setting))).length > 0) { - cleared.push(MIGRATE_ENV_FILE); - } - - const key = setting.configKey as keyof MigrationEntry | undefined; - if (key && saved[key] !== undefined) { - const { [key]: _cleared, ...rest } = saved; - await saveSettings(rest); - cleared.push("this project's settings"); - } - - if (cleared.length === 0) { - log.info( - `\`${name}\` is not set here. A value coming from the app's own env files or the shell has ` + - "to be removed there — run `clerk migrate settings` to see which is supplying it.", - ); - return; - } - - log.success(`Cleared \`${name}\` from ${cleared.join(" and ")}.`); -} - -export async function clear(options: SettingsClearOptions = {}, name?: string): Promise { - if (name !== undefined) return clearOne(name, options); - - const saved = await loadSettings(); - const hadConfig = Object.keys(saved).length > 0; - - if (!options.yes) { - if (isAgent() || !isHuman()) { - throwUsageError( - "`clerk migrate settings clear` forgets this project's settings and every credential in " + - `${MIGRATE_ENV_FILE}, and cannot prompt here. Pass -y to confirm.`, - undefined, - undefined, - [ - { - command: "clerk migrate settings clear -y", - description: "Forget them all without prompting", - }, - ], - ); - } - - if (hadConfig) warnAboutUndo(saved); - const proceed = await confirm({ - message: "Clear this project's migration settings?", - default: false, - }); - if (!proceed) throwUserAbort(); - } - - if (hadConfig) await saveSettings({}); - const dropped = await clearMigrateEnvValues(ENV_VARS); - - if (!hadConfig && dropped.length === 0) { - log.info("No migration settings to clear for this project."); - return; - } - - if (hadConfig) log.success("Cleared the saved transformer and file."); - if (dropped.length > 0) { - log.success( - `Removed ${dropped.length} credential${dropped.length === 1 ? "" : "s"} from ${MIGRATE_ENV_FILE}.`, - ); - } -} diff --git a/packages/cli-core/src/commands/migrate/settings/index.ts b/packages/cli-core/src/commands/migrate/settings/index.ts deleted file mode 100644 index 833e0335c..000000000 --- a/packages/cli-core/src/commands/migrate/settings/index.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { createArgument, InvalidArgumentError } from "@commander-js/extra-typings"; -import type { Command } from "@commander-js/extra-typings"; -import { clear } from "./clear.ts"; -import { list } from "./list.ts"; -import { SETTING_NAMES, suggestSettingName } from "./registry.ts"; -import { set } from "./set.ts"; - -const settings = { clear, list, set }; - -/** - * The `` argument both `set` and `clear` take. - * - * `.choices()` is what drives tab-completion and the help output's choice list, - * but it is implemented as a `parseArg` that throws before the action runs — so - * the friendlier "Unknown setting" errors inside `set.ts` and `clear.ts` are - * unreachable from the CLI, and a one-character miss like `logs-dir` gets only - * the full list back. Wrapping that parser keeps the completion metadata and - * puts the near miss first, where a reader scanning eight names would not find - * it. - */ -function settingNameArgument` | `[${string}]`>( - spec: S, - description: string, -) { - const argument = createArgument(spec, description).choices(SETTING_NAMES); - const rejectUnlessAllowed = argument.parseArg; - - // Whether the value is allowed stays Commander's question — asking it here - // too would be a second copy of the rule, free to disagree with the first. - // This only adds to the answer when the answer is no. - argument.parseArg = (value: string, previous: T): T => { - try { - return rejectUnlessAllowed?.(value, previous) as T; - } catch (error) { - const suggestion = suggestSettingName(value); - if (!suggestion) throw error; - throw new InvalidArgumentError( - `Did you mean "${suggestion}"? Allowed choices are ${SETTING_NAMES.join(", ")}.`, - ); - } - }; - - return argument; -} - -/** - * Registers `settings list|set|clear` under the `migrate` group. - * - * Noun-verb like every other group in the tree, and listing is the default - * because it is the read-only one — a bare `clerk migrate settings` should show, - * never change. - */ -export function registerMigrateSettings( - migrateCommand: Command<[], Record>, -): void { - const settingsCommand = migrateCommand - .command("settings") - .description("Inspect and change this project's saved migration settings") - .setExamples([ - { - command: "clerk migrate settings", - description: "Show every setting and where it resolves from", - }, - { - command: "clerk migrate settings set firebase-signer-key abc123", - description: "Save a credential to the gitignored .env.clerk-migrate", - }, - { - command: "clerk migrate settings clear firebase-signer-key", - description: "Forget one setting", - }, - { command: "clerk migrate settings clear -y", description: "Forget this project's settings" }, - ]); - - settingsCommand - .command("list", { isDefault: true }) - .description("Show each setting, its value and which source supplied it") - .option("--json", "Output as JSON") - .setExamples([ - { command: "clerk migrate settings list", description: "Credentials shown redacted" }, - { command: "clerk migrate settings list --json", description: "Machine-readable listing" }, - ]) - .action(async (_opts, cmd) => - settings.list(cmd.optsWithGlobals() as Parameters[0]), - ); - - settingsCommand - .command("set") - .description("Set one setting for this project") - .addArgument(settingNameArgument("", "Setting to change")) - .addArgument(createArgument("", "New value")) - .setExamples([ - { - command: "clerk migrate settings set transformer firebase", - description: "Remember the source platform", - }, - { - command: "clerk migrate settings set firebase-signer-key abc123", - description: "Write a credential to .env.clerk-migrate", - }, - ]) - .action(async (name, value) => settings.set(name, value)); - - settingsCommand - .command("clear") - .description("Forget one saved setting, or every setting and saved credential") - .addArgument(settingNameArgument("[name]", "Setting to clear; omit to clear them all")) - .option("-y, --yes", "Skip the confirmation prompt") - .setExamples([ - { command: "clerk migrate settings clear", description: "Clear everything after confirming" }, - { - command: "clerk migrate settings clear file", - description: "Forget only the remembered export file", - }, - { command: "clerk migrate settings clear -y", description: "Clear without prompting" }, - ]) - .action(async (name, _opts, cmd) => - settings.clear(cmd.optsWithGlobals() as Parameters[0], name), - ); -} diff --git a/packages/cli-core/src/commands/migrate/settings/list.ts b/packages/cli-core/src/commands/migrate/settings/list.ts deleted file mode 100644 index 747a179f5..000000000 --- a/packages/cli-core/src/commands/migrate/settings/list.ts +++ /dev/null @@ -1,155 +0,0 @@ -/** - * `clerk migrate settings` — what a run in this project would pick up, and - * where each value is coming from. - * - * The source column is the point. A migration reads from flags, the - * environment, two of the app's env files and the CLI's config; when a run uses - * a stale value, the question is never "what is it" but "which of those is - * winning". Credentials are redacted, so this is safe to paste into an issue. - * - * Laid out like the CLI's other listings — `migrate logs list` and `migrate - * transformers list`: a line or two of orientation, the table, then a count. - * It closes with next steps, the way `mcp list` and `whoami` do, because a - * listing is where someone lands before they know what to type. Those are for - * humans; the full command surface stays in `--help`. - */ - -import { cyan, dim } from "../../../lib/color.ts"; -import { log } from "../../../lib/log.ts"; -import { NEXT_STEPS, printNextSteps } from "../../../lib/next-steps.ts"; -import { findMigrateEnvValue } from "../lib/env-file.ts"; -import { loadSettings } from "../lib/settings.ts"; -import { displayValue, envNames, SETTINGS, type SettingDef } from "./registry.ts"; - -export type SettingsListOptions = { - json?: boolean; -}; - -interface ResolvedSetting { - setting: SettingDef; - value?: string; - source?: string; -} - -/** - * Names the variable as well as the file when an alias supplied the value. - * - * `.env.local` alone would be a half-answer for a setting that has four - * accepted spellings: the reader has to know *which* line in that file the run - * is reading before they can change it. An exported variable already carries - * its name in `source`. - */ -function describeSource(setting: SettingDef, located: { name: string; source: string }): string { - if (located.name === setting.envVar || located.source.startsWith(located.name)) { - return located.source; - } - return `${located.source} (${located.name})`; -} - -async function resolveAll(): Promise { - const saved = await loadSettings(); - - return Promise.all( - SETTINGS.map(async (setting): Promise => { - if (setting.store === "config") { - // `log-dir` is remembered in the config but yields to an environment - // value, so the environment has to be checked first here too — a - // listing that shows the remembered path while the run reads another - // is the one thing the source column exists to prevent. - if (setting.envVar) { - const located = await findMigrateEnvValue(envNames(setting)); - if (located) { - return { setting, value: located.value, source: describeSource(setting, located) }; - } - } - - const value = saved[setting.configKey as keyof typeof saved]; - return value === undefined - ? { setting } - : { setting, value: String(value), source: "clerk config" }; - } - - const located = await findMigrateEnvValue(envNames(setting)); - return located - ? { setting, value: located.value, source: describeSource(setting, located) } - : { setting }; - }), - ); -} - -function toJson(resolved: ResolvedSetting[]) { - return resolved.map(({ setting, value, source }) => ({ - name: setting.name, - store: setting.store, - description: setting.description, - // Redacted here too: `--json` is what gets piped into a log or a ticket. - value: value === undefined ? null : displayValue(setting, value), - set: value !== undefined, - secret: Boolean(setting.secret), - source: source ?? null, - })); -} - -/** - * Pads to a visible width, then colours. - * - * Colouring first and padding after would count the ANSI escape bytes towards - * the width and pull every later column left by however many they took. - */ -function column(text: string, width: number, paint: (value: string) => string): string { - return paint(text) + " ".repeat(Math.max(0, width - text.length)); -} - -export async function list(options: SettingsListOptions = {}): Promise { - const resolved = await resolveAll(); - - if (options.json) { - log.data(JSON.stringify(toJson(resolved), null, 2)); - return; - } - - // An unset value leaves the column empty rather than filling it with a - // placeholder: the source column already reads "not set" on the same row, and - // an empty cell is what makes the settings that do have a value stand out. - const cells = resolved.map(({ setting, value, source }) => ({ - setting, - name: setting.name, - value: value === undefined ? "" : displayValue(setting, value), - unset: value === undefined, - source: source ?? "not set", - })); - - const width = (header: string, pick: (cell: (typeof cells)[number]) => string) => - Math.max(header.length, ...cells.map((cell) => pick(cell).length)) + 2; - - const nameWidth = width("SETTING", (c) => c.name); - const valueWidth = width("VALUE", (c) => c.value); - const sourceWidth = width("SOURCE", (c) => c.source); - - log.info("A migration run in this directory picks these up unless a flag overrides them."); - log.info("Each setting is named after the `clerk migrate import` flag it stands in for."); - log.blank(); - - log.info( - column("SETTING", nameWidth, dim) + - column("VALUE", valueWidth, dim) + - column("SOURCE", sourceWidth, dim) + - dim("DESCRIPTION"), - ); - - for (const cell of cells) { - log.info( - column(cell.name, nameWidth, cyan) + - column(cell.value, valueWidth, (value) => value) + - column(cell.source, sourceWidth, dim) + - dim(cell.setting.description), - ); - } - - const set = cells.filter((cell) => !cell.unset).length; - log.blank(); - log.info(`${set} of ${cells.length} settings set. Credentials are shown redacted.`); - log.blank(); - - printNextSteps(NEXT_STEPS.MIGRATE_SETTINGS); -} diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts deleted file mode 100644 index f3eb1ffc7..000000000 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ /dev/null @@ -1,211 +0,0 @@ -/** - * What `clerk migrate settings` can show and change. - * - * Two stores, split by what the value is rather than by which command wrote it: - * - * - **config** — what this project last migrated. Not secret, not per-machine - * secret material, and useless to anyone but the CLI, so it lives in the - * CLI's own config file keyed by project. - * - **env** — credentials. They go to `.env.clerk-migrate`, which is - * gitignored on write and hand-editable, because a credential belongs - * somewhere the user can rotate it without the CLI's help. - * - * A setting is listed here exactly once; `list`, `set` and `clear` all read - * this table rather than each keeping their own idea of what exists. - */ - -import { REDACTED } from "../../../lib/constants.ts"; - -export type SettingStore = "config" | "env"; - -export interface SettingDef { - /** - * What the user types: `clerk migrate settings set `. - * - * Kebab-case, and identical to the `migrate import` flag it backs. A setting and - * its flag are the same knob reached two ways, so `firebase-signer-key` here - * and `--firebase-signer-key` there must not drift into two spellings the - * user has to learn separately. Sentence-case prose belongs in - * `description`, which is what the list renders alongside it. - */ - name: string; - store: SettingStore; - description: string; - /** - * The environment variable this setting is read from at run time. - * - * Required for an `env` setting, which lives nowhere else. A `config` - * setting may also declare one, meaning "remembered here, but an environment - * value wins" — `log-dir` is that shape, so an operator can pin a directory - * per shell without disturbing what the project remembers. - */ - envVar?: string; - /** - * Other variables accepted for the same setting, read only when - * {@link envVar} is absent. - * - * Firebase hands its four scrypt parameters over as `base64_signer_key`, - * `rounds` and friends, and every guide — including Clerk's own standalone - * migration script — tells the reader to paste them into `.env` under those - * names. Someone who did that has the values the CLI needs, spelled the way - * the source platform spells them, and a listing that reports "not set" is - * wrong about the project rather than strict about it. - * - * Prefixed names still win, and the listing names the variable it read, so a - * generic `ROUNDS` that means something else in the app is visible rather - * than silent. - */ - envAliases?: string[]; - /** For `config` settings, the key on the saved migration entry. */ - configKey?: "transformer" | "file" | "skipUnsupportedProviders" | "logDir"; - /** Redact when displaying — the value is a credential. */ - secret?: boolean; - /** Reject a value the run would only fail on later. */ - validate?: (value: string) => string | undefined; -} - -const positiveInteger = (value: string): string | undefined => { - const parsed = Number(value); - return Number.isInteger(parsed) && parsed > 0 ? undefined : "Expected a positive whole number"; -}; - -const boolean = (value: string): string | undefined => - ["true", "false"].includes(value) ? undefined : "Expected true or false"; - -const path = (value: string): string | undefined => - value.trim().length > 0 ? undefined : "Expected a directory path"; - -export const SETTINGS: SettingDef[] = [ - { - name: "transformer", - store: "config", - configKey: "transformer", - description: "Source platform the export came from", - }, - { - name: "file", - store: "config", - configKey: "file", - description: "Export file to import users from", - }, - { - name: "skip-unsupported-providers", - store: "config", - configKey: "skipUnsupportedProviders", - description: "Skip users with no provider enabled in Clerk (Supabase)", - validate: boolean, - }, - { - name: "log-dir", - store: "config", - configKey: "logDir", - envVar: "CLERK_MIGRATE_LOG_DIR", - description: "Directory migration logs are written to", - validate: path, - }, - { - name: "firebase-signer-key", - store: "env", - envVar: "CLERK_FIREBASE_SIGNER_KEY", - envAliases: ["FIREBASE_BASE64_SIGNER_KEY", "BASE64_SIGNER_KEY"], - description: "Firebase base64 signer key", - secret: true, - }, - { - name: "firebase-salt-separator", - store: "env", - envVar: "CLERK_FIREBASE_SALT_SEPARATOR", - envAliases: ["FIREBASE_BASE64_SALT_SEPARATOR", "BASE64_SALT_SEPARATOR"], - description: "Firebase base64 salt separator", - }, - { - name: "firebase-rounds", - store: "env", - envVar: "CLERK_FIREBASE_ROUNDS", - envAliases: ["FIREBASE_ROUNDS", "ROUNDS"], - description: "Firebase scrypt rounds", - validate: positiveInteger, - }, - { - name: "firebase-mem-cost", - store: "env", - envVar: "CLERK_FIREBASE_MEM_COST", - envAliases: ["FIREBASE_MEM_COST", "MEM_COST"], - description: "Firebase scrypt memory cost", - validate: positiveInteger, - }, -]; - -export const SETTING_NAMES = SETTINGS.map((setting) => setting.name); - -export function findSetting(name: string): SettingDef | undefined { - return SETTINGS.find((setting) => setting.name === name); -} - -/** Levenshtein distance, iterative over a single row. */ -function distance(a: string, b: string): number { - const row = Array.from({ length: b.length + 1 }, (_, i) => i); - - for (let i = 1; i <= a.length; i++) { - let diagonal = row[0] as number; - row[0] = i; - for (let j = 1; j <= b.length; j++) { - const above = row[j] as number; - row[j] = Math.min( - above + 1, - (row[j - 1] as number) + 1, - diagonal + (a[i - 1] === b[j - 1] ? 0 : 1), - ); - diagonal = above; - } - } - - return row[b.length] as number; -} - -/** - * The setting a misspelling was probably reaching for. - * - * Every setting name is a compound of short words — `log-dir`, `firebase-mem-cost` - * — so the misses that matter are a pluralised segment or a transposed pair, - * not a different word entirely. One edit per three characters keeps - * `logs-dir` pointing at `log-dir` without letting an unrelated name match - * something and send the reader off after it. - * - * @returns The closest name within that budget, or `undefined` when nothing is - * close enough to be worth naming. - */ -export function suggestSettingName(name: string): string | undefined { - const budget = Math.max(1, Math.floor(name.length / 3)); - - let best: { name: string; distance: number } | undefined; - for (const candidate of SETTING_NAMES) { - const gap = distance(name, candidate); - if (gap <= budget && (!best || gap < best.distance)) best = { name: candidate, distance: gap }; - } - - return best?.name; -} - -/** - * Every variable an `env` setting answers to, highest priority first. - * - * One list, read by both the listing and the run, so `clerk migrate settings` - * can never show a value the import would ignore. - */ -export function envNames(setting: SettingDef): string[] { - return [setting.envVar as string, ...(setting.envAliases ?? [])]; -} - -/** - * The display value for a setting: withheld entirely when it is a credential. - * - * {@link REDACTED} is what `clerk users create --dry-run` already prints for a - * password, so a credential reads the same wherever the CLI declines to show - * one. Head-and-tail (`aVer…3456`) would say *which* key is set, but the source - * column answers that, and a partial value is one the reader has to recognise - * as partial. - */ -export function displayValue(setting: SettingDef, value: string): string { - return setting.secret ? REDACTED : value; -} diff --git a/packages/cli-core/src/commands/migrate/settings/set.ts b/packages/cli-core/src/commands/migrate/settings/set.ts deleted file mode 100644 index 1b0e1d997..000000000 --- a/packages/cli-core/src/commands/migrate/settings/set.ts +++ /dev/null @@ -1,48 +0,0 @@ -/** - * `clerk migrate settings set ` — change one setting. - * - * Which store it lands in is a property of the setting, not a flag: a - * credential always goes to `.env.clerk-migrate`, project state always goes to - * the CLI config. Letting the caller choose would mean a signer key could be - * put somewhere that is not gitignored. - */ - -import { throwUsageError } from "../../../lib/errors.ts"; -import { log } from "../../../lib/log.ts"; -import { writeMigrateEnvValues } from "../lib/env-file.ts"; -import { loadSettings, saveSettings } from "../lib/settings.ts"; -import { displayValue, findSetting, SETTING_NAMES } from "./registry.ts"; - -export async function set(name: string, value: string): Promise { - const setting = findSetting(name); - if (!setting) { - throwUsageError( - `Unknown setting "${name}". Valid names: ${SETTING_NAMES.join(", ")}.`, - undefined, - undefined, - [ - { - command: "clerk migrate settings", - description: "List the settings and their current values", - }, - ], - ); - } - - const invalid = setting.validate?.(value); - if (invalid) throwUsageError(`Invalid value for ${name}: ${invalid}.`); - - if (setting.store === "env") { - const file = await writeMigrateEnvValues({ [setting.envVar as string]: value }); - log.success(`Set \`${name}\` in ${file} (gitignored).`); - return; - } - - const saved = await loadSettings(); - await saveSettings({ - ...saved, - [setting.configKey as string]: - setting.configKey === "skipUnsupportedProviders" ? value === "true" : value, - }); - log.success(`Set \`${name}\` to ${displayValue(setting, value)} for this project.`); -} diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts deleted file mode 100644 index 90838c5f3..000000000 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ /dev/null @@ -1,341 +0,0 @@ -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { _setConfigDir } from "../../../lib/config.ts"; -import { setMode } from "../../../mode.ts"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { MIGRATE_ENV_FILE } from "../lib/env-file.ts"; -import { loadSettings, saveSettings } from "../lib/settings.ts"; -import { clear } from "./clear.ts"; -import { list } from "./list.ts"; -import { displayValue, findSetting, suggestSettingName } from "./registry.ts"; -import { set } from "./set.ts"; - -const captured = useCaptureLog(); - -let workDir: string; -let configDir: string; -let originalCwd: string; - -const envFileContent = () => fs.readFileSync(path.join(workDir, MIGRATE_ENV_FILE), "utf-8"); - -beforeAll(() => { - originalCwd = process.cwd(); - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-cmd-"))); - configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-cfg-")); - _setConfigDir(configDir); - process.chdir(workDir); -}); - -afterAll(() => { - _setConfigDir(undefined); - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); - fs.rmSync(configDir, { recursive: true, force: true }); -}); - -beforeEach(() => { - fs.rmSync(path.join(configDir, "config.json"), { force: true }); - fs.rmSync(path.join(workDir, MIGRATE_ENV_FILE), { force: true }); - fs.rmSync(path.join(workDir, ".gitignore"), { force: true }); -}); - -afterEach(() => { - process.exitCode = 0; -}); - -describe("displayValue", () => { - const signerKey = findSetting("firebase-signer-key")!; - - // No part of the value, at any length — the same `[REDACTED]` that - // `clerk users create --dry-run` prints for a password. - test.each([["short"], ["0123456789"], ["aVeryLongSignerKeyValue123456"]])( - "withholds the credential %p entirely", - (value) => { - expect(displayValue(signerKey, value)).toBe("[REDACTED]"); - }, - ); - - test("shows a setting that is not a credential", () => { - expect(displayValue(findSetting("transformer")!, "firebase")).toBe("firebase"); - }); -}); - -describe("set", () => { - test("writes a credential to the gitignored env file, not the CLI config", async () => { - await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); - - expect(envFileContent()).toContain("CLERK_FIREBASE_SIGNER_KEY=aVeryLongSignerKeyValue123456"); - expect(await loadSettings()).toEqual({}); - expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toContain(MIGRATE_ENV_FILE); - }); - - test("writes project state to the CLI config, not the env file", async () => { - await set("transformer", "firebase"); - - expect(await loadSettings()).toEqual({ transformer: "firebase" }); - expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); - }); - - test("keeps the settings it is not changing", async () => { - await saveSettings({ transformer: "clerk", file: "users.json" }); - await set("file", "other.json"); - - expect(await loadSettings()).toEqual({ transformer: "clerk", file: "other.json" }); - }); - - test("stores a boolean setting as a boolean", async () => { - await set("skip-unsupported-providers", "true"); - expect(await loadSettings()).toEqual({ skipUnsupportedProviders: true }); - }); - - test.each([ - ["firebase-rounds", "zero", /positive whole number/], - ["skip-unsupported-providers", "yes", /true or false/], - ])("rejects an invalid value for %s", async (name, value, message) => { - await expect(set(name, value)).rejects.toThrow(message); - }); - - test("names the valid settings when given an unknown one", async () => { - await expect(set("nope", "x")).rejects.toThrow(/firebase-signer-key/); - }); - - // A run would fail on it later; failing at write time keeps the bad value out - // of the file entirely. - test("writes nothing when the value is rejected", async () => { - await expect(set("firebase-rounds", "-1")).rejects.toThrow(); - expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); - }); -}); - -describe("list", () => { - test("names the source each value resolved from", async () => { - await set("transformer", "firebase"); - await set("firebase-salt-separator", "Bw=="); - captured.clear(); - - await list(); - - expect(captured.err).toContain("clerk config"); - expect(captured.err).toContain(MIGRATE_ENV_FILE); - }); - - test("redacts a credential but not the rest", async () => { - await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); - await set("transformer", "firebase"); - captured.clear(); - - await list(); - - expect(captured.err).toContain("[REDACTED]"); - expect(captured.err).not.toContain("aVeryLongSignerKeyValue123456"); - expect(captured.err).toContain("firebase"); - }); - - // --json is what gets piped into a ticket or a CI log. - test("redacts in JSON output too", async () => { - await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); - captured.clear(); - - await list({ json: true }); - - expect(captured.out).not.toContain("aVeryLongSignerKeyValue123456"); - expect(JSON.parse(captured.out)).toContainEqual( - expect.objectContaining({ name: "firebase-signer-key", value: "[REDACTED]", secret: true }), - ); - }); - - // The names are kebab-case because they mirror the `migrate import` flags; the - // description column is what makes the list readable. - test("explains each setting in prose", async () => { - await list(); - - expect(captured.err).toContain("Source platform the export came from"); - expect(captured.err).toContain("Export file to import users from"); - }); - - test("carries the description into JSON too", async () => { - await list({ json: true }); - - expect(JSON.parse(captured.out)).toContainEqual( - expect.objectContaining({ name: "file", description: "Export file to import users from" }), - ); - }); - - // Colouring before padding counts the ANSI bytes towards the column width, - // which pulls later columns left on exactly the rows that have a value. - test("starts the description at one column, set or not", async () => { - await set("transformer", "supabase"); - captured.clear(); - - await list(); - - // eslint-disable-next-line no-control-regex - const plain = captured.err.replaceAll(/\u001B\[\d+m/g, ""); - const columnOf = (description: string) => - plain - .split("\n") - .find((row) => row.includes(description)) - ?.indexOf(description); - - expect(columnOf("Source platform the export came from")).toBe( - columnOf("Export file to import users from") as number, - ); - }); - - test("marks everything as unset in a fresh project", async () => { - await list({ json: true }); - expect(JSON.parse(captured.out).every((entry: { set: boolean }) => !entry.set)).toBe(true); - }); - - // A listing is where someone lands before they know what to type, so it - // closes by naming the two commands that change what it just showed — - // the same next-steps block `mcp list` and `whoami` end on. - test("closes with next steps", async () => { - setMode("human"); - await list(); - setMode("agent"); - - expect(captured.err).toContain("clerk migrate settings set "); - expect(captured.err).toContain("clerk migrate settings clear"); - }); - - test("counts how many are set", async () => { - await set("transformer", "firebase"); - captured.clear(); - - await list(); - - expect(captured.err).toContain("1 of 8 settings set"); - }); - - // Firebase's own names for these, and what every guide tells you to paste - // into `.env`. Reporting "not set" for a value the import would read is the - // listing being wrong about the project rather than strict about it. - test("reads a credential written under the name Firebase uses", async () => { - fs.writeFileSync(path.join(workDir, ".env.local"), "ROUNDS=8\n"); - captured.clear(); - - await list(); - - fs.rmSync(path.join(workDir, ".env.local")); - // Named alongside the file: `ROUNDS` may mean something else in this app. - expect(captured.err).toContain(".env.local (ROUNDS)"); - }); -}); - -describe("clear", () => { - test("empties both stores", async () => { - await set("transformer", "firebase"); - await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); - - await clear({ yes: true }); - - expect(await loadSettings()).toEqual({}); - expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); - }); - - test("says so rather than claiming to have cleared nothing", async () => { - await clear({ yes: true }); - expect(captured.err).toContain("No migration settings to clear"); - }); - - test("leaves settings the migration does not own", async () => { - fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "OTHER=keep\n"); - await set("firebase-rounds", "8"); - - await clear({ yes: true }); - - expect(envFileContent()).toBe("OTHER=keep\n"); - }); - - // It destroys credentials, like `logs clean` destroys logs and `migrate - // delete` destroys users — and those two both refuse rather than assume. - // Proceeding here because nobody could be asked is the one reading of - // silence that cannot be undone. - test("refuses rather than assuming when it cannot prompt", async () => { - await set("transformer", "firebase"); - await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); - - await expect(clear({})).rejects.toThrow(/cannot prompt here\. Pass -y to confirm/); - - expect(await loadSettings()).toMatchObject({ transformer: "firebase" }); - expect(envFileContent()).toContain("CLERK_FIREBASE_SIGNER_KEY"); - }); -}); - -describe("clear ", () => { - test("drops one config setting and keeps the rest", async () => { - await set("transformer", "firebase"); - await set("file", "users.json"); - - await clear({ yes: true }, "file"); - - expect(await loadSettings()).toEqual({ transformer: "firebase" }); - }); - - test("drops one credential and keeps the rest of the env file", async () => { - await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); - await set("firebase-rounds", "8"); - - await clear({ yes: true }, "firebase-signer-key"); - - expect(envFileContent()).toContain("CLERK_FIREBASE_ROUNDS=8"); - expect(envFileContent()).not.toContain("CLERK_FIREBASE_SIGNER_KEY"); - }); - - // Clearing only the prefixed name would report success and leave the next run - // reading the alias. - test("drops every spelling the setting answers to", async () => { - fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "ROUNDS=8\nOTHER=keep\n"); - - await clear({ yes: true }, "firebase-rounds"); - - expect(envFileContent()).toBe("OTHER=keep\n"); - }); - - test("says so when the setting was not set here", async () => { - await clear({ yes: true }, "firebase-rounds"); - expect(captured.err).toContain("firebase-rounds"); - expect(captured.err).toContain("is not set here"); - - captured.clear(); - await clear({ yes: true }, "transformer"); - expect(captured.err).toContain("transformer"); - expect(captured.err).toContain("is not set here"); - }); - - // `log-dir` is remembered in the config but yields to an env var, so half a - // clear would report success and leave the run reading the same directory. - test("clears a setting that lives in both stores", async () => { - fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "CLERK_MIGRATE_LOG_DIR=./env-logs\n"); - await saveSettings({ logDir: "./saved-logs", transformer: "firebase" }); - - await clear({ yes: true }, "log-dir"); - - expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); - expect(await loadSettings()).toEqual({ transformer: "firebase" }); - }); - - test("rejects a name that is not a setting", async () => { - await expect(clear({ yes: true }, "nope")).rejects.toThrow(/Unknown setting "nope"/); - }); -}); - -describe("suggestSettingName", () => { - // `.choices()` rejects before the action runs, so this is the only thing - // standing between a one-character miss and a bare list of eight names. - test.each([ - ["logs-dir", "log-dir"], - ["log_dir", "log-dir"], - ["firebase-round", "firebase-rounds"], - ["transfomer", "transformer"], - ])("%s -> %s", (typo, expected) => { - expect(suggestSettingName(typo)).toBe(expected); - }); - - test.each(["banana", "secret", ""])("says nothing for %p", (unrelated) => { - expect(suggestSettingName(unrelated)).toBeUndefined(); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts index cee841da4..02f7f3306 100644 --- a/packages/cli-core/src/commands/migrate/wizard.test.ts +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -27,7 +27,6 @@ mock.module("../../lib/prompts.ts", () => ({ })); const { runWizard, throwAgentFlagsRequired } = await import("./wizard.ts"); -const { saveSettings } = await import("./lib/settings.ts"); const { _setConfigDir } = await import("../../lib/config.ts"); let workDir: string; @@ -98,41 +97,6 @@ describe("transformer picker", () => { }); }); -describe("defaults from the previous run", () => { - test("pre-selects the last transformer and pre-fills the last file", async () => { - await saveSettings({ transformer: "supabase", file: "other.csv" }); - mockSelect.mockResolvedValue("supabase"); - mockText.mockResolvedValue("other.csv"); - - await runWizard({}); - - expect(selectCall(0)?.default).toBe("supabase"); - expect(textCall(0)?.default).toBe("other.csv"); - }); - - test("offers no default when nothing has been saved", async () => { - mockSelect.mockResolvedValue("clerk"); - mockText.mockResolvedValue("users.json"); - - await runWizard({}); - - expect(selectCall(0)?.default).toBeUndefined(); - expect(textCall(0)?.default).toBeUndefined(); - }); - - // A saved key from a build that has since dropped that transformer would - // otherwise pre-select a value the picker cannot offer. - test("ignores a saved transformer that is no longer registered", async () => { - await saveSettings({ transformer: "okta" }); - mockSelect.mockResolvedValue("clerk"); - mockText.mockResolvedValue("users.json"); - - await runWizard({}); - - expect(selectCall(0)?.default).toBeUndefined(); - }); -}); - describe("file prompt validation", () => { const validate = async () => { mockSelect.mockResolvedValue("clerk"); diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index 122173046..b2ca866c1 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -2,9 +2,7 @@ * The interactive path behind a bare `clerk migrate import`. * * Ported from the standalone migration-tool's `src/migrate/cli.ts` interactive - * flow. The platform and file are pre-filled from the previous run, so a repeat - * migration is mostly pressing enter. Firebase's hash parameters are not: the - * signer key is a secret, and the CLI does not keep those. + * flow. * * Agent mode never reaches here — `run` raises a usage error naming the flags * instead, because an agent cannot answer a prompt. @@ -15,7 +13,6 @@ import { select } from "../../lib/listage.ts"; import { log } from "../../lib/log.ts"; import { text } from "../../lib/prompts.ts"; import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebase-hash.ts"; -import { loadSettings } from "./lib/settings.ts"; import { fileExists, getFileType } from "./lib/transform.ts"; import { transformers } from "./transformers/registry.ts"; import type { FirebaseHashConfig } from "./types.ts"; @@ -32,7 +29,7 @@ function hint(description: string): string { return firstSentence.length > 96 ? `${firstSentence.slice(0, 93)}...` : firstSentence; } -async function pickTransformer(defaultKey: string | undefined): Promise { +async function pickTransformer(): Promise { // Built from the registry, so a new platform appears here with no second // place to update. return select({ @@ -42,14 +39,12 @@ async function pickTransformer(defaultKey: string | undefined): Promise value: entry.key, description: hint(entry.description), })), - default: defaultKey && transformers.some((t) => t.key === defaultKey) ? defaultKey : undefined, }); } -async function askFile(defaultFile: string | undefined): Promise { +async function askFile(): Promise { return text({ message: "Path to the exported user file (JSON or CSV)", - default: defaultFile, validate: (value) => { const file = value?.trim(); if (!file) return "A file path is required"; @@ -71,9 +66,7 @@ async function askFirebaseHashConfig(): Promise log.info( "Firebase password hashes need the project's hash parameters. Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", ); - log.info( - "Set CLERK_FIREBASE_SIGNER_KEY, CLERK_FIREBASE_SALT_SEPARATOR, CLERK_FIREBASE_ROUNDS and CLERK_FIREBASE_MEM_COST to skip these prompts on the next run.", - ); + log.info("Pass the four --firebase-* flags to skip these prompts on the next run."); const signerKey = ( await text({ @@ -120,22 +113,13 @@ export async function runWizard( firebaseHashConfig?: FirebaseHashConfig; } & FirebaseHashFlags, ): Promise { - const saved = await loadSettings(); - - const transformer = provided.transformer ?? (await pickTransformer(saved.transformer)); - const file = provided.file ?? (await askFile(saved.file)); + const transformer = provided.transformer ?? (await pickTransformer()); + const file = provided.file ?? (await askFile()); let firebaseHashConfig = provided.firebaseHashConfig; if (transformer === "firebase" && !firebaseHashConfig) { - // Looked up here rather than before the picker: until the platform is - // chosen there is no reason to read Firebase's variables at all, and a - // migration from anywhere else must not see them. - firebaseHashConfig = await resolveFirebaseHashConfig(provided, "firebase"); - } - if (transformer === "firebase" && !firebaseHashConfig) { - // Prompted, never prefilled: the signer key is a secret the CLI does not - // keep, so there is nothing to offer back. - firebaseHashConfig = await askFirebaseHashConfig(); + firebaseHashConfig = + resolveFirebaseHashConfig(provided, "firebase") ?? (await askFirebaseHashConfig()); } return { transformer, file, ...(firebaseHashConfig ? { firebaseHashConfig } : {}) }; diff --git a/packages/cli-core/src/lib/config.test.ts b/packages/cli-core/src/lib/config.test.ts index faeeeb806..03d2f4eae 100644 --- a/packages/cli-core/src/lib/config.test.ts +++ b/packages/cli-core/src/lib/config.test.ts @@ -11,9 +11,6 @@ const { clearAuth, getProfile, setProfile, - getMigrationEntry, - setMigrationEntry, - getProjectKey, listProfiles, resolveProfile, resolveInstanceId, @@ -92,42 +89,6 @@ describe("config", () => { expect(await getAuth()).toBeUndefined(); }); - test("setMigrationEntry and getMigrationEntry", async () => { - expect(await getMigrationEntry("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/projects/my-app")).toBeUndefined(); - await setMigrationEntry("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/projects/my-app", { transformer: "clerk", file: "users.json" }); - expect(await getMigrationEntry("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/projects/my-app")).toEqual({ - transformer: "clerk", - file: "users.json", - }); - expect(await getMigrationEntry("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/projects/other")).toBeUndefined(); - }); - - // readConfig rebuilds the document field by field, so a key it does not know - // about is dropped on the next write rather than merely ignored. - // readConfig rebuilds the document field by field, so a section it does not - // know about is dropped on the next write rather than merely ignored. - test("migrations survive a write to another section", async () => { - await setMigrationEntry("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/projects/my-app", { transformer: "clerk" }); - await setProfile("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/projects/my-app", { - workspaceId: "org_abc", - appId: "app_def", - instances: { development: "ins_ghi" }, - }); - - expect(await getMigrationEntry("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/projects/my-app")).toEqual({ transformer: "clerk" }); - }); - - test("getProjectKey prefers the linked profile's key over the directory", async () => { - expect(await getProjectKey("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/projects/unlinked")).toBe("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/projects/unlinked"); - - await setProfile("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/projects/linked", { - workspaceId: "org_abc", - appId: "app_def", - instances: { development: "ins_ghi" }, - }); - expect(await getProjectKey("/projects/linked/src")).toBe("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/projects/linked"); - }); - test("setProfile and getProfile", async () => { const profile = { workspaceId: "org_abc", diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index 4d84ed82d..41943b990 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -50,21 +50,11 @@ interface RelayEntry { token: string; } -/** What `clerk migrate import` last imported for a project, and how. */ -interface MigrationEntry { - transformer?: string; - file?: string; - skipUnsupportedProviders?: boolean; - /** Where this project's migration logs are written. Absent until chosen. */ - logDir?: string; -} - interface ClerkConfig { environment?: string; auth?: Record; profiles: Record; relay?: Record; - migrations?: Record; machineUuid?: string; telemetryNoticeShown?: boolean; telemetryDisabled?: boolean; @@ -103,13 +93,6 @@ function migrateRawConfig(raw: Record): ClerkConfig { config.relay = relay; } - // Not validated per entry the way `relay` is: every field is optional, so - // there is no key whose absence marks an entry as junk. A malformed one costs - // a remembered default, not a failed run. - if (raw.migrations && typeof raw.migrations === "object" && !Array.isArray(raw.migrations)) { - config.migrations = raw.migrations as Record; - } - if (raw.auth && typeof raw.auth === "object") { const auth = raw.auth as Record; if (typeof auth.userId === "string") { @@ -231,18 +214,6 @@ export async function setRelayEntry(key: string, entry: RelayEntry): Promise { - const config = await readConfig(); - return config.migrations?.[key]; -} - -export async function setMigrationEntry(key: string, entry: MigrationEntry): Promise { - const config = await readConfig(); - if (!config.migrations) config.migrations = {}; - config.migrations[key] = entry; - await writeConfig(config); -} - /** Persistent random machine id for telemetry. Generated on first use. */ export async function ensureMachineUuid(): Promise { const config = await readConfig(); @@ -337,20 +308,6 @@ export async function resolveProfile(cwd: string): Promise< return undefined; } -/** - * The key a per-project record (e.g. `migrations`) is filed under. - * - * Prefers the linked profile's own key so the record sits beside the profile it - * belongs to, and so it survives `clerk link` being re-run from a subdirectory. - * Falls back to the git remote and then the directory, because a project that - * has never been linked still deserves to be remembered. - */ -export async function getProjectKey(cwd: string): Promise { - const resolved = await resolveProfile(cwd); - if (resolved) return resolved.path; - return (await getGitNormalizedRemote(cwd)) ?? cwd; -} - const INSTANCE_ALIASES: Record = { dev: "development", development: "development", @@ -490,4 +447,4 @@ export async function resolveAppContext( }; } -export type { Auth, Profile, ClerkConfig, MigrationEntry, AppContextOptions }; +export type { Auth, Profile, ClerkConfig, AppContextOptions }; diff --git a/packages/cli-core/src/lib/constants.ts b/packages/cli-core/src/lib/constants.ts index e7b757ac7..9e7c1c02f 100644 --- a/packages/cli-core/src/lib/constants.ts +++ b/packages/cli-core/src/lib/constants.ts @@ -55,15 +55,3 @@ export const NPM_REGISTRY_URL = "https://registry.npmjs.org/"; /** Event ingestion endpoint (telemetry-service worker → BigQuery). */ export const DEFAULT_TELEMETRY_ENDPOINT = "https://clerk-telemetry.com/v1/event"; export const TELEMETRY_TIMEOUT_MS = 1000; - -// ── Redaction ───────────────────────────────────────────────────────────── - -/** - * What a withheld secret displays as, everywhere the CLI shows one. - * - * Square brackets rather than a mask or a truncation: a row of dots or a - * head-and-tail (`aVer…3456`) reads as a value, and the reader has to work out - * that it is not one. Used by `clerk users create --dry-run` and by - * `clerk migrate settings`. - */ -export const REDACTED = "[REDACTED]"; diff --git a/packages/cli-core/src/lib/dotenv.ts b/packages/cli-core/src/lib/dotenv.ts index a2e0c2ec7..2b5d0cb20 100644 --- a/packages/cli-core/src/lib/dotenv.ts +++ b/packages/cli-core/src/lib/dotenv.ts @@ -28,76 +28,6 @@ export async function findExistingEnvFile(cwd: string, fallback: string): Promis return fallback; } -/** - * The env files read back when resolving a value, as opposed to written to. - * - * Deliberately shorter than {@link ENV_FILE_CANDIDATES}: the runtime has - * already loaded every `.env*` variant it recognises into `process.env`, which - * {@link findEnvValue} checks first. This list only has to cover the case where - * the CLI's own process did not load the file — a different cwd at startup, or - * a runtime with no dotenv support. - */ -const ENV_FILES = [".env", ".env.local"]; - -export interface FindEnvValueOptions { - /** Injectable in tests; defaults to the real environment. */ - env?: Record; - /** Lowest priority first — a later file overrides an earlier one. */ - files?: readonly string[]; -} - -export interface LocatedEnvValue { - value: string; - /** Which of `names` supplied it — the caller may have passed several aliases. */ - name: string; - /** Where it came from, for `--verbose` (`CLERK_SECRET_KEY env var`, `.env.local`). */ - source: string; -} - -/** - * Looks for a value under any of `names`, in the order the app itself would - * resolve one: the environment first, then env files with a later file - * overriding an earlier one. - * - * This is the CLI's one way to read a project-level setting. Reading - * `process.env` directly instead skips the file fallback and reports no source, - * so a command that does it cannot explain where its input came from. - */ -export async function findEnvValue( - cwd: string, - names: string[], - options: FindEnvValueOptions = {}, -): Promise { - const { env = process.env, files = ENV_FILES } = options; - - for (const name of new Set(names)) { - const value = env[name]; - if (value) return { value, name, source: `${name} env var` }; - } - - // Priority is by name, not by position: the framework-specific name beats - // the generic fallback even when the generic one appears later in the same - // file. Within one name, a later file still overrides an earlier one. - const foundByName = new Map(); - for (const envFile of files) { - const file = Bun.file(join(cwd, envFile)); - if (!(await file.exists())) continue; - - for (const line of parseEnvFile(await file.text())) { - if (line.type !== "entry" || !line.value) continue; - if (names.includes(line.key)) { - foundByName.set(line.key, { value: line.value, name: line.key, source: envFile }); - } - } - } - - for (const name of names) { - const located = foundByName.get(name); - if (located) return located; - } - return undefined; -} - export type EnvLine = | { type: "comment"; raw: string } | { type: "blank" } diff --git a/packages/cli-core/src/lib/git.ts b/packages/cli-core/src/lib/git.ts index 697a57de8..34fced039 100644 --- a/packages/cli-core/src/lib/git.ts +++ b/packages/cli-core/src/lib/git.ts @@ -104,9 +104,9 @@ export function normalizeGitRemoteUrl(raw: string): string { /** * Adds `entry` to the project's `.gitignore` unless it is already listed. * - * The CLI writes files into a user's repository that must not be committed — - * the keyless breadcrumb, and the migration settings file. Creating one without - * this is how a live credential ends up in a tracked file. + * The CLI writes files into a user's repository that must not be committed, + * such as the keyless breadcrumb. Creating one without this is how a live + * credential ends up in a tracked file. */ export async function ensureGitignoreEntry(cwd: string, entry: string): Promise { const gitignorePath = join(cwd, ".gitignore"); diff --git a/packages/cli-core/src/lib/keyless-target.ts b/packages/cli-core/src/lib/keyless-target.ts index 6eca393b1..99a63edbb 100644 --- a/packages/cli-core/src/lib/keyless-target.ts +++ b/packages/cli-core/src/lib/keyless-target.ts @@ -11,7 +11,7 @@ import { join } from "node:path"; import { bapiRequest } from "./bapi.ts"; import { resolveAppContext, resolveProfile } from "./config.ts"; import { getStoredSession, hasAccountCredentials, type OAuthSession } from "./credential-store.ts"; -import { findEnvValue } from "./dotenv.ts"; +import { parseEnvFile } from "./dotenv.ts"; import { CliError, ERROR_CODE, throwUsageError } from "./errors.ts"; import { decodePublishableKey } from "./fapi.ts"; import { detectPublishableKeyName, detectSecretKeyName } from "./framework.ts"; @@ -38,6 +38,8 @@ export type InstanceTarget = | { kind: "account"; ctx: AccountContext; label: string } | { kind: "keyless"; keyless: KeylessTarget; label: string }; +const ENV_FILES = [".env", ".env.local"]; + /** * Where the Clerk SDKs park the keys for a keyless app they created themselves * (running `next dev` with no keys configured). Shape: @@ -69,6 +71,45 @@ export async function readSdkKeylessApp( } } +interface LocatedKey { + value: string; + source: string; +} + +/** + * Looks for a key under any of `names`, in the order the app itself would + * resolve one: the environment first, then env files with a later file + * overriding an earlier one. + */ +async function findKeyInProject(cwd: string, names: string[]): Promise { + for (const name of new Set(names)) { + const value = process.env[name]; + if (value) return { value, source: `${name} env var` }; + } + + // Priority is by name, not by position: the framework-specific name beats + // the generic fallback even when the generic one appears later in the same + // file. Within one name, a later file still overrides an earlier one. + const foundByName = new Map(); + for (const envFile of ENV_FILES) { + const file = Bun.file(join(cwd, envFile)); + if (!(await file.exists())) continue; + + for (const line of parseEnvFile(await file.text())) { + if (line.type !== "entry" || !line.value) continue; + if (names.includes(line.key)) { + foundByName.set(line.key, { value: line.value, source: envFile }); + } + } + } + + for (const name of names) { + const located = foundByName.get(name); + if (located) return located; + } + return undefined; +} + /** * The instance secret key a keyless project keeps locally. Falls back to the * keys an SDK created for itself, which it only does when nothing else supplies @@ -76,7 +117,7 @@ export async function readSdkKeylessApp( */ export async function findLocalSecretKey(cwd: string): Promise { const names = [await detectSecretKeyName(cwd), "CLERK_SECRET_KEY"]; - const located = await findEnvValue(cwd, names); + const located = await findKeyInProject(cwd, names); const found = located ? { secretKey: located.value, source: located.source } @@ -95,7 +136,7 @@ async function sdkKeylessTarget(cwd: string): Promise /** The publishable key a keyless project holds locally, when one can be found. */ export async function findLocalPublishableKey(cwd: string): Promise { const names = [await detectPublishableKeyName(cwd), "CLERK_PUBLISHABLE_KEY"]; - const located = await findEnvValue(cwd, names); + const located = await findKeyInProject(cwd, names); return located?.value ?? (await readSdkKeylessApp(cwd))?.publishableKey; } diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index 642a0dd7c..5b3fa33a7 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -71,21 +71,11 @@ export const NEXT_STEPS = { "Run `clerk apps list` to see your other applications", "Run `clerk config pull` to inspect the live configuration of this instance", ], - MIGRATE_DONE: [ - "Run `clerk migrate logs list` to inspect the import log", - "Run `clerk migrate delete` to undo this migration", - ], + MIGRATE_DONE: ["Run `clerk migrate logs list` to inspect the import log"], // `logs list` only names the file; after a partial import the operator needs // the failures themselves, which live one line per user in that file. MIGRATE_DONE_WITH_ERRORS: (logFile: string) => [ `Run \`grep '"status":"error"' ${logFile}\` to see every user that failed and why`, - "Run `clerk migrate delete` to undo this migration", - ], - MIGRATE_DELETE: ["Run `clerk migrate logs list` to inspect the deletion log"], - MIGRATE_SETTINGS: [ - "Run `clerk migrate settings set ` to change one", - "Run `clerk migrate settings clear ` to forget one", - "Run `clerk migrate settings clear` to forget them all, credentials included", ], } as const; diff --git a/packages/cli-core/src/lib/users.ts b/packages/cli-core/src/lib/users.ts index 5d9345f18..d43a767e6 100644 --- a/packages/cli-core/src/lib/users.ts +++ b/packages/cli-core/src/lib/users.ts @@ -1,8 +1,8 @@ import { bapiRequest } from "./bapi.ts"; -import { REDACTED } from "./constants.ts"; import { ERROR_CODE, throwUsageError } from "./errors.ts"; const USERS_INVALID_JSON_MESSAGE = "User payload must be a JSON object."; +const REDACTED = "[REDACTED]"; const DIRECT_REDACT_KEYS = new Set(["password", "code"]); const OBJECT_REDACT_KEYS = new Set(["private_metadata", "unsafe_metadata"]); diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index 7cb412560..794dd8b79 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -1,8 +1,7 @@ import { Writable } from "node:stream"; -import { afterAll, afterEach, beforeAll, beforeEach, type spyOn } from "bun:test"; +import { afterEach, beforeEach, type spyOn } from "bun:test"; import { type CapturedLogs, setActiveCapture } from "../../lib/log.ts"; import { setUiOutput } from "../../lib/ui.ts"; -import { _resetLogDir } from "../../commands/migrate/lib/logger.ts"; export function capturedOutput(spy: ReturnType): string { return spy.mock.calls.map((c: unknown[]) => c[0]).join("\n"); @@ -147,9 +146,6 @@ export const configStubs = { listProfiles: noop, getRelayEntry: noop, setRelayEntry: noop, - getMigrationEntry: noop, - setMigrationEntry: noop, - getProjectKey: async () => "", resolveProfile: noop, resolveProfileOrAutolink: noop, resolveInstanceId: () => ({ id: "", label: "" }), @@ -252,32 +248,3 @@ type FetchImpl = (input: string | URL | Request, init?: RequestInit) => Promise< export function stubFetch(impl: FetchImpl): void { globalThis.fetch = impl as typeof fetch; } - -/** - * Settles the migration log directory for a whole test file. - * - * `migrate import`, `export` and `delete` ask a human where logs should go the - * first time a project runs one. A test that flips to human mode to exercise - * something else — next steps, a wizard — would stop on that question and, - * where `prompts.ts` is mocked, silently eat the answer meant for another - * prompt. Pinning the environment variable answers it before it is asked, the - * same way an operator who exported one never sees it. - */ -export function useMigrateLogDir(dir = "./logs"): void { - let original: string | undefined; - - beforeAll(() => { - original = process.env.CLERK_MIGRATE_LOG_DIR; - process.env.CLERK_MIGRATE_LOG_DIR = dir; - }); - - // Resolution is cached per process, and each test file runs under its own - // temporary cwd, so the cached absolute path has to go with it. - beforeEach(() => _resetLogDir()); - - afterAll(() => { - if (original === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; - else process.env.CLERK_MIGRATE_LOG_DIR = original; - _resetLogDir(); - }); -} From 83763892213213f6ed8c8830cc9f8a9474af4179 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 29 Sep 2026 17:41:13 -0400 Subject: [PATCH 054/141] feat(migrate): keep every run in a run store, and add `clerk migrate runs` Imports and exports kept their record in flat NDJSON logs, one per command type, under `./logs`, with nothing to say what a file was run against or whether it finished. The run store replaces that record. Each import and export is now a run, stored as a folder under `/.clerk/migrate/`: - `run.json` records what ran, against which instance, with which key, from which file (with its sha256), and how it ended. - `users.ndjson` holds one line per user outcome. The last line per source ID wins, so a continued run appends to it. - `lock` holds the writer's PID. A live PID refuses a second writer. A dead PID, or a missing finish time, shows the run as interrupted. The folder comes from `--runs-dir`, then `CLERK_MIGRATE_DIR`, then the project root. The project root is the linked profile's directory, then the git toplevel, then the cwd. The default location adds `.clerk/` to `.gitignore` first. The target's instance ID comes from `GET /v1/instance`, so every key source gives the same identity. `clerk migrate runs` lists every run. `clerk migrate runs ` shows one: its counts, error breakdown, and the failed and skipped users. Both accept `--json`, and an unknown ID exits 2. `migrate logs`, `lib/logger.ts` and `lib/log-files.ts` are removed. Validation failures, 429 retries and extra identifiers that failed to attach are now written to the user's run line instead of separate log entries. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 155 +++----- .../src/commands/migrate/export/auth0.test.ts | 12 +- .../src/commands/migrate/export/auth0.ts | 30 +- .../src/commands/migrate/export/authjs.ts | 23 +- .../src/commands/migrate/export/betterauth.ts | 26 +- .../src/commands/migrate/export/clerk.test.ts | 33 +- .../src/commands/migrate/export/clerk.ts | 28 +- .../migrate/export/db-exports.test.ts | 76 ++-- .../src/commands/migrate/export/db-options.ts | 2 + .../commands/migrate/export/firebase.test.ts | 11 +- .../src/commands/migrate/export/firebase.ts | 30 +- .../src/commands/migrate/export/index.ts | 7 + .../src/commands/migrate/export/shared.ts | 34 ++ .../src/commands/migrate/export/supabase.ts | 28 +- .../commands/migrate/export/workos.test.ts | 16 +- .../src/commands/migrate/export/workos.ts | 23 +- .../src/commands/migrate/import-users.test.ts | 54 +-- .../src/commands/migrate/import-users.ts | 88 ++--- .../src/commands/migrate/index.test.ts | 37 +- .../cli-core/src/commands/migrate/index.ts | 28 +- .../commands/migrate/lib/log-files.test.ts | 203 ---------- .../src/commands/migrate/lib/log-files.ts | 153 -------- .../src/commands/migrate/lib/logger.test.ts | 139 ------- .../src/commands/migrate/lib/logger.ts | 131 ------- .../commands/migrate/lib/run-store.test.ts | 171 ++++++++ .../src/commands/migrate/lib/run-store.ts | 370 ++++++++++++++++++ .../src/commands/migrate/lib/target.ts | 116 ++++++ .../commands/migrate/lib/transform.test.ts | 37 +- .../src/commands/migrate/lib/transform.ts | 61 +-- .../src/commands/migrate/logs/clean.ts | 72 ---- .../src/commands/migrate/logs/convert.ts | 128 ------ .../src/commands/migrate/logs/index.ts | 74 ---- .../src/commands/migrate/logs/list.ts | 125 ------ .../migrate/logs/logs-interactive.test.ts | 201 ---------- .../src/commands/migrate/logs/logs.test.ts | 288 -------------- .../src/commands/migrate/readme.test.ts | 10 +- .../commands/migrate/run-interactive.test.ts | 2 +- .../cli-core/src/commands/migrate/run.test.ts | 55 ++- packages/cli-core/src/commands/migrate/run.ts | 116 ++++-- .../src/commands/migrate/runs.test.ts | 93 +++++ .../cli-core/src/commands/migrate/runs.ts | 221 +++++++++++ .../migrate/transformers/transformers.test.ts | 21 +- .../cli-core/src/commands/migrate/types.ts | 67 +--- packages/cli-core/src/lib/next-steps.ts | 10 +- 44 files changed, 1557 insertions(+), 2048 deletions(-) delete mode 100644 packages/cli-core/src/commands/migrate/lib/log-files.test.ts delete mode 100644 packages/cli-core/src/commands/migrate/lib/log-files.ts delete mode 100644 packages/cli-core/src/commands/migrate/lib/logger.test.ts delete mode 100644 packages/cli-core/src/commands/migrate/lib/logger.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/run-store.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/run-store.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/target.ts delete mode 100644 packages/cli-core/src/commands/migrate/logs/clean.ts delete mode 100644 packages/cli-core/src/commands/migrate/logs/convert.ts delete mode 100644 packages/cli-core/src/commands/migrate/logs/index.ts delete mode 100644 packages/cli-core/src/commands/migrate/logs/list.ts delete mode 100644 packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts delete mode 100644 packages/cli-core/src/commands/migrate/logs/logs.test.ts create mode 100644 packages/cli-core/src/commands/migrate/runs.test.ts create mode 100644 packages/cli-core/src/commands/migrate/runs.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 216744f2f..b8597f5b5 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -76,6 +76,7 @@ clerk migrate import -y --transformer clerk --file users.json | `--firebase-rounds ` | Firebase scrypt rounds | | `--firebase-mem-cost ` | Firebase scrypt memory cost | | `-y, --yes` | Skip the confirmation prompt | +| `--runs-dir ` | Where runs are kept (see [Runs](#clerk-migrate-runs)) | Plus the targeting flags from the table above: `--secret-key`, `--app` and `--instance`. @@ -83,8 +84,8 @@ Plus the targeting flags from the table above: `--secret-key`, `--app` and `--transformer` and `--file` are required. Omitting either fails with a usage error that names the valid values. -Failures do not stop the run: each user's outcome is written to the log and the -import continues. A `429` backs off — honouring `Retry-After` when the response +Failures do not stop the run: each user's outcome is written to the +[run](#clerk-migrate-runs) and the import continues. A `429` backs off — honouring `Retry-After` when the response carries it — and retries up to 5 times before the user is recorded as failed. The command exits non-zero if any user failed. @@ -277,8 +278,9 @@ flag's absence means "development" — the resolved key decides, through `--secret-key`, `--app`, `CLERK_SECRET_KEY`, the keyless project and the linked profile in that order. -Every export also writes `logs/export-.log`, so `migrate logs list` -sees it alongside imports and deletions. +Every export is also a [run](#clerk-migrate-runs), with one line per exported +user, so `clerk migrate runs` lists it alongside imports. Every export takes +`--runs-dir ` to keep that run somewhere else. #### Three platforms export no passwords @@ -471,98 +473,58 @@ and every 10 pages during the user fetch. `withSpinner` hands a no-op to anything that is not a TTY, so without this an agent exporting a large tenant would see nothing at all until the run finished. -### `clerk migrate logs` +### `clerk migrate runs` -Everything that touches the local log directory — `./logs` unless -`CLERK_MIGRATE_LOG_DIR` says otherwise. Noun-verb like every -other group in the CLI (`config pull`, `users list`), rather than the standalone -tool's `clean-logs`/`convert-logs`, which were npm script names. +Every import and export is a **run**, and the run store is the one place +`clerk migrate` keeps state. `runs` reads it. ```sh -clerk migrate logs # defaults to list -clerk migrate logs list --json -clerk migrate logs clean -y -clerk migrate logs convert --all -clerk migrate logs convert import-2026-01-01T12-00-00.log +clerk migrate runs # every run, newest first +clerk migrate runs 20260929-141502-a1b2 # one run in full +clerk migrate runs --json ``` -| Subcommand | Takes | Description | -| -------------- | ------------------ | ----------------------------------------------- | -| `logs list` | `--json` | File, type, date, size and entry count per file | -| `logs clean` | `-y, --yes` | Delete the `.log` files in the log directory | -| `logs convert` | `[file…]`, `--all` | NDJSON → a JSON array, written as `.json` | +| Flag | Description | +| ------------------- | ----------------------------- | +| `[run-id]` | Show one run instead of all | +| `--json` | The same data, on stdout | +| `--runs-dir ` | Read runs from somewhere else | -All three read the directory through one shared enumerator, which is what makes -`logs list` nearly free. +It prints the runs folder first. The listing shows each run's ID, date, kind, +status, target, file and counts. `runs ` adds the error breakdown and the +users that failed or were skipped, with the path to the full record. An unknown +ID exits 2. -#### `logs list` +#### Where runs are kept -The default, because listing is read-only and therefore safe to run by -accident. Reports each file's name, type, date, size and entry count, newest -first; `--json` gives an agent the same data without parsing NDJSON. +The first of these that is set: -``` -Each log represents a user export, user import, or a user delete run. -Each log consists of a single NDJSON entry per user. - -FILE TYPE DATE SIZE ENTRIES -import-2026-02-01T09-14-22.log import Feb 1, 2026 at 4:14 AM 4.1 KB 120 -delete-2026-01-30T17-02-51.log delete Jan 30, 2026 at 12:02 PM 612 B 18 - -2 log files in ./logs - -Log types: - export One entry per user pulled from the source platform. - import One entry per user created in Clerk, with any error. - delete One entry per user removed from Clerk, with any error. -``` - -A kind is the name of the command that wrote it — `migrate import` writes -`import-.log` — so a listing points straight at the run behind each -line. The legend is fixed rather than derived from what happens to be present, -because "what else could be here" is the other half of the question. - -The older `migration-` and `user-deletion-` names, written by the standalone -tool and by earlier CLI builds, still classify as `import` and `delete`, so a -directory of old logs lists and converts unchanged. - -The filename leads, because it is what `logs convert` and `logs clean` talk -about. The date column renders the filename's UTC stamp in the reader's own -zone — "which run was that" is a question about local time; `--json` keeps the -raw stamp. +1. `--runs-dir ` +2. `CLERK_MIGRATE_DIR` +3. `/.clerk/migrate/` -The directory is printed relative (`./logs`) when it sits under the current -directory and absolute when it does not, so the path can be pasted either way. +The project root is the linked profile's directory, then the git toplevel, then +the current directory. Writing to the default location adds `.clerk/` to the +project's `.gitignore` first, because run files carry user data. -Says so plainly when the log directory is empty or absent. +#### What a run holds -#### `logs clean` +Each run is a folder named for its ID, `YYYYMMDD-HHmmss-xxxx`: -Destructive, so the confirmation is not optional: interactive runs prompt -(defaulting to **no**), and non-interactive or agent runs must pass `-y` rather -than being allowed to assume. Deletes `.log` files only — converted `.json` -output is left alone. +| File | Contents | +| -------------- | -------------------------------------------------------------------------------------------------------- | +| `run.json` | Kind, status, start and finish times, the target, the source, the file and its sha256, and the counts | +| `users.ndjson` | One line per user outcome: `sourceId`, `clerkId`, `status`, and `reason`, `error` or `code` when present | +| `lock` | The PID of the process writing the run, while it runs | -#### `logs convert` +A user's status is `created`, `failed`, `skipped` or `exported`. The last line +for each `sourceId` wins. A `429` retry, an extra email or phone that did not +attach, and a validation failure all land in `error`. -Turns NDJSON into a JSON array for spreadsheet or database analysis, written -alongside the original as `.json`. The original is left in place. - -Takes file positionals or `--all`; given neither, an interactive terminal -offers a multiselect and an agent gets a usage error naming both alternatives. - -A malformed line is reported with its line number and skipped, and the -remaining entries still convert: - -``` -import-2026-01-01T12-00-00.log:2 is not valid JSON and was skipped — … -1 malformed line skipped. -``` - -That beats failing the whole file: a run killed mid-write leaves one truncated -final line, and the hundreds of complete entries before it are still worth -having. It also beats dropping the line silently, which would leave a JSON -array that looks complete. +A run is `partial` when any user failed or was skipped, and `complete` +otherwise. A run whose process died, or that never recorded a finish time, +lists as `interrupted`. A lock held by a live process refuses a second writer +with exit 2. ## Transformers @@ -973,21 +935,15 @@ those are reachable through has no route for any of these settings, so ## Artifacts -Both are written relative to the **current working directory**, not to the -CLI's config directory, because they describe "which file am I migrating" -rather than "which project is linked here". +| Path | Contents | +| ------------------------------------------ | --------------------------------------------------- | +| `//` | One [run](#what-a-run-holds) per import or export | +| `./exports/-export-.json` | The export itself, unless `--output` says otherwise | -| Path | Contents | -| ------------------------------------------ | --------------------------------------------------------------------- | -| `./logs/export-.log` | NDJSON: one line per exported user | -| `./logs/import-.log` | NDJSON: one line per user, plus validation failures and retry notices | -| `./exports/-export-.json` | The export itself, unless `--output` says otherwise | +`users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C +still leaves a complete record of everything already processed. -Log writes are synchronous appends, so a run interrupted with Ctrl-C still -leaves a complete record of everything already processed. Use the last -successful `userId` in that log with `--resume-after` to continue. - -### Why the logs are NDJSON +### Why `users.ndjson` is NDJSON One JSON object per line, rather than one JSON array per file. A migration is a long append-only stream, and that format is the one that survives it: @@ -997,20 +953,15 @@ long append-only stream, and that format is the one that survives it: - **Crash-safe.** Kill the process at any point and every line already written is still valid. A truncated array is not parseable at all. - **Streamable.** `tail -f` shows a long import progressing live, and analysis - reads line by line instead of loading a million-user log into memory. + reads line by line instead of loading a million-user record into memory. Which is also why it greps usefully without any tooling: ```sh -grep '"status":"success"' logs/import-2026-01-01T12-00-00.log | wc -l -grep '"userId":"user_123"' logs/import-2026-01-01T12-00-00.log +grep '"status":"created"' .clerk/migrate/20260929-141502-a1b2/users.ndjson | wc -l +grep '"sourceId":"user_123"' .clerk/migrate/20260929-141502-a1b2/users.ndjson ``` -The trade-off is that spreadsheets, databases and most JSON tooling want an -array. That is what `clerk migrate logs convert` is for — convert when you need -to open a log in Excel or hand it to someone who should not have to know what -NDJSON is. The original `.log` stays put. - ## API Endpoints | Method | Path | Used by | diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index 8355a23f7..1a1c47931 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -4,8 +4,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; +import type { UserLine } from "../lib/run-store.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { getLogDir } from "../lib/logger.ts"; import { buildAuth0Export, exportAuth0, @@ -43,7 +43,7 @@ beforeEach(() => { // leaked "human" from an earlier test stops a later one on the destination prompt. setMode("agent"); requests = []; - fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); }); @@ -261,10 +261,11 @@ describe("mapAuth0UserToExport", () => { }); describe("buildAuth0Export", () => { - test("counts coverage and logs each user", () => { + test("counts coverage and records each user", () => { + const lines: UserLine[] = []; const { users, coverage } = buildAuth0Export( [auth0User(0), auth0User(1, { given_name: undefined })], - "2026-01-01T00:00:00", + (line) => lines.push(line), ); expect(users).toHaveLength(2); @@ -272,8 +273,7 @@ describe("buildAuth0Export", () => { expect(byLabel["have an email address"]).toBe(2); expect(byLabel["have a first name"]).toBe(1); - const logged = fs.readdirSync(getLogDir()); - expect(logged[0]).toMatch(/^export-/); + expect(lines.map((line) => line.status)).toEqual(["exported", "exported"]); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 260abe1eb..49fbd206a 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -21,9 +21,15 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt, text } from "../../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import type { UserLine } from "../lib/run-store.ts"; import { withInputRetry } from "../lib/input-retry.ts"; -import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; +import { + finishExportRun, + reportExport, + resolveOutputPath, + startExportRun, + writeExportOutput, +} from "./shared.ts"; const PAGE_SIZE = 100; @@ -41,6 +47,8 @@ export type ExportAuth0Options = { clientId?: string; clientSecret?: string; output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; }; export type Auth0Credentials = { @@ -288,7 +296,10 @@ export type Auth0ExportResult = { coverage: { label: string; count: number }[]; }; -export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0ExportResult { +export function buildAuth0Export( + users: Auth0User[], + record: (line: UserLine) => void = () => {}, +): Auth0ExportResult { const exported: Record[] = []; const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0 }; @@ -304,9 +315,9 @@ export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0Exp if (mapped.family_name) counts.lastName++; if (mapped.phone_number) counts.phone++; - exportLogger({ userId, status: "success" }, dateTime); + record({ sourceId: userId, status: "exported" }); } catch (error) { - exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + record({ sourceId: userId, status: "skipped", error: (error as Error).message }); } } @@ -328,8 +339,6 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { const destination = await resolveOutputPath("auth0", options.output); await withGutter("Exporting users from Auth0", async () => { - const dateTime = getDateTimeStamp(); - // Only Auth0 can say whether these three go together, and whether the // application carries the `read:users` scope, so a rejected set is asked // for again here. @@ -346,15 +355,20 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { fetchAllAuth0Users({ credentials, token, spinner }), ); - const { users: exported, coverage } = buildAuth0Export(users, dateTime); + const run = await startExportRun(options, { platform: "auth0" }); + + const { users: exported, coverage } = buildAuth0Export(users, run.append); const outputPath = writeExportOutput(exported, destination); + const record = finishExportRun(run, outputPath); + reportExport({ platform: "auth0", userCount: exported.length, outputPath, coverage, transformerKey: "auth0", + runId: record.id, }); if (exported.length > 0) { diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 51628be7f..207be1278 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -13,9 +13,15 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { log } from "../../../lib/log.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import type { UserLine } from "../lib/run-store.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; +import { + finishExportRun, + reportExport, + resolveOutputPath, + startExportRun, + writeExportOutput, +} from "./shared.ts"; import { promptDbUrl, resolveDbUrl, @@ -74,7 +80,7 @@ export async function fetchAuthJsUsers( : new Error(`No Auth.js user table found. Tried ${TABLE_CANDIDATES.join(", ")}.`); } -export function buildAuthJsExport(rows: AuthJsRow[], dateTime: string) { +export function buildAuthJsExport(rows: AuthJsRow[], record: (line: UserLine) => void = () => {}) { const users: Record[] = []; const counts = { email: 0, emailVerified: 0, name: 0 }; @@ -98,7 +104,7 @@ export function buildAuthJsExport(rows: AuthJsRow[], dateTime: string) { } users.push(user); - exportLogger({ userId, status: "success" }, dateTime); + record({ sourceId: userId, status: "exported" }); } return { @@ -124,8 +130,6 @@ export async function exportAuthJs(options: DbExportOptions): Promise { const destination = await resolveOutputPath("authjs", options.output); await withGutter("Exporting users from Auth.js", async () => { - const dateTime = getDateTimeStamp(); - const { value: { rows, table }, } = await withInputRetry( @@ -138,15 +142,20 @@ export async function exportAuthJs(options: DbExportOptions): Promise { ); log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); - const { users, coverage } = buildAuthJsExport(rows, dateTime); + const run = await startExportRun(options, { platform: "authjs" }); + + const { users, coverage } = buildAuthJsExport(rows, run.append); const outputPath = writeExportOutput(users, destination); + const record = finishExportRun(run, outputPath); + reportExport({ platform: "authjs", userCount: users.length, outputPath, coverage, transformerKey: "authjs", + runId: record.id, }); if (users.length > 0) { diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 4e8adf089..ceeacbdb0 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -17,9 +17,15 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import type { UserLine } from "../lib/run-store.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; +import { + finishExportRun, + reportExport, + resolveOutputPath, + startExportRun, + writeExportOutput, +} from "./shared.ts"; import { promptDbUrl, resolveDbUrl, @@ -121,7 +127,10 @@ const FIELD_ALIASES: Record = { updatedAt: "updated_at", }; -export function buildBetterAuthExport(rows: BetterAuthRow[], dateTime: string) { +export function buildBetterAuthExport( + rows: BetterAuthRow[], + record: (line: UserLine) => void = () => {}, +) { const users: Record[] = []; const counts = { email: 0, emailVerified: 0, password: 0, name: 0, username: 0, phone: 0 }; @@ -142,7 +151,7 @@ export function buildBetterAuthExport(rows: BetterAuthRow[], dateTime: string) { if (row.phoneNumber) counts.phone++; users.push(user); - exportLogger({ userId, status: "success" }, dateTime); + record({ sourceId: userId, status: "exported" }); } return { @@ -171,8 +180,6 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const destination = await resolveOutputPath("betterauth", options.output); await withGutter("Exporting users from Better Auth", async () => { - const dateTime = getDateTimeStamp(); - const { value: { rows, plugins }, } = await withInputRetry( @@ -194,15 +201,20 @@ export async function exportBetterAuth(options: DbExportOptions): Promise : "No plugin columns detected; exporting the core user fields.", ); - const { users, coverage } = buildBetterAuthExport(rows, dateTime); + const run = await startExportRun(options, { platform: "betterauth" }); + + const { users, coverage } = buildBetterAuthExport(rows, run.append); const outputPath = writeExportOutput(users, destination); + const record = finishExportRun(run, outputPath); + reportExport({ platform: "betterauth", userCount: users.length, outputPath, coverage, transformerKey: "betterauth", + runId: record.id, }); }); } diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 75e1f0565..282a8b2bb 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -3,8 +3,8 @@ import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import type { UserLine } from "../lib/run-store.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { getLogDir } from "../lib/logger.ts"; import { buildClerkExport, exportClerk, @@ -37,7 +37,7 @@ beforeEach(() => { // leaked "human" from an earlier test stops a later one on the destination prompt. setMode("agent"); requests = []; - fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); }); @@ -203,10 +203,10 @@ describe("fetchAllClerkUsers", () => { describe("buildClerkExport", () => { test("counts coverage per field", () => { - const { coverage } = buildClerkExport( - [user({ id: "u1", first_name: "Ada", password_enabled: true }), user({ id: "u2" })], - "2026-01-01T00:00:00", - ); + const { coverage } = buildClerkExport([ + user({ id: "u1", first_name: "Ada", password_enabled: true }), + user({ id: "u2" }), + ]); const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); expect(byLabel["have an email address"]).toBe(2); @@ -214,21 +214,14 @@ describe("buildClerkExport", () => { expect(byLabel["have a password (not exportable — see below)"]).toBe(1); }); - test("logs one NDJSON line per exported user", () => { - buildClerkExport([user({ id: "u1" }), user({ id: "u2" })], "2026-01-01T00:00:00"); - - const entries = fs - .readdirSync(getLogDir()) - .flatMap((name) => fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n")) - .map((line) => JSON.parse(line) as Record); + test("records one line per exported user", () => { + const lines: UserLine[] = []; + buildClerkExport([user({ id: "u1" }), user({ id: "u2" })], (line) => lines.push(line)); - expect(entries).toHaveLength(2); - expect(entries[0]).toEqual({ userId: "u1", status: "success" }); - }); - - test("writes the export log where `logs list` will find it", () => { - buildClerkExport([user()], "2026-01-01T12:00:00"); - expect(fs.readdirSync(getLogDir())[0]).toBe("export-2026-01-01T12-00-00.log"); + expect(lines).toEqual([ + { sourceId: "u1", status: "exported" }, + { sourceId: "u2", status: "exported" }, + ]); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 22305feed..8a1d99f08 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -17,10 +17,16 @@ import { bapiRequest } from "../../../lib/bapi.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import type { UserLine } from "../lib/run-store.ts"; import { retryOn429 } from "../lib/retry.ts"; import { resolveClerkSource } from "./clerk-source.ts"; -import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; +import { + finishExportRun, + reportExport, + resolveOutputPath, + startExportRun, + writeExportOutput, +} from "./shared.ts"; /** BAPI's maximum page size for `GET /v1/users`. */ const PAGE_SIZE = 500; @@ -30,6 +36,8 @@ export type ExportClerkOptions = { secretKey?: string; app?: string; instance?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; }; type BapiIdentifier = { @@ -187,7 +195,10 @@ export type ClerkExportResult = { }; /** Maps every user and counts what the export actually contains. */ -export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExportResult { +export function buildClerkExport( + users: BapiUser[], + record: (line: UserLine) => void = () => {}, +): ClerkExportResult { const exported: Record[] = []; const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0, password: 0 }; @@ -203,9 +214,9 @@ export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExpo if (mapped.primary_phone_number) counts.phone++; if (user.password_enabled) counts.password++; - exportLogger({ userId: user.id, status: "success" }, dateTime); + record({ sourceId: user.id, status: "exported" }); } catch (error) { - exportLogger({ userId: user.id, status: "error", error: (error as Error).message }, dateTime); + record({ sourceId: user.id, status: "skipped", error: (error as Error).message }); } } @@ -235,16 +246,16 @@ export async function exportClerk(options: ExportClerkOptions): Promise { const destination = await resolveOutputPath("clerk", options.output); await withGutter("Exporting users from Clerk", async () => { - const dateTime = getDateTimeStamp(); - log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); const users = await withSpinner("Fetching users from Clerk...", async (spinner) => fetchAllClerkUsers({ secretKey: source.secretKey, spinner }), ); - const { users: exported, coverage } = buildClerkExport(users, dateTime); + const run = await startExportRun(options, { platform: "clerk", appLabel: source.target }); + const { users: exported, coverage } = buildClerkExport(users, run.append); const outputPath = writeExportOutput(exported, destination); + const record = finishExportRun(run, outputPath); reportExport({ platform: "clerk", @@ -252,6 +263,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { outputPath, coverage, transformerKey: "clerk", + runId: record.id, }); if (exported.length > 0) { diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 7cfc62a9a..677762c86 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -13,9 +13,9 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; +import type { UserLine } from "../lib/run-store.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { createDbClient, type DbClient } from "../lib/db.ts"; -import { getLogDir } from "../lib/logger.ts"; import { buildAuthJsExport, buildAuthJsQuery, exportAuthJs, fetchAuthJsUsers } from "./authjs.ts"; import { buildBetterAuthExport, @@ -49,7 +49,7 @@ afterAll(() => { }); beforeEach(() => { - fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); }); @@ -207,22 +207,19 @@ describe("authjs export", () => { }); test("treats email_verified as a nullable timestamp, not a boolean", () => { - const { users } = buildAuthJsExport( - [ - { id: "a", email: "a@x.dev", email_verified: "2024-01-15" }, - { id: "b", email: "b@x.dev", email_verified: null }, - ], - "2026-01-01T00:00:00", - ); + const { users } = buildAuthJsExport([ + { id: "a", email: "a@x.dev", email_verified: "2024-01-15" }, + { id: "b", email: "b@x.dev", email_verified: null }, + ]); expect(users[0]?.email_verified).toBe("2024-01-15"); expect("email_verified" in (users[1] ?? {})).toBe(false); }); test("counts coverage", () => { - const { coverage } = buildAuthJsExport( - [{ id: "a", email: "a@x.dev", name: "A", email_verified: "2024-01-01" }, { id: "b" }], - "2026-01-01T00:00:00", - ); + const { coverage } = buildAuthJsExport([ + { id: "a", email: "a@x.dev", name: "A", email_verified: "2024-01-01" }, + { id: "b" }, + ]); const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); expect(byLabel["have an email address"]).toBe(1); expect(byLabel["have a verified email"]).toBe(1); @@ -295,10 +292,9 @@ describe("betterauth export", () => { }); test("renames camelCase columns onto what the transformer reads", () => { - const { users } = buildBetterAuthExport( - [{ id: "u1", emailVerified: 1, phoneNumber: "+1555", createdAt: "2025-01-01" }], - "2026-01-01T00:00:00", - ); + const { users } = buildBetterAuthExport([ + { id: "u1", emailVerified: 1, phoneNumber: "+1555", createdAt: "2025-01-01" }, + ]); expect(users[0]).toMatchObject({ user_id: "u1", email_verified: 1, @@ -324,53 +320,41 @@ describe("betterauth export", () => { describe("supabase export", () => { test("serializes timestamps the transformer can parse", () => { - const { users } = buildSupabaseExport( - [{ id: "u1", email: "a@x.dev", created_at: new Date("2024-01-01T00:00:00Z") }], - "2026-01-01T00:00:00", - ); + const { users } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", created_at: new Date("2024-01-01T00:00:00Z") }, + ]); expect(users[0]?.created_at).toBe("2024-01-01T00:00:00.000Z"); }); test("omits null columns rather than exporting them", () => { - const { users } = buildSupabaseExport( - [{ id: "u1", email: "a@x.dev", phone: null, last_name: null }], - "2026-01-01T00:00:00", - ); + const { users } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", phone: null, last_name: null }, + ]); expect("phone" in (users[0] ?? {})).toBe(false); expect("last_name" in (users[0] ?? {})).toBe(false); }); test("counts the password hashes, the reason this reads the database", () => { - const { coverage } = buildSupabaseExport( - [ - { id: "u1", email: "a@x.dev", encrypted_password: "$2b$10$x" }, - { id: "u2", email: "b@x.dev" }, - ], - "2026-01-01T00:00:00", - ); + const { coverage } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", encrypted_password: "$2b$10$x" }, + { id: "u2", email: "b@x.dev" }, + ]); const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); expect(byLabel["have a password hash"]).toBe(1); }); test("keeps raw_app_meta_data, which --skip-unsupported-providers reads", () => { - const { users } = buildSupabaseExport( - [{ id: "u1", email: "a@x.dev", raw_app_meta_data: { providers: ["discord"] } }], - "2026-01-01T00:00:00", - ); + const { users } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", raw_app_meta_data: { providers: ["discord"] } }, + ]); expect(users[0]?.raw_app_meta_data).toEqual({ providers: ["discord"] }); }); - test("logs one NDJSON line per exported user", () => { - buildSupabaseExport([{ id: "u1" }, { id: "u2" }], "2026-01-01T12:00:00"); + test("records one line per exported user", () => { + const lines: UserLine[] = []; + buildSupabaseExport([{ id: "u1" }, { id: "u2" }], (line) => lines.push(line)); - const written = fs.readdirSync(getLogDir()); - expect(written[0]).toBe("export-2026-01-01T12-00-00.log"); - expect( - fs - .readFileSync(path.join(getLogDir(), written[0] as string), "utf-8") - .trim() - .split("\n"), - ).toHaveLength(2); + expect(lines).toHaveLength(2); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 3baeee59a..7eadfce7c 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -15,6 +15,8 @@ import { detectDbType, isLibsqlUrl, redactConnectionString, type DbPlatform } fr export type DbExportOptions = { dbUrl?: string; output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; }; export type ResolveConfig = { diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index de8234467..aae629866 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -5,8 +5,8 @@ import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import { setAssumeYes } from "../lib/assume-yes.ts"; +import type { UserLine } from "../lib/run-store.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { getLogDir } from "../lib/logger.ts"; import { buildFirebaseExport, exportFirebase, @@ -72,7 +72,7 @@ beforeEach(() => { setMode("agent"); requests = []; delete process.env.FIREBASE_AUTH_EMULATOR_HOST; - fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); }); @@ -357,17 +357,18 @@ describe("mapFirebaseUserToExport", () => { }); describe("buildFirebaseExport", () => { - test("counts coverage and logs each user", () => { + test("counts coverage and records each user", () => { + const lines: UserLine[] = []; const { users, coverage } = buildFirebaseExport( [fbUser(0), { localId: "fb1", phoneNumber: "+1555" }], - "2026-01-01T12:00:00", + (line) => lines.push(line), ); expect(users).toHaveLength(2); const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); expect(byLabel["have a password hash"]).toBe(1); expect(byLabel["have a phone number"]).toBe(1); - expect(fs.readdirSync(getLogDir())[0]).toBe("export-2026-01-01T12-00-00.log"); + expect(lines).toHaveLength(2); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index be65f9ea4..6400e547c 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -32,9 +32,15 @@ import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import type { UserLine } from "../lib/run-store.ts"; import { withInputRetry } from "../lib/input-retry.ts"; -import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; +import { + finishExportRun, + reportExport, + resolveOutputPath, + startExportRun, + writeExportOutput, +} from "./shared.ts"; /** Identity Toolkit's maximum for `accounts:batchGet`. */ const PAGE_SIZE = 1000; @@ -50,6 +56,8 @@ const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/firebase"; export type ExportFirebaseOptions = { serviceAccount?: string; output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; }; export type ServiceAccount = { @@ -426,7 +434,10 @@ export function mapFirebaseUserToExport(user: FirebaseUser): Record void = () => {}, +) { const exported: Record[] = []; const counts = { email: 0, verified: 0, password: 0, name: 0, phone: 0 }; @@ -442,9 +453,9 @@ export function buildFirebaseExport(users: FirebaseUser[], dateTime: string) { if (mapped.displayName) counts.name++; if (mapped.phoneNumber) counts.phone++; - exportLogger({ userId, status: "success" }, dateTime); + record({ sourceId: userId, status: "exported" }); } catch (error) { - exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + record({ sourceId: userId, status: "skipped", error: (error as Error).message }); } } @@ -507,8 +518,6 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { - const dateTime = getDateTimeStamp(); - // Only Google can say whether a well-formed key is still a valid one, so a // revoked or deleted key fails here and is asked for again. const { value: token, input: account } = await withInputRetry( @@ -526,15 +535,20 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise entry.label.includes("password"))?.count ?? 0; diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 506654063..a6a7f1bdb 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -10,6 +10,7 @@ import { exportFirebase } from "./firebase.ts"; import { exportSupabase } from "./supabase.ts"; import { exportWorkOs } from "./workos.ts"; import type { DbExportOptions } from "./db-options.ts"; +import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "../lib/run-store.ts"; import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; /** @@ -99,6 +100,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record handlers.picker(cmd.optsWithGlobals() as Record), ); @@ -110,6 +112,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Where to write the export, relative to the current directory") .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) .option("--secret-key ", "Backend API secret key to use") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") @@ -137,6 +140,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Machine-to-machine application client secret") .option("-o, --output ", "Where to write the export, relative to the current directory") .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) .setExamples([ { command: @@ -160,6 +164,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Path to a service account key JSON file") .option("-o, --output ", "Where to write the export, relative to the current directory") .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) .setExamples([ { command: "clerk migrate export firebase --service-account ./service-account.json", @@ -186,6 +191,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Postgres, MySQL, libsql/Turso or SQLite connection string") .option("-o, --output ", "Where to write the export, relative to the current directory") .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) .setExamples([ { command: `clerk migrate export ${platform.key} --db-url "${platform.example}"`, diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index 7c4dc1614..363b29513 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -17,6 +17,14 @@ import { log } from "../../../lib/log.ts"; import { text } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; +import { + resolveRunsDir, + sha256File, + startRun, + type Run, + type RunRecord, + type RunTarget, +} from "../lib/run-store.ts"; /** * `YYYYMMDD-HHmm`, local time — ISO 8601 basic format, minus seconds. @@ -95,6 +103,26 @@ export async function resolveOutputPath(platform: string, output?: string): Prom return chosen.trim(); } +/** + * Starts the export run that records each user as it is exported. + * + * Started once the users are in hand, so a rejected credential leaves no run + * behind. + */ +export async function startExportRun( + options: { runsDir?: string }, + target: RunTarget, +): Promise { + const runsDir = await resolveRunsDir(options.runsDir, { write: true }); + return startRun(runsDir, { kind: "export", target, source: target.platform }); +} + +/** Records the written file on the run, and finishes it. */ +export function finishExportRun(run: Run, outputPath: string): RunRecord { + run.update({ file: { path: outputPath, sha256: sha256File(outputPath) } }); + return run.finish(); +} + /** * Writes the export, creating any missing parent directories. * @@ -142,6 +170,8 @@ export type ExportSummary = { sections?: ExportSection[]; /** The transformer that reads this file, for the "what next" line. */ transformerKey: string; + /** The export run that recorded each user. */ + runId: string; }; /** @@ -157,6 +187,7 @@ export function reportExport(summary: ExportSummary): void { log.blank(); if (summary.userCount === 0) { log.warn(`No users found to export. Wrote an empty file to ${summary.outputPath}.`); + log.info(dim(`Run ${summary.runId}`)); return; } @@ -175,6 +206,9 @@ export function reportExport(summary: ExportSummary): void { log.success( `Exported ${summary.userCount} user${summary.userCount === 1 ? "" : "s"} to ${summary.outputPath}`, ); + log.info( + dim(`Run ${summary.runId}. See each user with \`clerk migrate runs ${summary.runId}\`.`), + ); log.blank(); for (const line of formatImportCommand( diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 20972154e..5f9f86654 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -12,9 +12,15 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import type { UserLine } from "../lib/run-store.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; +import { + finishExportRun, + reportExport, + resolveOutputPath, + startExportRun, + writeExportOutput, +} from "./shared.ts"; import { promptDbUrl, resolveDbUrl, @@ -74,7 +80,10 @@ export async function fetchSupabaseUsers(client: DbClient): Promise(EXPORT_QUERY); } -export function buildSupabaseExport(rows: SupabaseRow[], dateTime: string) { +export function buildSupabaseExport( + rows: SupabaseRow[], + record: (line: UserLine) => void = () => {}, +) { const users: Record[] = []; const counts = { email: 0, emailConfirmed: 0, password: 0, phone: 0, firstName: 0, lastName: 0 }; @@ -90,9 +99,9 @@ export function buildSupabaseExport(rows: SupabaseRow[], dateTime: string) { if (row.first_name) counts.firstName++; if (row.last_name) counts.lastName++; - exportLogger({ userId, status: "success" }, dateTime); + record({ sourceId: userId, status: "exported" }); } catch (error) { - exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + record({ sourceId: userId, status: "skipped", error: (error as Error).message }); } } @@ -122,8 +131,6 @@ export async function exportSupabase(options: DbExportOptions): Promise { const destination = await resolveOutputPath("supabase", options.output); await withGutter("Exporting users from Supabase", async () => { - const dateTime = getDateTimeStamp(); - const { value: rows } = await withInputRetry( dbUrl, async () => promptDbUrl(SUPABASE_DB), @@ -133,15 +140,20 @@ export async function exportSupabase(options: DbExportOptions): Promise { ), ); - const { users, coverage } = buildSupabaseExport(rows, dateTime); + const run = await startExportRun(options, { platform: "supabase" }); + + const { users, coverage } = buildSupabaseExport(rows, run.append); const outputPath = writeExportOutput(users, destination); + const record = finishExportRun(run, outputPath); + reportExport({ platform: "supabase", userCount: users.length, outputPath, coverage, transformerKey: "supabase", + runId: record.id, }); if (users.length > 0) { diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts index 9dbf2edbd..419217163 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.test.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -3,8 +3,8 @@ import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import type { UserLine } from "../lib/run-store.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { getLogDir } from "../lib/logger.ts"; import { setAssumeYes } from "../lib/assume-yes.ts"; import { buildIdentityReport, @@ -50,7 +50,7 @@ beforeEach(() => { // "human" from an earlier test stops a later one on the destination prompt. setMode("agent"); requests = []; - fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); }); @@ -347,10 +347,11 @@ describe("mapWorkOsUserToExport", () => { }); describe("buildWorkOsExport", () => { - test("counts coverage and logs each user", () => { + test("counts coverage and records each user", () => { + const lines: UserLine[] = []; const { users, coverage } = buildWorkOsExport( [workosUser(0), workosUser(1, { first_name: undefined })], - "2026-01-01T00:00:00", + (line) => lines.push(line), ); expect(users).toHaveLength(2); @@ -358,13 +359,12 @@ describe("buildWorkOsExport", () => { expect(byLabel["have an email address"]).toBe(2); expect(byLabel["have a first name"]).toBe(1); - const logged = fs.readdirSync(getLogDir()); - expect(logged[0]).toMatch(/^export-/); + expect(lines.map((line) => line.status)).toEqual(["exported", "exported"]); }); // Always shown, always zero: seeing it before the import is the point. test("reports the password row even though it can only ever be zero", () => { - const { coverage } = buildWorkOsExport([workosUser(0)], "2026-01-01T00:00:00"); + const { coverage } = buildWorkOsExport([workosUser(0)]); expect(coverage.at(-1)).toEqual({ label: "have a password (WorkOS returns none)", count: 0, @@ -376,7 +376,7 @@ describe("buildWorkOsExport", () => { test("keeps providers out of the coverage table", () => { const { coverage } = buildWorkOsExport( [workosUser(0)], - "2026-01-01T00:00:00", + undefined, new Map([["user_00", [{ provider: "GoogleOAuth" }]]]), ); expect(coverage.some((row) => row.label.toLowerCase().includes("oauth"))).toBe(false); diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index 1e96682f9..8d2060546 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -23,13 +23,15 @@ import { log } from "../../../lib/log.ts"; import { confirm, password as passwordPrompt } from "../../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import type { UserLine } from "../lib/run-store.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { createApiScheduler } from "../lib/scheduler.ts"; import { + finishExportRun, reportExport, resolveOutputPath, + startExportRun, writeExportOutput, type ExportSection, } from "./shared.ts"; @@ -69,6 +71,8 @@ export type ExportWorkOsOptions = { /** Unset means "ask"; `--no-with-identities` sets it to false. */ withIdentities?: boolean; output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; }; export type WorkOsUser = Record & { id?: string }; @@ -400,7 +404,7 @@ export type WorkOsExportResult = { export function buildWorkOsExport( users: WorkOsUser[], - dateTime: string, + record: (line: UserLine) => void = () => {}, identities?: Map, ): WorkOsExportResult { const exported: Record[] = []; @@ -417,9 +421,9 @@ export function buildWorkOsExport( if (mapped.last_name) counts.lastName++; if (mapped.metadata) counts.metadata++; - exportLogger({ userId, status: "success" }, dateTime); + record({ sourceId: userId, status: "exported" }); } catch (error) { - exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + record({ sourceId: userId, status: "skipped", error: (error as Error).message }); } } @@ -443,8 +447,6 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise const destination = await resolveOutputPath("workos", options.output); await withGutter("Exporting users from WorkOS", async () => { - const dateTime = getDateTimeStamp(); - // Only WorkOS can say whether the key is live, for the right environment, // and not revoked — so a rejected key is asked for again here. The page it // fetches is kept and reused, so proving the key costs no extra request. @@ -465,8 +467,14 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise ) : undefined; - const { users: exported, coverage } = buildWorkOsExport(users, dateTime, providers?.identities); + const run = await startExportRun(options, { platform: "workos" }); + const { users: exported, coverage } = buildWorkOsExport( + users, + run.append, + providers?.identities, + ); const outputPath = writeExportOutput(exported, destination); + const record = finishExportRun(run, outputPath); reportExport({ platform: "workos", @@ -477,6 +485,7 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise ? [buildIdentityReport(users, providers.identities, providers.failed)] : [], transformerKey: "workos", + runId: record.id, }); if (exported.length > 0) { diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index 68ed77a0e..e9a7b06ea 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -1,7 +1,4 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; import { BapiError } from "../../lib/errors.ts"; import { buildCreateUserBody, @@ -10,12 +7,11 @@ import { readRetryAfter, splitIdentifiers, } from "./import-users.ts"; -import { getLogFilePath } from "./lib/logger.ts"; import type { ResolvedLimits } from "./lib/instance.ts"; +import type { UserLine } from "./lib/run-store.ts"; import type { User } from "./types.ts"; const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurrencyLimit: 8 }; -const DATE_TIME = "2026-01-01T00:00:00"; const user = (overrides: Partial = {}): User => ({ userId: "u1", email: "a@x.dev", ...overrides }) as User; @@ -151,27 +147,22 @@ describe("normalizeErrorMessage", () => { }); describe("importUsers", () => { - let workDir: string; - let originalCwd: string; let originalFetch: typeof globalThis.fetch; let requests: { method: string; url: string; body: unknown }[]; + let lines: UserLine[]; + const record = (line: UserLine) => lines.push(line); beforeAll(() => { - originalCwd = process.cwd(); originalFetch = globalThis.fetch; - workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-import-")); - process.chdir(workDir); }); afterAll(() => { globalThis.fetch = originalFetch; - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); }); beforeEach(() => { requests = []; - fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); + lines = []; }); afterEach(() => { @@ -202,13 +193,6 @@ describe("importUsers", () => { headers, }); - const logEntries = () => - fs - .readFileSync(getLogFilePath("import", DATE_TIME), "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record); - test("creates each user and reports them as successful", async () => { stub(() => ok("user_created")); @@ -216,12 +200,12 @@ describe("importUsers", () => { users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], secretKey: "sk_test_x", limits: LIMITS, - dateTime: DATE_TIME, + record, }); expect(summary).toMatchObject({ totalProcessed: 2, successful: 2, failed: 0 }); expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); - expect(logEntries().filter((e) => e.status === "success")).toHaveLength(2); + expect(lines.filter((line) => line.status === "created")).toHaveLength(2); }); test("attaches additional and unverified identifiers after the user exists", async () => { @@ -237,7 +221,7 @@ describe("importUsers", () => { ], secretKey: "sk_test_x", limits: LIMITS, - dateTime: DATE_TIME, + record, }); const emails = requests.filter((r) => r.url.endsWith("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/v1/email_addresses")); @@ -248,7 +232,7 @@ describe("importUsers", () => { expect(requests.filter((r) => r.url.endsWith("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/v1/phone_numbers"))).toHaveLength(1); }); - test("logs a failed additional identifier without failing the user", async () => { + test("notes a failed additional identifier without failing the user", async () => { stub((url) => url.endsWith("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/v1/email_addresses") ? clerkError(422, "that email is taken") @@ -259,11 +243,13 @@ describe("importUsers", () => { users: [user({ email: ["a@x.dev", "b@x.dev"] })], secretKey: "sk_test_x", limits: LIMITS, - dateTime: DATE_TIME, + record, }); expect(summary).toMatchObject({ successful: 1, failed: 0 }); - expect(logEntries().some((e) => e.status === "additional_email_error")).toBe(true); + expect(lines).toHaveLength(1); + expect(lines[0]?.status).toBe("created"); + expect(lines[0]?.error).toContain("Failed to add additional email b@x.dev"); }); test("records a failed user and keeps going", async () => { @@ -275,13 +261,13 @@ describe("importUsers", () => { users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], secretKey: "sk_test_x", limits: LIMITS, - dateTime: DATE_TIME, + record, }); expect(summary.successful + summary.failed).toBe(2); expect(summary.failed).toBe(1); expect([...summary.errorBreakdown.values()]).toEqual([1]); - expect(logEntries().some((e) => e.status === "error" && e.code === "422")).toBe(true); + expect(lines.some((line) => line.status === "failed" && line.code === "422")).toBe(true); }); test("retries a 429 after the interval the server asked for", async () => { @@ -294,13 +280,15 @@ describe("importUsers", () => { users: [user()], secretKey: "sk_test_x", limits: LIMITS, - dateTime: DATE_TIME, + record, }); expect(summary).toMatchObject({ successful: 1, failed: 0 }); expect(performance.now() - started).toBeGreaterThanOrEqual(900); expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); - expect(logEntries().some((e) => e.status === "429_retry")).toBe(true); + expect(lines).toHaveLength(1); + expect(lines[0]).toMatchObject({ status: "created", clerkId: "user_ok" }); + expect(lines[0]?.error).toContain("Rate limit hit (429)"); }); test("gives up after the retry ceiling and records the user as failed", async () => { @@ -310,13 +298,13 @@ describe("importUsers", () => { users: [user()], secretKey: "sk_test_x", limits: LIMITS, - dateTime: DATE_TIME, + record, }); expect(summary).toMatchObject({ successful: 0, failed: 1 }); // One initial attempt plus MAX_RETRIES retries. expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(6); - expect(logEntries().some((e) => e.code === "429")).toBe(true); + expect(lines).toMatchObject([{ status: "failed", code: "429" }]); }, 20_000); test("carries the validation failure count into the summary", async () => { @@ -326,7 +314,7 @@ describe("importUsers", () => { users: [user()], secretKey: "sk_test_x", limits: LIMITS, - dateTime: DATE_TIME, + record, validationFailed: 4, }); diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 4676a7565..c29ccdad9 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -19,9 +19,9 @@ import { bapiRequest } from "../../lib/bapi.ts"; import { BapiError } from "../../lib/errors.ts"; import type { SpinnerControls } from "../../lib/spinner.ts"; -import { errorLogger, importLogger } from "./lib/logger.ts"; import type { ResolvedLimits } from "./lib/instance.ts"; import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; +import type { UserLine } from "./lib/run-store.ts"; import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; import type { ImportSummary, User } from "./types.ts"; @@ -171,18 +171,21 @@ export function buildCreateUserBody( type CreateContext = { secretKey: string; schedule: ApiScheduler; - dateTime: string; }; -/** Attaches one extra identifier, logging (but not rethrowing) any failure. */ +/** + * Attaches one extra identifier. + * + * @returns A note describing the failure, or `undefined` when it attached. + * Never throws: the user itself was already created. + */ async function attachIdentifier( ctx: CreateContext, - userId: string, clerkUserId: string, kind: "email" | "phone", value: string, verified: boolean, -): Promise { +): Promise { const path = kind === "email" ? "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/v1/email_addresses" : "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/v1/phone_numbers"; const body = kind === "email" @@ -198,31 +201,23 @@ async function attachIdentifier( body: JSON.stringify(body), }), ); + return undefined; } catch (error) { const label = `${verified ? "additional" : "unverified"} ${kind} ${value}`; - errorLogger( - { - userId, - status: `additional_${kind}_error`, - errors: [ - { - code: `additional_${kind}_failed`, - message: `Failed to add ${label}`, - longMessage: `Failed to add ${label}: ${(error as Error).message}`, - }, - ], - }, - ctx.dateTime, - ); + return `Failed to add ${label}: ${(error as Error).message}`; } } -/** Creates one user, then attaches any additional identifiers it carries. */ +/** + * Creates one user, then attaches any additional identifiers it carries. + * + * @returns The Clerk ID, and a note for each identifier that did not attach. + */ async function createUser( ctx: CreateContext, user: User, skipPasswordRequirement: boolean, -): Promise { +): Promise<{ clerkUserId: string; notes: string[] }> { const identifiers = splitIdentifiers(user); const response = await ctx.schedule(async () => @@ -238,29 +233,30 @@ async function createUser( // Extra identifiers are best-effort: a duplicate secondary email should not // undo a user who was otherwise imported successfully. - await Promise.all([ + const notes = await Promise.all([ ...identifiers.additionalEmails.map(async (email) => - attachIdentifier(ctx, user.userId, clerkUserId, "email", email, true), + attachIdentifier(ctx, clerkUserId, "email", email, true), ), ...identifiers.unverifiedEmails.map(async (email) => - attachIdentifier(ctx, user.userId, clerkUserId, "email", email, false), + attachIdentifier(ctx, clerkUserId, "email", email, false), ), ...identifiers.additionalPhones.map(async (phone) => - attachIdentifier(ctx, user.userId, clerkUserId, "phone", phone, true), + attachIdentifier(ctx, clerkUserId, "phone", phone, true), ), ...identifiers.unverifiedPhones.map(async (phone) => - attachIdentifier(ctx, user.userId, clerkUserId, "phone", phone, false), + attachIdentifier(ctx, clerkUserId, "phone", phone, false), ), ]); - return clerkUserId; + return { clerkUserId, notes: notes.filter((note): note is string => note !== undefined) }; } export type ImportUsersOptions = { users: User[]; secretKey: string; limits: ResolvedLimits; - dateTime: string; + /** Receives one line per user, as each one finishes. */ + record: (line: UserLine) => void; /** Allow users that carry no password. */ skipPasswordRequirement?: boolean; /** Carried into the summary so the report covers the whole file. */ @@ -279,7 +275,7 @@ export async function importUsers(options: ImportUsersOptions): Promise { + // One line per user, written once it has finished: a retry that succeeded + // and an extra email that did not attach are both part of that line's story. + const recordFailure = (userId: string, message: string, code: string, notes: string[]) => { failed++; processed++; const normalized = normalizeErrorMessage(message); errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); - importLogger({ userId, status: "error", error: message, code }, dateTime); + record({ sourceId: userId, status: "failed", error: [message, ...notes].join("; "), code }); progress(); }; const processUser = async (user: User): Promise => { + const retries: string[] = []; try { - const clerkUserId = await retryOn429( + const { clerkUserId, notes } = await retryOn429( async () => createUser(ctx, user, skipPasswordRequirement), - { - onRetry: ({ message }) => - errorLogger( - { - userId: user.userId, - status: "429_retry", - errors: [{ code: "rate_limit_retry", message, longMessage: message }], - }, - dateTime, - ), - }, + { onRetry: ({ message }) => retries.push(message) }, ); successful++; processed++; - importLogger({ userId: user.userId, status: "success", clerkUserId }, dateTime); + const error = [...notes, ...retries].join("; "); + record({ + sourceId: user.userId, + clerkId: clerkUserId, + status: "created", + ...(error ? { error } : {}), + }); progress(); } catch (error) { if (error instanceof RateLimitExceededError) { - recordFailure(user.userId, error.message, "429"); + recordFailure(user.userId, error.message, "429", retries); return; } const apiError = error as BapiError; const message = apiError.longMessage ?? apiError.message ?? "Unknown error"; - recordFailure(user.userId, message, String(apiError.status ?? "unknown")); + recordFailure(user.userId, message, String(apiError.status ?? "unknown"), retries); } }; diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index 2aea5a480..4e895c550 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -137,36 +137,29 @@ describe("registerMigrate", () => { ); }); - test.each([[["logs"]], [["logs", "list"]], [["logs", "clean"]], [["logs", "convert"]]])( - "registers migrate %p", - (names) => { - expect(findCommand(["migrate", ...names])).toBeDefined(); - }, - ); + test("registers runs with an optional run ID", () => { + const runs = findCommand(["migrate", "runs"]); + expect(runs?.registeredArguments[0]?.required).toBe(false); + expect(runs?.options.map((option) => option.long)).toEqual(["--json", "--runs-dir"]); + }); - // Listing is read-only, so it is safe as the default for a bare - // `clerk migrate logs`. - test("makes list the default logs subcommand", () => { - const logs = findCommand(["migrate", "logs"]) as unknown as { _defaultCommandName?: string }; - expect(logs._defaultCommandName).toBe("list"); + test("the logs group is gone: runs replaces it", () => { + expect(findCommand(["migrate", "logs"])).toBeUndefined(); }); + // Every command reads or writes the run store, so each one can be pointed + // somewhere else. test.each([ - [["logs", "list"], "--json"], - [["logs", "clean"], "--yes"], - [["logs", "convert"], "--all"], - ])("%s accepts %s", (names, flag) => { + [["import"]], + [["runs"]], + [["export"]], + ...exportPlatformKeys().map((platform) => [["export", platform]]), + ])("migrate %p accepts --runs-dir", (names) => { expect(findCommand(["migrate", ...names])?.options.map((option) => option.long)).toContain( - flag, + "--runs-dir", ); }); - test("logs convert takes variadic file positionals", () => { - const args = findCommand(["migrate", "logs", "convert"])?.registeredArguments; - expect(args?.[0]?.variadic).toBe(true); - expect(args?.[0]?.required).toBe(false); - }); - test("constrains --transformer to the registered transformers, for validation and completion", () => { const option = findCommand(["migrate", "import"])?.options.find( (o) => o.long === "--transformer", diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index 63b14c130..ec4719231 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -3,12 +3,13 @@ import type { Program } from "../../cli-program.ts"; import { parseIntegerOption } from "../../lib/option-parsers.ts"; import { setAssumeYes } from "./lib/assume-yes.ts"; import { registerMigrateExport } from "./export/index.ts"; -import { registerMigrateLogs } from "./logs/index.ts"; +import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; import { run } from "./run.ts"; +import { runs } from "./runs.ts"; import { list as transformersList } from "./transformers/list.ts"; import { transformerKeys } from "./transformers/registry.ts"; -const migrate = { run, transformersList }; +const migrate = { run, runs, transformersList }; export function registerMigrate(program: Program): void { const migrateCommand = program @@ -28,7 +29,7 @@ export function registerMigrate(program: Program): void { command: "clerk migrate export supabase", description: "Export users from Supabase, ready to import", }, - { command: "clerk migrate logs", description: "List the local migration logs" }, + { command: "clerk migrate runs", description: "List every migration run" }, { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, ]); @@ -80,6 +81,7 @@ export function registerMigrate(program: Program): void { .option("--secret-key ", "Backend API secret key to use") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) .setExamples([ { command: "clerk migrate import -y --transformer clerk --file users.json", @@ -104,6 +106,24 @@ export function registerMigrate(program: Program): void { registerMigrateExport(migrateCommand); + migrateCommand + .command("runs") + .description("List migration runs, or show one") + .argument("[run-id]", "A run to show in full") + .option("--json", "Output as JSON") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .setExamples([ + { command: "clerk migrate runs", description: "List every run, newest first" }, + { + command: "clerk migrate runs 20260929-141502-a1b2", + description: "Show one run: counts, errors and the users that did not make it", + }, + { command: "clerk migrate runs --json", description: "Machine-readable listing" }, + ]) + .action(async (runId, _opts, cmd) => + migrate.runs(runId, cmd.optsWithGlobals() as Parameters[1]), + ); + // A compiled binary has no source tree to grep, so the available mappings // need a command rather than only appearing in the interactive picker. const transformersCommand = migrateCommand @@ -138,6 +158,4 @@ export function registerMigrate(program: Program): void { cmd.optsWithGlobals() as Parameters[0], ), ); - - registerMigrateLogs(migrateCommand); } diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.test.ts b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts deleted file mode 100644 index cc7587453..000000000 --- a/packages/cli-core/src/commands/migrate/lib/log-files.test.ts +++ /dev/null @@ -1,203 +0,0 @@ -import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { classifyLogFile, findLogFile, formatSize, listLogFiles, readNdjson } from "./log-files.ts"; -import { getLogDir } from "./logger.ts"; - -let workDir: string; -let originalCwd: string; - -beforeAll(() => { - originalCwd = process.cwd(); - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logfiles-"))); - process.chdir(workDir); -}); - -afterAll(() => { - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); -}); - -beforeEach(() => { - fs.rmSync(getLogDir(), { recursive: true, force: true }); -}); - -/** Writes a log file with one NDJSON line per entry. */ -function writeLog(name: string, entries: unknown[]): string { - fs.mkdirSync(getLogDir(), { recursive: true }); - const filePath = path.join(getLogDir(), name); - fs.writeFileSync(filePath, entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n"); - return filePath; -} - -describe("classifyLogFile", () => { - test.each([ - ["import-2026-01-01T12-00-00.log", "import", "2026-01-01T12-00-00"], - ["delete-2026-01-01T12-00-00.log", "delete", "2026-01-01T12-00-00"], - ["export-2026-01-01T12-00-00.log", "export", "2026-01-01T12-00-00"], - // Written by the standalone tool and by earlier CLI builds. - ["migration-2026-01-01T12-00-00.log", "import", "2026-01-01T12-00-00"], - ["user-deletion-2026-01-01T12-00-00.log", "delete", "2026-01-01T12-00-00"], - ])("%s is a %s log from %s", (name, kind, timestamp) => { - expect(classifyLogFile(name)).toEqual({ kind: kind as never, timestamp }); - }); - - test.each([["random.log"], ["import.log"], ["notes.txt"]])( - "%s is unrecognized rather than a parse failure", - (name) => { - expect(classifyLogFile(name)).toEqual({ kind: "unknown", timestamp: "" }); - }, - ); -}); - -describe("listLogFiles", () => { - test("returns nothing when the directory does not exist", () => { - expect(fs.existsSync(getLogDir())).toBe(false); - expect(listLogFiles()).toEqual([]); - }); - - test("returns nothing when the directory is empty", () => { - fs.mkdirSync(getLogDir(), { recursive: true }); - expect(listLogFiles()).toEqual([]); - }); - - test("reports kind, timestamp, size and entry count per file", () => { - writeLog("import-2026-01-01T12-00-00.log", [{ userId: "u1" }, { userId: "u2" }]); - - const [file] = listLogFiles(); - expect(file).toMatchObject({ - name: "import-2026-01-01T12-00-00.log", - kind: "import", - timestamp: "2026-01-01T12-00-00", - entryCount: 2, - }); - expect(file?.sizeBytes).toBeGreaterThan(0); - }); - - test("ignores files that are not logs", () => { - writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); - fs.writeFileSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"), "[]"); - fs.writeFileSync(path.join(getLogDir(), "notes.txt"), "hi"); - - expect(listLogFiles().map((file) => file.name)).toEqual(["import-2026-01-01T12-00-00.log"]); - }); - - test("ignores subdirectories", () => { - fs.mkdirSync(path.join(getLogDir(), "nested.log"), { recursive: true }); - expect(listLogFiles()).toEqual([]); - }); - - test("returns the newest run first", () => { - writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); - writeLog("import-2026-03-01T12-00-00.log", [{ a: 1 }]); - writeLog("import-2026-02-01T12-00-00.log", [{ a: 1 }]); - - expect(listLogFiles().map((file) => file.timestamp)).toEqual([ - "2026-03-01T12-00-00", - "2026-02-01T12-00-00", - "2026-01-01T12-00-00", - ]); - }); - - // Sorting on the filename would put every "import-" ahead of every - // "migration-", regardless of when the runs actually happened. - test("orders by timestamp across log kinds, not by the name's prefix", () => { - writeLog("import-2026-01-30T17-02-51.log", [{ a: 1 }]); - writeLog("export-2026-02-01T09-14-22.log", [{ a: 1 }]); - - expect(listLogFiles().map((file) => file.kind)).toEqual(["export", "import"]); - }); - - test("sorts unrecognized names last", () => { - writeLog("something-else.log", [{ a: 1 }]); - writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); - - expect(listLogFiles().map((file) => file.name)).toEqual([ - "import-2026-01-01T12-00-00.log", - "something-else.log", - ]); - }); - - test("does not count blank lines as entries", () => { - fs.mkdirSync(getLogDir(), { recursive: true }); - fs.writeFileSync(path.join(getLogDir(), "migration-x.log"), '{"a":1}\n\n\n{"b":2}\n'); - expect(listLogFiles()[0]?.entryCount).toBe(2); - }); - - test("lists a log whose name does not match the convention", () => { - writeLog("something-else.log", [{ a: 1 }]); - expect(listLogFiles()[0]).toMatchObject({ kind: "unknown", timestamp: "", entryCount: 1 }); - }); -}); - -describe("findLogFile", () => { - beforeEach(() => { - writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); - }); - - test("finds a log by name", () => { - expect(findLogFile("import-2026-01-01T12-00-00.log")?.entryCount).toBe(1); - }); - - test("accepts a path and matches on the basename", () => { - expect(findLogFile("./logs/import-2026-01-01T12-00-00.log")?.entryCount).toBe(1); - }); - - test("returns nothing for a name that is not there", () => { - expect(findLogFile("migration-nope.log")).toBeUndefined(); - }); -}); - -describe("readNdjson", () => { - test("parses one entry per line", () => { - const file = writeLog("migration-a.log", [{ userId: "u1" }, { userId: "u2" }]); - const { entries, errors } = readNdjson(file); - - expect(entries).toEqual([{ userId: "u1" }, { userId: "u2" }]); - expect(errors).toEqual([]); - }); - - test("skips blank lines without reporting them", () => { - fs.mkdirSync(getLogDir(), { recursive: true }); - const file = path.join(getLogDir(), "migration-b.log"); - fs.writeFileSync(file, '\n{"a":1}\n \n{"b":2}\n\n'); - - const { entries, errors } = readNdjson(file); - expect(entries).toHaveLength(2); - expect(errors).toEqual([]); - }); - - // A run killed mid-write leaves one truncated line; the complete entries - // before it are still worth having, so the read reports rather than aborts. - test("reports a malformed line by number and keeps the rest", () => { - fs.mkdirSync(getLogDir(), { recursive: true }); - const file = path.join(getLogDir(), "migration-c.log"); - fs.writeFileSync(file, '{"a":1}\n{"b":\n{"c":3}\n'); - - const { entries, errors } = readNdjson(file); - expect(entries).toEqual([{ a: 1 }, { c: 3 }]); - expect(errors).toHaveLength(1); - expect(errors[0]?.line).toBe(2); - }); - - test("numbers lines from one, counting blanks", () => { - fs.mkdirSync(getLogDir(), { recursive: true }); - const file = path.join(getLogDir(), "migration-d.log"); - fs.writeFileSync(file, '\n\n{"a":1}\nnot json\n'); - - expect(readNdjson(file).errors[0]?.line).toBe(4); - }); -}); - -describe("formatSize", () => { - test.each([ - [0, "0 B"], - [512, "512 B"], - [1024, "1.0 KB"], - [1536, "1.5 KB"], - [1024 * 1024, "1.0 MB"], - ])("%i bytes reads as %s", (bytes, expected) => { - expect(formatSize(bytes)).toBe(expected); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.ts b/packages/cli-core/src/commands/migrate/lib/log-files.ts deleted file mode 100644 index af87c6712..000000000 --- a/packages/cli-core/src/commands/migrate/lib/log-files.ts +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Enumerating and reading the cwd-relative `./logs/` directory. - * - * The standalone migration-tool re-read the directory inside both of its log - * commands to build their pickers. `list`, `clean` and `convert` all share - * this instead, which is also what makes `logs list` nearly free. - */ - -import fs from "node:fs"; -import path from "node:path"; -import { getLogDir } from "./logger.ts"; - -/** - * The run that produced a log file, read from its filename prefix. - * - * The kinds are the command names — `migrate import` writes `import-*.log` — - * so a listing points straight at the command that produced each line. - */ -export type LogKind = "export" | "import" | "delete" | "unknown"; - -const FILENAME_PATTERN = /^(export|import|delete|migration|user-deletion)-(.+)\.log$/; - -/** - * `migration-` and `user-deletion-` are the names the standalone tool and - * earlier CLI builds wrote. They still classify, so a directory of older logs - * lists and converts rather than reading as "unknown". - */ -const KIND_BY_PREFIX: Record = { - export: "export", - import: "import", - delete: "delete", - migration: "import", - "user-deletion": "delete", -}; - -export type LogFile = { - name: string; - path: string; - kind: LogKind; - /** Timestamp as recorded in the filename, or `""` for an unrecognized name. */ - timestamp: string; - sizeBytes: number; - /** Non-empty NDJSON lines, malformed ones included. */ - entryCount: number; -}; - -export function classifyLogFile(name: string): { kind: LogKind; timestamp: string } { - const match = FILENAME_PATTERN.exec(name); - if (!match) return { kind: "unknown", timestamp: "" }; - return { kind: KIND_BY_PREFIX[match[1] as string] ?? "unknown", timestamp: match[2] as string }; -} - -function countEntries(filePath: string): number { - try { - return fs - .readFileSync(filePath, "utf-8") - .split("\n") - .filter((line) => line.trim().length > 0).length; - } catch { - // An unreadable file still belongs in the listing; its count is unknown. - return 0; - } -} - -/** - * Every `.log` file in `./logs/`, newest first. - * - * @returns An empty array when the directory is absent — "no logs yet" and "no - * logs directory" are the same thing to every caller. - */ -export function listLogFiles(): LogFile[] { - const dir = getLogDir(); - if (!fs.existsSync(dir)) return []; - - const files: LogFile[] = []; - for (const name of fs.readdirSync(dir)) { - if (!name.endsWith(".log")) continue; - - const filePath = path.join(dir, name); - let stats: fs.Stats; - try { - stats = fs.statSync(filePath); - } catch { - continue; - } - if (!stats.isFile()) continue; - - files.push({ - name, - path: filePath, - ...classifyLogFile(name), - sizeBytes: stats.size, - entryCount: countEntries(filePath), - }); - } - - // Sort on the timestamp, not the filename: the kind prefix sorts first in a - // filename comparison, which would interleave a run from January ahead of one - // from March purely because "import" > "export". Timestamps are - // ISO-ish and zero-padded, so lexical order is chronological. Names without - // one sort last, then alphabetically. - return files.sort( - (a, b) => b.timestamp.localeCompare(a.timestamp) || a.name.localeCompare(b.name), - ); -} - -/** Resolves a user-supplied name or path to a log file in `./logs/`. */ -export function findLogFile(nameOrPath: string): LogFile | undefined { - const wanted = path.basename(nameOrPath); - return listLogFiles().find((file) => file.name === wanted); -} - -export type NdjsonLineError = { - /** 1-indexed line number in the source file. */ - line: number; - message: string; -}; - -export type NdjsonReadResult = { - entries: unknown[]; - errors: NdjsonLineError[]; -}; - -/** - * Parses an NDJSON file line by line. - * - * Malformed lines are collected with their line numbers rather than aborting - * the read: a run killed mid-write leaves one truncated final line, and the - * hundreds of complete entries before it are still worth having. - */ -export function readNdjson(filePath: string): NdjsonReadResult { - const entries: unknown[] = []; - const errors: NdjsonLineError[] = []; - - const lines = fs.readFileSync(filePath, "utf-8").split("\n"); - for (const [index, line] of lines.entries()) { - if (line.trim().length === 0) continue; - try { - entries.push(JSON.parse(line)); - } catch (error) { - errors.push({ line: index + 1, message: (error as Error).message }); - } - } - - return { entries, errors }; -} - -/** Human-readable file size. */ -export function formatSize(bytes: number): string { - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; -} diff --git a/packages/cli-core/src/commands/migrate/lib/logger.test.ts b/packages/cli-core/src/commands/migrate/lib/logger.test.ts deleted file mode 100644 index 65000aaa9..000000000 --- a/packages/cli-core/src/commands/migrate/lib/logger.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { - errorLogger, - getDateTimeStamp, - getLogDir, - getLogFilePath, - importLogger, - validationLogger, -} from "./logger.ts"; - -const DATE_TIME = "2026-01-01T12:00:00"; - -let workDir: string; -let originalCwd: string; - -beforeAll(() => { - originalCwd = process.cwd(); - // realpath so the comparison against process.cwd() survives macOS's - // /var -> /private/var symlink. - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logger-"))); - process.chdir(workDir); -}); - -afterAll(() => { - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); -}); - -beforeEach(() => { - fs.rmSync(getLogDir(), { recursive: true, force: true }); -}); - -function readEntries(): Record[] { - return fs - .readFileSync(getLogFilePath("import", DATE_TIME), "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record); -} - -describe("log file paths", () => { - test("writes under the current working directory, not next to the binary", () => { - expect(getLogDir()).toBe(path.join(workDir, "logs")); - }); - - test("replaces the timestamp's colons so the name is valid on Windows", () => { - expect(path.basename(getLogFilePath("import", DATE_TIME))).toBe( - "import-2026-01-01T12-00-00.log", - ); - }); - - test("getDateTimeStamp drops milliseconds", () => { - expect(getDateTimeStamp()).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$/); - }); -}); - -describe("log writers", () => { - test("creates the logs directory on first write", () => { - expect(fs.existsSync(getLogDir())).toBe(false); - importLogger({ userId: "u1", status: "success", clerkUserId: "user_x" }, DATE_TIME); - expect(fs.existsSync(getLogDir())).toBe(true); - }); - - test("appends one NDJSON line per entry", () => { - importLogger({ userId: "u1", status: "success", clerkUserId: "user_x" }, DATE_TIME); - importLogger({ userId: "u2", status: "error", error: "boom", code: "422" }, DATE_TIME); - - const entries = readEntries(); - expect(entries).toHaveLength(2); - expect(entries[0]).toEqual({ userId: "u1", status: "success", clerkUserId: "user_x" }); - expect(entries[1]).toEqual({ userId: "u2", status: "error", error: "boom", code: "422" }); - }); - - test("writes one line per error in a failed payload", () => { - errorLogger( - { - userId: "u1", - status: "422", - errors: [ - { code: "a", message: "short a", longMessage: "long a" }, - { code: "b", message: "short b" }, - ], - }, - DATE_TIME, - ); - - const entries = readEntries(); - expect(entries).toHaveLength(2); - expect(entries[0]).toMatchObject({ type: "User Creation Error", error: "long a" }); - // Falls back to `message` when the API omitted a long form. - expect(entries[1]).toMatchObject({ error: "short b" }); - }); - - test("records validation failures in the same run log", () => { - validationLogger( - { error: "missing identifier", path: ["email"], userId: "u3", row: 4 }, - DATE_TIME, - ); - expect(readEntries()[0]).toEqual({ - userId: "u3", - status: "fail", - error: "missing identifier", - path: ["email"], - row: 4, - }); - }); -}); - -describe("resolving the log directory", () => { - let originalEnv: string | undefined; - - beforeAll(() => { - originalEnv = process.env.CLERK_MIGRATE_LOG_DIR; - }); - - afterAll(() => { - if (originalEnv === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; - else process.env.CLERK_MIGRATE_LOG_DIR = originalEnv; - }); - - beforeEach(() => { - delete process.env.CLERK_MIGRATE_LOG_DIR; - }); - - test("falls back to ./logs", () => { - expect(getLogDir()).toBe(path.join(workDir, "logs")); - }); - - test("reads CLERK_MIGRATE_LOG_DIR", () => { - process.env.CLERK_MIGRATE_LOG_DIR = "./audit"; - - expect(getLogFilePath("import", DATE_TIME)).toBe( - path.join(workDir, "audit", `import-${DATE_TIME.replace(/:/g, "-")}.log`), - ); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.ts b/packages/cli-core/src/commands/migrate/lib/logger.ts deleted file mode 100644 index 3d7ef5133..000000000 --- a/packages/cli-core/src/commands/migrate/lib/logger.ts +++ /dev/null @@ -1,131 +0,0 @@ -/** - * NDJSON migration logs. - * - * Ported from the standalone migration-tool's `src/logger.ts`, with one - * behavioural fix: logs are written relative to the current working directory - * rather than to `__dirname/../logs`. In a `bun build --compile` binary there - * is no source tree next to the executable, so the original path would land - * logs inside wherever the binary happens to live. - * - * Writes are synchronous appends so a run interrupted with Ctrl-C still leaves - * a complete record of everything already processed. - */ - -import fs from "node:fs"; -import path from "node:path"; -import { log } from "../../../lib/log.ts"; -import type { - DeleteLogEntry, - ErrorLog, - ErrorPayload, - ExportLogEntry, - ImportLogEntry, - ValidationErrorPayload, -} from "../types.ts"; - -/** Where logs go when `CLERK_MIGRATE_LOG_DIR` is not set. */ -export const DEFAULT_LOG_DIR = "./logs"; - -/** Absolute path of the log directory: `CLERK_MIGRATE_LOG_DIR`, else `./logs`. */ -export function getLogDir(): string { - return path.resolve(process.cwd(), process.env.CLERK_MIGRATE_LOG_DIR || DEFAULT_LOG_DIR); -} - -/** - * The log directory the way the user would type it from here. - * - * Relative (`./logs`) when it sits under the current directory, absolute when - * it does not — a path the reader can paste either way, without a home - * directory's worth of prefix on the common case. - */ -export function displayLogDir(): string { - const dir = getLogDir(); - const relative = path.relative(process.cwd(), dir); - if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) return dir; - return `.${path.sep}${relative}`; -} - -/** Absolute path of the log file a run with this timestamp writes to. */ -export function getLogFilePath(logFile: string, dateTime: string): string { - // Colons are illegal in Windows filenames, and the timestamp is an ISO string. - return path.join(getLogDir(), `${logFile}-${dateTime}.log`.replace(/:/g, "-")); -} - -/** ISO timestamp without milliseconds — the log-file name discriminator. */ -export function getDateTimeStamp(): string { - return new Date().toISOString().split(".")[0] ?? ""; -} - -function appendToLogFile(fullPath: string, entry: unknown): void { - try { - fs.mkdirSync(path.dirname(fullPath), { recursive: true }); - fs.appendFileSync(fullPath, `${JSON.stringify(entry)}\n`); - } catch (error) { - // A broken log destination must not abort an in-flight migration; the run - // is still making real progress against the API. - log.warn(`Could not write migration log: ${(error as Error).message}`); - } -} - -/** Writes each error in a failed API call as its own NDJSON line. */ -export function errorLogger(payload: ErrorPayload, dateTime: string): void { - for (const err of payload.errors) { - const entry: ErrorLog = { - type: "User Creation Error", - userId: payload.userId, - status: payload.status, - error: err.longMessage ?? err.message, - }; - appendToLogFile(getLogFilePath("import", dateTime), entry); - } -} - -/** Writes a user that failed schema validation before any API call. */ -export function validationLogger(payload: ValidationErrorPayload, dateTime: string): void { - appendToLogFile(getLogFilePath("import", dateTime), { - userId: payload.userId, - status: "fail" as const, - error: payload.error, - path: payload.path, - row: payload.row, - }); -} - -/** Writes the outcome of one import attempt. */ -export function importLogger(entry: ImportLogEntry, dateTime: string): void { - appendToLogFile(getLogFilePath("import", dateTime), entry); -} - -/** - * Writes the outcome of one deletion attempt. - * - * A separate `delete-` file rather than another line in the import log: undoing - * a migration is its own run, and mixing the two would make "what did this - * import do" unanswerable after an undo. - */ -export function deleteLogger(entry: DeleteLogEntry, dateTime: string): void { - appendToLogFile(getLogFilePath("delete", dateTime), entry); -} - -/** - * Writes the outcome of exporting one user. - * - * Its own `export-` file for the same reason deletes get theirs: an export is a - * distinct run, and `migrate logs list` reports each kind separately. - */ -export function exportLogger(entry: ExportLogEntry, dateTime: string): void { - appendToLogFile(getLogFilePath("export", dateTime), entry); -} - -/** Writes each error in a failed deletion as its own NDJSON line. */ -export function deleteErrorLogger(payload: ErrorPayload, dateTime: string): void { - for (const err of payload.errors) { - const entry: ErrorLog = { - type: "User Deletion Error", - userId: payload.userId, - status: payload.status, - error: err.longMessage ?? err.message, - }; - appendToLogFile(getLogFilePath("delete", dateTime), entry); - } -} diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts new file mode 100644 index 000000000..3f9ac5a3b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -0,0 +1,171 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { + continueRun, + latestUserLines, + listRuns, + newRunId, + readRun, + resolveRunsDir, + runState, + RUNS_DIR_ENV, + startRun, +} from "./run-store.ts"; + +let workDir: string; +let configDir: string; +let runsDir: string; +let originalEnv: string | undefined; + +beforeAll(() => { + originalEnv = process.env[RUNS_DIR_ENV]; + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-run-store-config-")); + _setConfigDir(configDir); +}); + +afterAll(() => { + if (originalEnv === undefined) delete process.env[RUNS_DIR_ENV]; + else process.env[RUNS_DIR_ENV] = originalEnv; + _setConfigDir(undefined); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + delete process.env[RUNS_DIR_ENV]; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-run-store-"))); + runsDir = path.join(workDir, "runs"); +}); + +afterEach(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +const init = { kind: "import" as const, target: { instanceId: "ins_1" }, source: "clerk" }; + +describe("resolveRunsDir", () => { + test("prefers --runs-dir, resolved against the cwd", async () => { + process.env[RUNS_DIR_ENV] = "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/from/env"; + expect(await resolveRunsDir("flag", { cwd: workDir })).toBe(path.join(workDir, "flag")); + }); + + test("falls back to CLERK_MIGRATE_DIR", async () => { + process.env[RUNS_DIR_ENV] = "from-env"; + expect(await resolveRunsDir(undefined, { cwd: workDir })).toBe(path.join(workDir, "from-env")); + }); + + test("defaults to .clerk/migrate in the project, gitignored only when writing", async () => { + expect(await resolveRunsDir(undefined, { cwd: workDir })).toBe( + path.join(workDir, ".clerk", "migrate"), + ); + expect(fs.existsSync(path.join(workDir, ".gitignore"))).toBe(false); + + await resolveRunsDir(undefined, { cwd: workDir, write: true }); + expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toBe(".clerk/\n"); + }); +}); + +describe("a run's life", () => { + test("IDs sort by start time and read as a date", () => { + expect(newRunId(new Date(2026, 8, 29, 14, 5, 2))).toMatch(/^20260929-140502-[0-9a-f]{4}$/); + }); + + test("starts running, holding a lock", () => { + const run = startRun(runsDir, init); + expect(readRun(runsDir, run.record.id)).toMatchObject({ status: "running", kind: "import" }); + expect(runState(runsDir, run.record)).toBe("running"); + }); + + test("finishes complete when every user made it", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + + const record = run.finish(); + + expect(record).toMatchObject({ status: "complete", counts: { total: 1, created: 1 } }); + expect(record.finishedAt).toBeDefined(); + expect(fs.existsSync(path.join(run.dir, "lock"))).toBe(false); + }); + + test.each([["failed"], ["skipped"]] as const)("finishes partial when a user was %s", (status) => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created" }); + run.append({ sourceId: "b", status }); + expect(run.finish().status).toBe("partial"); + }); + + test("counts each source ID by its last line", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "failed", error: "boom" }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + + expect(run.finish()).toMatchObject({ status: "complete", counts: { total: 1, created: 1 } }); + expect(latestUserLines(runsDir, run.record.id).get("a")?.clerkId).toBe("user_a"); + }); + + test("reads past a line a crash cut short", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created" }); + fs.appendFileSync(path.join(run.dir, "users.ndjson"), '{"sourceId":"b","sta'); + + expect([...latestUserLines(runsDir, run.record.id).keys()]).toEqual(["a"]); + }); +}); + +describe("locks and interruptions", () => { + // A PID no process can have. + const DEAD_PID = "2147483646"; + + test("a run whose process died is interrupted", () => { + const run = startRun(runsDir, init); + fs.writeFileSync(path.join(run.dir, "lock"), DEAD_PID); + expect(runState(runsDir, run.record)).toBe("interrupted"); + }); + + test("a run with no lock and no finish time is interrupted", () => { + const run = startRun(runsDir, init); + fs.rmSync(path.join(run.dir, "lock")); + expect(runState(runsDir, run.record)).toBe("interrupted"); + }); + + test("continuing takes the lock from a dead process and reopens the run", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created" }); + run.finish(); + + const again = continueRun(runsDir, readRun(runsDir, run.record.id)!); + expect(again.record).toMatchObject({ id: run.record.id, status: "running" }); + expect(again.record.finishedAt).toBeUndefined(); + again.append({ sourceId: "b", status: "created" }); + expect(again.finish().counts.total).toBe(2); + }); + + test("refuses a run another live process holds, with exit 2", () => { + const run = startRun(runsDir, init); + // PID 1 is always alive, and never this test. + fs.writeFileSync(path.join(run.dir, "lock"), "1"); + + expect(() => continueRun(runsDir, run.record)).toThrow(/in use by another process \(PID 1\)/); + }); +}); + +describe("listRuns", () => { + test("lists newest first and ignores folders that are not runs", () => { + const first = startRun(runsDir, init); + first.update({ startedAt: "2026-01-01T00:00:00.000Z" }); + const second = startRun(runsDir, { ...init, kind: "export" }); + second.update({ startedAt: "2026-02-01T00:00:00.000Z" }); + fs.mkdirSync(path.join(runsDir, "not-a-run")); + + expect(listRuns(runsDir).map((record) => record.id)).toEqual([ + second.record.id, + first.record.id, + ]); + }); + + test("is empty when the folder does not exist", () => { + expect(listRuns(path.join(workDir, "nowhere"))).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts new file mode 100644 index 000000000..460491088 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -0,0 +1,370 @@ +/** + * The run store: the one place `clerk migrate` keeps state. + * + * Every import, export and undo is a run. A run is a folder under the runs + * directory holding: + * + * - `run.json` — what ran, against what, from which file, and how it ended. + * - `users.ndjson` — one line per user outcome. The last line for a source ID + * wins, so a continued run appends rather than rewrites. + * - `lock` — the PID of the process working on the run. A live PID refuses a + * second writer; a dead one, or a run with no `finishedAt`, means the run + * was interrupted. + * + * `runs`, `undo`, re-runs and exports all read or write it, so there is no + * second record to drift out of step. + * + * Appends are synchronous so a run interrupted with Ctrl-C still leaves a + * complete record of everything already processed. + */ + +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { resolveProfile } from "../../../lib/config.ts"; +import { throwUsageError } from "../../../lib/errors.ts"; +import { ensureGitignoreEntry, getGitRepoRoot } from "../../../lib/git.ts"; +import { log } from "../../../lib/log.ts"; + +/** Overrides the runs directory, below `--runs-dir`. */ +export const RUNS_DIR_ENV = "CLERK_MIGRATE_DIR"; + +/** The `--runs-dir` option every migrate subcommand takes. */ +export const RUNS_DIR_FLAG = "--runs-dir "; +export const RUNS_DIR_DESCRIPTION = `Where migration runs are kept (default: .clerk/migrate in the project, or ${RUNS_DIR_ENV})`; + +export type RunKind = "import" | "undo" | "export"; +export type RunStatus = "running" | "complete" | "partial" | "undone"; +export type UserStatus = "created" | "failed" | "skipped" | "deleted" | "exported"; + +/** + * What a run acted on. For an import or undo, the Clerk instance. For an + * export, the source platform, plus the Clerk instance for `export clerk`. + */ +export type RunTarget = { + env?: string; + appId?: string; + appLabel?: string; + instanceId?: string; + instanceType?: string; + keySource?: string; + /** The platform an export read from. */ + platform?: string; +}; + +export type RunFile = { path: string; sha256: string }; + +export type RunCounts = Partial> & { total: number }; + +export type RunRecord = { + id: string; + kind: RunKind; + status: RunStatus; + startedAt: string; + finishedAt?: string; + target: RunTarget; + source?: string; + /** Content hash of a custom source, so an edited one is a different source. */ + sourceHash?: string; + file?: RunFile; + /** The export run an import read its file from. */ + fromExport?: string; + /** The import run an undo reverses. */ + undoes?: string; + /** The undo run that reversed this one. */ + undoneBy?: string; + counts: RunCounts; +}; + +export type UserLine = { + sourceId: string; + clerkId?: string; + status: UserStatus; + /** Why a user was skipped. */ + reason?: string; + error?: string; + code?: string; + passwordDropped?: boolean; +}; + +/** What `runs` shows for a run: its stored status, or how it stopped. */ +export type RunState = RunStatus | "interrupted"; + +const RUN_FILE = "run.json"; +const USERS_FILE = "users.ndjson"; +const LOCK_FILE = "lock"; + +// --- Location -------------------------------------------------------------- + +/** + * Where the project lives, for the default runs directory. + * + * A profile linked by directory names it outright. One linked by git remote + * or repository names the repository, whose root is the git toplevel. An + * unlinked directory outside git is its own project. + */ +async function projectRoot(cwd: string): Promise { + const profile = await resolveProfile(cwd); + if (profile?.resolvedVia === "directory") return profile.path; + return (await getGitRepoRoot(cwd)) ?? cwd; +} + +/** + * The runs directory: `--runs-dir`, then `CLERK_MIGRATE_DIR`, then + * `/.clerk/migrate`. + * + * @param options.write - The caller is about to write a run. The default + * location is gitignored first, because run files carry user data. + */ +export async function resolveRunsDir( + runsDir: string | undefined, + options: { write?: boolean; cwd?: string } = {}, +): Promise { + const cwd = options.cwd ?? process.cwd(); + if (runsDir) return path.resolve(cwd, runsDir); + + const fromEnv = process.env[RUNS_DIR_ENV]; + if (fromEnv) return path.resolve(cwd, fromEnv); + + const root = await projectRoot(cwd); + if (options.write) await ensureGitignoreEntry(root, ".clerk/"); + return path.join(root, ".clerk", "migrate"); +} + +/** The folder one run lives in. */ +export function runDir(runsDir: string, id: string): string { + return path.join(runsDir, id); +} + +// --- IDs and hashes -------------------------------------------------------- + +/** + * `YYYYMMDD-HHmmss-xxxx`, local time. + * + * Sorts by start time, reads as a date, and the random suffix keeps two runs + * started in the same second apart. + */ +export function newRunId(now: Date = new Date()): string { + const pad = (value: number) => String(value).padStart(2, "0"); + const date = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}`; + const time = `${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`; + return `${date}-${time}-${randomBytes(2).toString("hex")}`; +} + +/** Hex sha256 of a file's bytes. */ +export function sha256File(file: string): string { + return createHash("sha256").update(fs.readFileSync(file)).digest("hex"); +} + +// --- Locking --------------------------------------------------------------- + +function isPidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM: the process exists but belongs to someone else. + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +/** The PID holding the run's lock, when that process is still alive. */ +export function liveLockPid(runsDir: string, id: string): number | undefined { + let raw: string; + try { + raw = fs.readFileSync(path.join(runDir(runsDir, id), LOCK_FILE), "utf-8"); + } catch { + return undefined; + } + const pid = Number(raw.trim()); + return Number.isInteger(pid) && pid > 0 && isPidAlive(pid) ? pid : undefined; +} + +function acquireLock(runsDir: string, id: string): void { + const holder = liveLockPid(runsDir, id); + if (holder !== undefined && holder !== process.pid) { + throwUsageError( + `Run ${id} is in use by another process (PID ${holder}). Wait for it to finish, then try again.`, + ); + } + fs.writeFileSync(path.join(runDir(runsDir, id), LOCK_FILE), String(process.pid)); +} + +// --- Reading --------------------------------------------------------------- + +/** One run's record, or `undefined` when the folder holds no readable run. */ +export function readRun(runsDir: string, id: string): RunRecord | undefined { + try { + const parsed = JSON.parse( + fs.readFileSync(path.join(runDir(runsDir, id), RUN_FILE), "utf-8"), + ) as RunRecord; + return parsed && typeof parsed === "object" && parsed.id === id ? parsed : undefined; + } catch { + return undefined; + } +} + +/** Every line of a run's `users.ndjson`, in the order written. */ +export function readUserLines(runsDir: string, id: string): UserLine[] { + let raw: string; + try { + raw = fs.readFileSync(path.join(runDir(runsDir, id), USERS_FILE), "utf-8"); + } catch { + return []; + } + const lines: UserLine[] = []; + for (const line of raw.split("\n")) { + if (!line.trim()) continue; + try { + lines.push(JSON.parse(line) as UserLine); + } catch { + // A line cut short by a crash mid-write. Every earlier line still counts. + } + } + return lines; +} + +/** Each source ID's latest outcome, in first-seen order. */ +export function latestUserLines(runsDir: string, id: string): Map { + const latest = new Map(); + for (const line of readUserLines(runsDir, id)) latest.set(line.sourceId, line); + return latest; +} + +/** + * How a run stands now. A run that never finished and whose process is gone + * was interrupted, whatever its stored status says. + */ +export function runState(runsDir: string, record: RunRecord): RunState { + if (record.finishedAt) return record.status; + return liveLockPid(runsDir, record.id) === undefined ? "interrupted" : "running"; +} + +/** Every readable run, newest first. */ +export function listRuns(runsDir: string): RunRecord[] { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(runsDir, { withFileTypes: true }); + } catch { + return []; + } + return entries + .filter((entry) => entry.isDirectory()) + .map((entry) => readRun(runsDir, entry.name)) + .filter((record): record is RunRecord => record !== undefined) + .sort((a, b) => b.startedAt.localeCompare(a.startedAt) || b.id.localeCompare(a.id)); +} + +// --- Writing --------------------------------------------------------------- + +function writeRecord(runsDir: string, record: RunRecord): void { + const file = path.join(runDir(runsDir, record.id), RUN_FILE); + // Written whole and renamed into place, so a reader never sees half a file. + const temp = `${file}.tmp`; + fs.writeFileSync(temp, `${JSON.stringify(record, null, 2)}\n`); + fs.renameSync(temp, file); +} + +/** Totals by each source ID's latest status. */ +export function countLines(latest: Iterable): RunCounts { + const counts: RunCounts = { total: 0 }; + for (const line of latest) { + counts.total++; + counts[line.status] = (counts[line.status] ?? 0) + 1; + } + return counts; +} + +/** A run being written by this process. */ +export type Run = { + readonly runsDir: string; + readonly dir: string; + record: RunRecord; + /** Appends one user outcome. */ + append(line: UserLine): void; + /** Merges fields into `run.json`. */ + update(patch: Partial): void; + /** + * Counts the outcomes, settles the status and releases the lock. + * + * `partial` when any user failed or was skipped, `complete` otherwise. + */ + finish(): RunRecord; +}; + +function openRun(runsDir: string, record: RunRecord): Run { + const dir = runDir(runsDir, record.id); + const usersFile = path.join(dir, USERS_FILE); + + const run: Run = { + runsDir, + dir, + record, + append(line) { + try { + fs.appendFileSync(usersFile, `${JSON.stringify(line)}\n`); + } catch (error) { + // A broken destination must not abort an in-flight migration; the run + // is still making real progress against the API. + log.warn(`Could not write to ${usersFile}: ${(error as Error).message}`); + } + }, + update(patch) { + run.record = { ...run.record, ...patch }; + writeRecord(runsDir, run.record); + }, + finish() { + const counts = countLines(latestUserLines(runsDir, run.record.id).values()); + const unfinished = (counts.failed ?? 0) + (counts.skipped ?? 0); + run.update({ + counts, + status: unfinished > 0 ? "partial" : "complete", + finishedAt: new Date().toISOString(), + }); + fs.rmSync(path.join(dir, LOCK_FILE), { force: true }); + return run.record; + }, + }; + return run; +} + +export type StartRunInit = Omit; + +/** Creates a run folder, takes its lock and writes the first `run.json`. */ +export function startRun(runsDir: string, init: StartRunInit): Run { + const id = newRunId(); + fs.mkdirSync(runDir(runsDir, id), { recursive: true }); + acquireLock(runsDir, id); + + const record: RunRecord = { + id, + status: "running", + startedAt: new Date().toISOString(), + counts: { total: 0 }, + ...init, + }; + writeRecord(runsDir, record); + log.debug(`migrate: started ${init.kind} run ${id} in ${runsDir}`); + return openRun(runsDir, record); +} + +/** + * Reopens a finished or interrupted run to keep working on it. + * + * @throws UsageError when another live process holds its lock. + */ +export function continueRun(runsDir: string, record: RunRecord): Run { + acquireLock(runsDir, record.id); + const run = openRun(runsDir, record); + run.record = { ...record, status: "running" }; + delete run.record.finishedAt; + writeRecord(runsDir, run.record); + log.debug(`migrate: continuing ${record.kind} run ${record.id} in ${runsDir}`); + return run; +} + +/** Merges fields into a run this process is not writing, such as `undoneBy`. */ +export function patchRun(runsDir: string, id: string, patch: Partial): void { + const record = readRun(runsDir, id); + if (record) writeRecord(runsDir, { ...record, ...patch }); +} diff --git a/packages/cli-core/src/commands/migrate/lib/target.ts b/packages/cli-core/src/commands/migrate/lib/target.ts new file mode 100644 index 000000000..2465cf9f7 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/target.ts @@ -0,0 +1,116 @@ +/** + * The Clerk instance a migrate command acts on, and where its key came from. + * + * `resolveBapiSecretKey` picks the key. This names what that key points at: + * the instance ID and environment come from `GET /v1/instance`, so every key + * source yields the same identity. That identity is what the run store keys + * resumes and undos on, so an app label or an `--instance` alias is never + * enough on its own. + */ + +import { createHash } from "node:crypto"; +import { bapiRequest } from "../../../lib/bapi.ts"; +import { resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; +import { resolveAppContext } from "../../../lib/config.ts"; +import { resolveKeylessTarget } from "../../../lib/keyless-target.ts"; +import { log } from "../../../lib/log.ts"; +import { detectInstanceType } from "./instance.ts"; +import type { RunTarget } from "./run-store.ts"; + +export type TargetOptions = { + secretKey?: string; + app?: string; + instance?: string; +}; + +/** A resolved Clerk instance. Every field a run needs is present. */ +export type ClerkTarget = RunTarget & { + env: string; + instanceId: string; + instanceType: "dev" | "prod"; + keySource: string; +}; + +/** + * Which rung of `resolveBapiSecretKey`'s chain supplies the key, and the app + * it belongs to when that rung knows one. + * + * Mirrors the chain's order exactly. An exported `CLERK_SECRET_KEY` wins over + * a linked profile, so the profile's app is not named for it: that key may + * belong to any app at all. + */ +async function describeKeySource( + options: TargetOptions, +): Promise<{ keySource: string; appId?: string; appLabel?: string }> { + if (options.secretKey) return { keySource: "--secret-key" }; + + if (options.app) { + const ctx = await resolveAppContext({ app: options.app, instance: options.instance }); + return { keySource: "--app", appId: ctx.appId, appLabel: ctx.appLabel }; + } + + if (process.env.CLERK_SECRET_KEY) return { keySource: "CLERK_SECRET_KEY env var" }; + + const keyless = await resolveKeylessTarget({ instance: options.instance }); + if (keyless) { + return { keySource: `accountless app (${keyless.source})`, appLabel: "accountless app" }; + } + + const ctx = await resolveAppContext({ instance: options.instance }); + return { keySource: "linked profile", appId: ctx.appId, appLabel: ctx.appLabel }; +} + +/** + * The instance behind a key. + * + * Falls back to a hash of the key when `GET /v1/instance` cannot be read: the + * same key always addresses the same instance, so it still tells two + * instances apart, and nothing is sent anywhere. + */ +async function fetchInstanceIdentity( + secretKey: string, +): Promise<{ instanceId: string; env: string }> { + const fallbackEnv = detectInstanceType(secretKey) === "prod" ? "production" : "development"; + try { + const { body } = await bapiRequest({ method: "GET", path: "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/v1/instance", secretKey }); + const instance = body as { id?: unknown; environment_type?: unknown }; + if (typeof instance.id === "string" && instance.id.startsWith("ins_")) { + return { + instanceId: instance.id, + env: + typeof instance.environment_type === "string" ? instance.environment_type : fallbackEnv, + }; + } + } catch (error) { + log.debug(`migrate: could not read the instance behind the key: ${String(error)}`); + } + const digest = createHash("sha256").update(secretKey).digest("hex").slice(0, 16); + return { instanceId: `key_${digest}`, env: fallbackEnv }; +} + +/** Resolves the key, then names the instance it addresses. */ +export async function resolveClerkTarget( + options: TargetOptions, +): Promise<{ secretKey: string; target: ClerkTarget }> { + const secretKey = await resolveBapiSecretKey(options); + const source = await describeKeySource(options); + const { instanceId, env } = await fetchInstanceIdentity(secretKey); + + return { + secretKey, + target: { + env, + instanceId, + instanceType: detectInstanceType(secretKey), + ...source, + }, + }; +} + +/** One line naming a target, for prose: `My App (development, ins_123)`. */ +export function describeTarget(target: RunTarget): string { + if (target.platform && !target.instanceId) return target.platform; + const where = [target.env, target.instanceId].filter(Boolean).join(", "); + const name = target.appLabel ?? (target.platform ? `${target.platform}` : "instance"); + return where ? `${name} (${where})` : name; +} diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index efb9f1e52..3981d9f71 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -15,8 +15,6 @@ import { validatePreparedUsers, } from "./transform.ts"; -const DATE_TIME = "2026-01-01T00-00-00"; - let workDir: string; let originalCwd: string; @@ -136,20 +134,21 @@ describe("consolidateClerkIdentifiers", () => { describe("validatePreparedUsers", () => { test("keeps valid users and counts the rest", () => { - const result = validatePreparedUsers( - [{ userId: "u1", email: "a@x.dev" }, { userId: "u2" }, { userId: "u3", username: "carol" }], - DATE_TIME, - ); + const result = validatePreparedUsers([ + { userId: "u1", email: "a@x.dev" }, + { userId: "u2" }, + { userId: "u3", username: "carol" }, + ]); expect(result.users.map((user) => user.userId)).toEqual(["u1", "u3"]); expect(result.validationFailed).toBe(1); + expect(result.failures).toMatchObject([{ userId: "u2", row: 1 }]); }); test("aborts the whole run on an unknown password hasher", () => { expect(() => - validatePreparedUsers( - [{ userId: "u1", email: "a@x.dev", password: "d", passwordHasher: "rot13" }], - DATE_TIME, - ), + validatePreparedUsers([ + { userId: "u1", email: "a@x.dev", password: "d", passwordHasher: "rot13" }, + ]), ).toThrow(CliError); }); }); @@ -166,7 +165,6 @@ describe("transformUsers", () => { }, ], "clerk", - DATE_TIME, ); expect(validationFailed).toBe(0); expect(transformedData[0]).toMatchObject({ @@ -177,14 +175,9 @@ describe("transformUsers", () => { }); test("skips validation when asked, so analysis passes see every row", () => { - const { transformedData, validationFailed } = transformUsers( - [{ id: "u1" }], - "clerk", - DATE_TIME, - { - validate: false, - }, - ); + const { transformedData, validationFailed } = transformUsers([{ id: "u1" }], "clerk", { + validate: false, + }); expect(transformedData).toHaveLength(1); expect(validationFailed).toBe(0); }); @@ -196,7 +189,7 @@ describe("loadUsersFromFile", () => { path.join(workDir, "users.json"), JSON.stringify([{ id: "u1", primary_email_address: "a@x.dev" }]), ); - const { users } = await loadUsersFromFile("users.json", "clerk", DATE_TIME); + const { users } = await loadUsersFromFile("users.json", "clerk"); expect(users).toHaveLength(1); expect(users[0]?.userId).toBe("u1"); }); @@ -206,13 +199,13 @@ describe("loadUsersFromFile", () => { path.join(workDir, "users.csv"), 'id,primary_email_address,verified_email_addresses\nu2,a@x.dev,"a@x.dev,b@x.dev"\n', ); - const { users } = await loadUsersFromFile("users.csv", "clerk", DATE_TIME); + const { users } = await loadUsersFromFile("users.csv", "clerk"); expect(users[0]?.userId).toBe("u2"); expect(users[0]?.email).toEqual(["a@x.dev", "b@x.dev"]); }); test("rejects a JSON file that is not an array of users", async () => { fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); - await expect(loadUsersFromFile("wrapped.json", "clerk", DATE_TIME)).rejects.toThrow(CliError); + await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 9f1b4fe08..1c8561c85 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -19,10 +19,18 @@ import { type User, } from "../types.ts"; import { userSchema } from "../validator.ts"; -import { validationLogger } from "./logger.ts"; export type FileType = "application/json" | "text/csv"; +/** A user that failed schema validation before any API call was made. */ +export type ValidationFailure = { + /** The source ID, or `row-` when the row has none. */ + userId: string; + row: number; + error: string; + path: (string | number)[]; +}; + export type TransformOptions = { /** Set `false` to keep invalid rows, for analysis passes that count fields. */ validate?: boolean; @@ -294,17 +302,20 @@ export function consolidateClerkIdentifiers(user: Record): void // --- Validation ------------------------------------------------------------ /** - * Validates prepared users, logging each failure and dropping it from the run. + * Validates prepared users, dropping each failure from the run and returning + * it for the caller to record. * * An unrecognized `passwordHasher` is the one failure that aborts instead: * importing those users would store credentials nobody can ever sign in with, * and the fix is a one-word edit to the transformer. */ -export function validatePreparedUsers( - users: Record[], - dateTime: string, -): { users: User[]; validationFailed: number } { +export function validatePreparedUsers(users: Record[]): { + users: User[]; + validationFailed: number; + failures: ValidationFailure[]; +} { const validated: User[] = []; + const failures: ValidationFailure[] = []; let validationFailed = 0; for (let i = 0; i < users.length; i++) { @@ -335,18 +346,15 @@ export function validatePreparedUsers( ); } - validationLogger( - { - error: firstIssue.message, - path: firstIssue.path as (string | number)[], - userId: (user.userId as string) || `row-${i}`, - row: i, - }, - dateTime, - ); + failures.push({ + error: firstIssue.message, + path: firstIssue.path as (string | number)[], + userId: (user.userId as string) || `row-${i}`, + row: i, + }); } - return { users: validated, validationFailed }; + return { users: validated, validationFailed, failures }; } function addDefaultFields( @@ -368,9 +376,8 @@ function addDefaultFields( export function transformUsers( users: Record[], key: string, - dateTime: string, options: TransformOptions = {}, -): { transformedData: User[]; validationFailed: number } { +): { transformedData: User[]; validationFailed: number; failures: ValidationFailure[] } { const transformer = getTransformer(key); const context = options.context ?? {}; const transformed: Record[] = []; @@ -387,11 +394,15 @@ export function transformUsers( } if (options.validate === false) { - return { transformedData: transformed as User[], validationFailed: 0 }; + return { transformedData: transformed as User[], validationFailed: 0, failures: [] }; } - const result = validatePreparedUsers(transformed, dateTime); - return { transformedData: result.users, validationFailed: result.validationFailed }; + const result = validatePreparedUsers(transformed); + return { + transformedData: result.users, + validationFailed: result.validationFailed, + failures: result.failures, + }; } // --- File loading ---------------------------------------------------------- @@ -453,17 +464,15 @@ export async function readRawUsers(file: string, key: string): Promise { +): Promise<{ users: User[]; validationFailed: number; failures: ValidationFailure[] }> { const transformer = getTransformer(key); const raw = await readUsersFromFile(file, transformer); const withDefaults = addDefaultFields(raw, transformer); - const { transformedData, validationFailed } = transformUsers( + const { transformedData, validationFailed, failures } = transformUsers( withDefaults, key, - dateTime, options, ); - return { users: transformedData, validationFailed }; + return { users: transformedData, validationFailed, failures }; } diff --git a/packages/cli-core/src/commands/migrate/logs/clean.ts b/packages/cli-core/src/commands/migrate/logs/clean.ts deleted file mode 100644 index 3c609f5ad..000000000 --- a/packages/cli-core/src/commands/migrate/logs/clean.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * `clerk migrate logs clean` — delete the local log files. - * - * Ported from the standalone migration-tool's `src/clean-logs/index.ts`. - * - * Destructive, and it sits one word away from `clerk migrate delete`, which - * destroys something entirely different (users in a Clerk instance). So the - * confirmation is not optional: interactive runs prompt, and non-interactive - * ones must say `-y` rather than being allowed to assume. - */ - -import fs from "node:fs"; -import { throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; -import { log } from "../../../lib/log.ts"; -import { confirm } from "../../../lib/prompts.ts"; -import { withGutter } from "../../../lib/spinner.ts"; -import { isAgent, isHuman } from "../../../mode.ts"; -import { listLogFiles } from "../lib/log-files.ts"; -import { getLogDir } from "../lib/logger.ts"; - -export type LogsCleanOptions = { - yes?: boolean; -}; - -export async function clean(options: LogsCleanOptions = {}): Promise { - await withGutter("Cleaning migration logs", async () => { - const files = listLogFiles(); - - if (files.length === 0) { - log.info(`No migration logs to clean in ${getLogDir()}.`); - return; - } - - const label = `${files.length} log file${files.length === 1 ? "" : "s"}`; - - if (!options.yes) { - if (isAgent() || !isHuman()) { - throwUsageError( - `\`clerk migrate logs clean\` deletes ${label} from ${getLogDir()} and cannot prompt here. Pass -y to confirm.`, - undefined, - undefined, - [ - { - command: "clerk migrate logs clean -y", - description: "Delete every migration log without prompting", - }, - ], - ); - } - - const proceed = await confirm({ message: `Delete ${label}?`, default: false }); - if (!proceed) throwUserAbort(); - } - - let deleted = 0; - const failures: string[] = []; - - for (const file of files) { - try { - fs.unlinkSync(file.path); - deleted++; - } catch (error) { - failures.push(`${file.name}: ${(error as Error).message}`); - } - } - - for (const failure of failures) log.warn(`Could not delete ${failure}`); - - log.success(`Deleted ${deleted} log file${deleted === 1 ? "" : "s"}.`); - if (failures.length > 0) process.exitCode = 1; - }); -} diff --git a/packages/cli-core/src/commands/migrate/logs/convert.ts b/packages/cli-core/src/commands/migrate/logs/convert.ts deleted file mode 100644 index 33adf2bae..000000000 --- a/packages/cli-core/src/commands/migrate/logs/convert.ts +++ /dev/null @@ -1,128 +0,0 @@ -/** - * `clerk migrate logs convert` — NDJSON to a JSON array. - * - * Ported from the standalone migration-tool's `src/convert-logs/index.ts`, - * with two changes: files can be named as positionals or `--all` instead of - * only through a picker, and a malformed line is reported with its line number - * rather than aborting the whole file. - */ - -import fs from "node:fs"; -import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; -import { dim } from "../../../lib/color.ts"; -import { log } from "../../../lib/log.ts"; -import { multiselect } from "../../../lib/prompts.ts"; -import { withGutter } from "../../../lib/spinner.ts"; -import { isAgent, isHuman } from "../../../mode.ts"; -import { findLogFile, listLogFiles, readNdjson, type LogFile } from "../lib/log-files.ts"; -import { getLogDir } from "../lib/logger.ts"; - -export type LogsConvertOptions = { - all?: boolean; - files?: string[]; -}; - -/** The `.json` sibling a log converts into. */ -export function outputPathFor(file: LogFile): string { - return file.path.replace(/\.log$/, ".json"); -} - -/** - * Resolves which files to convert: explicit positionals, `--all`, or a - * multiselect when a human gave neither. - */ -async function resolveTargets(options: LogsConvertOptions): Promise { - const available = listLogFiles(); - - if (available.length === 0) { - log.info(`No migration logs to convert in ${getLogDir()}.`); - return []; - } - - if (options.files && options.files.length > 0) { - return options.files.map((name) => { - const found = findLogFile(name); - if (!found) { - throw new CliError(`No log file named ${name} in ${getLogDir()}.`, { - code: ERROR_CODE.FILE_NOT_FOUND, - }); - } - return found; - }); - } - - if (options.all) return available; - - if (isAgent() || !isHuman()) { - throwUsageError( - "`clerk migrate logs convert` needs a file to convert and cannot prompt here. Name one or more log files, or pass --all.", - undefined, - undefined, - [ - { command: "clerk migrate logs convert --all", description: "Convert every log file" }, - { - command: `clerk migrate logs convert ${available[0]?.name ?? "migration-....log"}`, - description: "Convert one log file", - }, - ], - ); - } - - const chosen = await multiselect({ - message: "Which log files should be converted to JSON?", - options: available.map((file) => ({ - value: file.name, - label: file.name, - hint: `${file.entryCount} entries`, - })), - }); - if (chosen.length === 0) throwUserAbort(); - - return available.filter((file) => chosen.includes(file.name)); -} - -export async function convert(options: LogsConvertOptions = {}): Promise { - // The multiselect lives inside the gutter so cancelling it closes with - // `└ Paused` rather than leaving a half-drawn frame. - await withGutter("Converting migration logs", async () => { - const targets = await resolveTargets(options); - if (targets.length === 0) return; - - let converted = 0; - let malformed = 0; - - for (const file of targets) { - const output = outputPathFor(file); - - try { - const { entries, errors } = readNdjson(file.path); - - // Reported per line, so a truncated final line from an interrupted run - // is visible rather than silently missing from the output. - for (const error of errors) { - malformed++; - log.warn( - `${file.name}:${error.line} is not valid JSON and was skipped — ${error.message}`, - ); - } - - fs.writeFileSync(output, JSON.stringify(entries, null, 2)); - converted++; - const count = `${entries.length} ${entries.length === 1 ? "entry" : "entries"}`; - log.info(`${file.name} → ${output.split("/").pop()} ${dim(`(${count})`)}`); - } catch (error) { - log.warn(`Could not convert ${file.name}: ${(error as Error).message}`); - process.exitCode = 1; - } - } - - if (converted > 0) { - log.success( - `Converted ${converted} log file${converted === 1 ? "" : "s"}. Originals left in place.`, - ); - } - if (malformed > 0) { - log.warn(`${malformed} malformed line${malformed === 1 ? "" : "s"} skipped.`); - } - }); -} diff --git a/packages/cli-core/src/commands/migrate/logs/index.ts b/packages/cli-core/src/commands/migrate/logs/index.ts deleted file mode 100644 index 640a88a36..000000000 --- a/packages/cli-core/src/commands/migrate/logs/index.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { createArgument } from "@commander-js/extra-typings"; -import type { Command } from "@commander-js/extra-typings"; -import { clean } from "./clean.ts"; -import { convert } from "./convert.ts"; -import { list } from "./list.ts"; - -const logs = { clean, convert, list }; - -/** - * Registers `logs list|clean|convert` under the `migrate` group. - * - * Noun-verb, matching every other group in the CLI (`config pull`, `users - * list`) rather than the standalone tool's `clean-logs`/`convert-logs`, which - * were npm script names. - */ -export function registerMigrateLogs(migrateCommand: Command<[], Record>): void { - const logsCommand = migrateCommand - .command("logs") - .description("Inspect, convert and clean up local migration logs") - .setExamples([ - { command: "clerk migrate logs", description: "List the local migration logs" }, - { command: "clerk migrate logs clean -y", description: "Delete every migration log" }, - { - command: "clerk migrate logs convert --all", - description: "Convert every log to a JSON array", - }, - ]); - - // Listing is read-only, so it is safe as the default for a bare - // `clerk migrate logs`. - logsCommand - .command("list", { isDefault: true }) - .description("List the migration log files") - .option("--json", "Output as JSON") - .setExamples([ - { command: "clerk migrate logs list", description: "Show type, timestamp, size and entries" }, - { command: "clerk migrate logs list --json", description: "Machine-readable listing" }, - ]) - .action(async (_opts, cmd) => - logs.list(cmd.optsWithGlobals() as Parameters[0]), - ); - - logsCommand - .command("clean") - .description("Delete the migration log files") - .option("-y, --yes", "Skip the confirmation prompt") - .setExamples([ - { command: "clerk migrate logs clean", description: "Delete after confirming" }, - { command: "clerk migrate logs clean -y", description: "Delete without prompting" }, - ]) - .action(async (_opts, cmd) => - logs.clean(cmd.optsWithGlobals() as Parameters[0]), - ); - - logsCommand - .command("convert") - .description("Convert NDJSON logs to JSON arrays for analysis") - .addArgument(createArgument("[file...]", "Log files to convert. Omit to pick interactively.")) - .option("--all", "Convert every log file") - .setExamples([ - { command: "clerk migrate logs convert --all", description: "Convert every log file" }, - { - command: "clerk migrate logs convert migration-2026-01-01T12-00-00.log", - description: "Convert one log file", - }, - { command: "clerk migrate logs convert", description: "Pick files interactively" }, - ]) - .action(async (files, _opts, cmd) => - logs.convert({ - ...(cmd.optsWithGlobals() as Parameters[0]), - files, - }), - ); -} diff --git a/packages/cli-core/src/commands/migrate/logs/list.ts b/packages/cli-core/src/commands/migrate/logs/list.ts deleted file mode 100644 index ae0f53928..000000000 --- a/packages/cli-core/src/commands/migrate/logs/list.ts +++ /dev/null @@ -1,125 +0,0 @@ -/** - * `clerk migrate logs list` — what is in `./logs/`. - * - * New in the CLI: the standalone tool enumerated the directory only to build - * its own pickers. Exposing it gives a human a "what did I just do" view and - * an agent a read-only way to inspect a migration without parsing NDJSON. - */ - -import { bold, cyan, dim } from "../../../lib/color.ts"; -import { log } from "../../../lib/log.ts"; -import { withGutter } from "../../../lib/spinner.ts"; -import { formatSize, listLogFiles, type LogFile, type LogKind } from "../lib/log-files.ts"; -import { displayLogDir } from "../lib/logger.ts"; - -/** Every kind a log file can be, and what one entry in it records. */ -const KIND_LEGEND: Record, string> = { - export: "One entry per user pulled from the source platform.", - import: "One entry per user created in Clerk, with any error.", - delete: "One entry per user removed from Clerk, with any error.", -}; - -const legendWidth = Math.max(...Object.keys(KIND_LEGEND).map((kind) => kind.length)) + 2; - -export type LogsListOptions = { - json?: boolean; -}; - -function toJson(files: LogFile[]) { - return files.map((file) => ({ - name: file.name, - kind: file.kind, - timestamp: file.timestamp, - size_bytes: file.sizeBytes, - entry_count: file.entryCount, - path: file.path, - })); -} - -/** - * The filename stamp as a date a human reads at a glance. - * - * The stamp is UTC (`getDateTimeStamp` is an ISO string with its colons swapped - * for filename-legal dashes), so it is parsed as UTC and rendered in the - * viewer's own zone — "which run was that" is a question about local time. - * Returns the raw stamp for anything unparseable rather than printing - * "Invalid Date". - */ -export function formatTimestamp(stamp: string): string { - if (!stamp) return ""; - const date = new Date(`${stamp.replace(/T(\d{2})-(\d{2})-(\d{2})$/, "T$1:$2:$3")}Z`); - if (Number.isNaN(date.getTime())) return stamp; - return date.toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short" }); -} - -export async function list(options: LogsListOptions = {}): Promise { - const files = listLogFiles(); - - if (options.json) { - log.data(JSON.stringify(toJson(files), null, 2)); - return; - } - - await withGutter("Listing migration logs", async () => { - if (files.length === 0) { - log.info(`No migration logs in ${displayLogDir()}.`); - return; - } - - const rows = files.map((file) => ({ - name: file.name, - kind: file.kind, - when: formatTimestamp(file.timestamp), - size: formatSize(file.sizeBytes), - entries: String(file.entryCount), - })); - - const width = (header: string, pick: (row: (typeof rows)[number]) => string) => - Math.max(header.length, ...rows.map((row) => pick(row).length)) + 2; - - /** - * Pads to the visible width, then colours. Colouring first would count the - * ANSI escape bytes towards the width and pull every later column left. - */ - const column = (text: string, size: number, paint: (value: string) => string) => - paint(text) + " ".repeat(Math.max(0, size - text.length)); - - const nameWidth = width("FILE", (row) => row.name); - const kindWidth = width("TYPE", (row) => row.kind); - const whenWidth = width("DATE", (row) => row.when); - const sizeWidth = width("SIZE", (row) => row.size); - - log.info("Each log represents a user export, user import, or a user delete run."); - log.info("Each log consists of a single NDJSON entry per user."); - log.blank(); - - log.info( - dim("FILE".padEnd(nameWidth)) + - dim("TYPE".padEnd(kindWidth)) + - dim("DATE".padEnd(whenWidth)) + - dim("SIZE".padEnd(sizeWidth)) + - dim("ENTRIES"), - ); - - for (const row of rows) { - log.info( - column(row.name, nameWidth, cyan) + - row.kind.padEnd(kindWidth) + - column(row.when || "—", whenWidth, row.when ? (value) => value : dim) + - column(row.size, sizeWidth, dim) + - row.entries, - ); - } - - log.blank(); - log.info(`${files.length} log file${files.length === 1 ? "" : "s"} in ${displayLogDir()}`); - log.blank(); - - // A listing only shows the kinds that happen to be present, so the legend - // is fixed: it also answers "what else could be here". - log.info(bold("Log types:")); - for (const [kind, description] of Object.entries(KIND_LEGEND)) { - log.info(` ${cyan(bold(kind))}${" ".repeat(legendWidth - kind.length)}${description}`); - } - }); -} diff --git a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts deleted file mode 100644 index 72da1cffc..000000000 --- a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts +++ /dev/null @@ -1,201 +0,0 @@ -/** - * The prompting half of `logs clean` and `logs convert`. - * - * Kept separate because `mock.module` registrations are process-lifetime, and - * `bun test --parallel` puts several files in each worker — so a mocked - * `prompts.ts` would leak into any file that later lands in the same worker and - * imports the real one. Human mode itself needs no mock: `setMode` is the - * supported override. - */ - -import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { getMode, setMode, type Mode } from "../../../mode.ts"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; - -type ConfirmPrompt = { message: string; default?: boolean }; -type MultiselectPrompt = { - message: string; - options: { value: string; label: string; hint?: string }[]; -}; - -const mockConfirm = mock(async (_config: ConfirmPrompt) => true); -const mockMultiselect = mock(async (_config: MultiselectPrompt) => [] as string[]); - -mock.module("../../../lib/prompts.ts", () => ({ - confirm: (config: ConfirmPrompt) => mockConfirm(config), - multiselect: (config: MultiselectPrompt) => mockMultiselect(config), - text: async () => "", - password: async () => "", - editor: async () => "{}", -})); - -const { clean } = await import("./clean.ts"); -const { convert } = await import("./convert.ts"); -const { UserAbortError } = await import("../../../lib/errors.ts"); -const { getLogDir } = await import("../lib/logger.ts"); - -let originalMode: Mode; - -const captured = useCaptureLog(); - -let workDir: string; -let originalCwd: string; - -const IMPORT = "import-2026-01-01T12-00-00.log"; -const DELETE = "delete-2026-02-01T12-00-00.log"; - -beforeAll(() => { - originalMode = getMode(); - setMode("human"); - originalCwd = process.cwd(); - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logs-int-"))); - process.chdir(workDir); -}); - -afterAll(() => { - setMode(originalMode); - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); -}); - -beforeEach(() => { - mockConfirm.mockReset(); - mockMultiselect.mockReset(); - mockConfirm.mockResolvedValue(true); - mockMultiselect.mockResolvedValue([]); - fs.rmSync(getLogDir(), { recursive: true, force: true }); - process.exitCode = 0; -}); - -function writeLog(name: string, entries: unknown[]): void { - fs.mkdirSync(getLogDir(), { recursive: true }); - fs.writeFileSync( - path.join(getLogDir(), name), - entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n", - ); -} - -describe("logs clean", () => { - test("prompts before deleting anything", async () => { - writeLog(IMPORT, [{ a: 1 }]); - writeLog(DELETE, [{ a: 1 }]); - - await clean(); - - expect(mockConfirm).toHaveBeenCalledTimes(1); - expect(mockConfirm.mock.calls[0]?.[0]?.message).toContain("2 log files"); - expect(fs.readdirSync(getLogDir())).toEqual([]); - }); - - // Deleting on a stray enter would be the wrong default for a destructive - // command sitting next to `clerk migrate delete`. - test("defaults the prompt to no", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await clean(); - - expect(mockConfirm.mock.calls[0]?.[0]?.default).toBe(false); - }); - - test("declining leaves every file in place", async () => { - writeLog(IMPORT, [{ a: 1 }]); - mockConfirm.mockResolvedValue(false); - - await expect(clean()).rejects.toThrow(UserAbortError); - - expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); - }); - - test("-y skips the prompt entirely", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await clean({ yes: true }); - - expect(mockConfirm).not.toHaveBeenCalled(); - expect(fs.readdirSync(getLogDir())).toEqual([]); - }); - - test("does not prompt when there is nothing to delete", async () => { - await clean(); - expect(mockConfirm).not.toHaveBeenCalled(); - }); -}); - -describe("logs convert", () => { - test("offers a multiselect when given neither files nor --all", async () => { - writeLog(IMPORT, [{ a: 1 }, { b: 2 }]); - writeLog(DELETE, [{ a: 1 }]); - mockMultiselect.mockResolvedValue([IMPORT]); - - await convert(); - - const options = mockMultiselect.mock.calls[0]?.[0]?.options; - expect(options?.map((option) => option.value)).toEqual([DELETE, IMPORT]); - expect(options?.[1]?.hint).toBe("2 entries"); - }); - - test("converts only what was selected", async () => { - writeLog(IMPORT, [{ a: 1 }]); - writeLog(DELETE, [{ a: 1 }]); - mockMultiselect.mockResolvedValue([IMPORT]); - - await convert(); - - expect(fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json"))).toEqual([ - "import-2026-01-01T12-00-00.json", - ]); - }); - - test("selecting nothing aborts without writing", async () => { - writeLog(IMPORT, [{ a: 1 }]); - mockMultiselect.mockResolvedValue([]); - - await expect(convert()).rejects.toThrow(UserAbortError); - - expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); - }); - - test("does not prompt when --all was passed", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await convert({ all: true }); - - expect(mockMultiselect).not.toHaveBeenCalled(); - expect(captured.err).toContain("Converted 1 log file"); - }); - - test("does not prompt when files were named", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await convert({ files: [IMPORT] }); - - expect(mockMultiselect).not.toHaveBeenCalled(); - }); -}); - -// withGutter turns a UserAbortError into `└ Paused`; a real failure would close -// with `└ Failed`. Declining a prompt is not a failure, so the two must not swap. -describe("cancelling inside the gutter", () => { - test("declining the logs clean confirm closes with Paused, not Failed", async () => { - writeLog(IMPORT, [{ a: 1 }]); - mockConfirm.mockResolvedValue(false); - - await expect(clean()).rejects.toThrow(UserAbortError); - - expect(captured.err).toContain("Paused"); - expect(captured.err).not.toContain("Failed"); - }); - - test("selecting nothing in the logs convert multiselect closes with Paused", async () => { - writeLog(IMPORT, [{ a: 1 }]); - mockMultiselect.mockResolvedValue([]); - - await expect(convert()).rejects.toThrow(UserAbortError); - - expect(captured.err).toContain("Paused"); - expect(captured.err).not.toContain("Failed"); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/logs/logs.test.ts b/packages/cli-core/src/commands/migrate/logs/logs.test.ts deleted file mode 100644 index 2e80ce5d5..000000000 --- a/packages/cli-core/src/commands/migrate/logs/logs.test.ts +++ /dev/null @@ -1,288 +0,0 @@ -import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { CliError } from "../../../lib/errors.ts"; -import { getMode, setMode, type Mode } from "../../../mode.ts"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { getLogDir } from "../lib/logger.ts"; -import { clean } from "./clean.ts"; -import { convert } from "./convert.ts"; -import { formatTimestamp, list } from "./list.ts"; - -const captured = useCaptureLog(); - -const ANSI_ESCAPE_PATTERN = new RegExp(String.raw`\u001b\[[0-9;]*m`, "g"); -const stripAnsi = (value: string) => value.replace(ANSI_ESCAPE_PATTERN, ""); - -let workDir: string; -let originalCwd: string; - -beforeAll(() => { - originalCwd = process.cwd(); - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logs-"))); - process.chdir(workDir); -}); - -afterAll(() => { - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); -}); - -beforeEach(() => { - fs.rmSync(getLogDir(), { recursive: true, force: true }); - process.exitCode = 0; -}); - -function writeLog(name: string, entries: unknown[]): void { - fs.mkdirSync(getLogDir(), { recursive: true }); - fs.writeFileSync( - path.join(getLogDir(), name), - entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n", - ); -} - -const IMPORT = "import-2026-01-01T12-00-00.log"; -const DELETE = "delete-2026-02-01T12-00-00.log"; - -describe("logs list", () => { - test("says so plainly when there is no logs directory", async () => { - await list(); - expect(captured.err).toContain("No migration logs in"); - }); - - test("says so plainly when the directory is empty", async () => { - fs.mkdirSync(getLogDir(), { recursive: true }); - await list(); - expect(captured.err).toContain("No migration logs in"); - }); - - test("reports file, type, date, size and entry count", async () => { - writeLog(IMPORT, [{ userId: "u1" }, { userId: "u2" }, { userId: "u3" }]); - - await list(); - - expect(captured.err).toContain("FILE"); - expect(captured.err).toContain("TYPE"); - expect(captured.err).toContain("DATE"); - expect(captured.err).toContain("SIZE"); - expect(captured.err).toContain("ENTRIES"); - expect(captured.err).toContain(IMPORT); - expect(captured.err).toContain("import"); - expect(captured.err).toContain(formatTimestamp("2026-01-01T12-00-00")); - expect(captured.err).toMatch(/\bB\b/); - expect(captured.err).toContain("3"); - }); - - // The filename stamp is for sorting and for `logs convert`; the column a - // human scans should read like a date. - test("shows the date rendered, not the raw filename stamp", async () => { - writeLog(IMPORT, [{ userId: "u1" }]); - - await list(); - - const dateColumn = stripAnsi(captured.err) - .split("\n") - .find((line) => line.includes(IMPORT)); - expect(dateColumn?.replace(IMPORT, "")).not.toContain("2026-01-01T12-00-00"); - }); - - test("reports the log directory relative to the current directory", async () => { - writeLog(IMPORT, [{ userId: "u1" }]); - - await list(); - - expect(stripAnsi(captured.err)).toContain(`1 log file in .${path.sep}logs`); - expect(captured.err).not.toContain(getLogDir()); - }); - - test("lists every log kind", async () => { - writeLog(IMPORT, [{ a: 1 }]); - writeLog(DELETE, [{ a: 1 }]); - - await list(); - - expect(captured.err).toContain("import"); - expect(captured.err).toContain("delete"); - expect(captured.err).toContain("2 log files"); - }); - - test("--json emits a machine-readable listing on stdout", async () => { - writeLog(IMPORT, [{ userId: "u1" }]); - - await list({ json: true }); - - const parsed = JSON.parse(captured.out) as Record[]; - expect(parsed).toHaveLength(1); - expect(parsed[0]).toMatchObject({ - name: IMPORT, - kind: "import", - timestamp: "2026-01-01T12-00-00", - entry_count: 1, - }); - }); - - test("--json emits an empty array rather than prose when there are no logs", async () => { - await list({ json: true }); - expect(JSON.parse(captured.out)).toEqual([]); - }); -}); - -describe("logs clean", () => { - test("says so plainly when there is nothing to clean", async () => { - await clean({ yes: true }); - expect(captured.err).toContain("No migration logs to clean"); - }); - - // Tests run non-TTY, which is the same signal an agent gives. - test("refuses without -y when it cannot prompt, and explains", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await expect(clean()).rejects.toThrow(/cannot prompt here.*Pass -y/s); - expect(fs.existsSync(path.join(getLogDir(), IMPORT))).toBe(true); - }); - - test("names how many files are at stake when it refuses", async () => { - writeLog(IMPORT, [{ a: 1 }]); - writeLog(DELETE, [{ a: 1 }]); - - await expect(clean()).rejects.toThrow(/2 log files/); - }); - - test("-y deletes the log files and reports the count", async () => { - writeLog(IMPORT, [{ a: 1 }]); - writeLog(DELETE, [{ a: 1 }]); - - await clean({ yes: true }); - - expect(fs.readdirSync(getLogDir())).toEqual([]); - expect(captured.err).toContain("Deleted 2 log files"); - }); - - test("leaves converted JSON output alone", async () => { - writeLog(IMPORT, [{ a: 1 }]); - fs.writeFileSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"), "[]"); - - await clean({ yes: true }); - - expect(fs.readdirSync(getLogDir())).toEqual(["import-2026-01-01T12-00-00.json"]); - }); -}); - -describe("logs convert", () => { - test("says so plainly when there is nothing to convert", async () => { - await convert({ all: true }); - expect(captured.err).toContain("No migration logs to convert"); - }); - - test("writes a JSON array alongside the original, leaving it intact", async () => { - writeLog(IMPORT, [{ userId: "u1" }, { userId: "u2" }]); - - await convert({ files: [IMPORT] }); - - const output = path.join(getLogDir(), "import-2026-01-01T12-00-00.json"); - expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([ - { userId: "u1" }, - { userId: "u2" }, - ]); - expect(fs.existsSync(path.join(getLogDir(), IMPORT))).toBe(true); - expect(captured.err).toContain("Originals left in place"); - }); - - test("--all converts every log file", async () => { - writeLog(IMPORT, [{ a: 1 }]); - writeLog(DELETE, [{ b: 2 }]); - - await convert({ all: true }); - - const written = fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json")); - expect(written.sort()).toEqual([ - "delete-2026-02-01T12-00-00.json", - "import-2026-01-01T12-00-00.json", - ]); - }); - - test("accepts a path and resolves it against ./logs/", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await convert({ files: [`./logs/${IMPORT}`] }); - - expect(fs.existsSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"))).toBe(true); - }); - - test("fails clearly on a file that is not there", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await expect(convert({ files: ["migration-nope.log"] })).rejects.toThrow(CliError); - }); - - // Silently dropping the line would leave a JSON array that looks complete. - test("reports a malformed line by number and converts the rest", async () => { - fs.mkdirSync(getLogDir(), { recursive: true }); - fs.writeFileSync(path.join(getLogDir(), IMPORT), '{"a":1}\n{"b":\n{"c":3}\n'); - - await convert({ files: [IMPORT] }); - - expect(captured.err).toContain(`${IMPORT}:2`); - expect(captured.err).toContain("1 malformed line skipped"); - - const output = path.join(getLogDir(), "import-2026-01-01T12-00-00.json"); - expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([{ a: 1 }, { c: 3 }]); - }); - - test("refuses without a target when it cannot prompt, naming the alternatives", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await expect(convert()).rejects.toThrow(/cannot prompt here/); - expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); - }); - - test("reports the entry count per converted file", async () => { - writeLog(IMPORT, [{ a: 1 }, { b: 2 }, { c: 3 }]); - - await convert({ all: true }); - - expect(captured.err).toContain("3 entries"); - }); -}); - -describe("human-mode frame", () => { - let originalMode: Mode; - - beforeAll(() => { - originalMode = getMode(); - setMode("human"); - }); - - afterAll(() => { - setMode(originalMode); - }); - - test("logs list wraps its output in an intro/outro gutter", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await list(); - - expect(captured.err).toContain("\u250c"); - expect(captured.err).toContain("Listing migration logs"); - expect(captured.err).toContain("\u2514"); - expect(captured.err).toContain("Done"); - }); - - test("--json stays outside the gutter, on stdout only", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await list({ json: true }); - - expect(JSON.parse(captured.out)).toHaveLength(1); - expect(captured.err).not.toContain("\u250c"); - }); - - test("a failure inside logs convert closes with Failed and still throws", async () => { - writeLog(IMPORT, [{ a: 1 }]); - - await expect(convert({ files: ["nope.log"] })).rejects.toThrow(CliError); - - expect(captured.err).toContain("Failed"); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index 150bf3713..69a570367 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -1,8 +1,8 @@ /** * Keeps README.md and the command tree honest about each other. * - * This README documents six export platforms, six transformers and three log - * subcommands across ~600 lines. Checking it by eye at review time does not + * This README documents seven export platforms, seven transformers and the + * run store across ~1000 lines. Checking it by eye at review time does not * scale, and a doc that names a flag the binary rejects is worse than no doc: * the reader trusts it and gets a usage error. * @@ -70,9 +70,9 @@ function resolve(tokens: string[]): { command: Command; rest: string[] } { /** * The flags a command accepts, including those of a default subcommand. * - * `migrate logs` and `migrate transformers` register their `list` `isDefault`, - * so Commander hands it everything after the group name. The documented - * spelling is `clerk migrate logs --json`, and this has to see the same flags + * `migrate transformers` registers its `list` `isDefault`, so Commander hands + * it everything after the group name. The documented spelling is + * `clerk migrate transformers --json`, and this has to see the same flags * Commander does or every such example reads as unsupported. */ function flagsOf(command: Command): string[] { diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index c8b0308e9..10c4a3669 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -122,7 +122,7 @@ beforeEach(() => { mockSelect.mockResolvedValue("clerk"); mockText.mockResolvedValue("export.json"); mockMultiselect.mockResolvedValue([]); - fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 7e531b1e7..5a74e7457 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -9,7 +9,7 @@ import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; // Every test below names its own `--secret-key`, which short-circuits the // signed-in check — except the one that asserts what happens without it. mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); -import { getLogDir } from "./lib/logger.ts"; +import { latestUserLines, listRuns } from "./lib/run-store.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; import { applyResumeAfter, explainErrors, run, validateRunOptions } from "./run.ts"; import type { User } from "./types.ts"; @@ -18,6 +18,9 @@ let workDir: string; let configDir: string; let originalCwd: string; +/** Where runs land for a project rooted at `workDir`. */ +const runsDir = () => path.join(workDir, ".clerk", "migrate"); + const users = (...ids: string[]): User[] => ids.map((userId) => ({ userId }) as User); beforeAll(() => { @@ -104,7 +107,7 @@ describe("run", () => { beforeEach(() => { requests = []; delete process.env.CLERK_MIGRATE_RATE_LIMIT; - fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(runsDir(), { recursive: true, force: true }); fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { @@ -155,19 +158,38 @@ describe("run", () => { expect(captured.err).toContain("Imported:"); }); - test("writes a timestamped NDJSON log for the run", async () => { + test("records the run in the project's run store", async () => { await run(baseOptions); - const logs = fs.readdirSync(getLogDir()); - expect(logs).toHaveLength(1); - expect(logs[0]).toMatch(/^import-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + const [record, ...rest] = listRuns(runsDir()); + expect(rest).toHaveLength(0); + expect(record).toMatchObject({ + kind: "import", + status: "complete", + source: "clerk", + counts: { total: 2, created: 2 }, + target: { keySource: "--secret-key", instanceType: "dev" }, + }); + expect(record?.id).toMatch(/^\d{8}-\d{6}-[0-9a-f]{4}$/); + expect(record?.file?.path).toBe(path.join(workDir, "export.json")); + expect(record?.file?.sha256).toMatch(/^[0-9a-f]{64}$/); + + const lines = [...latestUserLines(runsDir(), record!.id).values()]; + expect(lines.map((line) => [line.sourceId, line.status, line.clerkId])).toEqual([ + ["u1", "created", "user_created"], + ["u2", "created", "user_created"], + ]); + }); - const entries = fs - .readFileSync(path.join(getLogDir(), logs[0] as string), "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record); - expect(entries.filter((e) => e.status === "success")).toHaveLength(2); + test("gitignores the project's .clerk folder before writing a run", async () => { + await run(baseOptions); + expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toContain(".clerk/"); + }); + + test("--runs-dir puts the run somewhere else", async () => { + await run({ ...baseOptions, runsDir: "elsewhere" }); + expect(listRuns(path.join(workDir, "elsewhere"))).toHaveLength(1); + expect(listRuns(runsDir())).toHaveLength(0); }); test("--require-password imports only the users that have one", async () => { @@ -185,13 +207,20 @@ describe("run", () => { expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u2"]); }); - test("logs validation failures and imports the rest", async () => { + test("records validation failures in the run and imports the rest", async () => { fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify([...export2, { id: "u3" }])); await run(baseOptions); expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); expect(captured.err).toContain("1 user failed validation"); + + const [record] = listRuns(runsDir()); + expect(record).toMatchObject({ status: "partial", counts: { created: 2, failed: 1 } }); + expect(latestUserLines(runsDir(), record!.id).get("u3")).toMatchObject({ + status: "failed", + code: "validation", + }); }); /** Makes `GET /v1/users/count` report an instance that already holds users. */ diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index e3b6386a5..2af871585 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -12,9 +12,9 @@ * the missing flags for an agent, which cannot answer a prompt. */ -import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; import { bold, dim, green, red, yellow } from "../../lib/color.ts"; import { resolveProfile } from "../../lib/config.ts"; +import path from "node:path"; import { hasAccountCredentials } from "../../lib/credential-store.ts"; import { AUTH_ERROR_REASON, @@ -57,14 +57,20 @@ import { type SettingChange, } from "./lib/modify-settings.ts"; import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; -import { getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; +import { resolveRunsDir, sha256File, startRun, type RunRecord } from "./lib/run-store.ts"; import { countSocialProviders, findDisabledProviders, findUsersWithOnlyDisabledProviders, readSupabaseRows, } from "./lib/supabase-providers.ts"; -import { fileExists, getFileType, loadUsersFromFile } from "./lib/transform.ts"; +import { describeTarget, resolveClerkTarget } from "./lib/target.ts"; +import { + fileExists, + getFileType, + loadUsersFromFile, + resolveImportFilePath, +} from "./lib/transform.ts"; import { loadCustomTransformer } from "./transformers/load-custom.ts"; import { registerCustomTransformer, transformerKeys } from "./transformers/registry.ts"; import type { ImportSummary, User } from "./types.ts"; @@ -85,6 +91,8 @@ export type MigrateRunOptions = { transformerFile?: string; /** Supabase: drop users whose only social provider is disabled in Clerk. */ skipUnsupportedProviders?: boolean; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; } & FirebaseHashFlags; /** @@ -233,7 +241,8 @@ export function explainErrors(errors: Iterable, instanceType: InstanceTy function formatSummary( summary: ImportSummary, - logFile: string, + run: RunRecord, + runFolder: string, instanceType: InstanceType, ): string { const inFile = summary.totalProcessed + summary.validationFailed; @@ -255,7 +264,7 @@ function formatSummary( lines.push("", note); } } - lines.push("", dim(`Log: ${logFile}`)); + lines.push("", dim(`Run ${run.id}: ${runFolder}`)); return lines.join("\n"); } @@ -319,16 +328,18 @@ async function confirmDevUserLimit( * records per-user providers. If the instance's configuration cannot be read, * nobody is dropped: a failed lookup must not be mistaken for "no providers * are enabled". + * + * @returns The source IDs to skip. */ -async function skipDisabledProviderUsers( - users: User[], +async function findDisabledProviderUsers( file: string, transformer: string, secretKey: string, -): Promise { +): Promise> { + const none = new Set(); if (transformer !== "supabase") { log.warn(`--skip-unsupported-providers only applies to supabase exports; ignoring.`); - return users; + return none; } const settings = await withSpinner("Checking enabled providers...", async () => @@ -339,14 +350,14 @@ async function skipDisabledProviderUsers( log.warn( "Could not read the instance's enabled providers; importing every user. Re-run with --verbose for details.", ); - return users; + return none; } const rows = await readSupabaseRows(file); const disabled = findDisabledProviders(rows, enabled, toClerkStrategy); if (disabled.length === 0) { log.info("Every provider in this export is enabled in Clerk; no users skipped."); - return users; + return none; } const { excludedIds, byProvider } = findUsersWithOnlyDisabledProviders(rows, disabled); @@ -354,7 +365,7 @@ async function skipDisabledProviderUsers( log.info( `${disabled.join(", ")} not enabled in Clerk, but every user has another way to sign in; none skipped.`, ); - return users; + return none; } const breakdown = Object.entries(byProvider) @@ -364,7 +375,7 @@ async function skipDisabledProviderUsers( `--skip-unsupported-providers: skipping ${excludedIds.size} user${excludedIds.size === 1 ? "" : "s"} whose only provider is not enabled in Clerk (${breakdown}).`, ); - return users.filter((user) => !excludedIds.has(user.userId)); + return excludedIds; } type ReportInput = { @@ -653,16 +664,15 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const firebaseHashConfig = resolveFirebaseHashConfig(options, transformer); await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { - const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); - const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); + const { secretKey, target } = await resolveClerkTarget(options); const limits = resolveLimits(secretKey); - const dateTime = getDateTimeStamp(); - const logFile = getLogFilePath("import", dateTime); - const { users: loaded, validationFailed } = await withSpinner( - `Loading users from ${file}...`, - async () => - loadUsersFromFile(file, transformer, dateTime, { context: { firebaseHashConfig } }), + const { + users: loaded, + validationFailed, + failures, + } = await withSpinner(`Loading users from ${file}...`, async () => + loadUsersFromFile(file, transformer, { context: { firebaseHashConfig } }), ); let users = applyResumeAfter(loaded, options.resumeAfter); @@ -670,8 +680,16 @@ export async function run(rawOptions: MigrateRunOptions): Promise { log.info(`Resuming after ${options.resumeAfter} (${loaded.length - users.length} skipped).`); } + // Users left out on purpose. Recorded as skipped, so the run says who they + // were rather than only how many. + const skipped: { user: User; reason: string }[] = []; + if (options.skipUnsupportedProviders) { - users = await skipDisabledProviderUsers(users, file, transformer, secretKey); + const excluded = await findDisabledProviderUsers(file, transformer, secretKey); + for (const user of users.filter((candidate) => excluded.has(candidate.userId))) { + skipped.push({ user, reason: "only provider is not enabled in Clerk" }); + } + users = users.filter((user) => !excluded.has(user.userId)); } if (options.requirePassword) { @@ -682,17 +700,50 @@ export async function run(rawOptions: MigrateRunOptions): Promise { `--require-password: skipping ${dropped} user${dropped === 1 ? "" : "s"} without a password.`, ); } + for (const user of users.filter((candidate) => !candidate.password)) { + skipped.push({ user, reason: "no password (--require-password)" }); + } users = withPassword; } if (validationFailed > 0) { log.warn( - `${validationFailed} user${validationFailed === 1 ? "" : "s"} failed validation and will be skipped. See ${logFile}.`, + `${validationFailed} user${validationFailed === 1 ? "" : "s"} failed validation and will be skipped.`, ); } + const runsDir = await resolveRunsDir(options.runsDir, { write: true }); + const beginRun = () => { + const filePath = resolveImportFilePath(file); + const run = startRun(runsDir, { + kind: "import", + target, + source: transformer, + file: { path: filePath, sha256: sha256File(filePath) }, + }); + for (const failure of failures) { + run.append({ + sourceId: failure.userId, + status: "failed", + error: `${failure.error} (${failure.path.join(".") || "user"}, row ${failure.row + 1})`, + code: "validation", + }); + } + for (const { user, reason } of skipped) { + run.append({ sourceId: user.userId, status: "skipped", reason }); + } + return run; + }; + if (users.length === 0) { log.warn("No users left to import."); + // Still a run: the record of who failed validation, and why, is the one + // thing this attempt produced. + if (failures.length > 0 || skipped.length > 0) { + const record = beginRun().finish(); + log.info(dim(`Run ${record.id}: ${path.join(runsDir, record.id)}`)); + process.exitCode = 1; + } return; } @@ -701,12 +752,9 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ? await confirmDevUserLimit(users.length, secretKey, Boolean(options.yes)) : 0; - // `target` already carries the instance's environment ("My App - // (development)"), so the detected type is only worth spelling out when - // there is no app context to name — an explicit `--secret-key`. log.info( `Importing ${users.length} user${users.length === 1 ? "" : "s"} via the ${transformer} transformer into ` + - `${target ?? `the resolved instance (${limits.instanceType})`}.`, + `${describeTarget(target)}.`, ); await showReadinessReport({ @@ -734,26 +782,28 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (!proceed) throwUserAbort(); } + const run = beginRun(); const summary = await withSpinner(`Importing users: [0/${users.length}]...`, async (spinner) => importUsers({ users, secretKey, limits, - dateTime, + record: run.append, skipPasswordRequirement: !options.requirePassword, validationFailed, spinner, }), ); + const record = run.finish(); - log.info(formatSummary(summary, logFile, limits.instanceType)); + log.info(formatSummary(summary, record, run.dir, limits.instanceType)); // Offered even when some users failed: a partial import is exactly when - // reading the log and knowing how to undo it matters most. When users did - // fail, the per-user record of *why* leads, since the breakdown above only - // counts each error and never names who hit it. + // reading the per-user record matters most. const steps = - summary.failed > 0 ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(logFile) : NEXT_STEPS.MIGRATE_DONE; + summary.failed > 0 + ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(record.id) + : NEXT_STEPS.MIGRATE_DONE(record.id); setNextSteps(steps); printAgentNextSteps(steps); diff --git a/packages/cli-core/src/commands/migrate/runs.test.ts b/packages/cli-core/src/commands/migrate/runs.test.ts new file mode 100644 index 000000000..f5e5b0de6 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/runs.test.ts @@ -0,0 +1,93 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { startRun } from "./lib/run-store.ts"; +import { runs } from "./runs.ts"; + +const captured = useCaptureLog(); + +let runsDir: string; + +beforeEach(() => { + runsDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-runs-"))); +}); + +afterEach(() => { + fs.rmSync(runsDir, { recursive: true, force: true }); +}); + +function partialImport() { + const run = startRun(runsDir, { + kind: "import", + target: { appLabel: "My App", env: "development", instanceId: "ins_1" }, + source: "clerk", + file: { path: "/tmp/users.json", sha256: "abc" }, + }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + run.append({ sourceId: "b", status: "failed", error: "That email address is taken." }); + run.append({ sourceId: "c", status: "failed", error: "That email address is taken." }); + run.append({ sourceId: "d", status: "skipped", reason: "no password (--require-password)" }); + return run.finish(); +} + +describe("runs", () => { + test("names the runs folder, then lists each run", async () => { + const record = partialImport(); + + await runs(undefined, { runsDir }); + + expect(captured.err).toContain(`Runs folder: ${runsDir}`); + expect(captured.err).toContain(record.id); + expect(captured.err).toContain("My App (development, ins_1)"); + expect(captured.err).toContain("1 created, 2 failed, 1 skipped"); + }); + + test("says so when there are no runs", async () => { + await runs(undefined, { runsDir }); + expect(captured.err).toContain("No migration runs yet."); + }); + + test("--json lists the runs on stdout, with each one's state", async () => { + const record = partialImport(); + + await runs(undefined, { runsDir, json: true }); + + const parsed = JSON.parse(captured.out) as { runsDir: string; runs: { id: string }[] }; + expect(parsed).toMatchObject({ runsDir, runs: [{ id: record.id, state: "partial" }] }); + }); +}); + +describe("runs ", () => { + test("shows the counts, the error breakdown and who did not make it", async () => { + const record = partialImport(); + + await runs(record.id, { runsDir }); + + expect(captured.err).toContain("partial"); + expect(Bun.stripANSI(captured.err)).toContain("2 users: That email address is taken."); + expect(captured.err).toContain("Failed (2)"); + expect(captured.err).toContain("Skipped (1)"); + expect(captured.err).toContain("no password (--require-password)"); + expect(captured.err).toContain(path.join(runsDir, record.id, "users.ndjson")); + }); + + test("--json returns the run, its errors, and the failed and skipped users", async () => { + const record = partialImport(); + + await runs(record.id, { runsDir, json: true }); + + const parsed = JSON.parse(captured.out) as Record; + expect(parsed).toMatchObject({ + run: { id: record.id, state: "partial", counts: { created: 1, failed: 2, skipped: 1 } }, + errors: [{ error: "That email address is taken.", count: 2 }], + failed: [{ sourceId: "b" }, { sourceId: "c" }], + skipped: [{ sourceId: "d" }], + }); + }); + + test("an unknown ID is a usage error", async () => { + await expect(runs("nope", { runsDir })).rejects.toThrow(/No run `nope`/); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/runs.ts b/packages/cli-core/src/commands/migrate/runs.ts new file mode 100644 index 000000000..a8d802052 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/runs.ts @@ -0,0 +1,221 @@ +/** + * `clerk migrate runs [run-id]` — what every import, export and undo did. + * + * Reads the run store and nothing else. `runs` alone lists every run, newest + * first. `runs ` shows one: its header, counts, the error breakdown, the + * users that did not make it, and the command to run next. + */ + +import path from "node:path"; +import { bold, dim, green, red, yellow } from "../../lib/color.ts"; +import { throwUsageError } from "../../lib/errors.ts"; +import { log } from "../../lib/log.ts"; +import { normalizeErrorMessage } from "./import-users.ts"; +import { + latestUserLines, + listRuns, + readRun, + resolveRunsDir, + runDir, + runState, + type RunRecord, + type RunState, + type UserLine, +} from "./lib/run-store.ts"; +import { describeTarget } from "./lib/target.ts"; + +export type RunsOptions = { + json?: boolean; + runsDir?: string; +}; + +/** How many failed or skipped users `runs ` prints before pointing at the file. */ +const USER_LIST_LIMIT = 20; + +function formatDate(iso: string): string { + const date = new Date(iso); + if (Number.isNaN(date.getTime())) return iso; + return date.toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short" }); +} + +function colorState(state: RunState): string { + if (state === "complete") return green(state); + if (state === "partial" || state === "interrupted") return yellow(state); + if (state === "undone") return dim(state); + return state; +} + +/** The counts that matter for a run's kind, in the order they are read. */ +export function formatCounts(record: RunRecord): string { + const counts = record.counts; + const order = + record.kind === "export" + ? (["exported", "skipped"] as const) + : record.kind === "undo" + ? (["deleted", "failed"] as const) + : (["created", "failed", "skipped"] as const); + const parts = order + .filter((status) => (counts[status] ?? 0) > 0 || status === order[0]) + .map((status) => `${counts[status] ?? 0} ${status}`); + return parts.join(", "); +} + +function fileLabel(record: RunRecord): string { + if (!record.file) return ""; + const relative = path.relative(process.cwd(), record.file.path); + return relative.startsWith("..") ? record.file.path : relative; +} + +function pad(rows: string[][]): string[] { + const widths = rows[0]!.map((_, column) => + Math.max(...rows.map((row) => Bun.stringWidth(row[column] ?? ""))), + ); + return rows.map((row) => + row + .map((cell, column) => + column === row.length - 1 + ? cell + : cell + " ".repeat(widths[column]! - Bun.stringWidth(cell)), + ) + .join(" ") + .trimEnd(), + ); +} + +/** Error messages grouped the way the import summary groups them. */ +function errorBreakdown(lines: UserLine[]): { error: string; count: number }[] { + const counts = new Map(); + for (const line of lines) { + if (line.status !== "failed" || !line.error) continue; + const key = normalizeErrorMessage(line.error); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return [...counts].map(([error, count]) => ({ error, count })).sort((a, b) => b.count - a.count); +} + +function listJson(runsDir: string, records: RunRecord[]) { + return { + runsDir, + runs: records.map((record) => ({ ...record, state: runState(runsDir, record) })), + }; +} + +function printList(runsDir: string, records: RunRecord[]): void { + log.info(`Runs folder: ${runsDir}`); + log.blank(); + + if (records.length === 0) { + log.info("No migration runs yet."); + return; + } + + const rows = [ + ["RUN", "DATE", "KIND", "STATUS", "TARGET", "FILE", "RESULT"].map((header) => bold(header)), + ...records.map((record) => [ + record.id, + formatDate(record.startedAt), + record.kind, + colorState(runState(runsDir, record)), + describeTarget(record.target), + fileLabel(record), + formatCounts(record), + ]), + ]; + for (const line of pad(rows)) log.info(line); + log.blank(); + log.info(dim(`${records.length} run${records.length === 1 ? "" : "s"}`)); +} + +function showJson(runsDir: string, record: RunRecord) { + const lines = [...latestUserLines(runsDir, record.id).values()]; + return { + runsDir, + run: { ...record, state: runState(runsDir, record) }, + errors: errorBreakdown(lines), + failed: lines.filter((line) => line.status === "failed"), + skipped: lines.filter((line) => line.status === "skipped"), + }; +} + +function printUsers(title: string, lines: UserLine[], usersFile: string): void { + if (lines.length === 0) return; + log.blank(); + log.info(bold(`${title} (${lines.length})`)); + for (const line of lines.slice(0, USER_LIST_LIMIT)) { + const why = line.reason ?? line.error ?? ""; + log.info(` ${line.sourceId}${why ? dim(` ${why}`) : ""}`); + } + if (lines.length > USER_LIST_LIMIT) { + log.info(dim(` …and ${lines.length - USER_LIST_LIMIT} more in ${usersFile}`)); + } +} + +function printRun(runsDir: string, record: RunRecord): void { + const state = runState(runsDir, record); + const dir = runDir(runsDir, record.id); + const usersFile = path.join(dir, "users.ndjson"); + const lines = [...latestUserLines(runsDir, record.id).values()]; + + log.info(`Runs folder: ${runsDir}`); + log.blank(); + const header: [string, string | undefined][] = [ + ["Run", record.id], + ["Kind", record.kind], + ["Status", colorState(state)], + ["Started", formatDate(record.startedAt)], + ["Finished", record.finishedAt ? formatDate(record.finishedAt) : undefined], + ["Target", describeTarget(record.target)], + ["Key from", record.target.keySource], + ["Source", record.source], + ["File", record.file?.path], + ["From export", record.fromExport], + ["Undoes", record.undoes], + ["Undone by", record.undoneBy], + ["Result", formatCounts(record)], + ]; + const shown = header.filter((entry): entry is [string, string] => Boolean(entry[1])); + const width = Math.max(...shown.map(([label]) => label.length)) + 2; + for (const [label, value] of shown) log.info(`${bold(label.padEnd(width))}${value}`); + + const errors = errorBreakdown(lines); + if (errors.length > 0) { + log.blank(); + log.info(bold("Error breakdown")); + for (const { error, count } of errors) { + log.info(` ${red(String(count))} ${count === 1 ? "user" : "users"}: ${error}`); + } + } + + printUsers( + "Failed", + lines.filter((line) => line.status === "failed"), + usersFile, + ); + printUsers( + "Skipped", + lines.filter((line) => line.status === "skipped"), + usersFile, + ); + + log.blank(); + log.info(dim(`Every outcome: ${usersFile}`)); +} + +export async function runs(id: string | undefined, options: RunsOptions = {}): Promise { + const runsDir = await resolveRunsDir(options.runsDir); + + if (id === undefined) { + const records = listRuns(runsDir); + if (options.json) log.data(JSON.stringify(listJson(runsDir, records), null, 2)); + else printList(runsDir, records); + return; + } + + const record = readRun(runsDir, id); + if (!record) { + throwUsageError(`No run \`${id}\` in ${runsDir}. Run \`clerk migrate runs\` to list them.`); + } + + if (options.json) log.data(JSON.stringify(showJson(runsDir, record), null, 2)); + else printRun(runsDir, record); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts b/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts index b0005a471..533e96f7b 100644 --- a/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts +++ b/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts @@ -3,14 +3,11 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; -import { getLogDir } from "../lib/logger.ts"; import { loadUsersFromFile, transformUsers } from "../lib/transform.ts"; import type { FirebaseHashConfig } from "../types.ts"; import { getTransformer, transformerKeys, transformers } from "./registry.ts"; import { isVerified } from "./shared.ts"; -const DATE_TIME = "2026-01-01T00:00:00"; - const FIREBASE_HASH: FirebaseHashConfig = { base64_signer_key: "SIGNERKEY==", base64_salt_separator: "Bw==", @@ -39,11 +36,11 @@ async function load(key: string, records: unknown, ext = "json", context = {}) { path.join(workDir, file), typeof records === "string" ? records : JSON.stringify(records), ); - return loadUsersFromFile(file, key, DATE_TIME, { context }); + return loadUsersFromFile(file, key, { context }); } const one = (key: string, record: Record, context = {}) => - transformUsers([record], key, DATE_TIME, { validate: false, context }).transformedData[0] as + transformUsers([record], key, { validate: false, context }).transformedData[0] as | Record | undefined; @@ -414,11 +411,9 @@ describe("invalid records", () => { ]; test.each(INVALID)( - "%s logs a user with no identifier instead of crashing", + "%s reports a user with no identifier instead of crashing", async (key, record) => { - fs.rmSync(getLogDir(), { recursive: true, force: true }); - - const { users, validationFailed } = await load(key, [ + const { users, validationFailed, failures } = await load(key, [ record, { ...record, ...identifierFor(key) }, ]); @@ -426,13 +421,7 @@ describe("invalid records", () => { expect(validationFailed).toBe(1); expect(users).toHaveLength(1); - const logged = fs - .readdirSync(getLogDir()) - .flatMap((name) => - fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n"), - ) - .map((line) => JSON.parse(line) as Record); - expect(logged.some((entry) => entry.status === "fail")).toBe(true); + expect(failures).toHaveLength(1); }, ); diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts index 904267037..6ca7c2dda 100644 --- a/packages/cli-core/src/commands/migrate/types.ts +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -1,10 +1,7 @@ /** * Shared types for `clerk migrate`. * - * Ported from the standalone migration-tool's `src/types.ts`. The Clerk API - * error shape is declared locally rather than imported from `@clerk/types`, - * because this command family talks to BAPI through `lib/bapi.ts` instead of - * `@clerk/backend`. + * Ported from the standalone migration-tool's `src/types.ts`. */ import type * as z from "zod"; @@ -46,68 +43,6 @@ export type User = z.infer; /** Union of all registered transformer keys (e.g. `"clerk"`). */ export type TransformerKey = string; -/** - * One error entry as returned in a Clerk API error response body. - * - * Local mirror of `@clerk/types`' `ClerkAPIError` covering only the fields the - * migration logs read. - */ -export type ClerkApiError = { - code: string; - message: string; - longMessage?: string; -}; - -/** A failed user-creation attempt, as handed to the error logger. */ -export type ErrorPayload = { - userId: string; - status: string; - errors: ClerkApiError[]; -}; - -/** A user that failed schema validation before any API call was made. */ -export type ValidationErrorPayload = { - error: string; - path: (string | number)[]; - userId: string; - row: number; -}; - -/** A formatted error line as written to the NDJSON log. */ -export type ErrorLog = { - type: string; - userId: string; - status: string; - error: string | undefined; -}; - -/** One import attempt as written to the NDJSON log. */ -export type ImportLogEntry = { - userId: string; - status: "success" | "error"; - clerkUserId?: string; - error?: string; - code?: string; -}; - -/** One exported user as written to the NDJSON log. */ -export type ExportLogEntry = { - /** The source platform's ID for this user. */ - userId: string; - status: "success" | "error"; - error?: string; -}; - -/** One deletion attempt as written to the NDJSON log. */ -export type DeleteLogEntry = { - /** The source platform's ID — the Clerk user's `external_id`. */ - userId: string; - clerkUserId?: string; - status: "success" | "error"; - error?: string; - code?: string; -}; - /** Totals for a completed import run. */ export type ImportSummary = { totalProcessed: number; diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index 5b3fa33a7..ed11c8277 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -71,11 +71,11 @@ export const NEXT_STEPS = { "Run `clerk apps list` to see your other applications", "Run `clerk config pull` to inspect the live configuration of this instance", ], - MIGRATE_DONE: ["Run `clerk migrate logs list` to inspect the import log"], - // `logs list` only names the file; after a partial import the operator needs - // the failures themselves, which live one line per user in that file. - MIGRATE_DONE_WITH_ERRORS: (logFile: string) => [ - `Run \`grep '"status":"error"' ${logFile}\` to see every user that failed and why`, + MIGRATE_DONE: (runId: string) => [ + `Run \`clerk migrate runs ${runId}\` to see what happened to each user`, + ], + MIGRATE_DONE_WITH_ERRORS: (runId: string) => [ + `Run \`clerk migrate runs ${runId}\` to see every user that failed and why`, ], } as const; From 560cf284cac85ada5e1b3377424b6d62367caedb Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 29 Sep 2026 17:44:39 -0400 Subject: [PATCH 055/141] feat(migrate): add `clerk migrate undo ` `undo` deletes the users an import run created. It deletes only by the Clerk IDs recorded in that run's `users.ndjson`, so a user the import did not create is never in scope. It refuses with exit 2 when the resolved key addresses a different instance than the run imported into, naming both. It also refuses an export or undo run, and a run that has already been undone. It first shows how many users it will delete, and how many of them have signed in since the import (from `last_sign_in_at`). It needs consent: a yes at the prompt, or `--yes`. Without either, it prints that preview and exits 2 with the command to run. `--dry-run` stops after the preview, and `--json` never prompts. Deletes use the import's scheduler and 429 backoff. A user already gone from the instance counts as deleted. The undo is its own run, with `undoes: `. The import is marked `undone` only when every user is gone. A partial undo exits 1, and running `undo` again continues the same undo run. `lib/user-lookup.ts` adds batched `GET /v1/users` lookups (100 values per call, through the scheduler). `runs ` and the import's next steps now point at `undo`. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 54 ++- .../src/commands/migrate/index.test.ts | 15 + .../cli-core/src/commands/migrate/index.ts | 34 +- .../src/commands/migrate/lib/retry.ts | 2 +- .../src/commands/migrate/lib/target.ts | 7 + .../src/commands/migrate/lib/user-lookup.ts | 71 +++ .../cli-core/src/commands/migrate/runs.ts | 14 + .../src/commands/migrate/undo.test.ts | 236 ++++++++++ .../cli-core/src/commands/migrate/undo.ts | 411 ++++++++++++++++++ packages/cli-core/src/lib/next-steps.ts | 2 + 10 files changed, 838 insertions(+), 8 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/user-lookup.ts create mode 100644 packages/cli-core/src/commands/migrate/undo.test.ts create mode 100644 packages/cli-core/src/commands/migrate/undo.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index b8597f5b5..f91bb982b 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -473,9 +473,51 @@ and every 10 pages during the user fetch. `withSpinner` hands a no-op to anything that is not a TTY, so without this an agent exporting a large tenant would see nothing at all until the run finished. +### `clerk migrate undo` + +Deletes the users an import run created. The import run is the whole record of +what to delete: every source ID whose latest line is `created`, by the Clerk ID +recorded beside it. Nothing is matched by searching the instance, so a user the +import did not create is never in scope. + +```sh +clerk migrate undo 20260929-141502-a1b2 --dry-run # preview, delete nothing +clerk migrate undo 20260929-141502-a1b2 # confirms first +clerk migrate undo 20260929-141502-a1b2 --yes # no prompt +``` + +| Flag | Description | +| ------------------- | -------------------------------------------------------- | +| `` | The import run to undo | +| `--dry-run` | Show the preview and delete nothing | +| `-y, --yes` | Delete without prompting | +| `--json` | Output as JSON. Never prompts, so deleting needs `--yes` | +| `--runs-dir ` | Read runs from somewhere else | + +Plus the targeting flags: `--secret-key`, `--app` and `--instance`. + +It prints the target first, then a preview: how many users will be deleted, and +how many of them have signed in since the import (from each user's +`last_sign_in_at`). Nothing is deleted without consent: a yes at the prompt, or +`--yes`. Without either — an agent, a non-TTY run, or `--json` — it prints the +preview and exits 2 with the command to run. + +It refuses with exit 2, and deletes nothing, when: + +- the resolved key addresses a different instance than the run imported into + (the error names both) +- the run is an export or undo run +- the run has already been undone + +Deletes go through the same scheduler and `429` backoff as the import. A user +already gone from the instance counts as deleted. The undo is a run of its own, +`kind: "undo"` with `undoes: `. The import is marked `undone` only when +every user is deleted. A partial undo exits 1, and running `undo` again retries +the users that failed, in the same undo run. + ### `clerk migrate runs` -Every import and export is a **run**, and the run store is the one place +Every import, export and undo is a **run**, and the run store is the one place `clerk migrate` keeps state. `runs` reads it. ```sh @@ -517,7 +559,7 @@ Each run is a folder named for its ID, `YYYYMMDD-HHmmss-xxxx`: | `users.ndjson` | One line per user outcome: `sourceId`, `clerkId`, `status`, and `reason`, `error` or `code` when present | | `lock` | The PID of the process writing the run, while it runs | -A user's status is `created`, `failed`, `skipped` or `exported`. The last line +A user's status is `created`, `failed`, `skipped`, `deleted` or `exported`. The last line for each `sourceId` wins. A `429` retry, an extra email or phone that did not attach, and a validation failure all land in `error`. @@ -935,10 +977,10 @@ those are reachable through has no route for any of these settings, so ## Artifacts -| Path | Contents | -| ------------------------------------------ | --------------------------------------------------- | -| `//` | One [run](#what-a-run-holds) per import or export | -| `./exports/-export-.json` | The export itself, unless `--output` says otherwise | +| Path | Contents | +| ------------------------------------------ | ------------------------------------------------------- | +| `//` | One [run](#what-a-run-holds) per import, export or undo | +| `./exports/-export-.json` | The export itself, unless `--output` says otherwise | `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index 4e895c550..4456f9439 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -143,6 +143,20 @@ describe("registerMigrate", () => { expect(runs?.options.map((option) => option.long)).toEqual(["--json", "--runs-dir"]); }); + test("registers undo with a required run ID and its flags", () => { + const undo = findCommand(["migrate", "undo"]); + expect(undo?.registeredArguments[0]?.required).toBe(true); + expect(undo?.options.map((option) => option.long)).toEqual([ + "--dry-run", + "--yes", + "--json", + "--secret-key", + "--app", + "--instance", + "--runs-dir", + ]); + }); + test("the logs group is gone: runs replaces it", () => { expect(findCommand(["migrate", "logs"])).toBeUndefined(); }); @@ -152,6 +166,7 @@ describe("registerMigrate", () => { test.each([ [["import"]], [["runs"]], + [["undo"]], [["export"]], ...exportPlatformKeys().map((platform) => [["export", platform]]), ])("migrate %p accepts --runs-dir", (names) => { diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index ec4719231..b0dd498a8 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -6,10 +6,11 @@ import { registerMigrateExport } from "./export/index.ts"; import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; import { run } from "./run.ts"; import { runs } from "./runs.ts"; +import { undo } from "./undo.ts"; import { list as transformersList } from "./transformers/list.ts"; import { transformerKeys } from "./transformers/registry.ts"; -const migrate = { run, runs, transformersList }; +const migrate = { run, runs, undo, transformersList }; export function registerMigrate(program: Program): void { const migrateCommand = program @@ -30,6 +31,10 @@ export function registerMigrate(program: Program): void { description: "Export users from Supabase, ready to import", }, { command: "clerk migrate runs", description: "List every migration run" }, + { + command: "clerk migrate undo 20260929-141502-a1b2", + description: "Delete the users an import created", + }, { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, ]); @@ -106,6 +111,33 @@ export function registerMigrate(program: Program): void { registerMigrateExport(migrateCommand); + // Flat, not under a noun group: this is the one command in the tree that + // destroys data in Clerk, and it is worth keeping short and prominent. + migrateCommand + .command("undo") + .description("Delete the users an import run created") + .argument("", "The import run to undo (see `clerk migrate runs`)") + .option("--dry-run", "Show what would be deleted, and delete nothing") + .option("-y, --yes", "Delete without prompting") + .option("--json", "Output as JSON; never prompts, so pair it with --yes to delete") + .option("--secret-key ", "Backend API secret key to use") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .setExamples([ + { + command: "clerk migrate undo 20260929-141502-a1b2 --dry-run", + description: "Preview what would be deleted", + }, + { + command: "clerk migrate undo 20260929-141502-a1b2 --yes", + description: "Delete without prompting", + }, + ]) + .action(async (runId, _opts, cmd) => + migrate.undo(runId, cmd.optsWithGlobals() as Parameters[1]), + ); + migrateCommand .command("runs") .description("List migration runs, or show one") diff --git a/packages/cli-core/src/commands/migrate/lib/retry.ts b/packages/cli-core/src/commands/migrate/lib/retry.ts index a01de49a2..add3ae203 100644 --- a/packages/cli-core/src/commands/migrate/lib/retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/retry.ts @@ -1,5 +1,5 @@ /** - * Rate-limit backoff, shared by `migrate import` and `migrate delete`. + * Rate-limit backoff, shared by `migrate import` and `migrate undo`. * * Both walk the whole user set through BAPI and hit the same limits, so they * back off identically rather than approximately: extracting this is what diff --git a/packages/cli-core/src/commands/migrate/lib/target.ts b/packages/cli-core/src/commands/migrate/lib/target.ts index 2465cf9f7..2ac8c1576 100644 --- a/packages/cli-core/src/commands/migrate/lib/target.ts +++ b/packages/cli-core/src/commands/migrate/lib/target.ts @@ -10,6 +10,7 @@ import { createHash } from "node:crypto"; import { bapiRequest } from "../../../lib/bapi.ts"; +import { dim } from "../../../lib/color.ts"; import { resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; import { resolveAppContext } from "../../../lib/config.ts"; import { resolveKeylessTarget } from "../../../lib/keyless-target.ts"; @@ -114,3 +115,9 @@ export function describeTarget(target: RunTarget): string { const name = target.appLabel ?? (target.platform ? `${target.platform}` : "instance"); return where ? `${name} (${where})` : name; } + +/** The header every command that acts on an instance prints first. */ +export function printTarget(target: RunTarget): void { + log.info(`Target: ${describeTarget(target)}`); + if (target.keySource) log.info(dim(`Key from: ${target.keySource}`)); +} diff --git a/packages/cli-core/src/commands/migrate/lib/user-lookup.ts b/packages/cli-core/src/commands/migrate/lib/user-lookup.ts new file mode 100644 index 000000000..394749541 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/user-lookup.ts @@ -0,0 +1,71 @@ +/** + * Batched `GET /v1/users` lookups by a list of values. + * + * BAPI filters on up to 100 values per call and leaves out any it does not + * find, so the work is proportional to the list rather than to the instance. + * Every call goes through the run's scheduler and backs off on a 429, exactly + * as the writes do. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import type { SpinnerControls } from "../../../lib/spinner.ts"; +import { retryOn429 } from "./retry.ts"; +import type { ApiScheduler } from "./scheduler.ts"; + +/** BAPI accepts at most 100 values per filter on `GET /v1/users`. */ +export const LOOKUP_BATCH = 100; + +/** The fields of a BAPI user these lookups read. */ +export type LookedUpUser = { + id: string; + external_id?: string | null; + last_sign_in_at?: number | null; + email_addresses?: { email_address?: string }[]; + phone_numbers?: { phone_number?: string }[]; +}; + +export type LookupFilter = "user_id" | "external_id" | "email_address" | "phone_number"; + +/** Splits `items` into chunks of at most `size`. */ +export function batch(items: T[], size: number): T[][] { + const batches: T[][] = []; + for (let i = 0; i < items.length; i += size) batches.push(items.slice(i, i + size)); + return batches; +} + +/** Every user matching any of `values` on `filter`. */ +export async function lookupUsers(options: { + filter: LookupFilter; + values: string[]; + secretKey: string; + schedule: ApiScheduler; + spinner?: SpinnerControls; + label?: string; +}): Promise { + const batches = batch([...new Set(options.values)], LOOKUP_BATCH); + let done = 0; + + const pages = await Promise.all( + batches.map(async (values) => { + const params = new URLSearchParams(); + params.set("limit", String(LOOKUP_BATCH)); + for (const value of values) params.append(options.filter, value); + + const response = await retryOn429(async () => + options.schedule(async () => + bapiRequest({ + method: "GET", + path: `/v1/users?${params.toString()}`, + secretKey: options.secretKey, + }), + ), + ); + done++; + options.spinner?.update(`${options.label ?? "Looking up users"}: ${done}/${batches.length}`); + const users = response.body; + return Array.isArray(users) ? (users as LookedUpUser[]) : []; + }), + ); + + return pages.flat().filter((user) => typeof user.id === "string"); +} diff --git a/packages/cli-core/src/commands/migrate/runs.ts b/packages/cli-core/src/commands/migrate/runs.ts index a8d802052..5dd2fe6e4 100644 --- a/packages/cli-core/src/commands/migrate/runs.ts +++ b/packages/cli-core/src/commands/migrate/runs.ts @@ -93,6 +93,18 @@ function errorBreakdown(lines: UserLine[]): { error: string; count: number }[] { return [...counts].map(([error, count]) => ({ error, count })).sort((a, b) => b.count - a.count); } +/** What to run after reading about this run. */ +export function nextCommands(record: RunRecord, state: RunState): string[] { + if (state === "running") return []; + if (record.kind === "import" && state !== "undone") { + return [`Run \`clerk migrate undo ${record.id}\` to delete the users it created`]; + } + if (record.kind === "undo" && state !== "complete" && record.undoes) { + return [`Run \`clerk migrate undo ${record.undoes}\` to retry the users that failed`]; + } + return []; +} + function listJson(runsDir: string, records: RunRecord[]) { return { runsDir, @@ -131,6 +143,7 @@ function showJson(runsDir: string, record: RunRecord) { return { runsDir, run: { ...record, state: runState(runsDir, record) }, + next: nextCommands(record, runState(runsDir, record)), errors: errorBreakdown(lines), failed: lines.filter((line) => line.status === "failed"), skipped: lines.filter((line) => line.status === "skipped"), @@ -199,6 +212,7 @@ function printRun(runsDir: string, record: RunRecord): void { log.blank(); log.info(dim(`Every outcome: ${usersFile}`)); + for (const step of nextCommands(record, state)) log.info(` → ${step}`); } export async function runs(id: string | undefined, options: RunsOptions = {}): Promise { diff --git a/packages/cli-core/src/commands/migrate/undo.test.ts b/packages/cli-core/src/commands/migrate/undo.test.ts new file mode 100644 index 000000000..b17cf3c92 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/undo.test.ts @@ -0,0 +1,236 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { EXIT_CODE, type CliError } from "../../lib/errors.ts"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { latestUserLines, listRuns, readRun, startRun, type RunRecord } from "./lib/run-store.ts"; +import { undo } from "./undo.ts"; + +const captured = useCaptureLog(); + +let runsDir: string; +let originalFetch: typeof globalThis.fetch; +let requests: { method: string; url: string }[]; +/** Clerk IDs the stubbed instance still holds, with each one's last sign-in. */ +let instanceUsers: Map; +/** Clerk IDs whose DELETE fails with a 500. */ +let failing: Set; + +const IMPORT_STARTED = "2026-09-01T00:00:00.000Z"; +const AFTER_IMPORT = Date.parse("2026-09-02T00:00:00.000Z"); + +beforeAll(() => { + originalFetch = globalThis.fetch; +}); + +afterAll(() => { + globalThis.fetch = originalFetch; +}); + +beforeEach(() => { + runsDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-undo-"))); + requests = []; + failing = new Set(); + instanceUsers = new Map([ + ["user_a", null], + ["user_b", AFTER_IMPORT], + ]); + + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input.toString()); + const method = init?.method ?? "GET"; + requests.push({ method, url: url.toString() }); + + if (url.pathname === "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/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (method === "GET" && url.pathname === "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/v1/users") { + const ids = url.searchParams.getAll("user_id"); + return Response.json( + ids + .filter((id) => instanceUsers.has(id)) + .map((id) => ({ id, last_sign_in_at: instanceUsers.get(id) })), + ); + } + if (method === "DELETE") { + const id = url.pathname.split("/").pop() as string; + if (failing.has(id)) { + return Response.json({ errors: [{ code: "boom", message: "boom" }] }, { status: 500 }); + } + instanceUsers.delete(id); + return Response.json({ id, deleted: true }); + } + return new Response("unexpected", { status: 500 }); + }) as typeof fetch; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + process.exitCode = 0; + fs.rmSync(runsDir, { recursive: true, force: true }); +}); + +/** An import into `ins_1` that created two users and failed one. */ +function importRun(overrides: Partial = {}): RunRecord { + const run = startRun(runsDir, { + kind: "import", + target: { instanceId: "ins_1", env: "development" }, + source: "clerk", + }); + run.update({ startedAt: IMPORT_STARTED }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + run.append({ sourceId: "b", status: "created", clerkId: "user_b" }); + run.append({ sourceId: "c", status: "failed", error: "taken" }); + const record = run.finish(); + if (Object.keys(overrides).length > 0) run.update(overrides); + return { ...record, ...overrides }; +} + +const options = { secretKey: "sk_test_x", runsDir: "" }; +const withDir = (extra: Record = {}) => ({ ...options, runsDir, ...extra }); +const deletes = () => requests.filter((request) => request.method === "DELETE"); + +describe("refusals, all exit 2 and delete nothing", () => { + const exitCodeOf = async (promise: Promise) => { + const error = (await promise.catch((caught: unknown) => caught)) as CliError; + return error.exitCode; + }; + + test("an unknown run", async () => { + expect(await exitCodeOf(undo("nope", withDir({ yes: true })))).toBe(EXIT_CODE.USAGE); + }); + + test("a run that is not an import", async () => { + const run = startRun(runsDir, { kind: "export", target: { platform: "auth0" } }); + run.finish(); + await expect(undo(run.record.id, withDir({ yes: true }))).rejects.toThrow( + /is an export run\. Only an import run can be undone/, + ); + expect(deletes()).toHaveLength(0); + }); + + test("a run already undone", async () => { + const record = importRun({ status: "undone", undoneBy: "20260901-000000-beef" }); + await expect(undo(record.id, withDir({ yes: true }))).rejects.toThrow( + /already undone by run 20260901-000000-beef/, + ); + }); + + test("a key that addresses a different instance, naming both", async () => { + const record = importRun({ target: { instanceId: "ins_other", env: "production" } }); + await expect(undo(record.id, withDir({ yes: true }))).rejects.toThrow( + /imported into instance \(production, ins_other\).*addresses instance \(development, ins_1\)/s, + ); + expect(deletes()).toHaveLength(0); + }); + + test("no consent where nobody can be asked: the preview, then the command", async () => { + const record = importRun(); + + const error = (await undo(record.id, withDir()).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Pass --yes to confirm"); + expect(captured.err).toContain("Will delete 2 users"); + expect(deletes()).toHaveLength(0); + }); +}); + +describe("--dry-run", () => { + test("previews the deletes and how many users signed in since, and deletes nothing", async () => { + const record = importRun(); + + await undo(record.id, withDir({ dryRun: true })); + + expect(captured.err).toContain("Target: instance (development, ins_1)"); + expect(captured.err).toContain("Will delete 2 users"); + expect(captured.err).toContain("1 of them has signed in since the import"); + expect(deletes()).toHaveLength(0); + expect(listRuns(runsDir)).toHaveLength(1); + }); + + test("--json returns the target, the run and the preview", async () => { + const record = importRun(); + + await undo(record.id, withDir({ dryRun: true, json: true })); + + expect(JSON.parse(captured.out)).toMatchObject({ + target: { instanceId: "ins_1", keySource: "--secret-key" }, + run: { id: record.id }, + preview: { toDelete: 2, signedInSince: 1, alreadyGone: 0, alreadyDeleted: 0 }, + dryRun: true, + }); + }); +}); + +describe("deleting", () => { + test("deletes what the import created, records an undo run, and marks the import undone", async () => { + const record = importRun(); + + await undo(record.id, withDir({ yes: true })); + + expect( + deletes() + .map((request) => new URL(request.url).pathname) + .sort(), + ).toEqual(["/v1/users/user_a", "/v1/users/user_b"]); + const undoRun = listRuns(runsDir).find((candidate) => candidate.kind === "undo"); + expect(undoRun).toMatchObject({ + status: "complete", + undoes: record.id, + counts: { deleted: 2 }, + }); + expect(readRun(runsDir, record.id)).toMatchObject({ + status: "undone", + undoneBy: undoRun?.id, + }); + expect(process.exitCode).toBe(0); + }); + + test("counts a user already gone from the instance as deleted, without asking Clerk", async () => { + instanceUsers.delete("user_a"); + const record = importRun(); + + await undo(record.id, withDir({ yes: true })); + + expect(deletes()).toHaveLength(1); + expect(readRun(runsDir, record.id)?.status).toBe("undone"); + }); + + test("a failed delete leaves the undo partial, exits 1, and a re-run continues it", async () => { + failing.add("user_b"); + const record = importRun(); + + await undo(record.id, withDir({ yes: true })); + + expect(process.exitCode).toBe(1); + const [partial] = listRuns(runsDir).filter((candidate) => candidate.kind === "undo"); + expect(partial).toMatchObject({ status: "partial", counts: { deleted: 1, failed: 1 } }); + expect(readRun(runsDir, record.id)?.status).toBe("partial"); + + failing.clear(); + requests = []; + process.exitCode = 0; + await undo(record.id, withDir({ yes: true })); + + // Only the user that failed is retried, in the same undo run. + expect(deletes().map((request) => new URL(request.url).pathname)).toEqual(["/v1/users/user_b"]); + const undoRuns = listRuns(runsDir).filter((candidate) => candidate.kind === "undo"); + expect(undoRuns).toHaveLength(1); + expect(undoRuns[0]).toMatchObject({ id: partial?.id, status: "complete" }); + expect(latestUserLines(runsDir, partial!.id).get("b")?.status).toBe("deleted"); + expect(readRun(runsDir, record.id)?.status).toBe("undone"); + }); + + test("--json --yes returns the undo run and the result", async () => { + const record = importRun(); + + await undo(record.id, withDir({ yes: true, json: true })); + + expect(JSON.parse(captured.out)).toMatchObject({ + run: { kind: "undo", undoes: record.id, status: "complete" }, + result: { deleted: 2, failed: 0, errors: [] }, + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/undo.ts b/packages/cli-core/src/commands/migrate/undo.ts new file mode 100644 index 000000000..8b6aa4b7f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/undo.ts @@ -0,0 +1,411 @@ +/** + * `clerk migrate undo ` — delete the users an import created. + * + * The import run is the whole record of what to delete: every source ID whose + * latest line is `created`, by the Clerk ID recorded next to it. Nothing is + * matched by searching the instance, so nothing the import did not create is + * ever in scope. + * + * Refuses (exit 2) rather than guessing when: + * - the resolved key addresses a different instance than the run imported into + * - the run is not an import + * - the run has already been undone + * + * The undo is a run of its own (`kind: "undo"`, `undoes: `). The import is + * marked `undone` only once every user is gone. A partial undo exits 1, and + * running `undo` again continues the same undo run. + */ + +import { bapiRequest } from "../../lib/bapi.ts"; +import { bold, dim, green, red } from "../../lib/color.ts"; +import { BapiError, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; +import { log } from "../../lib/log.ts"; +import { confirm } from "../../lib/prompts.ts"; +import { withSpinner, type SpinnerControls } from "../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../mode.ts"; +import { normalizeErrorMessage } from "./import-users.ts"; +import { resolveLimits, type ResolvedLimits } from "./lib/instance.ts"; +import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; +import { + continueRun, + latestUserLines, + listRuns, + patchRun, + readRun, + resolveRunsDir, + runState, + startRun, + type Run, + type RunRecord, +} from "./lib/run-store.ts"; +import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; +import { describeTarget, printTarget, resolveClerkTarget, type ClerkTarget } from "./lib/target.ts"; +import { lookupUsers } from "./lib/user-lookup.ts"; + +export type UndoOptions = { + dryRun?: boolean; + yes?: boolean; + json?: boolean; + secretKey?: string; + app?: string; + instance?: string; + runsDir?: string; +}; + +/** One user the import created, still to be deleted. */ +export type UndoUser = { sourceId: string; clerkId: string }; + +export type UndoPreview = { + /** Users that will be deleted. */ + toDelete: number; + /** Of those, how many have signed in since the import started. */ + signedInSince: number; + /** Users the import created that are already gone from the instance. */ + alreadyGone: number; + /** Users an earlier attempt at this undo already deleted. */ + alreadyDeleted: number; +}; + +const plural = (count: number, word: string) => `${count} ${word}${count === 1 ? "" : "s"}`; + +/** Loads the import run, refusing anything that cannot be undone. */ +function readImportRun(runsDir: string, runId: string): RunRecord { + const record = readRun(runsDir, runId); + if (!record) { + throwUsageError(`No run \`${runId}\` in ${runsDir}. Run \`clerk migrate runs\` to list them.`); + } + if (record.kind !== "import") { + throwUsageError( + `Run ${runId} is an ${record.kind} run. Only an import run can be undone.`, + undefined, + undefined, + [{ command: "clerk migrate runs", description: "Find the import run to undo" }], + ); + } + if (record.status === "undone") { + throwUsageError(`Run ${runId} was already undone by run ${record.undoneBy ?? "(unknown)"}.`); + } + if (runState(runsDir, record) === "running") { + throwUsageError(`Run ${runId} is still running. Wait for it to finish, then undo it.`); + } + return record; +} + +/** Refuses to delete from an instance the import did not write to. */ +function assertSameInstance(record: RunRecord, target: ClerkTarget): void { + if (record.target.instanceId === target.instanceId) return; + throwUsageError( + `Run ${record.id} imported into ${describeTarget(record.target)}, but the resolved key ` + + `addresses ${describeTarget(target)}. Nothing was deleted.\n` + + "Target the instance the import used with --secret-key, --app or --instance.", + ); +} + +/** The latest undo of this import that has not finished its job, if any. */ +function findOpenUndo(runsDir: string, importId: string): RunRecord | undefined { + return listRuns(runsDir).find( + (candidate) => + candidate.kind === "undo" && + candidate.undoes === importId && + runState(runsDir, candidate) !== "complete", + ); +} + +/** Every user the import created, minus those an earlier undo already deleted. */ +function usersToDelete( + runsDir: string, + record: RunRecord, + openUndo: RunRecord | undefined, +): { users: UndoUser[]; alreadyDeleted: number } { + const deleted = new Set(); + if (openUndo) { + for (const line of latestUserLines(runsDir, openUndo.id).values()) { + if (line.status === "deleted") deleted.add(line.sourceId); + } + } + + const users: UndoUser[] = []; + for (const line of latestUserLines(runsDir, record.id).values()) { + if (line.status !== "created" || !line.clerkId) continue; + if (deleted.has(line.sourceId)) continue; + users.push({ sourceId: line.sourceId, clerkId: line.clerkId }); + } + return { users, alreadyDeleted: deleted.size }; +} + +/** + * Reads each user back from the instance, for the preview. + * + * @returns The users still there, and how many of them have signed in since + * the import started — the ones whose deletion someone will notice. + */ +async function inspectUsers(options: { + users: UndoUser[]; + since: string; + secretKey: string; + schedule: ApiScheduler; + spinner?: SpinnerControls; +}): Promise<{ present: UndoUser[]; gone: UndoUser[]; signedInSince: number }> { + const found = await lookupUsers({ + filter: "user_id", + values: options.users.map((user) => user.clerkId), + secretKey: options.secretKey, + schedule: options.schedule, + spinner: options.spinner, + label: "Checking the imported users", + }); + + const byId = new Map(found.map((user) => [user.id, user])); + const since = Date.parse(options.since); + const present = options.users.filter((user) => byId.has(user.clerkId)); + const gone = options.users.filter((user) => !byId.has(user.clerkId)); + const signedInSince = present.filter((user) => { + const lastSignIn = byId.get(user.clerkId)?.last_sign_in_at; + return typeof lastSignIn === "number" && lastSignIn > since; + }).length; + + return { present, gone, signedInSince }; +} + +export type UndoSummary = { + deleted: number; + failed: number; + errorBreakdown: Map; +}; + +/** Deletes each user, rate-limited and 429-retried exactly as the import is. */ +async function deleteUsers(options: { + users: UndoUser[]; + secretKey: string; + limits: ResolvedLimits; + run: Run; + spinner?: SpinnerControls; +}): Promise { + const { users, secretKey, limits, run, spinner } = options; + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + const errorBreakdown = new Map(); + let processed = 0; + let deleted = 0; + let failed = 0; + + const progress = () => + spinner?.update( + `Deleting users: [${processed}/${users.length}] (${deleted} deleted, ${failed} failed)...`, + ); + + // A failure on one user must not stop the rest: a half-undone import with + // no record of which half is far worse than a reported failure. + const deleteOne = async (user: UndoUser): Promise => { + const retries: string[] = []; + try { + await retryOn429( + async () => + schedule(async () => + bapiRequest({ method: "DELETE", path: `/v1/users/${user.clerkId}`, secretKey }), + ), + { onRetry: ({ message }) => retries.push(message) }, + ); + deleted++; + run.append({ + sourceId: user.sourceId, + clerkId: user.clerkId, + status: "deleted", + ...(retries.length > 0 ? { error: retries.join("; ") } : {}), + }); + } catch (error) { + // Gone already, by the dashboard or another undo: the goal is met. + if (error instanceof BapiError && error.status === 404) { + deleted++; + run.append({ + sourceId: user.sourceId, + clerkId: user.clerkId, + status: "deleted", + reason: "already deleted", + }); + } else { + failed++; + const message = + error instanceof RateLimitExceededError + ? error.message + : ((error as BapiError).longMessage ?? (error as Error).message ?? "Unknown error"); + const code = + error instanceof RateLimitExceededError + ? "429" + : String((error as BapiError).status ?? "unknown"); + const normalized = normalizeErrorMessage(message); + errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); + run.append({ + sourceId: user.sourceId, + clerkId: user.clerkId, + status: "failed", + error: [message, ...retries].join("; "), + code, + }); + } + } + processed++; + progress(); + }; + + progress(); + await Promise.all(users.map(deleteOne)); + return { deleted, failed, errorBreakdown }; +} + +function printPreview(record: RunRecord, preview: UndoPreview): void { + const are = (count: number) => (count === 1 ? "is" : "are"); + log.blank(); + log.info( + `${bold(`Will delete ${plural(preview.toDelete, "user")}`)} created by import run ${record.id}.`, + ); + if (preview.signedInSince > 0) { + const who = + preview.signedInSince === 1 ? "1 of them has" : `${preview.signedInSince} of them have`; + log.warn(`${who} signed in since the import. Deleting removes their accounts and sessions.`); + } + if (preview.alreadyGone > 0) { + log.info( + dim( + `${plural(preview.alreadyGone, "user")} the import created ${are(preview.alreadyGone)} already gone.`, + ), + ); + } + if (preview.alreadyDeleted > 0) { + log.info( + dim( + `${plural(preview.alreadyDeleted, "user")} ${preview.alreadyDeleted === 1 ? "was" : "were"} deleted by an earlier attempt.`, + ), + ); + } +} + +/** The run header shared by every outcome, human and JSON alike. */ +function jsonResult( + target: ClerkTarget, + record: RunRecord, + preview: UndoPreview, + extra: Record = {}, +) { + return { target, run: record, preview, ...extra }; +} + +export async function undo(runId: string, options: UndoOptions = {}): Promise { + const runsDir = await resolveRunsDir(options.runsDir, { write: !options.dryRun }); + const record = readImportRun(runsDir, runId); + + const { secretKey, target } = await resolveClerkTarget(options); + if (!options.json) printTarget(target); + assertSameInstance(record, target); + + const limits = resolveLimits(secretKey); + const openUndo = findOpenUndo(runsDir, record.id); + const { users, alreadyDeleted } = usersToDelete(runsDir, record, openUndo); + + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + const { present, gone, signedInSince } = + users.length > 0 + ? await withSpinner("Checking the imported users...", async (spinner) => + inspectUsers({ users, since: record.startedAt, secretKey, schedule, spinner }), + ) + : { present: [], gone: [], signedInSince: 0 }; + + const preview: UndoPreview = { + toDelete: present.length, + signedInSince, + alreadyGone: gone.length, + alreadyDeleted, + }; + + if (!options.json) printPreview(record, preview); + + const command = `clerk migrate undo ${record.id} --yes`; + const nothingLeft = present.length === 0 && gone.length === 0; + + if (options.dryRun) { + if (options.json) { + log.data(JSON.stringify(jsonResult(target, record, preview, { dryRun: true }), null, 2)); + } else { + log.blank(); + log.info(dim(`Dry run: nothing was deleted. Run \`${command}\` to delete them.`)); + } + return; + } + + // Rule 1: nothing is deleted without consent — `--yes`, or a yes at a + // prompt. `--json` never prompts. + if (!nothingLeft && !options.yes) { + const canAsk = !options.json && isHuman() && !isAgent(); + if (!canAsk) { + if (options.json) { + log.data( + JSON.stringify(jsonResult(target, record, preview, { consent: "required" }), null, 2), + ); + } + throwUsageError( + `\`clerk migrate undo\` permanently deletes ${plural(present.length, "user")} and needs consent. Pass --yes to confirm.`, + undefined, + undefined, + [{ command, description: "Delete the users this import created" }], + ); + } + const proceed = await confirm({ + message: `Permanently delete ${plural(present.length, "user")}?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + const run = openUndo + ? continueRun(runsDir, openUndo) + : startRun(runsDir, { kind: "undo", target, undoes: record.id, source: record.source }); + + // Users the import created that are no longer there: the undo's goal for + // them is already met, so they count as deleted without another request. + for (const user of gone) { + run.append({ ...user, status: "deleted", reason: "not found in the instance" }); + } + + const summary = + present.length > 0 + ? await withSpinner(`Deleting users: [0/${present.length}]...`, async (spinner) => + deleteUsers({ users: present, secretKey, limits, run, spinner }), + ) + : { deleted: 0, failed: 0, errorBreakdown: new Map() }; + + const undoRecord = run.finish(); + if (undoRecord.status === "complete") { + patchRun(runsDir, record.id, { status: "undone", undoneBy: undoRecord.id }); + } + + if (options.json) { + log.data( + JSON.stringify( + jsonResult(target, undoRecord, preview, { + result: { + deleted: summary.deleted, + failed: summary.failed, + errors: [...summary.errorBreakdown].map(([error, count]) => ({ error, count })), + }, + }), + null, + 2, + ), + ); + } else { + log.blank(); + log.info(`${bold("Deleted:")} ${green(String(summary.deleted))}`); + log.info(`${bold("Failed:")} ${red(String(summary.failed))}`); + for (const [error, count] of summary.errorBreakdown) { + log.info(` ${plural(count, "user")}: ${error}`); + } + log.blank(); + log.info(dim(`Undo run ${undoRecord.id}: ${run.dir}`)); + if (undoRecord.status === "complete") { + log.success(`Import run ${record.id} is undone.`); + } else { + log.info(`Run \`clerk migrate undo ${record.id}\` again to retry the users that failed.`); + } + } + + if (undoRecord.status !== "complete") process.exitCode = 1; +} diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index ed11c8277..74f10aba4 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -73,9 +73,11 @@ export const NEXT_STEPS = { ], MIGRATE_DONE: (runId: string) => [ `Run \`clerk migrate runs ${runId}\` to see what happened to each user`, + `Run \`clerk migrate undo ${runId}\` to delete the users it created`, ], MIGRATE_DONE_WITH_ERRORS: (runId: string) => [ `Run \`clerk migrate runs ${runId}\` to see every user that failed and why`, + `Run \`clerk migrate undo ${runId}\` to delete the users it created`, ], } as const; From e9e75b641af2b711e794b17ea8f6f8a43fd970f0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 29 Sep 2026 17:51:25 -0400 Subject: [PATCH 056/141] feat(migrate): exports write a self-describing file into their run Every export already recorded a run. Its file now lands in that run's folder as `export.json` by default, and `-o` still writes it elsewhere. Exports no longer ask where to save, so `-y` no longer needs `--output`. The file is now an envelope: `{ clerkMigrate: 1, source, exportedAt, runId, firebase?, users }`. The import reads it: - `clerk migrate import ` takes the file that run wrote, and records `fromExport` on the import run. The argument can also be a file path. - An envelope names its own source, so no `--transformer` is needed. A `--transformer` that contradicts it exits 2. - A bare array, a CSV, or Firebase's own `{ users: [...] }` still needs `--transformer`. - The Firebase export saves the project's hash parameters in the envelope. The import reads the `--firebase-*` flags first, then the envelope. Exports print `clerk migrate import ` as the next step. Every export takes `--json`: it prints `{ target, run, output, users, coverage, next }` to stdout and never prompts. Across `migrate`, `--json` runs the command in agent mode, where every prompt already stops with a usage error instead. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 131 ++++----- .../src/commands/migrate/export/auth0.test.ts | 29 +- .../src/commands/migrate/export/auth0.ts | 26 +- .../src/commands/migrate/export/authjs.ts | 24 +- .../src/commands/migrate/export/betterauth.ts | 24 +- .../src/commands/migrate/export/clerk.test.ts | 31 +- .../src/commands/migrate/export/clerk.ts | 24 +- .../migrate/export/db-exports.test.ts | 7 +- .../src/commands/migrate/export/db-options.ts | 2 + .../commands/migrate/export/firebase.test.ts | 78 ++--- .../src/commands/migrate/export/firebase.ts | 74 +++-- .../src/commands/migrate/export/index.ts | 50 ++-- .../commands/migrate/export/shared.test.ts | 206 +++++-------- .../src/commands/migrate/export/shared.ts | 270 +++++++----------- .../src/commands/migrate/export/supabase.ts | 24 +- .../commands/migrate/export/workos.test.ts | 36 ++- .../src/commands/migrate/export/workos.ts | 26 +- .../src/commands/migrate/index.test.ts | 39 ++- .../cli-core/src/commands/migrate/index.ts | 17 +- .../src/commands/migrate/lib/export-file.ts | 51 ++++ .../src/commands/migrate/lib/run-store.ts | 3 + .../src/commands/migrate/lib/transform.ts | 17 ++ .../src/commands/migrate/readme.test.ts | 4 +- .../cli-core/src/commands/migrate/run.test.ts | 110 ++++++- packages/cli-core/src/commands/migrate/run.ts | 75 ++++- 25 files changed, 710 insertions(+), 668 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/export-file.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index f91bb982b..7cd4d5eb1 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -60,11 +60,13 @@ Reads an exported user file, maps it onto Clerk's user schema, validates every record, and creates the users through the Backend API. ```sh +clerk migrate import 20260929-141502-a1b2 -y # an export run clerk migrate import -y --transformer clerk --file users.json ``` | Flag | Description | | --------------------------------------- | --------------------------------------------------------------- | +| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | | `-t, --transformer ` | Source platform the file came from (see below) | | `--transformer-file ` | A transformer you wrote, for a platform with no built-in | | `-f, --file ` | Path to the export. `.json` or `.csv` | @@ -81,8 +83,14 @@ clerk migrate import -y --transformer clerk --file users.json Plus the targeting flags from the table above: `--secret-key`, `--app` and `--instance`. -`--transformer` and `--file` are required. Omitting either fails with a usage -error that names the valid values. +The file is the positional argument or `--file`, not both. An export run ID +stands for the file that run wrote, and the import records it as `fromExport`. + +A file `clerk migrate export` wrote carries its source, so it needs no +`--transformer`. A `--transformer` that contradicts it exits 2. Any other file +— a bare JSON array, a CSV, Firebase's own `{ "users": [...] }` — needs +`--transformer`, and omitting it fails with a usage error that names the valid +values. Failures do not stop the run: each user's outcome is written to the [run](#clerk-migrate-runs) and the import continues. A `429` backs off — honouring `Retry-After` when the response @@ -174,42 +182,44 @@ having been told not to. | `firebase` | Firebase Identity Toolkit | `--transformer firebase` | | `workos` | WorkOS User Management API | `--transformer workos` | -Every export asks where to save the file before it starts, proposing -`./exports/-export-.json`. Press enter to take it, -or type over it to save somewhere else — the proposal is prefilled, so it is -one prompt rather than a confirm and a path question. - -The stamp is ISO 8601 basic format in local time, to the minute: it goes in a -name people read off the screen and tab-complete, and it means a second export -never silently overwrites the first. - -`--output` answers that prompt up front and skips it, as does agent mode, which -takes the proposed path. `--output` resolves against the **current directory**, -like every other path flag here. - -`-y` does neither: it **fails**, naming `--output` and handing back the whole -command with the proposed path already in it, to run again. This is the one -prompt whose default cannot be undone by re-running — a file written where -nobody chose it has to be found and moved, and the second run writes a second -copy. Every other question `-y` silences has a default that costs nothing to -land on. Agent mode keeps defaulting even when it also passes `-y`, since there -was no prompt on that path to suppress. - -The question comes before any users are fetched, so a long export can be left -unattended rather than stalling on a prompt with everything held in memory. - -| Flag | Platforms | Description | -| -------------------------- | ---------------------------------- | ----------------------------------------------------------- | -| `-o, --output ` | all | Where to write the export | -| `-y, --yes` | all | Do not prompt: require `--output`, fail on a bad credential | -| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL, libsql/Turso or SQLite connection string | -| `--service-account ` | `firebase` | Path to a service account key JSON file | -| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | -| `--client-id ` | `auth0` | Machine-to-machine application client ID | -| `--client-secret ` | `auth0` | Machine-to-machine application client secret | -| `--api-key ` | `workos` | WorkOS secret API key, the one starting `sk_` | -| `--with-identities` | `workos` | Also record each user's OAuth providers | -| `--no-with-identities` | `workos` | Skip the OAuth provider fan-out without being asked | +Every export is a [run](#clerk-migrate-runs), and the file lands in the run +folder as `export.json`. `--output` writes it somewhere else instead, +resolved against the **current directory** like every other path flag here; +the run still records where. Nothing is asked about where the file goes. + +The file is an envelope around the users: + +```json +{ + "clerkMigrate": 1, + "source": "clerk", + "exportedAt": "2026-09-29T14:15:02.000Z", + "runId": "20260929-141502-a1b2", + "users": [ … ] +} +``` + +`source` is what lets `clerk migrate import ` run with no +`--transformer`. A Firebase export adds `firebase`, the project's hash +parameters, so the import needs no `--firebase-*` flags. + +`--json` prints the result on stdout instead — `{ target, run, output, users, +coverage, next }` — and never prompts, so a missing credential exits 2 naming +the flag to pass. + +| Flag | Platforms | Description | +| -------------------------- | ---------------------------------- | --------------------------------------------------------- | +| `-o, --output ` | all | Write the export here instead of the run folder | +| `-y, --yes` | all | Do not prompt: fail on a bad credential | +| `--json` | all | Print the result as JSON; never prompts | +| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL, libsql/Turso or SQLite connection string | +| `--service-account ` | `firebase` | Path to a service account key JSON file | +| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | +| `--client-id ` | `auth0` | Machine-to-machine application client ID | +| `--client-secret ` | `auth0` | Machine-to-machine application client secret | +| `--api-key ` | `workos` | WorkOS secret API key, the one starting `sk_` | +| `--with-identities` | `workos` | Also record each user's OAuth providers | +| `--no-with-identities` | `workos` | Skip the OAuth provider fan-out without being asked | `export clerk` also takes the targeting flags — it reads from a Clerk instance, so it resolves a key the same way `clerk migrate import` does, with one extra @@ -255,32 +265,28 @@ Field coverage ! 1/3 have a username ! 2/3 have a password (not exportable — see below) -Exported 3 users to /project/exports/clerk-export-20260817-1432.json +Exported 3 users to /project/.clerk/migrate/20260929-141502-a1b2/export.json +Run 20260929-141502-a1b2. See each user with `clerk migrate runs 20260929-141502-a1b2`. Import them with: - clerk migrate import --transformer clerk --file exports/clerk-export-20260817-1432.json + clerk migrate import 20260929-141502-a1b2 Imports into whichever instance the resolved secret key belongs to. For production, add `--instance prod` or use a production secret key. - Add `-y` to skip the import confirmation prompt. ``` The import command prints through the same channel as the coverage table rather than the gutter's **Next steps** outro, which is human-only — an agent -would otherwise be told what was exported and never how to import it. `-y` is -carried across from the export that was given it, and replaced by the hint line -above when it was not: on import `-y` also waves through the -development-instance user-limit warning, so it is not a flag to suggest to -someone who never asked for it. +would otherwise be told what was exported and never how to import it. There is one command, not a development and a production variant, because no flag's absence means "development" — the resolved key decides, through `--secret-key`, `--app`, `CLERK_SECRET_KEY`, the keyless project and the linked profile in that order. -Every export is also a [run](#clerk-migrate-runs), with one line per exported -user, so `clerk migrate runs` lists it alongside imports. Every export takes -`--runs-dir ` to keep that run somewhere else. +The export run has one line per exported user, so `clerk migrate runs` lists it +alongside imports. Every export takes `--runs-dir ` to keep that run +somewhere else. #### Three platforms export no passwords @@ -373,21 +379,13 @@ appears in output. Firebase's scrypt is a modified variant, so a digest is worthless without the project's four hash parameters. The export **reads them from the project** and -prints the exact import command: +saves them in the export file's envelope, so the import needs nothing more: ``` Password hash parameters -Read from the project. Import with: - clerk migrate import -y --transformer firebase --file exports/firebase-export.json --firebase-signer-key "…" --firebase-salt-separator "…" --firebase-rounds 8 --firebase-mem-cost 14 +Read from the project and saved in the export file, so the import needs nothing more. ``` -On one line however long it gets: this prints inside the gutter, which prefixes -every line given to it with `│`. Split over lines with backslash continuations, -that character lands in the middle of the command and is copied along with it — -the shell then reads each one as another argument and rejects the import. A line -that wraps on screen carries no such character and pastes back as what was -printed. - Reading the config needs a broader role than listing users, so if it is denied the export still succeeds and points at **Authentication → Users → (⋮) → Password hash parameters** instead. An export with no password hashes says so @@ -717,8 +715,11 @@ naming what is missing. A partial set produces a well-formed digest that verifies against nothing, so users would import successfully and then be unable to sign in. -They are read from the flags only. The signer key is a Firebase secret, and the -CLI stores none of them. +The flags are read first, then the export file's envelope, which carries the +parameters when `clerk migrate export firebase` could read them from the +project. The flags win, so a rotated key can be passed without re-exporting. +The envelope sits in the gitignored run folder, since the signer key is a +Firebase secret. An export with no password hashes needs no parameters at all. @@ -977,10 +978,10 @@ those are reachable through has no route for any of these settings, so ## Artifacts -| Path | Contents | -| ------------------------------------------ | ------------------------------------------------------- | -| `//` | One [run](#what-a-run-holds) per import, export or undo | -| `./exports/-export-.json` | The export itself, unless `--output` says otherwise | +| Path | Contents | +| --------------------------------- | ------------------------------------------------------- | +| `//` | One [run](#what-a-run-holds) per import, export or undo | +| `//export.json` | An export's envelope, unless `--output` says otherwise | `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index 1a1c47931..c4c33f2ea 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -277,11 +277,19 @@ describe("buildAuth0Export", () => { }); }); -/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +/** The envelope the one export run in this project wrote. */ function onlyExportFile(): string { - const entries = fs.readdirSync(path.join(workDir, "exports")); + const dir = path.join(workDir, ".clerk", "migrate"); + const entries = fs.readdirSync(dir); expect(entries).toHaveLength(1); - return path.join(workDir, "exports", entries[0] as string); + return path.join(dir, entries[0] as string, "export.json"); +} + +/** The users inside that envelope. */ +function exportedUsers(): Record[] { + return ( + JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as { users: Record[] } + ).users; } describe("exportAuth0", () => { @@ -289,13 +297,10 @@ describe("exportAuth0", () => { stubAuth0([[auth0User(0)], []]); await exportAuth0({ ...CREDENTIALS }); - - // Stamped to the minute, so a second export does not overwrite the first. - expect(path.basename(onlyExportFile())).toMatch(/^auth0-export-\d{8}-\d{4}\.json$/); - const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< - string, - unknown - >[]; + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toMatchObject({ + source: "auth0", + }); + const written = exportedUsers(); expect(written[0]?.user_id).toBe("auth0|a0"); expect(captured.err).toContain("Field coverage"); }); @@ -307,13 +312,11 @@ describe("exportAuth0", () => { const originalMode = getMode(); setMode("human"); try { - // --output answers the destination prompt, which human mode would - // otherwise stop on. await exportAuth0({ ...CREDENTIALS, output: "exports/mine.json" }); } finally { setMode(originalMode); } - expect(captured.err).toContain("migrate import --transformer auth0 --file exports/mine.json"); + expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); }); test("--output controls the destination", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 49fbd206a..cdc33b3da 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -23,13 +23,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spin import { isAgent, isHuman } from "../../../mode.ts"; import type { UserLine } from "../lib/run-store.ts"; import { withInputRetry } from "../lib/input-retry.ts"; -import { - finishExportRun, - reportExport, - resolveOutputPath, - startExportRun, - writeExportOutput, -} from "./shared.ts"; +import { finishExport, startExportRun } from "./shared.ts"; const PAGE_SIZE = 100; @@ -49,6 +43,8 @@ export type ExportAuth0Options = { output?: string; /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; }; export type Auth0Credentials = { @@ -336,8 +332,6 @@ export function buildAuth0Export( export async function exportAuth0(options: ExportAuth0Options): Promise { const resolved = await resolveAuth0Credentials(options); - const destination = await resolveOutputPath("auth0", options.output); - await withGutter("Exporting users from Auth0", async () => { // Only Auth0 can say whether these three go together, and whether the // application carries the `read:users` scope, so a rejected set is asked @@ -356,20 +350,8 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { ); const run = await startExportRun(options, { platform: "auth0" }); - const { users: exported, coverage } = buildAuth0Export(users, run.append); - const outputPath = writeExportOutput(exported, destination); - - const record = finishExportRun(run, outputPath); - - reportExport({ - platform: "auth0", - userCount: exported.length, - outputPath, - coverage, - transformerKey: "auth0", - runId: record.id, - }); + finishExport({ run, options, users: exported, coverage }); if (exported.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 207be1278..90a36712e 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -15,13 +15,7 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { log } from "../../../lib/log.ts"; import type { UserLine } from "../lib/run-store.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { - finishExportRun, - reportExport, - resolveOutputPath, - startExportRun, - writeExportOutput, -} from "./shared.ts"; +import { finishExport, startExportRun } from "./shared.ts"; import { promptDbUrl, resolveDbUrl, @@ -127,8 +121,6 @@ const AUTHJS_DB = { export async function exportAuthJs(options: DbExportOptions): Promise { const dbUrl = await resolveDbUrl(options, AUTHJS_DB); - const destination = await resolveOutputPath("authjs", options.output); - await withGutter("Exporting users from Auth.js", async () => { const { value: { rows, table }, @@ -143,20 +135,8 @@ export async function exportAuthJs(options: DbExportOptions): Promise { log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); const run = await startExportRun(options, { platform: "authjs" }); - const { users, coverage } = buildAuthJsExport(rows, run.append); - const outputPath = writeExportOutput(users, destination); - - const record = finishExportRun(run, outputPath); - - reportExport({ - platform: "authjs", - userCount: users.length, - outputPath, - coverage, - transformerKey: "authjs", - runId: record.id, - }); + finishExport({ run, options, users, coverage }); if (users.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index ceeacbdb0..2398f7c58 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -19,13 +19,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { - finishExportRun, - reportExport, - resolveOutputPath, - startExportRun, - writeExportOutput, -} from "./shared.ts"; +import { finishExport, startExportRun } from "./shared.ts"; import { promptDbUrl, resolveDbUrl, @@ -177,8 +171,6 @@ const BETTERAUTH_DB = { export async function exportBetterAuth(options: DbExportOptions): Promise { const dbUrl = await resolveDbUrl(options, BETTERAUTH_DB); - const destination = await resolveOutputPath("betterauth", options.output); - await withGutter("Exporting users from Better Auth", async () => { const { value: { rows, plugins }, @@ -202,19 +194,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise ); const run = await startExportRun(options, { platform: "betterauth" }); - const { users, coverage } = buildBetterAuthExport(rows, run.append); - const outputPath = writeExportOutput(users, destination); - - const record = finishExportRun(run, outputPath); - - reportExport({ - platform: "betterauth", - userCount: users.length, - outputPath, - coverage, - transformerKey: "betterauth", - runId: record.id, - }); + finishExport({ run, options, users, coverage }); }); } diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 282a8b2bb..6c54ed80e 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -225,11 +225,19 @@ describe("buildClerkExport", () => { }); }); -/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +/** The envelope the one export run in this project wrote. */ function onlyExportFile(): string { - const entries = fs.readdirSync(path.join(workDir, "exports")); + const dir = path.join(workDir, ".clerk", "migrate"); + const entries = fs.readdirSync(dir); expect(entries).toHaveLength(1); - return path.join(workDir, "exports", entries[0] as string); + return path.join(dir, entries[0] as string, "export.json"); +} + +/** The users inside that envelope. */ +function exportedUsers(): Record[] { + return ( + JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as { users: Record[] } + ).users; } describe("exportClerk", () => { @@ -237,13 +245,10 @@ describe("exportClerk", () => { stubPages([[user({ id: "u1", first_name: "Ada" })], []]); await exportClerk({ secretKey: "sk_test_x" }); - - // Stamped to the minute, so a second export does not overwrite the first. - expect(path.basename(onlyExportFile())).toMatch(/^clerk-export-\d{8}-\d{4}\.json$/); - const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< - string, - unknown - >[]; + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toMatchObject({ + source: "clerk", + }); + const written = exportedUsers(); expect(written).toHaveLength(1); expect(written[0]?.id).toBe("u1"); expect(captured.err).toContain("Field coverage"); @@ -257,13 +262,11 @@ describe("exportClerk", () => { const originalMode = getMode(); setMode("human"); try { - // --output answers the destination prompt, which human mode would - // otherwise stop on. await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); } finally { setMode(originalMode); } - expect(captured.err).toContain("migrate import --transformer clerk --file exports/mine.json"); + expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); }); test("--output controls the destination, relative to the working directory", async () => { @@ -289,7 +292,7 @@ describe("exportClerk", () => { await exportClerk({ secretKey: "sk_test_x" }); expect(captured.err).toContain("No users found to export"); - expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toEqual([]); + expect(exportedUsers()).toEqual([]); }); test("an empty export warns but does not suggest importing it", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 8a1d99f08..63873ff43 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -20,13 +20,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spin import type { UserLine } from "../lib/run-store.ts"; import { retryOn429 } from "../lib/retry.ts"; import { resolveClerkSource } from "./clerk-source.ts"; -import { - finishExportRun, - reportExport, - resolveOutputPath, - startExportRun, - writeExportOutput, -} from "./shared.ts"; +import { finishExport, startExportRun } from "./shared.ts"; /** BAPI's maximum page size for `GET /v1/users`. */ const PAGE_SIZE = 500; @@ -38,6 +32,8 @@ export type ExportClerkOptions = { instance?: string; /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; }; type BapiIdentifier = { @@ -243,8 +239,6 @@ export async function exportClerk(options: ExportClerkOptions): Promise { instance: options.instance, }); - const destination = await resolveOutputPath("clerk", options.output); - await withGutter("Exporting users from Clerk", async () => { log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); @@ -254,17 +248,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { const run = await startExportRun(options, { platform: "clerk", appLabel: source.target }); const { users: exported, coverage } = buildClerkExport(users, run.append); - const outputPath = writeExportOutput(exported, destination); - const record = finishExportRun(run, outputPath); - - reportExport({ - platform: "clerk", - userCount: exported.length, - outputPath, - coverage, - transformerKey: "clerk", - runId: record.id, - }); + finishExport({ run, options, users: exported, coverage }); if (exported.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 677762c86..9fc918236 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -229,7 +229,8 @@ describe("authjs export", () => { await exportAuthJs({ dbUrl: authJsDb("User"), output: "authjs.json" }); const written = JSON.parse(fs.readFileSync(path.join(workDir, "authjs.json"), "utf-8")); - expect(written).toHaveLength(2); + expect(written).toMatchObject({ clerkMigrate: 1, source: "authjs" }); + expect(written.users).toHaveLength(2); expect(captured.err).toContain("Read 2 rows from"); expect(captured.err).toContain("stores no passwords"); }); @@ -309,7 +310,9 @@ describe("betterauth export", () => { await exportBetterAuth({ dbUrl: file, output: "ba.json" }); expect(captured.err).toContain("Detected plugin columns: username"); - expect(JSON.parse(fs.readFileSync(path.join(workDir, "ba.json"), "utf-8"))).toHaveLength(1); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "ba.json"), "utf-8")).users).toHaveLength( + 1, + ); }); test("says so plainly when no plugins are in use", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 7eadfce7c..a5ef0fda3 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -17,6 +17,8 @@ export type DbExportOptions = { output?: string; /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; }; export type ResolveConfig = { diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index aae629866..64909dcd5 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -4,7 +4,6 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; -import { setAssumeYes } from "../lib/assume-yes.ts"; import type { UserLine } from "../lib/run-store.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { @@ -404,61 +403,39 @@ describe("fetchHashConfig", () => { describe("formatHashConfigGuidance", () => { const config = { signerKey: "KEY==", saltSeparator: "Bw==", rounds: 8, memoryCost: 14 }; - test("prints the exact import command when the parameters are known", () => { - const text = formatHashConfigGuidance(config, "exports/firebase-export.json", 3).join("\n"); - expect(text).toContain('--firebase-signer-key "KEY=="'); - expect(text).toContain('--firebase-salt-separator "Bw=="'); - expect(text).toContain("--firebase-rounds 8 --firebase-mem-cost 14"); - }); - - // The command is printed inside the gutter, which prefixes every line it is - // given with `│`. Split over lines, that character lands mid-command and is - // copied with it — the shell then reads each one as another argument and - // rejects the import. - test("keeps the command on one line, so it can be copied out of the gutter", () => { - const [command] = formatHashConfigGuidance(config, "out.json", 3).slice(-1); - expect(command).not.toContain("\n"); - expect(command).not.toContain("\\"); - }); - - // The shared import block carries `-y` across from the export; this command - // is the one a Firebase operator actually copies, so it has to agree. - test("carries -y across from the export that was given it", () => { - setAssumeYes(true); - try { - expect(formatHashConfigGuidance(config, "out.json", 3).join("\n")).toContain( - "clerk migrate import -y --transformer firebase", - ); - } finally { - setAssumeYes(false); - } - }); - - test("leaves -y out when the export was not given it", () => { - expect(formatHashConfigGuidance(config, "out.json", 3).join("\n")).toContain( - "clerk migrate import --transformer firebase", - ); + // They are in the envelope now, so the import needs no flags for them. + test("says the parameters travel in the export file when the project gave them", () => { + const text = formatHashConfigGuidance(config, 3).join("\n"); + expect(text).toContain("saved in the export file"); + expect(text).not.toContain("--firebase-signer-key"); }); test("says where to find them when the project would not say", () => { - const text = formatHashConfigGuidance(null, "out.json", 3).join("\n"); + const text = formatHashConfigGuidance(null, 3).join("\n"); expect(text).toContain("Password hash parameters"); expect(text).toContain("Authentication → Users"); + expect(text).toContain("--firebase-signer-key"); }); // Nothing to configure, so nothing to tell them to configure. test("says nothing is needed when the export has no hashes", () => { - expect(formatHashConfigGuidance(null, "out.json", 0).join("\n")).toContain( - "no hash parameters are needed", - ); + expect(formatHashConfigGuidance(null, 0).join("\n")).toContain("no hash parameters are needed"); }); }); -/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +/** The envelope the one export run in this project wrote. */ function onlyExportFile(): string { - const entries = fs.readdirSync(path.join(workDir, "exports")); + const dir = path.join(workDir, ".clerk", "migrate"); + const entries = fs.readdirSync(dir); expect(entries).toHaveLength(1); - return path.join(workDir, "exports", entries[0] as string); + return path.join(dir, entries[0] as string, "export.json"); +} + +/** The users inside that envelope. */ +function exportedUsers(): Record[] { + return ( + JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as { users: Record[] } + ).users; } describe("exportFirebase", () => { @@ -468,13 +445,10 @@ describe("exportFirebase", () => { }); await exportFirebase({ serviceAccount: "./sa.json" }); - - // Stamped to the minute, so a second export does not overwrite the first. - expect(path.basename(onlyExportFile())).toMatch(/^firebase-export-\d{8}-\d{4}\.json$/); - const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< - string, - unknown - >[]; + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toMatchObject({ + source: "firebase", + }); + const written = exportedUsers(); expect(written).toHaveLength(2); expect(captured.err).toContain("Field coverage"); expect(captured.err).toContain("demo-fb project"); @@ -487,15 +461,11 @@ describe("exportFirebase", () => { const originalMode = getMode(); setMode("human"); try { - // --output answers the destination prompt, which human mode would - // otherwise stop on. await exportFirebase({ serviceAccount: "./sa.json", output: "exports/mine.json" }); } finally { setMode(originalMode); } - expect(captured.err).toContain( - "migrate import --transformer firebase --file exports/mine.json", - ); + expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); }); test("--output controls the destination", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 6400e547c..4e94d2e9f 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -31,16 +31,10 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; -import { isAssumeYes } from "../lib/assume-yes.ts"; import type { UserLine } from "../lib/run-store.ts"; +import type { FirebaseHashConfig } from "../types.ts"; import { withInputRetry } from "../lib/input-retry.ts"; -import { - finishExportRun, - reportExport, - resolveOutputPath, - startExportRun, - writeExportOutput, -} from "./shared.ts"; +import { finishExport, startExportRun } from "./shared.ts"; /** Identity Toolkit's maximum for `accounts:batchGet`. */ const PAGE_SIZE = 1000; @@ -58,6 +52,8 @@ export type ExportFirebaseOptions = { output?: string; /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; }; export type ServiceAccount = { @@ -471,10 +467,14 @@ export function buildFirebaseExport( }; } -/** The exact `migrate import` invocation, with the project's own parameters. */ +/** + * What the reader needs to know about the hash parameters. + * + * When they were read, they are already in the export file, so the import + * needs nothing extra. When they were not, the import has to be given them. + */ export function formatHashConfigGuidance( config: HashConfig | null, - outputPath: string, passwordCount: number, ): string[] { if (passwordCount === 0) { @@ -486,7 +486,7 @@ export function formatHashConfigGuidance( bold("Password hash parameters"), "This export carries password hashes, which Clerk can only verify with the project's", "scrypt parameters. Find them in the Firebase console under", - "Authentication → Users → (⋮) → Password hash parameters, then pass:", + "Authentication → Users → (⋮) → Password hash parameters, then pass them to the import:", dim( " --firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost", ), @@ -495,28 +495,25 @@ export function formatHashConfigGuidance( return [ bold("Password hash parameters"), - "Read from the project. Import with:", - // One line, however long. Inside the gutter every line printed here is - // prefixed with `│`, and backslash continuations put that character in the - // middle of the command — copied along with it, and rejected by the shell - // as three extra arguments. A line that wraps on screen has no such - // character in it and pastes back as what was printed. - dim( - ` clerk migrate import ${isAssumeYes() ? "-y " : ""}--transformer firebase --file ${outputPath}` + - ` --firebase-signer-key "${config.signerKey}"` + - ` --firebase-salt-separator "${config.saltSeparator}"` + - ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, - ), + dim("Read from the project and saved in the export file, so the import needs nothing more."), ]; } +/** The hash parameters in the shape the import reads them. */ +export function toFirebaseHashConfig(config: HashConfig): FirebaseHashConfig { + return { + base64_signer_key: config.signerKey, + base64_salt_separator: config.saltSeparator, + rounds: config.rounds, + mem_cost: config.memoryCost, + }; +} + export async function exportFirebase(options: ExportFirebaseOptions): Promise { // Read and validate before anything reaches the network, so a wrong file // fails in a second rather than after an auth round-trip. const resolved = await resolveServiceAccount(options); - const destination = await resolveOutputPath("firebase", options.output); - await withGutter("Exporting users from Firebase", async () => { // Only Google can say whether a well-formed key is still a valid one, so a // revoked or deleted key fails here and is asked for again. @@ -536,27 +533,24 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise entry.label.includes("password"))?.count ?? 0; + const hashConfig = passwordCount > 0 ? await fetchHashConfig(account, token) : null; - reportExport({ - platform: "firebase", - userCount: exported.length, - outputPath, + finishExport({ + run, + options, + users: exported, coverage, - transformerKey: "firebase", - runId: record.id, + ...(hashConfig ? { firebase: toFirebaseHashConfig(hashConfig) } : {}), }); - const passwordCount = coverage.find((entry) => entry.label.includes("password"))?.count ?? 0; - const hashConfig = passwordCount > 0 ? await fetchHashConfig(account, token) : null; - - log.blank(); - for (const line of formatHashConfigGuidance(hashConfig, outputPath, passwordCount)) { - log.info(line); + if (!options.json) { + log.blank(); + for (const line of formatHashConfigGuidance(hashConfig, passwordCount)) log.info(line); } }); } diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index a6a7f1bdb..3c7736a03 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -91,8 +91,8 @@ export function registerMigrateExport(migrateCommand: Command<[], Record handlers.picker(cmd.optsWithGlobals() as Record), ); exportCommand .command("clerk") - .description( - "Export users from a Clerk instance (default: ./exports/clerk-export-.json)", - ) - .option("-o, --output ", "Where to write the export, relative to the current directory") - .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .description("Export users from a Clerk instance") + .option("-o, --output ", "Write the export here instead of the run folder") + .option("-y, --yes", "Do not prompt: fail on a rejected credential") .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") .option("--secret-key ", "Backend API secret key to use") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") @@ -132,15 +132,14 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)", - ) + .description("Export users from an Auth0 tenant") .option("--domain ", "Auth0 tenant domain, e.g. my-tenant.us.auth0.com") .option("--client-id ", "Machine-to-machine application client ID") .option("--client-secret ", "Machine-to-machine application client secret") - .option("-o, --output ", "Where to write the export, relative to the current directory") - .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .option("-o, --output ", "Write the export here instead of the run folder") + .option("-y, --yes", "Do not prompt: fail on a rejected credential") .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") .setExamples([ { command: @@ -158,13 +157,12 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)", - ) + .description("Export users from a Firebase project") .option("--service-account ", "Path to a service account key JSON file") - .option("-o, --output ", "Where to write the export, relative to the current directory") - .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .option("-o, --output ", "Write the export here instead of the run folder") + .option("-y, --yes", "Do not prompt: fail on a rejected credential") .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") .setExamples([ { command: "clerk migrate export firebase --service-account ./service-account.json", @@ -177,21 +175,20 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)", - ) + .description("Export users from a WorkOS tenant") .option("--api-key ", "WorkOS secret API key, the one starting `sk_`") .option( "--with-identities", "Also record each user's OAuth providers — one extra request per user", ) .option("--no-with-identities", "Skip the OAuth provider fan-out without being asked") - .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("-o, --output ", "Write the export here instead of the run folder") .option( "-y, --yes", - "Do not prompt: require --output, fail on a rejected credential, and assume --with-identities", + "Do not prompt: fail on a rejected credential, and assume --with-identities", ) .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") .setExamples([ { command: "clerk migrate export workos --api-key sk_…", @@ -211,13 +208,12 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)`, - ) + .description(platform.summary) .option("--db-url ", "Postgres, MySQL, libsql/Turso or SQLite connection string") - .option("-o, --output ", "Where to write the export, relative to the current directory") - .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .option("-o, --output ", "Write the export here instead of the run folder") + .option("-y, --yes", "Do not prompt: fail on a rejected credential") .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") .setExamples([ { command: `clerk migrate export ${platform.key} --db-url "${platform.example}"`, diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts index 43cf7281f..f1e7f7912 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.test.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -1,149 +1,109 @@ -import { beforeEach, describe, expect, mock, test } from "bun:test"; -import { type CliError, ERROR_CODE, EXIT_CODE } from "../../../lib/errors.ts"; - -const mockText = mock(); -mock.module("../../../lib/prompts.ts", () => ({ - text: (...args: unknown[]) => mockText(...args), -})); - -let human = true; -mock.module("../../../mode.ts", () => ({ - isHuman: () => human, - isAgent: () => !human, - getMode: () => (human ? "human" : "agent"), - setMode: () => {}, -})); - -const { defaultOutputPath, formatImportCommand, outputStamp, resolveOutputPath } = - await import("./shared.ts"); -const { setAssumeYes } = await import("../lib/assume-yes.ts"); +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { readEnvelope } from "../lib/export-file.ts"; +import { latestUserLines, readRun } from "../lib/run-store.ts"; +import { finishExport, formatImportCommand, startExportRun } from "./shared.ts"; -beforeEach(() => { - human = true; - setAssumeYes(false); - mockText.mockReset(); -}); +const captured = useCaptureLog(); -describe("outputStamp", () => { - // Local time, and no seconds: this ends up in a filename someone reads off - // the screen and types back. - test("stamps to the minute", () => { - expect(outputStamp(new Date(2026, 7, 17, 14, 32, 59))).toBe("20260817-1432"); - }); +let runsDir: string; +let originalCwd: string; - test("pads single-digit months, days, hours and minutes", () => { - expect(outputStamp(new Date(2026, 0, 3, 9, 5, 0))).toBe("20260103-0905"); - }); +beforeEach(() => { + originalCwd = process.cwd(); + runsDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-export-shared-"))); + process.chdir(runsDir); }); -describe("defaultOutputPath", () => { - test("names the platform and the stamp, under exports/", () => { - expect(defaultOutputPath("clerk", new Date(2026, 7, 17, 14, 32))).toBe( - "exports/clerk-export-20260817-1432.json", - ); - }); - - // Two exports of the same platform an hour apart must not collide. - test("gives two runs different names", () => { - expect(defaultOutputPath("auth0", new Date(2026, 7, 17, 14, 32))).not.toBe( - defaultOutputPath("auth0", new Date(2026, 7, 17, 15, 32)), - ); - }); +afterEach(() => { + process.chdir(originalCwd); + fs.rmSync(runsDir, { recursive: true, force: true }); }); -describe("resolveOutputPath", () => { - test("--output is an answer already given", async () => { - expect(await resolveOutputPath("clerk", "somewhere/mine.json")).toBe("somewhere/mine.json"); - expect(mockText).not.toHaveBeenCalled(); - }); +const users = [{ id: "u1", email: "a@x.dev" }]; +const coverage = [{ label: "have an email address", count: 1 }]; - // One prompt, not a confirm plus a path question: the proposal is prefilled, - // so enter accepts it and typing replaces it. - test("prefills the proposed path so enter accepts it", async () => { - mockText.mockImplementation(async (config: { default: string }) => config.default); +describe("finishExport", () => { + test("writes the envelope into the run folder by default", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + run.append({ sourceId: "u1", status: "exported" }); - const chosen = await resolveOutputPath("clerk"); + const { record, outputPath } = finishExport({ run, options: {}, users, coverage }); - expect(chosen).toMatch(/^exports\/clerk-export-\d{8}-\d{4}\.json$/); - expect(mockText).toHaveBeenCalledTimes(1); - expect(mockText.mock.calls[0]?.[0]).toMatchObject({ message: "Save the export to:" }); + expect(outputPath).toBe(path.join(runsDir, record.id, "export.json")); + expect(readEnvelope(outputPath)).toMatchObject({ + clerkMigrate: 1, + source: "supabase", + runId: record.id, + users, + }); + expect(readRun(runsDir, record.id)).toMatchObject({ + kind: "export", + status: "complete", + file: { path: outputPath }, + }); + expect(latestUserLines(runsDir, record.id).get("u1")?.status).toBe("exported"); }); - test("takes a path typed over the proposal, trimmed", async () => { - mockText.mockResolvedValue(" ../elsewhere/users.json "); + test("--output writes somewhere else, and the run still records where", async () => { + const run = await startExportRun({ runsDir }, { platform: "auth0" }); - expect(await resolveOutputPath("firebase")).toBe("../elsewhere/users.json"); - }); - - test("agent mode takes the proposed path without asking", async () => { - human = false; + const { record, outputPath } = finishExport({ + run, + options: { output: "mine/users.json" }, + users, + coverage, + }); - expect(await resolveOutputPath("supabase")).toMatch( - /^exports\/supabase-export-\d{8}-\d{4}\.json$/, - ); - expect(mockText).not.toHaveBeenCalled(); + expect(outputPath).toBe(path.join(runsDir, "mine", "users.json")); + expect(readRun(runsDir, record.id)?.file?.path).toBe(outputPath); }); - // The one prompt whose default cannot be undone by running the command - // again: a file at a path nobody chose has to be found and moved, and a - // second run writes a second copy. So `-y` fails here rather than guessing. - describe("with -y", () => { - beforeEach(() => setAssumeYes(true)); - - test("fails rather than prompting or defaulting", async () => { - await expect(resolveOutputPath("supabase")).rejects.toThrow( - /needs an export location and will not prompt for one with -y/, - ); - expect(mockText).not.toHaveBeenCalled(); - }); + test("carries Firebase's hash parameters to the import", async () => { + const run = await startExportRun({ runsDir }, { platform: "firebase" }); + const firebase = { + base64_signer_key: "k", + base64_salt_separator: "s", + rounds: 8, + mem_cost: 14, + }; - test("is a usage error, so the exit code says what to fix", async () => { - const error = (await resolveOutputPath("supabase").catch((e: unknown) => e)) as CliError; + const { outputPath } = finishExport({ run, options: {}, users, coverage, firebase }); - expect(error.code).toBe(ERROR_CODE.USAGE_ERROR); - expect(error.exitCode).toBe(EXIT_CODE.USAGE); - }); + expect(readEnvelope(outputPath)?.firebase).toEqual(firebase); + }); - // The whole point of failing instead of defaulting: the error has to hand - // back a line that runs, or it has cost the operator the run for nothing. - test("hands back the command to re-run, proposed path and all", async () => { - const error = (await resolveOutputPath("supabase").catch((e: unknown) => e)) as CliError; + test("prints the import command by run ID", async () => { + const run = await startExportRun({ runsDir }, { platform: "clerk" }); - expect(error.examples?.[0]?.command).toMatch( - /^clerk migrate export supabase -y --output exports\/supabase-export-\d{8}-\d{4}\.json$/, - ); - }); + const { record } = finishExport({ run, options: {}, users, coverage }); - test("names the platform that was actually run", async () => { - const error = (await resolveOutputPath("firebase").catch((e: unknown) => e)) as CliError; + expect(captured.err).toContain(`clerk migrate import ${record.id}`); + }); - expect(error.message).toContain("`clerk migrate export firebase`"); - }); + test("--json returns the result on stdout instead", async () => { + const run = await startExportRun({ runsDir }, { platform: "clerk" }); - test("stays quiet when --output already answered it", async () => { - expect(await resolveOutputPath("clerk", "somewhere/mine.json")).toBe("somewhere/mine.json"); - }); + const { record, outputPath } = finishExport({ run, options: { json: true }, users, coverage }); - // An agent passes `-y` reflexively and has no prompt to suppress, so the - // flag must not turn a working export into a usage error there. - test("still defaults in agent mode", async () => { - human = false; - - expect(await resolveOutputPath("supabase")).toMatch( - /^exports\/supabase-export-\d{8}-\d{4}\.json$/, - ); + expect(JSON.parse(captured.out)).toMatchObject({ + run: { id: record.id, kind: "export" }, + output: outputPath, + users: 1, + next: `clerk migrate import ${record.id}`, }); + expect(captured.err).not.toContain("Field coverage"); }); }); describe("formatImportCommand", () => { - const stripAnsi = (value: string): string => value.replace(/\u001b\[[0-9;]*m/g, ""); - const render = () => stripAnsi(formatImportCommand("supabase", "exports/mine.json").join("\n")); + const render = () => Bun.stripANSI(formatImportCommand("20260929-141502-a1b2").join("\n")); - test("names the transformer and the file just written", () => { - expect(render()).toContain( - "clerk migrate import --transformer supabase --file exports/mine.json", - ); + test("names the export run, which carries its own source", () => { + expect(render()).toContain("clerk migrate import 20260929-141502-a1b2"); }); // The instance comes from the resolved key, so there is no flag whose @@ -151,18 +111,4 @@ describe("formatImportCommand", () => { test("says how to reach production", () => { expect(render()).toContain("--instance prod"); }); - - test("offers -y when the export was not given it", () => { - expect(render()).toContain("Add `-y` to skip the import confirmation prompt."); - }); - - // Carried across rather than always printed: on import `-y` also waves - // through the development-instance user-limit warning. - test("carries -y across from the export that was given it", () => { - setAssumeYes(true); - - const text = render(); - expect(text).toContain("clerk migrate import -y --transformer supabase"); - expect(text).not.toContain("Add `-y`"); - }); }); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index 363b29513..a0ac34fd5 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -1,22 +1,18 @@ /** - * Shared plumbing for the export modules: where the file lands, and what the - * user is told about it. + * Shared plumbing for the export modules: the run that records each user, + * where the file lands, and what the user is told about it. * - * Ported from the standalone migration-tool's `src/lib/export.ts`, with one - * behavioural change: `--output` resolves against the **current working - * directory**, the way every other path flag in this CLI does. The original - * resolved a relative `--output` inside `exports/`, so `--output ./here.json` - * silently wrote to `exports/here.json`. + * Every export is a run. The file lands in that run's folder as + * `export.json` unless `--output` names somewhere else, and `--output` + * resolves against the **current working directory**, the way every other + * path flag in this CLI does. */ import fs from "node:fs"; import path from "node:path"; import { dim, green, yellow } from "../../../lib/color.ts"; -import { throwUsageError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; -import { text } from "../../../lib/prompts.ts"; -import { isHuman } from "../../../mode.ts"; -import { isAssumeYes } from "../lib/assume-yes.ts"; +import { ENVELOPE_VERSION, type ExportEnvelope } from "../lib/export-file.ts"; import { resolveRunsDir, sha256File, @@ -25,83 +21,16 @@ import { type RunRecord, type RunTarget, } from "../lib/run-store.ts"; - -/** - * `YYYYMMDD-HHmm`, local time — ISO 8601 basic format, minus seconds. - * - * Basic throughout rather than `2026-08-17-1954`, which mixes the extended - * date form with the basic time form and leaves the trailing group looking - * like a fourth date component. One separator, and it sorts lexically. - * - * Seconds are dropped on purpose. This lands in a filename people read off the - * screen, type back and tab-complete, and two exports of the same platform - * inside one minute is not an accident anyone has by surprise. - * - * Local rather than UTC because the only reader is the person who just ran the - * command, deciding which of two files is the one they meant. - */ -export function outputStamp(now: Date = new Date()): string { - const pad = (value: number) => String(value).padStart(2, "0"); - const date = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}`; - return `${date}-${pad(now.getHours())}${pad(now.getMinutes())}`; -} - -/** Where an export lands when `--output` is not given. */ -export function defaultOutputPath(platform: string, now?: Date): string { - return path.join("exports", `${platform}-export-${outputStamp(now)}.json`); -} - -/** - * Settles where the file lands, before the export runs. - * - * Asked up front rather than at write time so a long export can be left - * unattended — coming back to a stalled prompt with every user held in memory - * and nothing on disk is the worse half of that trade. - * - * One prompt, not a confirm followed by a path prompt: the proposed path is - * prefilled, so Enter accepts it and typing replaces it. - * - * `--output` is an answer already given, and agent mode has nobody to ask, so - * it takes the proposal. - * - * `-y` is neither: somebody is there, and they said not to ask. It fails - * instead of defaulting, because this is the one prompt whose default cannot - * be undone by running the command again — a file written to a path nobody - * chose has to be found and moved, and a second run writes a second copy. - * Silencing that question is what `--output` is for, so the error hands over - * the exact line, proposed path and all. (The log-directory question does take - * its default under `-y`: `./logs` is where the reader would look anyway, and - * nothing is saved.) - */ -export async function resolveOutputPath(platform: string, output?: string): Promise { - if (output) return output; - - const proposed = defaultOutputPath(platform); - // Ordered so agent mode keeps defaulting even when it also passes `-y`: - // there was never a prompt on that path to suppress. - if (!isHuman()) return proposed; - - if (isAssumeYes()) { - throwUsageError( - `\`clerk migrate export ${platform}\` needs an export location and will not prompt for one with -y.\nPass --output, then run it again.`, - undefined, - undefined, - [ - { - command: `clerk migrate export ${platform} -y --output ${proposed}`, - description: "Re-run with the proposed path", - }, - ], - ); - } - - const chosen = await text({ - message: "Save the export to:", - default: proposed, - validate: (value) => (value?.trim() ? undefined : "A path is required"), - }); - return chosen.trim(); -} +import type { FirebaseHashConfig } from "../types.ts"; + +/** What every export command takes on top of its own credentials. */ +export type ExportCommonOptions = { + /** Where to write the file, instead of the run folder. */ + output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; + json?: boolean; +}; /** * Starts the export run that records each user as it is exported. @@ -110,29 +39,27 @@ export async function resolveOutputPath(platform: string, output?: string): Prom * behind. */ export async function startExportRun( - options: { runsDir?: string }, + options: ExportCommonOptions, target: RunTarget, ): Promise { const runsDir = await resolveRunsDir(options.runsDir, { write: true }); return startRun(runsDir, { kind: "export", target, source: target.platform }); } -/** Records the written file on the run, and finishes it. */ -export function finishExportRun(run: Run, outputPath: string): RunRecord { - run.update({ file: { path: outputPath, sha256: sha256File(outputPath) } }); - return run.finish(); +/** Where the file lands: `--output`, or `export.json` in the run folder. */ +export function exportPath(run: Run, output: string | undefined): string { + return output ? path.resolve(process.cwd(), output) : path.join(run.dir, "export.json"); } /** - * Writes the export, creating any missing parent directories. + * Writes the envelope, creating any missing parent directories. * - * @returns The absolute path written, for reporting. + * @returns The absolute path written. */ -export function writeExportOutput(users: unknown[], outputFile: string): string { - const resolved = path.resolve(process.cwd(), outputFile); - fs.mkdirSync(path.dirname(resolved), { recursive: true }); - fs.writeFileSync(resolved, JSON.stringify(users, null, 2)); - return resolved; +export function writeExportFile(file: string, envelope: ExportEnvelope): string { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(envelope, null, 2)); + return file; } export type CoverageField = { label: string; count: number }; @@ -161,92 +88,103 @@ export function formatFieldCoverage(fields: CoverageField[], total: number): str */ export type ExportSection = { title: string; rows: string[] }; -export type ExportSummary = { - platform: string; - userCount: number; - outputPath: string; +/** + * The import command for an export run, and what it will target. + * + * One command rather than a development and a production variant, because + * there is no flag whose absence means "development": the key decides, through + * `--secret-key`, `--app`, `CLERK_SECRET_KEY`, the keyless project and the + * linked profile in that order. A line labelled "development" would be wrong + * for anyone holding `CLERK_SECRET_KEY=sk_live_…`, which is the reader who can + * least afford it. So the note names what picks the instance instead. + */ +export function formatImportCommand(runId: string): string[] { + return [ + "Import them with:", + dim(` clerk migrate import ${runId}`), + "", + dim(" Imports into whichever instance the resolved secret key belongs to."), + dim(" For production, add `--instance prod` or use a production secret key."), + ]; +} + +export type FinishExportInput = { + run: Run; + options: ExportCommonOptions; + users: Record[]; coverage: CoverageField[]; /** Extra blocks, printed under the coverage table in order. */ sections?: ExportSection[]; - /** The transformer that reads this file, for the "what next" line. */ - transformerKey: string; - /** The export run that recorded each user. */ - runId: string; + /** Firebase's hash parameters, carried to the import in the envelope. */ + firebase?: FirebaseHashConfig; }; +export type FinishedExport = { record: RunRecord; outputPath: string }; + /** - * Reports the coverage table and the import command that reads the file. + * Writes the envelope, finishes the run, and reports it. * - * The command prints through `log.info`, alongside the coverage table, rather - * than being handed back for `setNextSteps`. The gutter's next-steps outro is - * human-only — `withGutter` and `printNextSteps` both return early for an agent - * or a non-TTY — and this is the one line that says what to do with the file - * just written. An agent that cannot see it has to guess the invocation. + * The import command prints through `log.info` rather than the gutter's + * next-steps outro, which is human-only: this is the one line that says what + * to do with the file, and an agent that cannot see it has to guess. `--json` + * returns the same facts on stdout instead. */ -export function reportExport(summary: ExportSummary): void { +export function finishExport(input: FinishExportInput): FinishedExport { + const { run, options, users, coverage } = input; + const platform = run.record.target.platform ?? run.record.source ?? ""; + const outputPath = writeExportFile(exportPath(run, options.output), { + clerkMigrate: ENVELOPE_VERSION, + source: run.record.source ?? platform, + exportedAt: new Date().toISOString(), + runId: run.record.id, + ...(input.firebase ? { firebase: input.firebase } : {}), + users, + }); + run.update({ file: { path: outputPath, sha256: sha256File(outputPath) } }); + const record = run.finish(); + const next = `clerk migrate import ${record.id}`; + + if (options.json) { + log.data( + JSON.stringify( + { + target: record.target, + run: record, + output: outputPath, + users: users.length, + coverage, + ...(input.sections?.length ? { sections: input.sections } : {}), + next, + }, + null, + 2, + ), + ); + return { record, outputPath }; + } + log.blank(); - if (summary.userCount === 0) { - log.warn(`No users found to export. Wrote an empty file to ${summary.outputPath}.`); - log.info(dim(`Run ${summary.runId}`)); - return; + if (users.length === 0) { + log.warn(`No users found to export. Wrote an empty file to ${outputPath}.`); + log.info(dim(`Run ${record.id}`)); + return { record, outputPath }; } log.info("Field coverage"); - for (const line of formatFieldCoverage(summary.coverage, summary.userCount)) { - log.info(line); - } + for (const line of formatFieldCoverage(coverage, users.length)) log.info(line); - for (const section of summary.sections ?? []) { + for (const section of input.sections ?? []) { log.blank(); log.info(section.title); for (const row of section.rows) log.info(row); } log.blank(); - log.success( - `Exported ${summary.userCount} user${summary.userCount === 1 ? "" : "s"} to ${summary.outputPath}`, - ); - log.info( - dim(`Run ${summary.runId}. See each user with \`clerk migrate runs ${summary.runId}\`.`), - ); + log.success(`Exported ${users.length} user${users.length === 1 ? "" : "s"} to ${outputPath}`); + log.info(dim(`Run ${record.id}. See each user with \`clerk migrate runs ${record.id}\`.`)); log.blank(); - for (const line of formatImportCommand( - summary.transformerKey, - relativeIfInside(summary.outputPath), - )) { - log.info(line); - } -} - -/** - * The import command for the file just written, and what it will target. - * - * One command rather than a development and a production variant, because - * there is no flag whose absence means "development": the key decides, through - * `--secret-key`, `--app`, `CLERK_SECRET_KEY`, the keyless project and the - * linked profile in that order. A line labelled "development" would be wrong - * for anyone holding `CLERK_SECRET_KEY=sk_live_…`, which is the reader who can - * least afford it. So the note names what picks the instance instead. - * - * `-y` is carried across from this export rather than always printed: on - * import it also waves through the development-instance user-limit warning, so - * it is not a flag to suggest to someone who never asked for it. - */ -export function formatImportCommand(transformerKey: string, file: string): string[] { - const yes = isAssumeYes() ? "-y " : ""; - return [ - "Import them with:", - dim(` clerk migrate import ${yes}--transformer ${transformerKey} --file ${file}`), - "", - dim(" Imports into whichever instance the resolved secret key belongs to."), - dim(" For production, add `--instance prod` or use a production secret key."), - ...(isAssumeYes() ? [] : [dim(" Add `-y` to skip the import confirmation prompt.")]), - ]; -} + for (const line of formatImportCommand(record.id)) log.info(line); -/** Shortens a path for display when it sits under the working directory. */ -function relativeIfInside(absolute: string): string { - const relative = path.relative(process.cwd(), absolute); - return relative.startsWith("..") ? absolute : relative; + return { record, outputPath }; } diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 5f9f86654..bb6ea8481 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -14,13 +14,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { - finishExportRun, - reportExport, - resolveOutputPath, - startExportRun, - writeExportOutput, -} from "./shared.ts"; +import { finishExport, startExportRun } from "./shared.ts"; import { promptDbUrl, resolveDbUrl, @@ -128,8 +122,6 @@ const SUPABASE_DB = { export async function exportSupabase(options: DbExportOptions): Promise { const dbUrl = await resolveDbUrl(options, SUPABASE_DB); - const destination = await resolveOutputPath("supabase", options.output); - await withGutter("Exporting users from Supabase", async () => { const { value: rows } = await withInputRetry( dbUrl, @@ -141,20 +133,8 @@ export async function exportSupabase(options: DbExportOptions): Promise { ); const run = await startExportRun(options, { platform: "supabase" }); - const { users, coverage } = buildSupabaseExport(rows, run.append); - const outputPath = writeExportOutput(users, destination); - - const record = finishExportRun(run, outputPath); - - reportExport({ - platform: "supabase", - userCount: users.length, - outputPath, - coverage, - transformerKey: "supabase", - runId: record.id, - }); + finishExport({ run, options, users, coverage }); if (users.length > 0) { log.info( diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts index 419217163..2ac38570b 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.test.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -383,11 +383,19 @@ describe("buildWorkOsExport", () => { }); }); -/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +/** The envelope the one export run in this project wrote. */ function onlyExportFile(): string { - const entries = fs.readdirSync(path.join(workDir, "exports")); + const dir = path.join(workDir, ".clerk", "migrate"); + const entries = fs.readdirSync(dir); expect(entries).toHaveLength(1); - return path.join(workDir, "exports", entries[0] as string); + return path.join(dir, entries[0] as string, "export.json"); +} + +/** The users inside that envelope. */ +function exportedUsers(): Record[] { + return ( + JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as { users: Record[] } + ).users; } describe("exportWorkOs", () => { @@ -395,13 +403,10 @@ describe("exportWorkOs", () => { stubWorkOs([[workosUser(0)]]); await exportWorkOs({ apiKey: API_KEY }); - - // Stamped to the minute, so a second export does not overwrite the first. - expect(path.basename(onlyExportFile())).toMatch(/^workos-export-\d{8}-\d{4}\.json$/); - const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< - string, - unknown - >[]; + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toMatchObject({ + source: "workos", + }); + const written = exportedUsers(); expect(written[0]?.id).toBe("user_00"); expect(captured.err).toContain("Field coverage"); }); @@ -419,7 +424,7 @@ describe("exportWorkOs", () => { } finally { setMode(originalMode); } - expect(captured.err).toContain("migrate import --transformer workos --file exports/mine.json"); + expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); }); test("--output controls the destination", async () => { @@ -443,10 +448,11 @@ describe("exportWorkOs", () => { await exportWorkOs({ apiKey: API_KEY, output: "rich.json", withIdentities: true }); - const written = JSON.parse(fs.readFileSync(path.join(workDir, "rich.json"), "utf-8")) as Record< - string, - unknown - >[]; + const written = ( + JSON.parse(fs.readFileSync(path.join(workDir, "rich.json"), "utf-8")) as { + users: Record[]; + } + ).users; expect(written[0]?.identities).toEqual([{ provider: "GoogleOAuth", idp_id: "g1" }]); }); diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index 8d2060546..37fc531e0 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -27,14 +27,7 @@ import type { UserLine } from "../lib/run-store.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { createApiScheduler } from "../lib/scheduler.ts"; -import { - finishExportRun, - reportExport, - resolveOutputPath, - startExportRun, - writeExportOutput, - type ExportSection, -} from "./shared.ts"; +import { finishExport, startExportRun, type ExportSection } from "./shared.ts"; const API_BASE = "https://api.workos.com/user_management"; @@ -73,6 +66,8 @@ export type ExportWorkOsOptions = { output?: string; /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; }; export type WorkOsUser = Record & { id?: string }; @@ -444,8 +439,6 @@ export function buildWorkOsExport( export async function exportWorkOs(options: ExportWorkOsOptions): Promise { const resolved = await resolveWorkOsApiKey(options); - const destination = await resolveOutputPath("workos", options.output); - await withGutter("Exporting users from WorkOS", async () => { // Only WorkOS can say whether the key is live, for the right environment, // and not revoked — so a rejected key is asked for again here. The page it @@ -473,19 +466,14 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise run.append, providers?.identities, ); - const outputPath = writeExportOutput(exported, destination); - const record = finishExportRun(run, outputPath); - - reportExport({ - platform: "workos", - userCount: exported.length, - outputPath, + finishExport({ + run, + options, + users: exported, coverage, sections: providers ? [buildIdentityReport(users, providers.identities, providers.failed)] : [], - transformerKey: "workos", - runId: record.id, }); if (exported.length > 0) { diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index 4456f9439..a8bd3b24d 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -1,4 +1,8 @@ import { describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { getMode, setMode } from "../../mode.ts"; import { createProgram } from "../../cli-program.ts"; import { exportPlatformKeys } from "./export/registry.ts"; import { isAssumeYes, setAssumeYes } from "./lib/assume-yes.ts"; @@ -109,13 +113,19 @@ describe("registerMigrate", () => { ); }); - test("documents the default output location in help", () => { - expect(findCommand(["migrate", "export", "clerk"])?.description()).toContain( - "./exports/clerk-export-.json", - ); - expect(findCommand(["migrate", "export", "auth0"])?.description()).toContain( - "./exports/auth0-export-.json", - ); + test.each(exportPlatformKeys())( + "migrate export %s names the run folder as the default output", + (platform) => { + const output = findCommand(["migrate", "export", platform])?.options.find( + (option) => option.long === "--output", + ); + expect(output?.description).toContain("instead of the run folder"); + }, + ); + + test.each(exportPlatformKeys())("migrate export %s accepts --json", (platform) => { + const flags = findCommand(["migrate", "export", platform])?.options.map((o) => o.long); + expect(flags).toContain("--json"); }); test("makes list the default transformers subcommand", () => { @@ -228,6 +238,21 @@ describe("the migrate group's -y hook", () => { ); }); + // `--json` means nobody reads a prompt, and agent mode is how every prompt in + // this tree already knows to stand down. + test("--json runs the command in agent mode", async () => { + const original = getMode(); + const runsDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-json-")); + try { + setMode("human"); + await parse(["migrate", "runs", "--json", "--runs-dir", runsDir]); + expect(getMode()).toBe("agent"); + } finally { + setMode(original); + fs.rmSync(runsDir, { recursive: true, force: true }); + } + }); + test("records its absence, so a previous run cannot leak into this one", async () => { setAssumeYes(true); expect(await parse(["migrate", "export", "supabase", "--db-url", "./none.sqlite"])).toBe(false); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index b0dd498a8..fd70d35c0 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -1,6 +1,7 @@ import { createOption } from "@commander-js/extra-typings"; import type { Program } from "../../cli-program.ts"; import { parseIntegerOption } from "../../lib/option-parsers.ts"; +import { setMode } from "../../mode.ts"; import { setAssumeYes } from "./lib/assume-yes.ts"; import { registerMigrateExport } from "./export/index.ts"; import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; @@ -42,8 +43,14 @@ export function registerMigrate(program: Program): void { // export commands — so it is resolved once here rather than threaded // through every export handler. Hooks are inherited, so this fires for every // subcommand under `migrate`; one that declares no `-y` resolves to false. + // + // `--json` means nobody is reading a prompt, so it runs the command in agent + // mode: every prompt in this tree already stands down for an agent, with the + // usage error naming what to pass instead. migrateCommand.hook("preAction", (_thisCommand, actionCommand) => { - setAssumeYes(Boolean(actionCommand.opts().yes)); + const opts = actionCommand.opts(); + setAssumeYes(Boolean(opts.yes)); + if (opts.json) setMode("agent"); }); // Named, not `isDefault`. `import` and `export` are the two directions this @@ -57,6 +64,7 @@ export function registerMigrate(program: Program): void { migrateCommand .command("import") .description("Import users from an exported JSON or CSV file") + .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") .addOption( createOption( "-t, --transformer ", @@ -105,8 +113,11 @@ export function registerMigrate(program: Program): void { description: "Skip Supabase users whose only provider is not enabled in Clerk", }, ]) - .action(async (_opts, cmd) => - migrate.run(cmd.optsWithGlobals() as Parameters[0]), + .action(async (input, _opts, cmd) => + migrate.run({ + ...(cmd.optsWithGlobals() as Parameters[0]), + ...(input ? { input } : {}), + }), ); registerMigrateExport(migrateCommand); diff --git a/packages/cli-core/src/commands/migrate/lib/export-file.ts b/packages/cli-core/src/commands/migrate/lib/export-file.ts new file mode 100644 index 000000000..5d153db5c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/export-file.ts @@ -0,0 +1,51 @@ +/** + * The file `clerk migrate export` writes: the users, wrapped in an envelope + * that says where they came from. + * + * The envelope is what lets `clerk migrate import ` run with no + * `--source`: the source, and Firebase's hash parameters, travel with the + * users. A bare array, a CSV, or Firebase's own `{ users: [...] }` still + * import, but need the source named. + */ + +import fs from "node:fs"; +import type { FirebaseHashConfig } from "../types.ts"; + +export const ENVELOPE_VERSION = 1; + +export type ExportEnvelope = { + clerkMigrate: typeof ENVELOPE_VERSION; + source: string; + exportedAt: string; + runId: string; + firebase?: FirebaseHashConfig; + users: Record[]; +}; + +export function isEnvelope(value: unknown): value is ExportEnvelope { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const candidate = value as Partial; + return ( + candidate.clerkMigrate === ENVELOPE_VERSION && + typeof candidate.source === "string" && + Array.isArray(candidate.users) + ); +} + +/** + * The envelope in a file, when it has one. + * + * `undefined` for a CSV, a bare array, or anything unreadable: those are the + * shapes that need `--source`, and the load that follows reports what is + * actually wrong with a broken file. + */ +// ponytail: parses the file here and again when the users load; cache by path if exports get large enough to notice. +export function readEnvelope(file: string): ExportEnvelope | undefined { + if (!file.toLowerCase().endsWith(".json")) return undefined; + try { + const parsed: unknown = JSON.parse(fs.readFileSync(file, "utf-8")); + return isEnvelope(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 460491088..e5b897871 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -131,6 +131,9 @@ export async function resolveRunsDir( return path.join(root, ".clerk", "migrate"); } +/** The shape of a run ID, so one can be told apart from a file path. */ +export const RUN_ID_PATTERN = /^\d{8}-\d{6}-[0-9a-f]{4}$/; + /** The folder one run lives in. */ export function runDir(runsDir: string, id: string): string { return path.join(runsDir, id); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 1c8561c85..d82731f38 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -19,6 +19,7 @@ import { type User, } from "../types.ts"; import { userSchema } from "../validator.ts"; +import { isEnvelope } from "./export-file.ts"; export type FileType = "application/json" | "text/csv"; @@ -426,6 +427,22 @@ async function readUsersFromFile( const type = getFileType(file); let preExtracted: Record[] | undefined; + // An export's envelope already holds the users in the source's own shape, + // so there is nothing left for a pre-transform to unwrap. + if (type === "application/json") { + const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + if (isEnvelope(parsed)) return parsed.users; + if (!transformer.preTransform) { + if (!Array.isArray(parsed)) { + throw new CliError( + `Expected ${file} to contain a JSON array of users, got ${typeof parsed}.`, + { code: ERROR_CODE.INVALID_JSON }, + ); + } + return parsed as Record[]; + } + } + if (transformer.preTransform) { const result = await transformer.preTransform(filePath, type ?? ""); filePath = result.filePath; diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index 69a570367..2985008f8 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -115,8 +115,8 @@ describe("migrate README", () => { // Guards the extractor: a regex that silently matched nothing would make // every check below pass vacuously. test("finds the documented examples", () => { - expect(EXAMPLES.length).toBeGreaterThan(20); - expect(FLAG_USES.length).toBeGreaterThan(20); + expect(EXAMPLES.length).toBeGreaterThan(10); + expect(FLAG_USES.length).toBeGreaterThan(10); }); test.each(EXAMPLES)("`%s` resolves to a real command", (example) => { diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 5a74e7457..a5b951340 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -9,7 +9,7 @@ import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; // Every test below names its own `--secret-key`, which short-circuits the // signed-in check — except the one that asserts what happens without it. mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); -import { latestUserLines, listRuns } from "./lib/run-store.ts"; +import { latestUserLines, listRuns, startRun } from "./lib/run-store.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; import { applyResumeAfter, explainErrors, run, validateRunOptions } from "./run.ts"; import type { User } from "./types.ts"; @@ -181,6 +181,114 @@ describe("run", () => { ]); }); + describe("export envelopes", () => { + /** An export run whose envelope holds `users`, as `clerk migrate export` writes it. */ + function exportRun(source: string, rows: unknown[], extra: Record = {}) { + const run = startRun(runsDir(), { kind: "export", target: { platform: source }, source }); + const file = path.join(run.dir, "export.json"); + fs.writeFileSync( + file, + JSON.stringify({ + clerkMigrate: 1, + source, + exportedAt: "2026-09-01T00:00:00.000Z", + runId: run.record.id, + users: rows, + ...extra, + }), + ); + run.update({ file: { path: file, sha256: "x" } }); + return { record: run.finish(), file }; + } + + const { transformer: _transformer, file: _file, ...noSource } = baseOptions; + + test("imports by export run ID, with the source the envelope names", async () => { + const { record } = exportRun("clerk", export2); + + await run({ ...noSource, input: record.id }); + + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + const imported = listRuns(runsDir()).find((candidate) => candidate.kind === "import"); + expect(imported).toMatchObject({ source: "clerk", fromExport: record.id }); + }); + + test("imports an envelope file with no transformer named", async () => { + const { file } = exportRun("clerk", export2); + + await run({ ...noSource, file }); + + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + }); + + test("refuses a transformer that contradicts the envelope", async () => { + const { record } = exportRun("clerk", export2); + + await expect(run({ ...noSource, transformer: "auth0", input: record.id })).rejects.toThrow( + /exported from clerk, but the transformer named is auth0/, + ); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(0); + }); + + test("refuses a run ID that is not an export", async () => { + await run(baseOptions); + const [imported] = listRuns(runsDir()); + + await expect(run({ ...noSource, input: imported!.id })).rejects.toThrow( + /is an import run, which has no file to import/, + ); + }); + + test("reads Firebase's hash parameters from the envelope", async () => { + const firebase = { + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }; + const { record } = exportRun( + "firebase", + [{ localId: "f1", email: "f@x.dev", passwordHash: "HASH", salt: "SALT" }], + { firebase }, + ); + + await run({ ...noSource, input: record.id }); + + const created = requests.find((r) => r.url.endsWith("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/v1/users")); + expect(created?.body).toMatchObject({ + password_hasher: "scrypt_firebase", + password_digest: "HASH$SALT$SIGNER$Bw==$8$14", + }); + }); + + test("lets the --firebase-* flags override the envelope", async () => { + const { record } = exportRun( + "firebase", + [{ localId: "f1", email: "f@x.dev", passwordHash: "HASH", salt: "SALT" }], + { + firebase: { + base64_signer_key: "OLD", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }, + }, + ); + + await run({ + ...noSource, + input: record.id, + firebaseSignerKey: "NEW", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }); + + const created = requests.find((r) => r.url.endsWith("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/v1/users")); + expect((created!.body as { password_digest: string }).password_digest).toContain("$NEW$"); + }); + }); + test("gitignores the project's .clerk folder before writing a run", async () => { await run(baseOptions); expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toContain(".clerk/"); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 2af871585..273cdde6a 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -57,7 +57,15 @@ import { type SettingChange, } from "./lib/modify-settings.ts"; import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; -import { resolveRunsDir, sha256File, startRun, type RunRecord } from "./lib/run-store.ts"; +import { readEnvelope, type ExportEnvelope } from "./lib/export-file.ts"; +import { + readRun, + resolveRunsDir, + RUN_ID_PATTERN, + sha256File, + startRun, + type RunRecord, +} from "./lib/run-store.ts"; import { countSocialProviders, findDisabledProviders, @@ -79,6 +87,8 @@ import { login } from "../auth/login.ts"; import { link } from "../link/index.ts"; export type MigrateRunOptions = { + /** An export file, or the ID of the export run that wrote one. */ + input?: string; transformer?: string; file?: string; resumeAfter?: string; @@ -655,13 +665,73 @@ async function ensureImportTarget(options: MigrateRunOptions): Promise { } } +/** + * Settles which file to read: the positional argument or `--file`, where the + * argument may name the export run that wrote the file. + */ +async function resolveInput( + options: MigrateRunOptions, +): Promise<{ file?: string; fromExport?: string }> { + if (options.input && options.file) { + throwUsageError("Name the file once: either as the argument or with --file, not both."); + } + const value = options.input ?? options.file; + if (!value) return {}; + if (!RUN_ID_PATTERN.test(value) || fileExists(value)) return { file: value }; + + const runsDir = await resolveRunsDir(options.runsDir); + const record = readRun(runsDir, value); + if (!record) { + throwUsageError(`No run \`${value}\` in ${runsDir}. Run \`clerk migrate runs\` to list them.`); + } + if (record.kind !== "export" || !record.file) { + throwUsageError( + `Run ${value} is an ${record.kind} run, which has no file to import. Name an export run, or a file.`, + ); + } + return { file: record.file.path, fromExport: record.id }; +} + +/** + * The source an export file names for itself, checked against the one the + * flags name. + * + * @throws UsageError when the two disagree: importing an Auth0 export through + * the Supabase mapping would create users with the wrong fields. + */ +function applyEnvelope( + options: MigrateRunOptions, + envelope: ExportEnvelope | undefined, +): MigrateRunOptions { + if (!envelope) return options; + if (options.transformer && options.transformer !== envelope.source) { + throwUsageError( + `The file was exported from ${envelope.source}, but the transformer named is ${options.transformer}. ` + + "Drop the transformer: the file already says where it came from.", + ); + } + return { ...options, transformer: envelope.source }; +} + export async function run(rawOptions: MigrateRunOptions): Promise { await ensureImportTarget(rawOptions); rawOptions = await applyCustomTransformer(rawOptions); + + const input = await resolveInput(rawOptions); + rawOptions = { ...rawOptions, file: input.file, input: undefined }; + const envelope = + input.file && fileExists(input.file) + ? readEnvelope(resolveImportFilePath(input.file)) + : undefined; + rawOptions = applyEnvelope(rawOptions, envelope); + const options = await resolveMissingOptions(rawOptions); const { transformer, file } = validateRunOptions(options); - const firebaseHashConfig = resolveFirebaseHashConfig(options, transformer); + // The flags win, so a rotated key can be passed without re-exporting. + const firebaseHashConfig = + resolveFirebaseHashConfig(options, transformer) ?? + (transformer === "firebase" ? envelope?.firebase : undefined); await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { const { secretKey, target } = await resolveClerkTarget(options); @@ -720,6 +790,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { target, source: transformer, file: { path: filePath, sha256: sha256File(filePath) }, + ...(input.fromExport ? { fromExport: input.fromExport } : {}), }); for (const failure of failures) { run.append({ From 5e24e5ae13ac732186e6db8c2d1c12f9a6c111d1 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 29 Sep 2026 18:00:01 -0400 Subject: [PATCH 057/141] feat(migrate)!: rename transformers to sources, and say what each one carries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A transformer describes where a file came from, so it is now called a source: - `transformers/` → `sources/` - `TransformerRegistryEntry` → `SourceEntry` - `clerk migrate transformers list` → `clerk migrate sources [source]` - `-t/--transformer` and `--transformer-file` → `--source ` A `--source` value that starts with `./`, `../` or `/`, or ends in `.ts`, `.js` or `.mjs`, is loaded as a custom source. Anything else must be a built-in key; an unknown key exits 2 and lists the valid ones. An import run records a custom source's content hash as `sourceHash`, so an edited source is a different source. `--source` completes the built-in keys through `KNOWN_OPTION_VALUES`, since `.choices()` would reject a path. Every source now declares `carries`: a `{ level, note }` for passwords, MFA and metadata. A custom source must declare it too. `sources` lists each source with those three levels. `sources ` shows one source in full: its export command, a note for each level, where each field lands, its defaults and its caveats. No source copies social sign-ins, so there is no column for them; every source shows one shared account-linking note with the docs link instead. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/completion/__complete.ts | 12 ++ .../cli-core/src/commands/migrate/README.md | 191 ++++++++--------- .../commands/migrate/export/registry.test.ts | 4 +- .../src/commands/migrate/export/registry.ts | 20 +- .../src/commands/migrate/index.test.ts | 48 ++--- .../cli-core/src/commands/migrate/index.ts | 74 +++---- .../commands/migrate/lib/transform.test.ts | 6 +- .../src/commands/migrate/lib/transform.ts | 19 +- .../src/commands/migrate/readme.test.ts | 11 +- .../commands/migrate/run-interactive.test.ts | 10 +- .../cli-core/src/commands/migrate/run.test.ts | 124 +++++------ packages/cli-core/src/commands/migrate/run.ts | 156 +++++--------- .../{transformers => sources}/auth0.ts | 19 +- .../{transformers => sources}/authjs.ts | 19 +- .../{transformers => sources}/betterauth.ts | 19 +- .../{transformers => sources}/clerk.ts | 19 +- .../{transformers => sources}/firebase.ts | 16 +- .../src/commands/migrate/sources/list.test.ts | 167 +++++++++++++++ .../src/commands/migrate/sources/list.ts | 202 ++++++++++++++++++ .../load-custom.test.ts | 112 ++++++---- .../{transformers => sources}/load-custom.ts | 58 +++-- .../src/commands/migrate/sources/registry.ts | 128 +++++++++++ .../{transformers => sources}/shared.ts | 0 .../sources.test.ts} | 40 +++- .../{transformers => sources}/supabase.ts | 19 +- .../{transformers => sources}/workos.ts | 16 +- .../migrate/transformers/list.test.ts | 171 --------------- .../src/commands/migrate/transformers/list.ts | 120 ----------- .../commands/migrate/transformers/registry.ts | 76 ------- .../cli-core/src/commands/migrate/types.ts | 25 ++- .../src/commands/migrate/wizard.test.ts | 22 +- .../cli-core/src/commands/migrate/wizard.ts | 26 +-- .../src/test/integration/completion.test.ts | 12 ++ 33 files changed, 1094 insertions(+), 867 deletions(-) rename packages/cli-core/src/commands/migrate/{transformers => sources}/auth0.ts (70%) rename packages/cli-core/src/commands/migrate/{transformers => sources}/authjs.ts (70%) rename packages/cli-core/src/commands/migrate/{transformers => sources}/betterauth.ts (74%) rename packages/cli-core/src/commands/migrate/{transformers => sources}/clerk.ts (74%) rename packages/cli-core/src/commands/migrate/{transformers => sources}/firebase.ts (89%) create mode 100644 packages/cli-core/src/commands/migrate/sources/list.test.ts create mode 100644 packages/cli-core/src/commands/migrate/sources/list.ts rename packages/cli-core/src/commands/migrate/{transformers => sources}/load-custom.test.ts (59%) rename packages/cli-core/src/commands/migrate/{transformers => sources}/load-custom.ts (74%) create mode 100644 packages/cli-core/src/commands/migrate/sources/registry.ts rename packages/cli-core/src/commands/migrate/{transformers => sources}/shared.ts (100%) rename packages/cli-core/src/commands/migrate/{transformers/transformers.test.ts => sources/sources.test.ts} (92%) rename packages/cli-core/src/commands/migrate/{transformers => sources}/supabase.ts (83%) rename packages/cli-core/src/commands/migrate/{transformers => sources}/workos.ts (74%) delete mode 100644 packages/cli-core/src/commands/migrate/transformers/list.test.ts delete mode 100644 packages/cli-core/src/commands/migrate/transformers/list.ts delete mode 100644 packages/cli-core/src/commands/migrate/transformers/registry.ts diff --git a/packages/cli-core/src/commands/completion/__complete.ts b/packages/cli-core/src/commands/completion/__complete.ts index 7921e307e..642281f9e 100644 --- a/packages/cli-core/src/commands/completion/__complete.ts +++ b/packages/cli-core/src/commands/completion/__complete.ts @@ -1,4 +1,5 @@ import type { CommandUnknownOpts, Option } from "@commander-js/extra-typings"; +import { sources } from "../migrate/sources/registry.ts"; import { KNOWN_DASHBOARD_PATHS } from "../open/dashboard-paths.ts"; const DIRECTIVE = { @@ -33,6 +34,15 @@ const HTTP_METHOD_COMPLETIONS: Completion[] = [ { name: "DELETE", description: "Delete resource" }, ]; +/** + * The built-in migrate sources. `--source` also takes a path to a source you + * wrote, so it cannot use `.choices()`. + */ +const SOURCE_COMPLETIONS: Completion[] = sources.map((entry) => ({ + name: entry.key, + description: entry.label, +})); + /** * Hardcoded option-value completions for options that don't use Commander's `.choices()`. * Keys are the long or short flag (e.g., "--mode", "-X"). @@ -52,6 +62,7 @@ const KNOWN_OPTION_VALUES: Record = { { name: "latest", description: "Latest stable release" }, { name: "canary", description: "Latest canary (pre-release) build" }, ], + "--source": SOURCE_COMPLETIONS, "--for": [ { name: "orgs", description: "Organizations only" }, { name: "users", description: "Users only" }, @@ -66,6 +77,7 @@ const KNOWN_OPTION_VALUES: Record = { * to root, e.g. "open dashboard". */ const KNOWN_POSITIONAL_COMPLETIONS: Record = { + "migrate sources": SOURCE_COMPLETIONS, "open dashboard": KNOWN_DASHBOARD_PATHS.map((path) => ({ name: path, description: "Dashboard subpath", diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 7cd4d5eb1..bb694161a 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -38,7 +38,7 @@ demanding flags. clerk migrate import ``` -It picks the transformer from a list built off the registry, asks for the file, +It picks the source from a list built off the registry, asks for the file, and collects Firebase's hash parameters when they are needed. Anything already passed as a flag is not asked for. @@ -51,7 +51,7 @@ usage error naming exactly what to pass: ``` `clerk migrate import` is interactive and cannot prompt in agent mode. -Pass --transformer and --file . +Pass the file (or an export run ID) and --source . ``` ### `clerk migrate import` @@ -61,24 +61,23 @@ record, and creates the users through the Backend API. ```sh clerk migrate import 20260929-141502-a1b2 -y # an export run -clerk migrate import -y --transformer clerk --file users.json +clerk migrate import users.json --source clerk -y ``` -| Flag | Description | -| --------------------------------------- | --------------------------------------------------------------- | -| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | -| `-t, --transformer ` | Source platform the file came from (see below) | -| `--transformer-file ` | A transformer you wrote, for a platform with no built-in | -| `-f, --file ` | Path to the export. `.json` or `.csv` | -| `-r, --resume-after ` | Skip every user up to and including this **source** ID | -| `--require-password` | Import only users that carry a password digest | -| `--skip-unsupported-providers` | Supabase: skip users whose only social provider is off in Clerk | -| `--firebase-signer-key ` | Firebase base64 signer key | -| `--firebase-salt-separator ` | Firebase base64 salt separator | -| `--firebase-rounds ` | Firebase scrypt rounds | -| `--firebase-mem-cost ` | Firebase scrypt memory cost | -| `-y, --yes` | Skip the confirmation prompt | -| `--runs-dir ` | Where runs are kept (see [Runs](#clerk-migrate-runs)) | +| Flag | Description | +| --------------------------------------- | ---------------------------------------------------------------- | +| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | +| `--source ` | Where the file came from: a [source](#sources), or one you wrote | +| `-f, --file ` | Path to the export. `.json` or `.csv` | +| `-r, --resume-after ` | Skip every user up to and including this **source** ID | +| `--require-password` | Import only users that carry a password digest | +| `--skip-unsupported-providers` | Supabase: skip users whose only social provider is off in Clerk | +| `--firebase-signer-key ` | Firebase base64 signer key | +| `--firebase-salt-separator ` | Firebase base64 salt separator | +| `--firebase-rounds ` | Firebase scrypt rounds | +| `--firebase-mem-cost ` | Firebase scrypt memory cost | +| `-y, --yes` | Skip the confirmation prompt | +| `--runs-dir ` | Where runs are kept (see [Runs](#clerk-migrate-runs)) | Plus the targeting flags from the table above: `--secret-key`, `--app` and `--instance`. @@ -87,9 +86,9 @@ The file is the positional argument or `--file`, not both. An export run ID stands for the file that run wrote, and the import records it as `fromExport`. A file `clerk migrate export` wrote carries its source, so it needs no -`--transformer`. A `--transformer` that contradicts it exits 2. Any other file +`--source`. A `--source` that contradicts it exits 2. Any other file — a bare JSON array, a CSV, Firebase's own `{ "users": [...] }` — needs -`--transformer`, and omitting it fails with a usage error that names the valid +`--source`, and omitting it fails with a usage error that names the valid values. Failures do not stop the run: each user's outcome is written to the @@ -172,15 +171,15 @@ rest of the export continues against whichever credential worked. Agent mode and a non-TTY fail outright instead, having nobody to ask, and `-y` fails too, having been told not to. -| Platform | Source | Feeds | -| ------------ | -------------------------------- | -------------------------- | -| `clerk` | Clerk Backend API | `--transformer clerk` | -| `auth0` | Auth0 Management API | `--transformer auth0` | -| `supabase` | Supabase Postgres (`auth.users`) | `--transformer supabase` | -| `authjs` | Auth.js database | `--transformer authjs` | -| `betterauth` | Better Auth database | `--transformer betterauth` | -| `firebase` | Firebase Identity Toolkit | `--transformer firebase` | -| `workos` | WorkOS User Management API | `--transformer workos` | +| Platform | Source | Feeds | +| ------------ | -------------------------------- | --------------------- | +| `clerk` | Clerk Backend API | `--source clerk` | +| `auth0` | Auth0 Management API | `--source auth0` | +| `supabase` | Supabase Postgres (`auth.users`) | `--source supabase` | +| `authjs` | Auth.js database | `--source authjs` | +| `betterauth` | Better Auth database | `--source betterauth` | +| `firebase` | Firebase Identity Toolkit | `--source firebase` | +| `workos` | WorkOS User Management API | `--source workos` | Every export is a [run](#clerk-migrate-runs), and the file lands in the run folder as `export.json`. `--output` writes it somewhere else instead, @@ -200,7 +199,7 @@ The file is an envelope around the users: ``` `source` is what lets `clerk migrate import ` run with no -`--transformer`. A Firebase export adds `firebase`, the project's hash +`--source`. A Firebase export adds `firebase`, the project's hash parameters, so the import needs no `--firebase-*` flags. `--json` prints the result on stdout instead — `{ target, run, output, users, @@ -566,74 +565,65 @@ otherwise. A run whose process died, or that never recorded a finish time, lists as `interrupted`. A lock held by a live process refuses a second writer with exit 2. -## Transformers +## Sources -A transformer maps one platform's export onto Clerk's user schema. Adding a -platform is one file in `transformers/` plus one line in `transformers/registry.ts` — -`--transformer`'s accepted values and its tab-completion both read from that array. +A source maps one platform's export onto Clerk's user schema, and says what it +brings across. Adding a platform is one file in `sources/` plus one line in +`sources/registry.ts`; `--source`'s tab-completion reads from that array. -| Key | Source | Passwords | Notes | -| ------------ | ----------------------------- | ----------------- | ---------------------------------------------------------------- | -| `clerk` | Clerk Dashboard export | as exported | Instance to instance, e.g. development → production | -| `auth0` | Auth0 Export Users API | `bcrypt` | Hashes need a support request to Auth0; not in a standard export | -| `authjs` | Auth.js / NextAuth user table | none | Assumes `SELECT id, name, email, email_verified, created_at` | -| `betterauth` | Better Auth export | `bcrypt` | Reads the credential account's `password_hash` | -| `firebase` | `firebase auth:export` | `scrypt_firebase` | CSV or JSON; needs the four hash parameters below | -| `supabase` | Supabase `auth.users` export | `bcrypt` | Supports `--skip-unsupported-providers` | -| `workos` | WorkOS User Management API | none | No hasher default: WorkOS returns no digest to name one for | +| Key | Reads | Passwords | MFA | Metadata | +| ------------ | ----------------------------- | --------- | ------- | -------- | +| `clerk` | Clerk Dashboard export | partial | partial | yes | +| `auth0` | Auth0 Export Users API | partial | no | yes | +| `authjs` | Auth.js / NextAuth user table | no | no | no | +| `betterauth` | Better Auth export | yes | no | no | +| `firebase` | `firebase auth:export` | yes | no | no | +| `supabase` | Supabase `auth.users` export | yes | no | partial | +| `workos` | WorkOS User Management API | no | no | yes | -### `clerk migrate transformers list` +There is no column for social sign-ins, because no source copies them and none +needs to. Every source shows the same note instead: enable the same providers in +Clerk, and a user who signs in with one is linked to their imported account by +verified email. See +[account linking](https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking). -Which mappings are available. New in the CLI: the standalone tool's interactive -picker was the only place these appeared, which was fine when the user had the -source tree to grep. A compiled binary's users have neither. +### `clerk migrate sources` ```sh -clerk migrate transformers list -clerk migrate transformers list --json -clerk migrate transformers list --transformer-file ./my-transformer.ts +clerk migrate sources # every source, with what it carries +clerk migrate sources betterauth # one source in full +clerk migrate sources ./my-source.ts # a source you wrote +clerk migrate sources --json ``` -| Flag | Description | -| --------------------------- | --------------------------------- | -| `--json` | Output as JSON | -| `--transformer-file ` | Also list a transformer you wrote | +| Flag | Description | +| ---------- | ---------------------------------------------------------------- | +| `[source]` | A built-in key, or the path to a source you wrote, to show fully | +| `--json` | The same data, on stdout | -Each entry prints its key, the platform label, and what the transformer assumes -about the export — wrapped to the terminal, capped at 80 columns so two runs of -the same command lay out the same way. A backticked span is never broken across -lines. There is no intro/outro gutter: this reads a static registry rather than -running anything. +`sources` alone prints the table above. `sources ` shows one source in +full: its export command, what it carries with a note for each, where each +field lands (`encrypted_password → password`), its fixed defaults, and any +caveats. An unknown key exits 2 and lists the valid ones. There is no +intro/outro gutter: this reads a static registry rather than running anything. -``` -A transformer maps one platform's export onto the fields Clerk imports. Pass the -one your export came from as `--transformer `. - -Transformers: - clerk Clerk - Migrate between Clerk instances (e.g. development to production, or to - another Clerk application). Export your users from the Clerk Dashboard - first. +### `--source` - … +`clerk migrate import` takes `--source `: - workos WorkOS - Works with WorkOS's User Management API. WorkOS returns no password hashes, - so imported users sign in by reset or SSO. +- A value starting with `./`, `../` or `/`, or ending in `.ts`, `.js` or + `.mjs`, is loaded as a [custom source](#custom-sources). +- Anything else must be a built-in key. An unknown key exits 2 with the list of + valid keys. -7 built-in transformers -Migrating from something else? Write a transformer and pass --transformer-file. -``` +A file `clerk migrate export` wrote names its own source, so it needs none. -`--json` gives an agent the same data, including which source field each -transformer maps to `userId`. - -### Custom transformers (`--transformer-file`) +### Custom sources Migrating from a platform with no built-in, without recompiling the CLI: ```sh -clerk migrate import --transformer-file ./my-platform.ts --file users.json +clerk migrate import users.json --source ./my-platform.ts ``` The file lives in **your** project, not in the CLI, and is imported at runtime. @@ -652,6 +642,11 @@ export default { family: "lastName", pw_bcrypt: "password", }, + carries: { + passwords: { level: "yes", note: "bcrypt hashes from the pw_bcrypt column." }, + mfa: { level: "no", note: "Not exported." }, + metadata: { level: "no", note: "Not exported." }, + }, defaults: { passwordHasher: "bcrypt" }, postTransform: (user) => { if (!user.firstName) delete user.firstName; @@ -663,23 +658,23 @@ TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, `satisfies` and `as const` all work in a file the compiled binary imports. Plain `.js` works too. -`--transformer-file` and `--transformer` together is an error: both name a -transformer and there is no sensible precedence between the one you wrote and -the one we ship. +An import run records a custom source's key and a hash of the file, so an +edited source counts as a different source. #### Validation The file is code the CLI executes, so its shape is checked before use and rejected with the specific problem rather than crashing mid-pipeline: -| Problem | Message | -| ---------------------------------- | ---------------------------------------------------------------------------------------------------- | -| Path does not exist | `No transformer file at /abs/path.ts.` | -| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | -| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | -| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | -| `key` clashes with a built-in | `key is "clerk", which is already a built-in transformer` | -| A hook is not a function | `postTransform must be a function when present` | +| Problem | Message | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Path does not exist | `No source file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | +| A hook is not a function | `postTransform must be a function when present` | The `userId` check is the load-bearing one: without it the import would run to completion and create every user with no `external_id`, which is what makes a @@ -687,7 +682,7 @@ migration re-runnable. ### Verified vs unverified identifiers -Every platform records verification differently, and each transformer declares +Every platform records verification differently, and each source declares which style it uses. An identifier the source never confirmed is routed to `unverifiedEmailAddresses` / `unverifiedPhoneNumbers` rather than the primary field, because Clerk creates primary identifiers **verified** — sending an @@ -705,7 +700,7 @@ alongside each digest. Find them in the Firebase console under **Authentication → Users → (⋮) → Password hash parameters**. ```sh -clerk migrate import -y -t firebase -f users.json \ +clerk migrate import users.json --source firebase -y \ --firebase-signer-key --firebase-salt-separator \ --firebase-rounds 8 --firebase-mem-cost 14 ``` @@ -738,13 +733,13 @@ printed: a failed lookup must not be mistaken for "no providers are enabled". ## Schema fields -What a transformer maps _onto_. Every user is validated against this schema -before any request is made, so a field a transformer produces that is not listed +What a source maps _onto_. Every user is validated against this schema +before any request is made, so a field a source produces that is not listed here is silently dropped — Zod strips unknown keys — and never reaches Clerk. -Writing a custom transformer means targeting these names exactly. +Writing a custom source means targeting these names exactly. -The schema lives in `validator.ts`; adding a source platform means adding a -transformer, not editing it. +The schema lives in `validator.ts`; adding a platform means adding a source, +not editing it. **Required:** `userId` (`string`). It becomes the Clerk user's `external_id`, which is what makes a migration re-runnable. diff --git a/packages/cli-core/src/commands/migrate/export/registry.test.ts b/packages/cli-core/src/commands/migrate/export/registry.test.ts index f8471aa8b..fbdb8ec7b 100644 --- a/packages/cli-core/src/commands/migrate/export/registry.test.ts +++ b/packages/cli-core/src/commands/migrate/export/registry.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { transformerKeys } from "../transformers/registry.ts"; +import { sourceKeys } from "../sources/registry.ts"; import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; describe("export registry", () => { @@ -23,7 +23,7 @@ describe("export registry", () => { // The picker, the docs and the "what next" line all read this, so a typo // would send someone to a transformer that does not exist. test.each([...exportPlatforms])("$key names a real transformer", (entry) => { - expect(transformerKeys()).toContain(entry.transformerKey); + expect(sourceKeys()).toContain(entry.sourceKey); }); test.each([...exportPlatforms])("$key has something to run", (entry) => { diff --git a/packages/cli-core/src/commands/migrate/export/registry.ts b/packages/cli-core/src/commands/migrate/export/registry.ts index a5d5a6c15..385953b20 100644 --- a/packages/cli-core/src/commands/migrate/export/registry.ts +++ b/packages/cli-core/src/commands/migrate/export/registry.ts @@ -2,7 +2,7 @@ * Export registry. * * The picker behind a bare `clerk migrate export` is built from this array, so - * adding a platform is one file plus one entry — the same shape the transformer + * adding a platform is one file plus one entry — the same shape the source * registry uses. * * `run` takes no arguments on purpose: each platform resolves its own flags, @@ -22,8 +22,8 @@ export type ExportRegistryEntry = { key: string; label: string; description: string; - /** Which `--transformer` reads the file this export writes. */ - transformerKey: string; + /** Which source reads the file this export writes. */ + sourceKey: string; run: (options: Record) => Promise; }; @@ -32,49 +32,49 @@ export const exportPlatforms: ExportRegistryEntry[] = [ key: "clerk", label: "Clerk", description: "Another Clerk instance, e.g. development → production", - transformerKey: "clerk", + sourceKey: "clerk", run: async (options) => exportClerk(options), }, { key: "auth0", label: "Auth0", description: "An Auth0 tenant, via the Management API", - transformerKey: "auth0", + sourceKey: "auth0", run: async (options) => exportAuth0(options), }, { key: "supabase", label: "Supabase", description: "A Supabase Postgres database — includes password hashes", - transformerKey: "supabase", + sourceKey: "supabase", run: async (options) => exportSupabase(options), }, { key: "authjs", label: "Auth.js (NextAuth)", description: "An Auth.js database — Postgres, MySQL or SQLite", - transformerKey: "authjs", + sourceKey: "authjs", run: async (options) => exportAuthJs(options), }, { key: "firebase", label: "Firebase", description: "A Firebase project, via Identity Toolkit", - transformerKey: "firebase", + sourceKey: "firebase", run: async (options) => exportFirebase(options), }, { key: "betterauth", label: "Better Auth", description: "A Better Auth database — plugin columns detected automatically", - transformerKey: "betterauth", + sourceKey: "betterauth", run: async (options) => exportBetterAuth(options), }, { key: "workos", label: "WorkOS", description: "A WorkOS tenant, via the User Management API — no password hashes", - transformerKey: "workos", + sourceKey: "workos", run: async (options) => exportWorkOs(options), }, ]; diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index a8bd3b24d..cd617e397 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -6,7 +6,6 @@ import { getMode, setMode } from "../../mode.ts"; import { createProgram } from "../../cli-program.ts"; import { exportPlatformKeys } from "./export/registry.ts"; import { isAssumeYes, setAssumeYes } from "./lib/assume-yes.ts"; -import { transformerKeys } from "./transformers/registry.ts"; function findCommand(names: string[]) { let current = createProgram().commands.find((cmd) => cmd.name() === names[0]); @@ -35,7 +34,7 @@ describe("registerMigrate", () => { }); test.each([ - "--transformer", + "--source", "--file", "--resume-after", "--require-password", @@ -53,8 +52,18 @@ describe("registerMigrate", () => { expect(flags).toContain(flag); }); - test.each([[["transformers"]], [["transformers", "list"]]])("registers migrate %p", (names) => { - expect(findCommand(["migrate", ...names])).toBeDefined(); + test("registers sources with an optional source and --json", () => { + const sources = findCommand(["migrate", "sources"]); + expect(sources?.registeredArguments[0]?.required).toBe(false); + expect(sources?.options.map((option) => option.long)).toEqual(["--json"]); + }); + + test.each([[["transformers"]], [["transformers", "list"]]])("no longer registers %p", (names) => { + expect(findCommand(["migrate", ...names])).toBeUndefined(); + }); + + test.each(["--transformer", "--transformer-file"])("migrate import drops %s", (flag) => { + expect(findCommand(["migrate", "import"])?.options.map((o) => o.long)).not.toContain(flag); }); test.each([ @@ -128,25 +137,6 @@ describe("registerMigrate", () => { expect(flags).toContain("--json"); }); - test("makes list the default transformers subcommand", () => { - const group = findCommand(["migrate", "transformers"]) as unknown as { - _defaultCommandName?: string; - }; - expect(group._defaultCommandName).toBe("list"); - }); - - test.each(["--json", "--transformer-file"])("transformers list accepts %s", (flag) => { - expect(findCommand(["migrate", "transformers", "list"])?.options.map((o) => o.long)).toContain( - flag, - ); - }); - - test("migrate import accepts --transformer-file", () => { - expect(findCommand(["migrate", "import"])?.options.map((o) => o.long)).toContain( - "--transformer-file", - ); - }); - test("registers runs with an optional run ID", () => { const runs = findCommand(["migrate", "runs"]); expect(runs?.registeredArguments[0]?.required).toBe(false); @@ -185,16 +175,14 @@ describe("registerMigrate", () => { ); }); - test("constrains --transformer to the registered transformers, for validation and completion", () => { - const option = findCommand(["migrate", "import"])?.options.find( - (o) => o.long === "--transformer", - ); - // Tracks the registry so adding a platform needs no edit here. - expect(option?.argChoices).toEqual(transformerKeys()); + // It also takes a path, so it cannot use `.choices()`: completion offers the + // built-in keys through `KNOWN_OPTION_VALUES` instead. + test("--source accepts any value, so a path to a source you wrote gets through", () => { + const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === "--source"); + expect(option?.argChoices).toBeUndefined(); }); test.each([ - ["-t", "--transformer"], ["-f", "--file"], ["-r", "--resume-after"], ["-y", "--yes"], diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index fd70d35c0..43ec89376 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -1,4 +1,3 @@ -import { createOption } from "@commander-js/extra-typings"; import type { Program } from "../../cli-program.ts"; import { parseIntegerOption } from "../../lib/option-parsers.ts"; import { setMode } from "../../mode.ts"; @@ -8,10 +7,9 @@ import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; import { run } from "./run.ts"; import { runs } from "./runs.ts"; import { undo } from "./undo.ts"; -import { list as transformersList } from "./transformers/list.ts"; -import { transformerKeys } from "./transformers/registry.ts"; +import { list as sources } from "./sources/list.ts"; -const migrate = { run, runs, undo, transformersList }; +const migrate = { run, runs, undo, sources }; export function registerMigrate(program: Program): void { const migrateCommand = program @@ -20,11 +18,12 @@ export function registerMigrate(program: Program): void { .setExamples([ { command: "clerk migrate import", description: "Walk through an import interactively" }, { - command: "clerk migrate import -y --transformer clerk --file users.json", + command: "clerk migrate import users.json --source clerk -y", description: "Import users from a Clerk export", }, { - command: "clerk migrate import -y -t supabase -f users.json --skip-unsupported-providers", + command: + "clerk migrate import users.json --source supabase --skip-unsupported-providers -y", description: "Skip Supabase users whose provider is not enabled", }, { @@ -36,7 +35,7 @@ export function registerMigrate(program: Program): void { command: "clerk migrate undo 20260929-141502-a1b2", description: "Delete the users an import created", }, - { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { command: "clerk migrate sources", description: "What each source brings across" }, ]); // `-y` is read several layers down — by the credential-retry loop and the @@ -65,15 +64,9 @@ export function registerMigrate(program: Program): void { .command("import") .description("Import users from an exported JSON or CSV file") .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") - .addOption( - createOption( - "-t, --transformer ", - "Source platform the file was exported from", - ).choices(transformerKeys()), - ) .option( - "--transformer-file ", - "Path to a transformer you wrote, for a platform with no built-in", + "--source ", + "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`", ) .option("-f, --file ", "Path to the exported user data (JSON or CSV)") .option("-r, --resume-after ", "Skip every user up to and including this source ID") @@ -97,19 +90,24 @@ export function registerMigrate(program: Program): void { .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) .setExamples([ { - command: "clerk migrate import -y --transformer clerk --file users.json", + command: "clerk migrate import 20260929-141502-a1b2 -y", + description: "Import what an export run wrote", + }, + { + command: "clerk migrate import users.json --source clerk -y", description: "Import a Clerk Dashboard export", }, { - command: "clerk migrate import -y -t clerk -f users.csv --require-password", + command: "clerk migrate import users.csv --source clerk --require-password -y", description: "Import only the users that carry a password digest", }, { - command: "clerk migrate import -y -t clerk -f users.json -r user_2x9k", - description: "Resume a partial migration after the last imported user", + command: "clerk migrate import users.json --source ./my-source.ts -y", + description: "Import with a source you wrote", }, { - command: "clerk migrate import -y -t supabase -f users.json --skip-unsupported-providers", + command: + "clerk migrate import users.json --source supabase --skip-unsupported-providers -y", description: "Skip Supabase users whose only provider is not enabled in Clerk", }, ]) @@ -169,36 +167,20 @@ export function registerMigrate(program: Program): void { // A compiled binary has no source tree to grep, so the available mappings // need a command rather than only appearing in the interactive picker. - const transformersCommand = migrateCommand - .command("transformers") - .description("Inspect the available source-platform transformers") - .setExamples([ - { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, - { - command: "clerk migrate transformers list --json", - description: "Machine-readable, including each one's ID field", - }, - { - command: "clerk migrate transformers list --transformer-file ./my-transformer.ts", - description: "Include one you wrote", - }, - ]); - - transformersCommand - .command("list", { isDefault: true }) - .description("List the built-in transformers, and any loaded from a file") + migrateCommand + .command("sources") + .description("List the sources an import can read, or show one in full") + .argument("[source]", "A built-in source, or the path to a source you wrote") .option("--json", "Output as JSON") - .option("--transformer-file ", "Also list a transformer you wrote") .setExamples([ - { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { command: "clerk migrate sources", description: "What each source brings across" }, { - command: "clerk migrate transformers list --transformer-file ./my-transformer.ts", - description: "Include one you wrote", + command: "clerk migrate sources betterauth", + description: "Where each field lands, how to export, and caveats", }, + { command: "clerk migrate sources ./my-source.ts", description: "Check a source you wrote" }, ]) - .action(async (_opts, cmd) => - migrate.transformersList( - cmd.optsWithGlobals() as Parameters[0], - ), + .action(async (source, _opts, cmd) => + migrate.sources(source, cmd.optsWithGlobals() as Parameters[1]), ); } diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 3981d9f71..2f117da56 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -3,7 +3,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; -import clerkTransformer from "../transformers/clerk.ts"; +import clerkSource from "../sources/clerk.ts"; import { consolidateClerkIdentifiers, flattenObjectSelectively, @@ -65,7 +65,7 @@ describe("transformKeys", () => { expect( transformKeys( { id: "u1", primary_email_address: "a@example.com", extra: "kept" }, - clerkTransformer, + clerkSource, ), ).toEqual({ userId: "u1", email: "a@example.com", extra: "kept" }); }); @@ -75,7 +75,7 @@ describe("transformKeys", () => { ["stringified empty object", '"{}"'], ["null", null], ])("drops fields whose value is %s", (_label, value) => { - expect(transformKeys({ id: "u1", first_name: value }, clerkTransformer)).toEqual({ + expect(transformKeys({ id: "u1", first_name: value }, clerkSource)).toEqual({ userId: "u1", }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index d82731f38..b4e872dba 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -11,13 +11,8 @@ import fs from "node:fs"; import path from "node:path"; import csvParser from "csv-parser"; import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; -import { getTransformer } from "../transformers/registry.ts"; -import { - PASSWORD_HASHERS, - type TransformContext, - type TransformerRegistryEntry, - type User, -} from "../types.ts"; +import { getSource } from "../sources/registry.ts"; +import { PASSWORD_HASHERS, type TransformContext, type SourceEntry, type User } from "../types.ts"; import { userSchema } from "../validator.ts"; import { isEnvelope } from "./export-file.ts"; @@ -360,7 +355,7 @@ export function validatePreparedUsers(users: Record[]): { function addDefaultFields( users: Record[], - transformer: TransformerRegistryEntry, + transformer: SourceEntry, ): Record[] { if (!transformer.defaults) return users; return users.map((user) => ({ ...user, ...transformer.defaults })); @@ -379,7 +374,7 @@ export function transformUsers( key: string, options: TransformOptions = {}, ): { transformedData: User[]; validationFailed: number; failures: ValidationFailure[] } { - const transformer = getTransformer(key); + const transformer = getSource(key); const context = options.context ?? {}; const transformed: Record[] = []; @@ -421,7 +416,7 @@ async function readCsv(filePath: string): Promise[]> { async function readUsersFromFile( file: string, - transformer: TransformerRegistryEntry, + transformer: SourceEntry, ): Promise[]> { let filePath = resolveImportFilePath(file); const type = getFileType(file); @@ -470,7 +465,7 @@ async function readUsersFromFile( * users are transformed. */ export async function readRawUsers(file: string, key: string): Promise[]> { - return readUsersFromFile(file, getTransformer(key)); + return readUsersFromFile(file, getSource(key)); } /** @@ -483,7 +478,7 @@ export async function loadUsersFromFile( key: string, options: TransformOptions = {}, ): Promise<{ users: User[]; validationFailed: number; failures: ValidationFailure[] }> { - const transformer = getTransformer(key); + const transformer = getSource(key); const raw = await readUsersFromFile(file, transformer); const withDefaults = addDefaultFields(raw, transformer); const { transformedData, validationFailed, failures } = transformUsers( diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index 2985008f8..76181117b 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -1,8 +1,8 @@ /** * Keeps README.md and the command tree honest about each other. * - * This README documents seven export platforms, seven transformers and the - * run store across ~1000 lines. Checking it by eye at review time does not + * This README documents seven export platforms, seven sources and the run + * store across ~1000 lines. Checking it by eye at review time does not * scale, and a doc that names a flag the binary rejects is worse than no doc: * the reader trusts it and gets a usage error. * @@ -70,10 +70,9 @@ function resolve(tokens: string[]): { command: Command; rest: string[] } { /** * The flags a command accepts, including those of a default subcommand. * - * `migrate transformers` registers its `list` `isDefault`, so Commander hands - * it everything after the group name. The documented spelling is - * `clerk migrate transformers --json`, and this has to see the same flags - * Commander does or every such example reads as unsupported. + * A group that registers a subcommand `isDefault` hands it everything after + * the group name, so this has to see the same flags Commander does or every + * such example reads as unsupported. */ function flagsOf(command: Command): string[] { const own = command.options.flatMap( diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 10c4a3669..b6e69e48c 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -81,7 +81,7 @@ const EXPORT = [ { id: "u2", primary_email_address: "b@x.dev" }, ]; -const baseOptions = { transformer: "clerk", file: "export.json", secretKey: "sk_test_x" }; +const baseOptions = { source: "clerk", file: "export.json", secretKey: "sk_test_x" }; let originalPlatformKey: string | undefined; @@ -168,7 +168,7 @@ function stubInstanceSettings(settings: StubSettings) { const created = () => requests.filter((r) => r.url.endsWith("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/v1/users")); describe("the wizard fills in missing flags", () => { - test("bare `clerk migrate import` prompts for the transformer and file, then imports", async () => { + test("bare `clerk migrate import` prompts for the source and file, then imports", async () => { await run({ secretKey: "sk_test_x" }); expect(mockSelect).toHaveBeenCalledTimes(1); @@ -177,14 +177,14 @@ describe("the wizard fills in missing flags", () => { }); test("asks only for what the flags did not supply", async () => { - await run({ ...baseOptions, transformer: "clerk" }); + await run({ ...baseOptions, source: "clerk" }); expect(mockSelect).not.toHaveBeenCalled(); expect(mockText).not.toHaveBeenCalled(); }); - test("prompts for the file when only the transformer was passed", async () => { - await run({ transformer: "clerk", secretKey: "sk_test_x" }); + test("prompts for the file when only the source was passed", async () => { + await run({ source: "clerk", secretKey: "sk_test_x" }); expect(mockSelect).not.toHaveBeenCalled(); expect(mockText).toHaveBeenCalledTimes(1); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index a5b951340..b29b332c6 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -10,7 +10,7 @@ import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; // signed-in check — except the one that asserts what happens without it. mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); import { latestUserLines, listRuns, startRun } from "./lib/run-store.ts"; -import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; +import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; import { applyResumeAfter, explainErrors, run, validateRunOptions } from "./run.ts"; import type { User } from "./types.ts"; @@ -41,28 +41,23 @@ afterAll(() => { }); describe("validateRunOptions", () => { - test("accepts a transformer and an existing JSON file", () => { - expect(validateRunOptions({ transformer: "clerk", file: "users.json" })).toEqual({ - transformer: "clerk", + test("accepts a source and an existing JSON file", () => { + expect(validateRunOptions({ source: "clerk", file: "users.json" })).toEqual({ + source: "clerk", file: "users.json", }); }); test.each([ - ["no transformer", { file: "users.json" }, /--transformer/], - ["an unknown transformer", { transformer: "okta", file: "users.json" }, /Unknown transformer/], - ["no file", { transformer: "clerk" }, /--file/], - ["a missing file", { transformer: "clerk", file: "nope.json" }, /File not found/], - [ - "an unsupported extension", - { transformer: "clerk", file: "users.txt" }, - /Unsupported file type/, - ], + ["no source", { file: "users.json" }, /Missing --source/], + ["no file", { source: "clerk" }, /Missing the file to import/], + ["a missing file", { source: "clerk", file: "nope.json" }, /File not found/], + ["an unsupported extension", { source: "clerk", file: "users.txt" }, /Unsupported file type/], ])("rejects %s", (_label, options, message) => { expect(() => validateRunOptions(options)).toThrow(message); }); - test("names the valid transformers when one is missing", () => { + test("names the valid sources when one is missing", () => { expect(() => validateRunOptions({ file: "users.json" })).toThrow(/clerk/); }); }); @@ -126,7 +121,7 @@ describe("run", () => { }); const baseOptions = { - transformer: "clerk", + source: "clerk", file: "export.json", yes: true, secretKey: "sk_test_x", @@ -136,7 +131,7 @@ describe("run", () => { const previous = process.env.CLERK_SECRET_KEY; delete process.env.CLERK_SECRET_KEY; try { - await expect(run({ transformer: "clerk", file: "export.json", yes: true })).rejects.toThrow( + await expect(run({ source: "clerk", file: "export.json", yes: true })).rejects.toThrow( /Not logged in/, ); expect(requests).toHaveLength(0); @@ -201,7 +196,7 @@ describe("run", () => { return { record: run.finish(), file }; } - const { transformer: _transformer, file: _file, ...noSource } = baseOptions; + const { source: _source, file: _file, ...noSource } = baseOptions; test("imports by export run ID, with the source the envelope names", async () => { const { record } = exportRun("clerk", export2); @@ -213,7 +208,7 @@ describe("run", () => { expect(imported).toMatchObject({ source: "clerk", fromExport: record.id }); }); - test("imports an envelope file with no transformer named", async () => { + test("imports an envelope file with no source named", async () => { const { file } = exportRun("clerk", export2); await run({ ...noSource, file }); @@ -221,11 +216,11 @@ describe("run", () => { expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); }); - test("refuses a transformer that contradicts the envelope", async () => { + test("refuses a source that contradicts the envelope", async () => { const { record } = exportRun("clerk", export2); - await expect(run({ ...noSource, transformer: "auth0", input: record.id })).rejects.toThrow( - /exported from clerk, but the transformer named is auth0/, + await expect(run({ ...noSource, source: "auth0", input: record.id })).rejects.toThrow( + /exported from clerk, but --source names auth0/, ); expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(0); }); @@ -391,11 +386,11 @@ describe("run", () => { // Tests run non-TTY, so `isHuman()` is false and the wizard path is never // reached — the same guard an agent hits. - describe("without --transformer or --file", () => { + describe("without --source or a file", () => { test.each([ - [{}, /--transformer and --file /], - [{ transformer: "clerk" }, /--file /], - [{ file: "export.json" }, /--transformer /], + [{}, /the file \(or an export run ID\) and --source /], + [{ source: "clerk" }, /Pass the file \(or an export run ID\)\./], + [{ file: "export.json" }, /Pass --source \./], ])("names the missing flags rather than prompting (%p)", async (partial, expected) => { await expect(run({ ...partial, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( expected, @@ -502,7 +497,7 @@ describe("run", () => { ]), ); - await run({ ...baseOptions, transformer: "supabase", yes: false }); + await run({ ...baseOptions, source: "supabase", yes: false }); expect(captured.err).toContain("Social connections"); expect(captured.err).toContain("Discord"); @@ -529,7 +524,7 @@ describe("run", () => { ]), ); - await run({ ...baseOptions, transformer: "supabase", yes: false }); + await run({ ...baseOptions, source: "supabase", yes: false }); const social = captured.err.slice(captured.err.indexOf("Social connections")); expect(social).toContain("Discord"); @@ -538,12 +533,17 @@ describe("run", () => { }); }); - describe("--transformer-file", () => { + describe("--source ", () => { const CUSTOM = `export default { key: "myplatform", label: "My Platform", description: "Exports from My Platform.", transformer: { account_ref: "userId", contact_email: "email", given: "firstName", pw: "password" }, + carries: { + passwords: { level: "yes", note: "bcrypt." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, defaults: { passwordHasher: "bcrypt" }, postTransform: (user) => { if (!user.firstName) delete user.firstName; }, };`; @@ -566,15 +566,15 @@ describe("run", () => { }); afterEach(() => { - __resetCustomTransformersForTesting(); + __resetCustomSourcesForTesting(); }); const created = () => requests.filter((r) => r.url.endsWith("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/v1/users")); - test("imports through a user-authored transformer", async () => { + test("imports through a user-authored source", async () => { await run({ file: "export.json", - transformerFile: customFile, + source: customFile, yes: true, secretKey: "sk_test_x", }); @@ -584,13 +584,13 @@ describe("run", () => { "mp_2", ]); expect(captured.err).toContain("myplatform"); - expect(captured.err).toContain("transformer from"); + expect(captured.err).toContain("source from"); }); - test("applies the custom transformer's defaults and postTransform", async () => { + test("applies the custom source's defaults and postTransform", async () => { await run({ file: "export.json", - transformerFile: customFile, + source: customFile, yes: true, secretKey: "sk_test_x", }); @@ -601,17 +601,19 @@ describe("run", () => { expect("first_name" in (bodies[1] ?? {})).toBe(false); }); - // No sensible precedence between "the one you wrote" and "the one we ship". - test("conflicts with --transformer rather than picking one", async () => { + // An edited source is a different source, so the run records which one. + test("records the custom source's content hash on the run", async () => { + await run({ file: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + + const [record] = listRuns(runsDir()); + expect(record?.source).toBe("myplatform"); + expect(record?.sourceHash).toMatch(/^[0-9a-f]{64}$/); + }); + + test("an unknown built-in key is a usage error listing the valid ones", async () => { await expect( - run({ - transformer: "clerk", - file: "export.json", - transformerFile: customFile, - yes: true, - secretKey: "sk_test_x", - }), - ).rejects.toThrow(/both name a transformer. Pass one or the other/); + run({ file: "export.json", source: "okta", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/Unknown source "okta". Valid sources: clerk, auth0/); expect(created()).toHaveLength(0); }); @@ -619,11 +621,11 @@ describe("run", () => { await expect( run({ file: "export.json", - transformerFile: "./nope.ts", + source: "./nope.ts", yes: true, secretKey: "sk_test_x", }), - ).rejects.toThrow(/No transformer file at/); + ).rejects.toThrow(/No source file at/); expect(requests).toHaveLength(0); }); @@ -635,15 +637,15 @@ describe("run", () => { ); await expect( - run({ file: "export.json", transformerFile: bad, yes: true, secretKey: "sk_test_x" }), + run({ file: "export.json", source: bad, yes: true, secretKey: "sk_test_x" }), ).rejects.toThrow(/no source field maps to `userId`/); expect(requests).toHaveLength(0); }); - test("still requires --file", async () => { - await expect( - run({ transformerFile: customFile, yes: true, secretKey: "sk_test_x" }), - ).rejects.toThrow(/--file/); + test("still requires a file", async () => { + await expect(run({ source: customFile, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /Pass the file \(or an export run ID\)/, + ); }); }); @@ -688,7 +690,7 @@ describe("run", () => { async (key, records, externalId) => { fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(records)); - await run({ ...baseOptions, transformer: key }); + await run({ ...baseOptions, source: key }); const created = requests.filter((r) => r.url.endsWith("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/v1/users")); expect(created).toHaveLength(1); @@ -716,7 +718,7 @@ describe("run", () => { await run({ ...baseOptions, - transformer: "firebase", + source: "firebase", firebaseSignerKey: "SIGNER", firebaseSaltSeparator: "Bw==", firebaseRounds: 8, @@ -736,14 +738,14 @@ describe("run", () => { test("a partial firebase flag set fails before anything is read", async () => { await expect( - run({ ...baseOptions, transformer: "firebase", firebaseSignerKey: "SIGNER" }), + run({ ...baseOptions, source: "firebase", firebaseSignerKey: "SIGNER" }), ).rejects.toThrow(/--firebase-salt-separator/); expect(requests).toHaveLength(0); }); - test("an unknown transformer fails listing the valid keys", async () => { - await expect(run({ ...baseOptions, transformer: "okta" })).rejects.toThrow( - /Unknown transformer "okta".*clerk.*supabase/s, + test("an unknown source fails listing the valid keys", async () => { + await expect(run({ ...baseOptions, source: "okta" })).rejects.toThrow( + /Unknown source "okta".*clerk.*supabase/s, ); }); }); @@ -805,7 +807,7 @@ describe("run", () => { test("skips only the user whose sole provider is disabled", async () => { stubInstance({ oauth_google: { enabled: true }, oauth_discord: { enabled: false } }); - await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + await run({ ...baseOptions, source: "supabase", skipUnsupportedProviders: true }); expect(created()).toEqual(["sb_email", "sb_both"]); expect(captured.err).toContain("skipping 1 user "); @@ -815,7 +817,7 @@ describe("run", () => { test("imports everyone when the provider is enabled", async () => { stubInstance({ oauth_discord: { enabled: true } }); - await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + await run({ ...baseOptions, source: "supabase", skipUnsupportedProviders: true }); expect(created()).toHaveLength(3); }); @@ -825,13 +827,13 @@ describe("run", () => { test("imports everyone when the instance config cannot be read", async () => { stubInstance(null); - await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + await run({ ...baseOptions, source: "supabase", skipUnsupportedProviders: true }); expect(created()).toHaveLength(3); expect(captured.err).toContain("Could not read the instance's enabled providers"); }); - test("is a no-op with a warning on a non-supabase transformer", async () => { + test("is a no-op with a warning on a non-supabase source", async () => { fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); await run({ ...baseOptions, skipUnsupportedProviders: true }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 273cdde6a..a7b1cb2fa 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -79,8 +79,7 @@ import { loadUsersFromFile, resolveImportFilePath, } from "./lib/transform.ts"; -import { loadCustomTransformer } from "./transformers/load-custom.ts"; -import { registerCustomTransformer, transformerKeys } from "./transformers/registry.ts"; +import { resolveSource, sourceKeys } from "./sources/registry.ts"; import type { ImportSummary, User } from "./types.ts"; import { runWizard, throwAgentFlagsRequired } from "./wizard.ts"; import { login } from "../auth/login.ts"; @@ -89,7 +88,10 @@ import { link } from "../link/index.ts"; export type MigrateRunOptions = { /** An export file, or the ID of the export run that wrote one. */ input?: string; - transformer?: string; + /** A built-in source key, or the path to a source you wrote. */ + source?: string; + /** Content hash of a custom `--source`, set once it is loaded. */ + sourceHash?: string; file?: string; resumeAfter?: string; requirePassword?: boolean; @@ -97,8 +99,6 @@ export type MigrateRunOptions = { secretKey?: string; app?: string; instance?: string; - /** Path to a user-authored transformer, for a platform with no built-in. */ - transformerFile?: string; /** Supabase: drop users whose only social provider is disabled in Clerk. */ skipUnsupportedProviders?: boolean; /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ @@ -108,66 +108,36 @@ export type MigrateRunOptions = { /** * Validates the flags a run needs before anything is read or sent. * - * @returns The transformer key and file path, both guaranteed present. + * `--source` has already been resolved to a registered key by the time this + * runs, custom sources included. + * + * @returns The source key and file path, both guaranteed present. */ export function validateRunOptions(options: MigrateRunOptions): { - transformer: string; + source: string; file: string; } { - const valid = transformerKeys(); - - // A custom transformer has already been loaded and registered by the time - // this runs, so its key is resolvable even though it is not in `valid`. - if (options.transformerFile) { - if (!options.file) { - throwUsageError( - "Missing required option --file (path to a JSON or CSV export).", - undefined, - ERROR_CODE.USAGE_ERROR, - [ - { - command: - "clerk migrate import -y --transformer-file ./my-transformer.ts --file users.json", - description: "Import with a custom transformer", - }, - ], - ); - } - if (!fileExists(options.file)) { - throw new CliError(`File not found: ${options.file}`, { code: ERROR_CODE.FILE_NOT_FOUND }); - } - if (!getFileType(options.file)) { - throwUsageError(`Unsupported file type for ${options.file}. Provide a .json or .csv file.`); - } - return { transformer: options.transformer as string, file: options.file }; - } - - if (!options.transformer) { + if (!options.source) { throwUsageError( - `Missing required option --transformer. Valid values: ${valid.join(", ")}.`, + `Missing --source. Valid values: ${sourceKeys().join(", ")}, or the path to a source you wrote.`, undefined, ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate import -y --transformer clerk --file users.json", + command: "clerk migrate import users.json --source clerk -y", description: "Import a Clerk export", }, ], ); } - if (!valid.includes(options.transformer)) { - throwUsageError( - `Unknown transformer "${options.transformer}". Valid values: ${valid.join(", ")}.`, - ); - } if (!options.file) { throwUsageError( - "Missing required option --file (path to a JSON or CSV export).", + "Missing the file to import (a JSON or CSV export, or an export run ID).", undefined, ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate import -y --transformer clerk --file users.json", + command: "clerk migrate import users.json --source clerk -y", description: "Import a Clerk export", }, ], @@ -180,7 +150,7 @@ export function validateRunOptions(options: MigrateRunOptions): { throwUsageError(`Unsupported file type for ${options.file}. Provide a .json or .csv file.`); } - return { transformer: options.transformer, file: options.file }; + return { source: options.source, file: options.file }; } /** @@ -343,11 +313,11 @@ async function confirmDevUserLimit( */ async function findDisabledProviderUsers( file: string, - transformer: string, + source: string, secretKey: string, ): Promise> { const none = new Set(); - if (transformer !== "supabase") { + if (source !== "supabase") { log.warn(`--skip-unsupported-providers only applies to supabase exports; ignoring.`); return none; } @@ -391,7 +361,7 @@ async function findDisabledProviderUsers( type ReportInput = { users: User[]; file: string; - transformer: string; + source: string; secretKey: string; validationFailed: number; }; @@ -404,7 +374,7 @@ async function readFileSide(input: ReportInput) { // Only Supabase exports record per-user providers, so only they can be // cross-referenced against the instance's social connections. let providerCounts: Record | undefined; - if (input.transformer === "supabase") { + if (input.source === "supabase") { try { providerCounts = countSocialProviders(await readSupabaseRows(input.file)); } catch (error) { @@ -541,7 +511,7 @@ async function showReadinessReport( } /** - * Fills in a missing `--transformer`/`--file` interactively, or explains what + * Fills in a missing `--source`/file interactively, or explains what * to pass. * * Agent mode is the CLI's existing non-interactive signal, so an agent that @@ -549,22 +519,22 @@ async function showReadinessReport( * prompt it cannot answer. */ async function resolveMissingOptions(options: MigrateRunOptions): Promise { - const missing = { transformer: !options.transformer, file: !options.file }; - if (!missing.transformer && !missing.file) return options; + const missing = { source: !options.source, file: !options.file }; + if (!missing.source && !missing.file) return options; if (isAgent() || !isHuman()) { throwAgentFlagsRequired(missing); } - // Resolved before the prompt only when `--transformer firebase` was already + // Resolved before the prompt only when `--source firebase` was already // passed; otherwise the wizard picks the platform first and looks them up // itself, so a non-Firebase migration never reads them at all. - const firebaseHashConfig = resolveFirebaseHashConfig(options, options.transformer); + const firebaseHashConfig = resolveFirebaseHashConfig(options, options.source); const answers = await runWizard({ ...options, firebaseHashConfig }); return { ...options, - transformer: answers.transformer, + source: answers.source, file: answers.file, ...(answers.firebaseHashConfig ? { @@ -578,40 +548,22 @@ async function resolveMissingOptions(options: MigrateRunOptions): Promise { - if (!options.transformerFile) return options; - - // Both name a transformer, and there is no sensible precedence between "the - // one you wrote" and "the one we ship" — say so rather than picking. - if (options.transformer) { - throwUsageError( - "--transformer and --transformer-file both name a transformer. Pass one or the other.", - undefined, - undefined, - [ - { - command: - "clerk migrate import -y --transformer-file ./my-transformer.ts --file users.json", - description: "Use a transformer you wrote", - }, - { - command: "clerk migrate import -y --transformer clerk --file users.json", - description: "Use a built-in transformer", - }, - ], - ); +async function applySource(options: MigrateRunOptions): Promise { + if (!options.source) return options; + const resolved = await resolveSource(options.source); + if (resolved.path) { + log.info(`Loaded the \`${resolved.key}\` source from ${options.source}.`); } - - const custom = await loadCustomTransformer(options.transformerFile); - registerCustomTransformer(custom); - log.info(`Loaded the \`${custom.key}\` transformer from ${options.transformerFile}.`); - - return { ...options, transformer: custom.key }; + return { + ...options, + source: resolved.key, + ...(resolved.hash ? { sourceHash: resolved.hash } : {}), + }; } /** @@ -646,8 +598,7 @@ async function ensureImportTarget(options: MigrateRunOptions): Promise { examples: [ { command: "clerk auth login", description: "Sign in, then re-run the import" }, { - command: - "clerk migrate import -y --secret-key sk_test_... --transformer clerk --file users.json", + command: "clerk migrate import users.json --source clerk -y --secret-key sk_test_...", description: "Import without signing in", }, ], @@ -704,18 +655,18 @@ function applyEnvelope( envelope: ExportEnvelope | undefined, ): MigrateRunOptions { if (!envelope) return options; - if (options.transformer && options.transformer !== envelope.source) { + if (options.source && options.source !== envelope.source) { throwUsageError( - `The file was exported from ${envelope.source}, but the transformer named is ${options.transformer}. ` + - "Drop the transformer: the file already says where it came from.", + `The file was exported from ${envelope.source}, but --source names ${options.source}. ` + + "Drop --source: the file already says where it came from.", ); } - return { ...options, transformer: envelope.source }; + return { ...options, source: envelope.source }; } export async function run(rawOptions: MigrateRunOptions): Promise { await ensureImportTarget(rawOptions); - rawOptions = await applyCustomTransformer(rawOptions); + rawOptions = await applySource(rawOptions); const input = await resolveInput(rawOptions); rawOptions = { ...rawOptions, file: input.file, input: undefined }; @@ -727,11 +678,11 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const options = await resolveMissingOptions(rawOptions); - const { transformer, file } = validateRunOptions(options); + const { source, file } = validateRunOptions(options); // The flags win, so a rotated key can be passed without re-exporting. const firebaseHashConfig = - resolveFirebaseHashConfig(options, transformer) ?? - (transformer === "firebase" ? envelope?.firebase : undefined); + resolveFirebaseHashConfig(options, source) ?? + (source === "firebase" ? envelope?.firebase : undefined); await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { const { secretKey, target } = await resolveClerkTarget(options); @@ -742,7 +693,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { validationFailed, failures, } = await withSpinner(`Loading users from ${file}...`, async () => - loadUsersFromFile(file, transformer, { context: { firebaseHashConfig } }), + loadUsersFromFile(file, source, { context: { firebaseHashConfig } }), ); let users = applyResumeAfter(loaded, options.resumeAfter); @@ -755,7 +706,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const skipped: { user: User; reason: string }[] = []; if (options.skipUnsupportedProviders) { - const excluded = await findDisabledProviderUsers(file, transformer, secretKey); + const excluded = await findDisabledProviderUsers(file, source, secretKey); for (const user of users.filter((candidate) => excluded.has(candidate.userId))) { skipped.push({ user, reason: "only provider is not enabled in Clerk" }); } @@ -788,7 +739,8 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const run = startRun(runsDir, { kind: "import", target, - source: transformer, + source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), file: { path: filePath, sha256: sha256File(filePath) }, ...(input.fromExport ? { fromExport: input.fromExport } : {}), }); @@ -824,14 +776,14 @@ export async function run(rawOptions: MigrateRunOptions): Promise { : 0; log.info( - `Importing ${users.length} user${users.length === 1 ? "" : "s"} via the ${transformer} transformer into ` + + `Importing ${users.length} user${users.length === 1 ? "" : "s"} from ${source} into ` + `${describeTarget(target)}.`, ); await showReadinessReport({ users, file, - transformer, + source, secretKey, validationFailed, skipReport: Boolean(options.yes), diff --git a/packages/cli-core/src/commands/migrate/transformers/auth0.ts b/packages/cli-core/src/commands/migrate/sources/auth0.ts similarity index 70% rename from packages/cli-core/src/commands/migrate/transformers/auth0.ts rename to packages/cli-core/src/commands/migrate/sources/auth0.ts index c1595d139..9dca6253a 100644 --- a/packages/cli-core/src/commands/migrate/transformers/auth0.ts +++ b/packages/cli-core/src/commands/migrate/sources/auth0.ts @@ -1,4 +1,4 @@ -import type { TransformerRegistryEntry } from "../types.ts"; +import type { SourceEntry } from "../types.ts"; import { routeByVerification } from "./shared.ts"; /** @@ -12,11 +12,22 @@ import { routeByVerification } from "./shared.ts"; * be requested from Auth0 support. When present they are bcrypt (`$2a$`/`$2b$`, * 10 rounds), which is why `passwordHasher` defaults to `bcrypt`. */ -const auth0Transformer = { +const auth0Source = { key: "auth0", label: "Auth0", description: "Works with Auth0's Export Users API. Password hashes require a support request to Auth0.", + carries: { + passwords: { + level: "partial", + note: "Auth0 releases bcrypt hashes only through a support request. Add each as `passwordHash` before importing.", + }, + mfa: { level: "no", note: "Auth0 exports no MFA enrolments. Users enrol again in Clerk." }, + metadata: { + level: "yes", + note: "`user_metadata` → public metadata, `app_metadata` → private metadata.", + }, + }, transformer: { user_id: "userId", email: "email", @@ -38,6 +49,6 @@ const auth0Transformer = { defaults: { passwordHasher: "bcrypt" as const, }, -} satisfies TransformerRegistryEntry; +} satisfies SourceEntry; -export default auth0Transformer; +export default auth0Source; diff --git a/packages/cli-core/src/commands/migrate/transformers/authjs.ts b/packages/cli-core/src/commands/migrate/sources/authjs.ts similarity index 70% rename from packages/cli-core/src/commands/migrate/transformers/authjs.ts rename to packages/cli-core/src/commands/migrate/sources/authjs.ts index 3391b1804..abf7ed97f 100644 --- a/packages/cli-core/src/commands/migrate/transformers/authjs.ts +++ b/packages/cli-core/src/commands/migrate/sources/authjs.ts @@ -1,4 +1,4 @@ -import type { TransformerRegistryEntry } from "../types.ts"; +import type { SourceEntry } from "../types.ts"; import { routeByVerification, splitName } from "./shared.ts"; /** @@ -16,11 +16,22 @@ import { routeByVerification, splitName } from "./shared.ts"; * so users arrive without a digest and are imported with * `skip_password_requirement`. */ -const authjsTransformer = { +const authjsSource = { key: "authjs", label: "Auth.js (NextAuth)", description: "Assumes an export of `SELECT id, name, email, email_verified, created_at FROM users`. `name` is split into firstName and lastName.", + carries: { + passwords: { + level: "no", + note: "Auth.js is passwordless (OAuth and email links), so users arrive without one.", + }, + mfa: { level: "no", note: "Auth.js has no MFA of its own." }, + metadata: { + level: "no", + note: "Only `id`, `name`, `email`, `email_verified` and `created_at` are read.", + }, + }, transformer: { id: "userId", email: "email", @@ -33,6 +44,6 @@ const authjsTransformer = { routeByVerification(user, "email", "emailVerified", "timestamp"); splitName(user); }, -} satisfies TransformerRegistryEntry; +} satisfies SourceEntry; -export default authjsTransformer; +export default authjsSource; diff --git a/packages/cli-core/src/commands/migrate/transformers/betterauth.ts b/packages/cli-core/src/commands/migrate/sources/betterauth.ts similarity index 74% rename from packages/cli-core/src/commands/migrate/transformers/betterauth.ts rename to packages/cli-core/src/commands/migrate/sources/betterauth.ts index f32d2f31c..a98c7127a 100644 --- a/packages/cli-core/src/commands/migrate/transformers/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/sources/betterauth.ts @@ -1,4 +1,4 @@ -import type { TransformerRegistryEntry } from "../types.ts"; +import type { SourceEntry } from "../types.ts"; import { routeByVerification, splitName } from "./shared.ts"; /** @@ -12,11 +12,22 @@ import { routeByVerification, splitName } from "./shared.ts"; * no handling: the schema strips anything it does not declare. `banned` is the * exception, because that one *is* a Clerk field. */ -const betterAuthTransformer = { +const betterAuthSource = { key: "betterauth", label: "Better Auth", description: "Works with the Better Auth export. Supports bcrypt passwords and the admin plugin's banned flag.", + carries: { + passwords: { level: "yes", note: "bcrypt hashes from the credential account come across." }, + mfa: { + level: "no", + note: "The two-factor plugin's secrets are not exported. Users enrol again in Clerk.", + }, + metadata: { + level: "no", + note: "Plugin columns such as `role` have no Clerk equivalent and are left out.", + }, + }, transformer: { user_id: "userId", email: "email", @@ -41,6 +52,6 @@ const betterAuthTransformer = { defaults: { passwordHasher: "bcrypt" as const, }, -} satisfies TransformerRegistryEntry; +} satisfies SourceEntry; -export default betterAuthTransformer; +export default betterAuthSource; diff --git a/packages/cli-core/src/commands/migrate/transformers/clerk.ts b/packages/cli-core/src/commands/migrate/sources/clerk.ts similarity index 74% rename from packages/cli-core/src/commands/migrate/transformers/clerk.ts rename to packages/cli-core/src/commands/migrate/sources/clerk.ts index 8fb094839..20e6fe4c2 100644 --- a/packages/cli-core/src/commands/migrate/transformers/clerk.ts +++ b/packages/cli-core/src/commands/migrate/sources/clerk.ts @@ -1,4 +1,4 @@ -import type { TransformerRegistryEntry } from "../types.ts"; +import type { SourceEntry } from "../types.ts"; /** * Clerk → Clerk transformer, for moving users between Clerk instances @@ -6,11 +6,22 @@ import type { TransformerRegistryEntry } from "../types.ts"; * * Maps the Dashboard's user export format onto the import schema. */ -const clerkTransformer = { +const clerkSource = { key: "clerk", label: "Clerk", description: "Migrate between Clerk instances (e.g. development to production, or to another Clerk application). Export your users from the Clerk Dashboard first.", + carries: { + passwords: { + level: "partial", + note: "A Dashboard export carries each digest and its hasher. `clerk migrate export clerk` cannot: the Backend API never returns them.", + }, + mfa: { + level: "partial", + note: "TOTP secrets and backup codes come across from a Dashboard export only.", + }, + metadata: { level: "yes", note: "Public, private and unsafe metadata keep their places." }, + }, transformer: { id: "userId", primary_email_address: "email", @@ -40,6 +51,6 @@ const clerkTransformer = { create_organizations_limit: "createOrganizationsLimit", delete_self_enabled: "deleteSelfEnabled", }, -} satisfies TransformerRegistryEntry; +} satisfies SourceEntry; -export default clerkTransformer; +export default clerkSource; diff --git a/packages/cli-core/src/commands/migrate/transformers/firebase.ts b/packages/cli-core/src/commands/migrate/sources/firebase.ts similarity index 89% rename from packages/cli-core/src/commands/migrate/transformers/firebase.ts rename to packages/cli-core/src/commands/migrate/sources/firebase.ts index 9c9d09c6e..bf21a17fb 100644 --- a/packages/cli-core/src/commands/migrate/transformers/firebase.ts +++ b/packages/cli-core/src/commands/migrate/sources/firebase.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; -import type { PreTransformResult, TransformerRegistryEntry } from "../types.ts"; +import type { PreTransformResult, SourceEntry } from "../types.ts"; import { routeByVerification, splitName, toIsoDate } from "./shared.ts"; /** @@ -30,7 +30,7 @@ const FIREBASE_CSV_HEADERS = * * See https://clerk.com/docs/guides/development/migrating/firebase */ -const firebaseTransformer = { +const firebaseSource = { key: "firebase", label: "Firebase", description: @@ -65,6 +65,14 @@ const firebaseTransformer = { return { filePath }; }, + carries: { + passwords: { + level: "yes", + note: "scrypt hashes come across with the project's hash parameters, read by the export or passed as `--firebase-*`.", + }, + mfa: { level: "no", note: "Firebase exports no MFA enrolments. Users enrol again in Clerk." }, + metadata: { level: "no", note: "Custom claims are not exported." }, + }, transformer: { localId: "userId", email: "email", @@ -117,6 +125,6 @@ const firebaseTransformer = { defaults: { passwordHasher: "scrypt_firebase" as const, }, -} satisfies TransformerRegistryEntry; +} satisfies SourceEntry; -export default firebaseTransformer; +export default firebaseSource; diff --git a/packages/cli-core/src/commands/migrate/sources/list.test.ts b/packages/cli-core/src/commands/migrate/sources/list.test.ts new file mode 100644 index 000000000..a1fbbcde2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.test.ts @@ -0,0 +1,167 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { list, wrapText } from "./list.ts"; +import { ACCOUNT_LINKING_URL, sources } from "./registry.ts"; + +const captured = useCaptureLog(); + +/** `log.info` highlights backticked spans and the levels are coloured. */ +const plain = () => Bun.stripANSI(captured.err).replace(/\s+/g, " "); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-sources-"))); + process.chdir(workDir); + fs.writeFileSync( + path.join(workDir, "custom.ts"), + `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", mail: "email" }, + carries: { + passwords: { level: "partial", note: "Only for accounts made after 2020." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "yes", note: "All of it." }, + }, + };`, + ); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("sources", () => { + test.each([...sources])("lists $key with what it carries", async (entry) => { + await list(undefined); + const row = plain().match(new RegExp(`${entry.key} (\\w+) (\\w+) (\\w+)`)); + expect(row?.slice(1)).toEqual([ + entry.carries.passwords.level, + entry.carries.mfa.level, + entry.carries.metadata.level, + ]); + }); + + // No social column: every source shares one note. + test("prints the account-linking note once", async () => { + await list(undefined); + expect(captured.err.split(ACCOUNT_LINKING_URL)).toHaveLength(2); + }); + + // A compiled binary has no source tree to grep, so the way to extend it has + // to be discoverable from the list itself. + test("says how to add one", async () => { + await list(undefined); + expect(captured.err).toContain("--source ./my-source.ts"); + }); + + test("--json lists every built-in with its carries on stdout", async () => { + await list(undefined, { json: true }); + + const parsed = JSON.parse(captured.out) as { + sources: { key: string; carries: unknown }[]; + account_linking: string; + }; + expect(parsed.sources.map((entry) => entry.key)).toEqual(sources.map((entry) => entry.key)); + expect(parsed.sources[0]?.carries).toEqual(sources[0]?.carries); + expect(parsed.account_linking).toContain(ACCOUNT_LINKING_URL); + }); +}); + +describe("sources ", () => { + test("shows where each field lands, how to export, and what comes across", async () => { + await list("supabase"); + + expect(plain()).toContain("Export with `clerk migrate export supabase`"); + expect(plain()).toContain("encrypted_password → password"); + expect(plain()).toContain("Passwords yes"); + expect(plain()).toContain("passwordHasher is always"); + expect(captured.err).toContain(ACCOUNT_LINKING_URL); + }); + + test("shows a source you wrote, by its path", async () => { + await list("./custom.ts"); + + expect(plain()).toContain("myplatform My Platform (custom — ./custom.ts)"); + expect(plain()).toContain("mail → email"); + expect(plain()).toContain("Only for accounts made after 2020."); + }); + + test("--json returns the detail on stdout", async () => { + await list("auth0", { json: true }); + + expect(JSON.parse(captured.out)).toMatchObject({ + key: "auth0", + export_command: "clerk migrate export auth0", + fields: { user_id: "userId" }, + carries: { passwords: { level: "partial" } }, + }); + }); + + test("an unknown source is a usage error naming the valid ones", async () => { + const error = (await list("okta").catch((caught: unknown) => caught)) as CliError; + expect(error).toBeInstanceOf(CliError); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Valid sources: clerk, auth0"); + }); +}); + +describe("human-mode frame", () => { + let originalMode: Mode; + + beforeAll(() => { + originalMode = getMode(); + setMode("human"); + }); + + afterAll(() => { + setMode(originalMode); + }); + + // Reading a static registry is not a run: there is no progress to bracket, + // and the gutter's `│` would sit in front of every wrapped line. + test("prints no intro/outro gutter", async () => { + await list(undefined); + + expect(captured.err).not.toContain("┌"); + expect(captured.err).not.toContain("└"); + }); + + test("--json stays on stdout only", async () => { + await list(undefined, { json: true }); + + expect(() => JSON.parse(captured.out)).not.toThrow(); + expect(captured.err).toBe(""); + }); +}); + +describe("wrapText", () => { + test("breaks on whitespace within the width", () => { + expect(wrapText("one two three four", 9)).toEqual(["one two", "three", "four"]); + }); + + // `log.info` pairs backticks per line, so a span split across two lines + // leaves one unmatched backtick on each and colours the wrong half of both. + test("never breaks inside a backticked span", () => { + const lines = wrapText("Assumes an export of `SELECT id, name FROM users`.", 30); + + expect(lines).toContain("`SELECT id, name FROM users`."); + for (const line of lines) { + expect((line.match(/`/g) ?? []).length % 2).toBe(0); + } + }); + + test("gives an over-long word its own line rather than dropping it", () => { + expect(wrapText("short supercalifragilistic", 8)).toEqual(["short", "supercalifragilistic"]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts new file mode 100644 index 000000000..6955e37db --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -0,0 +1,202 @@ +/** + * `clerk migrate sources [source]` — which platforms an import can read, and + * what each one brings across. + * + * A compiled binary's users have no source tree to read the mappings in, so + * this is where they are shown. `sources` alone lists every source with its + * passwords, MFA and metadata at a glance. `sources ` shows one in + * full: what each field becomes, how to export from it, and its caveats. + */ + +import { bold, cyan, dim, green, red, yellow } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { exportPlatforms } from "../export/registry.ts"; +import type { Carry, CarryLevel, SourceEntry } from "../types.ts"; +import { ACCOUNT_LINKING_NOTE, resolveSource, sources as builtIns } from "./registry.ts"; + +export type SourcesOptions = { + json?: boolean; +}; + +const KINDS = [ + ["passwords", "Passwords"], + ["mfa", "MFA"], + ["metadata", "Metadata"], +] as const; + +/** + * Capped, not just measured: a description that rewrapped differently on every + * terminal makes two runs of the same command look like different output. 80 is + * the same width `--help` lays itself out at. + */ +const MAX_WIDTH = 80; + +function outputWidth(): number { + return Math.min(process.stderr.columns || MAX_WIDTH, MAX_WIDTH); +} + +/** + * A run of non-space characters, except that a backticked span counts as one + * character run even when it contains spaces. Keeps `SELECT a, b FROM users` + * whole: `log.info` pairs backticks per line, so a span broken across two lines + * leaves an unmatched backtick on each and colours the wrong half of both. + */ +const WORD = /(?:`[^`]*`|\S)+/g; + +/** + * Wraps on whitespace. Safe to measure raw because the backtick spans + * `log.info` highlights keep their backticks — the colour it adds is invisible + * to width, and nothing here is coloured before wrapping. + */ +export function wrapText(text: string, width: number): string[] { + const lines: string[] = []; + let line = ""; + + for (const word of text.match(WORD) ?? []) { + if (!line) line = word; + else if (line.length + 1 + word.length <= width) line += ` ${word}`; + else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + + return lines; +} + +function levelMark(level: CarryLevel): string { + if (level === "yes") return green("yes"); + if (level === "partial") return yellow("partial"); + return red("no"); +} + +/** The command that writes a file this source reads, when there is one. */ +function exportCommand(entry: SourceEntry): string | undefined { + const platform = exportPlatforms.find((candidate) => candidate.sourceKey === entry.key); + return platform ? `clerk migrate export ${platform.key}` : undefined; +} + +function toJson(entry: SourceEntry, custom?: string) { + return { + key: entry.key, + label: entry.label, + description: entry.description, + ...(custom ? { path: custom } : {}), + carries: entry.carries, + export_command: exportCommand(entry) ?? null, + fields: entry.transformer, + caveats: entry.caveats ?? [], + }; +} + +function printList(width: number): void { + for (const line of wrapText( + "A source maps one platform's export onto the fields Clerk imports. A file from " + + "`clerk migrate export` names its own; anything else takes `--source `.", + width, + )) { + log.info(line); + } + log.blank(); + + const keyWidth = Math.max(...builtIns.map((entry) => entry.key.length)) + 2; + log.info( + bold( + `${"SOURCE".padEnd(keyWidth)}${"PASSWORDS".padEnd(11)}${"MFA".padEnd(9)}${"METADATA".padEnd(10)}`, + ), + ); + for (const entry of builtIns) { + const cell = (carry: Carry, pad: number) => + levelMark(carry.level) + " ".repeat(Math.max(1, pad - carry.level.length)); + log.info( + `${cyan(entry.key.padEnd(keyWidth))}${cell(entry.carries.passwords, 11)}` + + `${cell(entry.carries.mfa, 9)}${levelMark(entry.carries.metadata.level)}`, + ); + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); + log.blank(); + log.info("Run `clerk migrate sources ` for what each field becomes."); + log.info("Migrating from something else? Write a source and pass --source ./my-source.ts."); +} + +function printDetail(entry: SourceEntry, width: number, custom?: string): void { + log.info(`${cyan(bold(entry.key))} ${entry.label}${custom ? dim(` (custom — ${custom})`) : ""}`); + for (const line of wrapText(entry.description, width)) log.info(line); + + const command = exportCommand(entry); + if (command) { + log.blank(); + log.info(`${bold("Export with")} \`${command}\``); + } + + log.blank(); + log.info(bold("What comes across")); + for (const [kind, label] of KINDS) { + const carry = entry.carries[kind]; + log.info(` ${label.padEnd(10)}${levelMark(carry.level)}`); + for (const line of wrapText(carry.note, width - 4)) log.info(` ${dim(line)}`); + } + + const fields = Object.entries(entry.transformer); + const fromWidth = Math.max(...fields.map(([from]) => from.length)) + 2; + log.blank(); + log.info(bold("Where each field lands")); + for (const [from, to] of fields) log.info(` ${from.padEnd(fromWidth)}→ ${to}`); + if (entry.defaults) { + for (const [field, value] of Object.entries(entry.defaults)) { + log.info(dim(` ${field} is always ${JSON.stringify(value)}`)); + } + } + + if (entry.caveats?.length) { + log.blank(); + log.info(bold("Caveats")); + for (const caveat of entry.caveats) { + const [first, ...rest] = wrapText(caveat, width - 4); + log.info(` - ${first ?? ""}`); + for (const line of rest) log.info(` ${line}`); + } + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); +} + +export async function list(key: string | undefined, options: SourcesOptions = {}): Promise { + const width = outputWidth(); + + if (key === undefined) { + if (options.json) { + log.data( + JSON.stringify( + { + sources: builtIns.map((entry) => toJson(entry)), + account_linking: ACCOUNT_LINKING_NOTE, + }, + null, + 2, + ), + ); + return; + } + printList(width); + return; + } + + // No gutter: this reads a static registry, it does not run anything. + const resolved = await resolveSource(key); + if (options.json) { + log.data( + JSON.stringify( + { ...toJson(resolved.entry, resolved.path), account_linking: ACCOUNT_LINKING_NOTE }, + null, + 2, + ), + ); + return; + } + printDetail(resolved.entry, width, resolved.path ? key : undefined); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts similarity index 59% rename from packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts rename to packages/cli-core/src/commands/migrate/sources/load-custom.test.ts index 5dff6ad0b..513457375 100644 --- a/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts @@ -3,8 +3,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; -import { loadCustomTransformer, validateTransformer } from "./load-custom.ts"; -import { __resetCustomTransformersForTesting } from "./registry.ts"; +import { loadCustomSource, validateSource } from "./load-custom.ts"; +import { __resetCustomSourcesForTesting } from "./registry.ts"; let workDir: string; let originalCwd: string; @@ -22,16 +22,16 @@ afterAll(() => { }); afterEach(() => { - __resetCustomTransformersForTesting(); + __resetCustomSourcesForTesting(); }); /** - * Writes a transformer file with a unique name. + * Writes a source file with a unique name. * * Names must not repeat: a dynamic `import()` caches by URL, so reusing one * would silently return the previous test's module. */ -function writeTransformer(source: string, ext = "ts"): string { +function writeSource(source: string, ext = "ts"): string { const name = `custom-${counter++}.${ext}`; fs.writeFileSync(path.join(workDir, name), source); return `./${name}`; @@ -42,11 +42,16 @@ const VALID = `export default { label: "My Platform", description: "Exports from My Platform.", transformer: { account_ref: "userId", contact_email: "email" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, };`; -describe("loadCustomTransformer", () => { - test("loads a user-authored TypeScript transformer", async () => { - const entry = await loadCustomTransformer(writeTransformer(VALID)); +describe("loadCustomSource", () => { + test("loads a user-authored TypeScript source", async () => { + const entry = await loadCustomSource(writeSource(VALID)); expect(entry).toMatchObject({ key: "myplatform", @@ -56,17 +61,17 @@ describe("loadCustomTransformer", () => { }); test("loads plain JavaScript too", async () => { - const entry = await loadCustomTransformer(writeTransformer(VALID, "js")); + const entry = await loadCustomSource(writeSource(VALID, "js")); expect(entry.key).toBe("myplatform"); }); // The file is the user's own code; the CLI must transpile whatever they wrote. test("transpiles TypeScript syntax the runtime has to strip", async () => { - const entry = await loadCustomTransformer( - writeTransformer(` - interface Entry { key: string; label: string; transformer: Record } + const entry = await loadCustomSource( + writeSource(` + interface Entry { key: string; label: string; transformer: Record; carries: unknown } const mapping = { my_id: "userId" } as const; - const custom: Entry = { key: "tsplatform", label: "TS", transformer: { ...mapping } }; + const custom: Entry = { key: "tsplatform", label: "TS", transformer: { ...mapping }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } }; export default custom satisfies Entry; `), ); @@ -74,10 +79,11 @@ describe("loadCustomTransformer", () => { }); test("carries the optional hooks through", async () => { - const entry = await loadCustomTransformer( - writeTransformer(`export default { + const entry = await loadCustomSource( + writeSource(`export default { key: "hooked", label: "Hooked", transformer: { id: "userId" }, + carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } }, defaults: { passwordHasher: "bcrypt" }, postTransform: (user) => { user.firstName = "set"; }, };`), @@ -90,61 +96,66 @@ describe("loadCustomTransformer", () => { }); test("supplies a description when the author omitted one", async () => { - const entry = await loadCustomTransformer( - writeTransformer( - `export default { key: "bare", label: "Bare", transformer: { id: "userId" } };`, + const entry = await loadCustomSource( + writeSource( + `export default { key: "bare", label: "Bare", transformer: { id: "userId" }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } };`, ), ); - expect(entry.description).toBe("Custom transformer"); + expect(entry.description).toBe("Custom source"); }); test("reports a path that is not there", async () => { - await expect(loadCustomTransformer("./nope.ts")).rejects.toThrow(/No transformer file at/); + await expect(loadCustomSource("./nope.ts")).rejects.toThrow(/No source file at/); }); test("reports a directory given instead of a file", async () => { fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); - await expect(loadCustomTransformer("./adir")).rejects.toThrow(/is a directory/); + await expect(loadCustomSource("./adir")).rejects.toThrow(/is a directory/); }); test("reports a file that does not parse, quoting the syntax error", async () => { - await expect( - loadCustomTransformer(writeTransformer("export default { key: ,,, }")), - ).rejects.toThrow(/Could not load/); + await expect(loadCustomSource(writeSource("export default { key: ,,, }"))).rejects.toThrow( + /Could not load/, + ); }); test("reports a file that throws while loading", async () => { await expect( - loadCustomTransformer(writeTransformer(`throw new Error("boom"); export default {};`)), + loadCustomSource(writeSource(`throw new Error("boom"); export default {};`)), ).rejects.toThrow(/Could not load .*boom/s); }); test("points at a named export when the default is missing", async () => { - const file = writeTransformer( + const file = writeSource( `export const myPlatform = { key: "x", label: "X", transformer: { a: "userId" } };`, ); - await expect(loadCustomTransformer(file)).rejects.toThrow( + await expect(loadCustomSource(file)).rejects.toThrow( /has no default export.*`myPlatform`.*did you mean `export default`/s, ); }); test("reports a missing default with no named exports to suggest", async () => { - await expect(loadCustomTransformer(writeTransformer("const unused = 1;"))).rejects.toThrow( + await expect(loadCustomSource(writeSource("const unused = 1;"))).rejects.toThrow( /has no default export\.$/m, ); }); }); -describe("validateTransformer", () => { +describe("validateSource", () => { const valid = { key: "myplatform", label: "My Platform", transformer: { account_ref: "userId" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, }; test("accepts a minimal valid entry", () => { - expect(validateTransformer(valid, "f.ts").key).toBe("myplatform"); + expect(validateSource(valid, "f.ts").key).toBe("myplatform"); }); test.each([ @@ -152,7 +163,7 @@ describe("validateTransformer", () => { ["a number default export", 42, /is number, not an object/], ["a string default export", "nope", /is string, not an object/], ])("rejects %s", (_label, value, expected) => { - expect(() => validateTransformer(value, "f.ts")).toThrow(expected); + expect(() => validateSource(value, "f.ts")).toThrow(expected); }); test.each([ @@ -163,11 +174,11 @@ describe("validateTransformer", () => { ["label", { ...valid, label: undefined }], ["label", { ...valid, label: "" }], ])("rejects a bad %s naming the field", (field, value) => { - expect(() => validateTransformer(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); }); test("rejects a non-string description", () => { - expect(() => validateTransformer({ ...valid, description: 7 }, "f.ts")).toThrow( + expect(() => validateSource({ ...valid, description: 7 }, "f.ts")).toThrow( /`description` must be a string/, ); }); @@ -178,19 +189,19 @@ describe("validateTransformer", () => { ["an array", { ...valid, transformer: [] }], ["a string", { ...valid, transformer: "id" }], ])("rejects a transformer mapping that is %s", (_label, value) => { - expect(() => validateTransformer(value, "f.ts")).toThrow(/`transformer`|`transformer\./); + expect(() => validateSource(value, "f.ts")).toThrow(/`transformer`|`transformer\./); }); test("names the offending entry when a mapping target is not a field name", () => { expect(() => - validateTransformer({ ...valid, transformer: { account_ref: "userId", bad: 7 } }, "f.ts"), + validateSource({ ...valid, transformer: { account_ref: "userId", bad: 7 } }, "f.ts"), ).toThrow(/`transformer.bad` must map to a Clerk field name, got number/); }); // Without it the import runs to completion and creates every user with no // external_id — which is what makes a migration reversible. test("rejects a mapping with no userId target", () => { - expect(() => validateTransformer({ ...valid, transformer: { a: "email" } }, "f.ts")).toThrow( + expect(() => validateSource({ ...valid, transformer: { a: "email" } }, "f.ts")).toThrow( /no source field maps to `userId`/, ); }); @@ -200,23 +211,40 @@ describe("validateTransformer", () => { ["preTransform", { ...valid, preTransform: "nope" }], ["postTransform", { ...valid, postTransform: 7 }], ])("rejects a %s of the wrong type", (field, value) => { - expect(() => validateTransformer(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); }); test.each([["clerk"], ["auth0"], ["supabase"]])( "rejects %s, which would shadow a built-in", (key) => { - expect(() => validateTransformer({ ...valid, key }, "f.ts")).toThrow( - /already a built-in transformer/, - ); + expect(() => + validateSource({ ...valid, key }, "f.ts", ["clerk", "auth0", "supabase"]), + ).toThrow(/already a built-in source/); }, ); + // What a source brings across is the first thing `sources` shows, and only + // its author knows it. + test("requires carries", () => { + expect(() => validateSource({ ...valid, carries: undefined }, "f.ts")).toThrow( + /`carries` must say what the source brings across/, + ); + }); + + test.each([ + ["a level that is not yes, no or partial", { level: "maybe", note: "x" }], + ["no note", { level: "yes" }], + ])("rejects a carries entry with %s", (_label, mfa) => { + expect(() => validateSource({ ...valid, carries: { ...valid.carries, mfa } }, "f.ts")).toThrow( + /`carries.mfa` must be/, + ); + }); + test("names the file in every message, so the author knows which one", () => { - expect(() => validateTransformer({}, "./their-file.ts")).toThrow(/\.\/their-file\.ts/); + expect(() => validateSource({}, "./their-file.ts")).toThrow(/\.\/their-file\.ts/); }); test("raises CliError, so the global handler formats it", () => { - expect(() => validateTransformer({}, "f.ts")).toThrow(CliError); + expect(() => validateSource({}, "f.ts")).toThrow(CliError); }); }); diff --git a/packages/cli-core/src/commands/migrate/transformers/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts similarity index 74% rename from packages/cli-core/src/commands/migrate/transformers/load-custom.ts rename to packages/cli-core/src/commands/migrate/sources/load-custom.ts index 1ac82a719..b7ac3e085 100644 --- a/packages/cli-core/src/commands/migrate/transformers/load-custom.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -1,10 +1,10 @@ /** - * Loading a user-authored transformer at runtime. + * Loading a user-authored source at runtime. * * In the standalone migration-tool, supporting a new platform meant adding a * file to `src/transformers/` and one line to the registry — the user had the * source tree. A compiled binary has neither a source tree to edit nor a way - * for an end user to rebuild it, so `--transformer-file` restores that + * for an end user to rebuild it, so `--source ` restores that * extensibility by importing a file from the user's own project instead. * * **Verified before this was built on:** a `bun build --compile` executable can @@ -21,13 +21,14 @@ import fs from "node:fs"; import path from "node:path"; import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; -import type { TransformerRegistryEntry } from "../types.ts"; -import { transformers } from "./registry.ts"; +import type { CarryLevel, SourceEntry } from "../types.ts"; const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; +const LEVELS: readonly CarryLevel[] = ["yes", "no", "partial"]; + function invalid(problem: string, file: string): never { - throw new CliError(`${file} is not a valid transformer: ${problem}`, { + throw new CliError(`${file} is not a valid source: ${problem}`, { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL, }); @@ -40,8 +41,13 @@ function invalid(problem: string, file: string): never { * author is writing this file by hand against a shape they cannot see. * * @param file - Path as the user typed it, for the error message. + * @param reservedKeys - The built-in keys, which a custom source may not reuse. */ -export function validateTransformer(value: unknown, file: string): TransformerRegistryEntry { +export function validateSource( + value: unknown, + file: string, + reservedKeys: readonly string[] = [], +): SourceEntry { if (value === null || typeof value !== "object") { invalid(`the default export is ${value === null ? "null" : typeof value}, not an object`, file); } @@ -94,42 +100,64 @@ export function validateTransformer(value: unknown, file: string): TransformerRe invalid("`defaults` must be an object when present", file); } + // What a source brings across is the first thing `sources` shows, and the + // author is the only one who knows it. + const carries = entry.carries as Record | undefined; + if (!carries || typeof carries !== "object" || Array.isArray(carries)) { + invalid( + "`carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note }", + file, + ); + } + for (const kind of ["passwords", "mfa", "metadata"] as const) { + const carry = carries[kind] as { level?: unknown; note?: unknown } | undefined; + if (!carry || !LEVELS.includes(carry.level as CarryLevel) || typeof carry.note !== "string") { + invalid( + `\`carries.${kind}\` must be { level: "yes" | "no" | "partial", note: string }`, + file, + ); + } + } + for (const hook of ["preTransform", "postTransform"] as const) { if (entry[hook] !== undefined && typeof entry[hook] !== "function") { invalid(`\`${hook}\` must be a function when present`, file); } } - if (transformers.some((builtIn) => builtIn.key === entry.key)) { + if (reservedKeys.includes(entry.key as string)) { invalid( - `\`key\` is "${String(entry.key)}", which is already a built-in transformer. Choose another key`, + `\`key\` is "${String(entry.key)}", which is already a built-in source. Choose another key`, file, ); } return { - ...(entry as unknown as TransformerRegistryEntry), - description: (entry.description as string | undefined) ?? "Custom transformer", + ...(entry as unknown as SourceEntry), + description: (entry.description as string | undefined) ?? "Custom source", }; } /** - * Imports and validates a user-authored transformer. + * Imports and validates a user-authored source. * * @throws CliError when the path is missing, the module fails to load, or the * exported value does not match the registry entry shape. */ -export async function loadCustomTransformer(file: string): Promise { +export async function loadCustomSource( + file: string, + reservedKeys: readonly string[] = [], +): Promise { const resolved = path.resolve(process.cwd(), file); if (!fs.existsSync(resolved)) { - throw new CliError(`No transformer file at ${resolved}.`, { + throw new CliError(`No source file at ${resolved}.`, { code: ERROR_CODE.FILE_NOT_FOUND, docsUrl: DOCS_URL, }); } if (fs.statSync(resolved).isDirectory()) { - throw new CliError(`${resolved} is a directory, not a transformer file.`, { + throw new CliError(`${resolved} is a directory, not a source file.`, { code: ERROR_CODE.USAGE_ERROR, }); } @@ -160,5 +188,5 @@ export async function loadCustomTransformer(file: string): Promise.ts` exporting a `SourceEntry`, + * then add it to the array below. + */ + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { throwUsageError } from "../../../lib/errors.ts"; +import type { SourceEntry } from "../types.ts"; +import auth0Source from "./auth0.ts"; +import authjsSource from "./authjs.ts"; +import betterAuthSource from "./betterauth.ts"; +import clerkSource from "./clerk.ts"; +import firebaseSource from "./firebase.ts"; +import { loadCustomSource } from "./load-custom.ts"; +import supabaseSource from "./supabase.ts"; +import workosSource from "./workos.ts"; + +export const sources: SourceEntry[] = [ + clerkSource, + auth0Source, + authjsSource, + betterAuthSource, + firebaseSource, + supabaseSource, + workosSource, +]; + +export const ACCOUNT_LINKING_URL = + "https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking"; + +/** + * What happens to social sign-ins, the same for every source. + * + * No export carries a user's OAuth connections, and none needs to: once the + * provider is enabled in Clerk, the user signs in with it and Clerk links the + * account to the imported user by verified email. + */ +export const ACCOUNT_LINKING_NOTE = + "Social sign-ins are not copied. Enable the same providers in Clerk, and a user who signs in " + + `with one is linked to their imported account by verified email. See ${ACCOUNT_LINKING_URL}`; + +/** + * A source loaded from a user's `--source ` for this invocation. + * + * Kept beside the built-ins rather than pushed into them, so the shipped list + * is never mutated. One invocation loads at most one, so this holding a + * single entry is the normal case; the array shape just avoids a special case + * in the lookups. + */ +const customSources: SourceEntry[] = []; + +export function registerCustomSource(entry: SourceEntry): void { + customSources.push(entry); +} + +/** Test-only: drops anything a previous test registered. */ +export function __resetCustomSourcesForTesting(): void { + customSources.length = 0; +} + +/** Built-ins plus whatever `--source ` loaded. */ +export function allSources(): SourceEntry[] { + return [...sources, ...customSources]; +} + +/** The built-in keys, for tab-completion and error messages. */ +export function sourceKeys(): string[] { + return sources.map((entry) => entry.key); +} + +/** + * Looks up a source by key, custom ones included. + * + * @throws Error when no source is registered under that key. + */ +export function getSource(key: string): SourceEntry { + const source = allSources().find((entry) => entry.key === key); + if (!source) { + throw new Error(`Source not found for key: ${key}`); + } + return source; +} + +/** A `--source` value that names a file rather than a built-in key. */ +export function isSourcePath(value: string): boolean { + return /^(\.\.?\/|\/)/.test(value) || /\.(ts|js|mjs)$/.test(value); +} + +/** A resolved `--source`: its key, and for a custom one, the file's content hash. */ +export type ResolvedSource = { key: string; entry: SourceEntry; path?: string; hash?: string }; + +/** + * Resolves `--source`: a built-in key, or a path to a source you wrote. + * + * A custom source is keyed by a hash of its file as well as its key, so an + * edited source is a different source when a re-run decides whether to + * continue an earlier import. + * + * @throws UsageError for an unknown key, listing the valid ones. + */ +export async function resolveSource(value: string): Promise { + if (isSourcePath(value)) { + const entry = await loadCustomSource(value, sourceKeys()); + registerCustomSource(entry); + const resolved = path.resolve(process.cwd(), value); + const hash = createHash("sha256").update(fs.readFileSync(resolved)).digest("hex"); + return { key: entry.key, entry, path: resolved, hash }; + } + + const entry = sources.find((candidate) => candidate.key === value); + if (!entry) { + throwUsageError( + `Unknown source "${value}". Valid sources: ${sourceKeys().join(", ")}.\n` + + "For a platform with no built-in, pass the path to a source you wrote, e.g. --source ./my-source.ts.", + undefined, + undefined, + [{ command: "clerk migrate sources", description: "List the built-in sources" }], + ); + } + return { key: entry.key, entry }; +} diff --git a/packages/cli-core/src/commands/migrate/transformers/shared.ts b/packages/cli-core/src/commands/migrate/sources/shared.ts similarity index 100% rename from packages/cli-core/src/commands/migrate/transformers/shared.ts rename to packages/cli-core/src/commands/migrate/sources/shared.ts diff --git a/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts similarity index 92% rename from packages/cli-core/src/commands/migrate/transformers/transformers.test.ts rename to packages/cli-core/src/commands/migrate/sources/sources.test.ts index 533e96f7b..e86424eb5 100644 --- a/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import { loadUsersFromFile, transformUsers } from "../lib/transform.ts"; import type { FirebaseHashConfig } from "../types.ts"; -import { getTransformer, transformerKeys, transformers } from "./registry.ts"; +import { getSource, isSourcePath, sourceKeys, sources } from "./registry.ts"; import { isVerified } from "./shared.ts"; const FIREBASE_HASH: FirebaseHashConfig = { @@ -20,7 +20,7 @@ let originalCwd: string; beforeAll(() => { originalCwd = process.cwd(); - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-transformers-"))); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-sources-"))); process.chdir(workDir); }); @@ -46,7 +46,7 @@ const one = (key: string, record: Record, context = {}) => describe("registry", () => { test("registers all seven platforms", () => { - expect(transformerKeys()).toEqual([ + expect(sourceKeys()).toEqual([ "clerk", "auth0", "authjs", @@ -57,17 +57,37 @@ describe("registry", () => { ]); }); - test.each([...transformers])("$key maps a source field to userId", (transformer) => { - expect(Object.values(transformer.transformer)).toContain("userId"); + test.each([...sources])("$key maps a source field to userId", (source) => { + expect(Object.values(source.transformer)).toContain("userId"); }); - test.each([...transformers])("$key carries a label and description", (transformer) => { - expect(transformer.label.length).toBeGreaterThan(0); - expect(transformer.description.length).toBeGreaterThan(0); + test.each([...sources])("$key carries a label and description", (source) => { + expect(source.label.length).toBeGreaterThan(0); + expect(source.description.length).toBeGreaterThan(0); + }); + + test.each([...sources])("$key says what it carries, with a note for each", (source) => { + for (const carry of Object.values(source.carries)) { + expect(["yes", "no", "partial"]).toContain(carry.level); + expect(carry.note.length).toBeGreaterThan(0); + } }); test("throws for an unregistered key", () => { - expect(() => getTransformer("okta")).toThrow(/Transformer not found/); + expect(() => getSource("okta")).toThrow(/Source not found/); + }); +}); + +describe("isSourcePath", () => { + test.each([["./mine.ts"], ["../up/mine.js"], ["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/abs/mine.mjs"], ["mine.ts"], ["dir/mine.js"]])( + "%s is a path", + (value) => { + expect(isSourcePath(value)).toBe(true); + }, + ); + + test.each([["clerk"], ["betterauth"], ["okta"]])("%s is a key", (value) => { + expect(isSourcePath(value)).toBe(false); }); }); @@ -196,7 +216,7 @@ describe("workos", () => { // No other transformer omits it. WorkOS never returns a digest, so naming a // hasher would imply a password column that cannot exist. test("names no password hasher, because WorkOS returns no hashes", () => { - expect(getTransformer("workos").defaults).toBeUndefined(); + expect(getSource("workos").defaults).toBeUndefined(); }); // The export carries these so whoever runs the migration can see who used diff --git a/packages/cli-core/src/commands/migrate/transformers/supabase.ts b/packages/cli-core/src/commands/migrate/sources/supabase.ts similarity index 83% rename from packages/cli-core/src/commands/migrate/transformers/supabase.ts rename to packages/cli-core/src/commands/migrate/sources/supabase.ts index 1efde4b6a..76091518a 100644 --- a/packages/cli-core/src/commands/migrate/transformers/supabase.ts +++ b/packages/cli-core/src/commands/migrate/sources/supabase.ts @@ -1,4 +1,4 @@ -import type { TransformerRegistryEntry } from "../types.ts"; +import type { SourceEntry } from "../types.ts"; import { routeByVerification, toIsoDate } from "./shared.ts"; /** @@ -20,11 +20,22 @@ function stripDiscriminator(value: unknown): string | undefined { return value.replace(DISCORD_DISCRIMINATOR, "").trim() || undefined; } -const supabaseTransformer = { +const supabaseSource = { key: "supabase", label: "Supabase", description: "Works with a Supabase `auth.users` export. Use --skip-unsupported-providers to drop users whose only social provider is not enabled in Clerk.", + carries: { + passwords: { level: "yes", note: "bcrypt `encrypted_password` hashes come across." }, + mfa: { + level: "no", + note: "Supabase MFA factors are not exported. Users enrol again in Clerk.", + }, + metadata: { + level: "partial", + note: "`raw_user_meta_data` → public metadata. `raw_app_meta_data` is not carried.", + }, + }, transformer: { id: "userId", email: "email", @@ -65,6 +76,6 @@ const supabaseTransformer = { defaults: { passwordHasher: "bcrypt" as const, }, -} satisfies TransformerRegistryEntry; +} satisfies SourceEntry; -export default supabaseTransformer; +export default supabaseSource; diff --git a/packages/cli-core/src/commands/migrate/transformers/workos.ts b/packages/cli-core/src/commands/migrate/sources/workos.ts similarity index 74% rename from packages/cli-core/src/commands/migrate/transformers/workos.ts rename to packages/cli-core/src/commands/migrate/sources/workos.ts index 9e5b8e349..bfa571b95 100644 --- a/packages/cli-core/src/commands/migrate/transformers/workos.ts +++ b/packages/cli-core/src/commands/migrate/sources/workos.ts @@ -1,4 +1,4 @@ -import type { TransformerRegistryEntry } from "../types.ts"; +import type { SourceEntry } from "../types.ts"; import { routeByVerification } from "./shared.ts"; /** @@ -17,11 +17,19 @@ import { routeByVerification } from "./shared.ts"; * WorkOS has no phone number and no username, which is why the map is short: * those fields have nothing to come from. */ -const workosTransformer = { +const workosSource = { key: "workos", label: "WorkOS", description: "Works with WorkOS's User Management API. WorkOS returns no password hashes, so imported users sign in by reset or SSO.", + carries: { + passwords: { + level: "no", + note: "WorkOS never returns password hashes. Users reset their password, or sign in with SSO.", + }, + mfa: { level: "no", note: "WorkOS returns TOTP secrets at enrolment only." }, + metadata: { level: "yes", note: "`metadata` → public metadata." }, + }, transformer: { id: "userId", email: "email", @@ -34,6 +42,6 @@ const workosTransformer = { postTransform: (user) => { routeByVerification(user, "email", "emailVerified", "boolean"); }, -} satisfies TransformerRegistryEntry; +} satisfies SourceEntry; -export default workosTransformer; +export default workosSource; diff --git a/packages/cli-core/src/commands/migrate/transformers/list.test.ts b/packages/cli-core/src/commands/migrate/transformers/list.test.ts deleted file mode 100644 index a297ec919..000000000 --- a/packages/cli-core/src/commands/migrate/transformers/list.test.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { CliError } from "../../../lib/errors.ts"; -import { getMode, setMode, type Mode } from "../../../mode.ts"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { list, wrapText } from "./list.ts"; -import { transformers } from "./registry.ts"; - -const captured = useCaptureLog(); - -const stripAnsi = (value: string) => value.replace(/\[[0-9;]*m/g, ""); - -let workDir: string; -let originalCwd: string; - -beforeAll(() => { - originalCwd = process.cwd(); - workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-tlist-"))); - process.chdir(workDir); - fs.writeFileSync( - path.join(workDir, "custom.ts"), - `export default { - key: "myplatform", - label: "My Platform", - description: "Exports from My Platform.", - transformer: { account_ref: "userId" }, - };`, - ); -}); - -afterAll(() => { - process.chdir(originalCwd); - fs.rmSync(workDir, { recursive: true, force: true }); -}); - -describe("human output", () => { - test.each([...transformers])("lists the $key transformer with its label", async (transformer) => { - await list(); - expect(captured.err).toContain(transformer.key); - expect(captured.err).toContain(transformer.label); - }); - - // Two normalizations: `log.info` auto-highlights backticked spans, so the - // rendered description carries colour codes the source string does not, and - // descriptions are wrapped to the terminal width across several indented - // lines. Collapsing whitespace compares the words, not the layout. - const collapse = (value: string) => stripAnsi(value).replace(/\s+/g, " "); - - test.each([...transformers])("includes the $key description", async (transformer) => { - await list(); - expect(collapse(captured.err)).toContain(collapse(transformer.description)); - }); - - test("counts the built-ins", async () => { - await list(); - expect(captured.err).toContain(`${transformers.length} built-in transformers`); - }); - - // A compiled binary has no source tree to grep, so the way to extend it has - // to be discoverable from the list itself. - test("says how to add one when none is loaded", async () => { - await list(); - expect(captured.err).toContain("--transformer-file"); - }); - - test("appends a custom transformer and names its source", async () => { - await list({ transformerFile: "./custom.ts" }); - - expect(captured.err).toContain("myplatform"); - expect(captured.err).toContain("custom — ./custom.ts"); - expect(captured.err).toContain("plus 1 loaded from --transformer-file"); - }); - - test("drops the how-to hint once one is loaded", async () => { - await list({ transformerFile: "./custom.ts" }); - expect(captured.err).not.toContain("Migrating from something else?"); - }); -}); - -describe("--json", () => { - test("emits every built-in on stdout", async () => { - await list({ json: true }); - - const parsed = JSON.parse(captured.out) as Record[]; - expect(parsed).toHaveLength(transformers.length); - expect(parsed.map((entry) => entry.key)).toEqual(transformers.map((entry) => entry.key)); - }); - - test("reports key, label, description and the userId source field", async () => { - await list({ json: true }); - - const parsed = JSON.parse(captured.out) as Record[]; - expect(parsed[0]).toMatchObject({ - key: "clerk", - label: "Clerk", - built_in: true, - maps_to_user_id: "id", - }); - }); - - test("marks a custom transformer as not built in", async () => { - await list({ json: true, transformerFile: "./custom.ts" }); - - const parsed = JSON.parse(captured.out) as Record[]; - expect(parsed.at(-1)).toMatchObject({ - key: "myplatform", - built_in: false, - source: "./custom.ts", - maps_to_user_id: "account_ref", - }); - }); -}); - -describe("a bad --transformer-file", () => { - test("fails rather than listing only the built-ins", async () => { - await expect(list({ transformerFile: "./nope.ts" })).rejects.toThrow(CliError); - }); -}); - -describe("human-mode frame", () => { - let originalMode: Mode; - - beforeAll(() => { - originalMode = getMode(); - setMode("human"); - }); - - afterAll(() => { - setMode(originalMode); - }); - - // Reading a static registry is not a run: there is no progress to bracket, - // and the gutter's `│` would sit in front of every wrapped line. - test("prints no intro/outro gutter", async () => { - await list(); - - expect(captured.err).not.toContain("┌"); - expect(captured.err).not.toContain("└"); - expect(stripAnsi(captured.err)).toContain("Transformers:"); - }); - - test("--json stays on stdout only", async () => { - await list({ json: true }); - - expect(() => JSON.parse(captured.out)).not.toThrow(); - expect(captured.err).toBe(""); - }); -}); - -describe("wrapText", () => { - test("breaks on whitespace within the width", () => { - expect(wrapText("one two three four", 9)).toEqual(["one two", "three", "four"]); - }); - - // `log.info` pairs backticks per line, so a span split across two lines - // leaves one unmatched backtick on each and colours the wrong half of both. - test("never breaks inside a backticked span", () => { - const lines = wrapText("Assumes an export of `SELECT id, name FROM users`.", 30); - - expect(lines).toContain("`SELECT id, name FROM users`."); - for (const line of lines) { - expect((line.match(/`/g) ?? []).length % 2).toBe(0); - } - }); - - test("gives an over-long word its own line rather than dropping it", () => { - expect(wrapText("short supercalifragilistic", 8)).toEqual(["short", "supercalifragilistic"]); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/transformers/list.ts b/packages/cli-core/src/commands/migrate/transformers/list.ts deleted file mode 100644 index 993ee4838..000000000 --- a/packages/cli-core/src/commands/migrate/transformers/list.ts +++ /dev/null @@ -1,120 +0,0 @@ -/** - * `clerk migrate transformers list` — which source platforms are available. - * - * New in the CLI. The standalone tool's interactive picker was the only place - * these were listed, which was fine when the user had the source tree to grep. - * A compiled binary's users have neither, so the list is a command. - */ - -import { bold, cyan } from "../../../lib/color.ts"; -import { log } from "../../../lib/log.ts"; -import type { TransformerRegistryEntry } from "../types.ts"; -import { loadCustomTransformer } from "./load-custom.ts"; -import { transformers } from "./registry.ts"; - -export type TransformersListOptions = { - json?: boolean; - transformerFile?: string; -}; - -type Listed = TransformerRegistryEntry & { builtIn: boolean; source?: string }; - -function toJson(entries: Listed[]) { - return entries.map((entry) => ({ - key: entry.key, - label: entry.label, - description: entry.description, - built_in: entry.builtIn, - ...(entry.source ? { source: entry.source } : {}), - maps_to_user_id: - Object.entries(entry.transformer).find(([, target]) => target === "userId")?.[0] ?? null, - })); -} - -/** - * Capped, not just measured: a description that rewrapped differently on every - * terminal makes two runs of the same command look like different output. 80 is - * the same width `--help` lays itself out at. - */ -const MAX_WIDTH = 80; - -function outputWidth(): number { - return Math.min(process.stderr.columns || MAX_WIDTH, MAX_WIDTH); -} - -/** - * A run of non-space characters, except that a backticked span counts as one - * character run even when it contains spaces. Keeps `SELECT a, b FROM users` - * whole: `log.info` pairs backticks per line, so a span broken across two lines - * leaves an unmatched backtick on each and colours the wrong half of both. - */ -const WORD = /(?:`[^`]*`|\S)+/g; - -/** - * Wraps on whitespace. Safe to measure raw because the backtick spans - * `log.info` highlights keep their backticks — the colour it adds is invisible - * to width, and nothing here is coloured before wrapping. - */ -export function wrapText(text: string, width: number): string[] { - const lines: string[] = []; - let line = ""; - - for (const word of text.match(WORD) ?? []) { - if (!line) line = word; - else if (line.length + 1 + word.length <= width) line += ` ${word}`; - else { - lines.push(line); - line = word; - } - } - if (line) lines.push(line); - - return lines; -} - -export async function list(options: TransformersListOptions = {}): Promise { - const entries: Listed[] = transformers.map((entry) => ({ ...entry, builtIn: true })); - - if (options.transformerFile) { - const custom = await loadCustomTransformer(options.transformerFile); - entries.push({ ...custom, builtIn: false, source: options.transformerFile }); - } - - if (options.json) { - log.data(JSON.stringify(toJson(entries), null, 2)); - return; - } - - const width = outputWidth(); - - // No gutter: this reads a static registry, it does not run anything. The - // frame belongs on `migrate import`, where there is progress to bracket. - for (const line of wrapText( - "A transformer maps one platform's export onto the fields Clerk imports. " + - "Pass the one your export came from as `--transformer `.", - width, - )) { - log.info(line); - } - log.blank(); - - log.info(bold("Transformers:")); - for (const entry of entries) { - const suffix = entry.builtIn ? "" : ` (custom — ${entry.source})`; - log.info(` ${cyan(bold(entry.key))} ${entry.label}${suffix}`); - for (const line of wrapText(entry.description, width - 4)) { - log.info(` ${line}`); - } - log.blank(); - } - - const custom = entries.length - transformers.length; - log.info( - `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + - (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), - ); - - if (custom === 0) { - log.info("Migrating from something else? Write a transformer and pass --transformer-file."); - } -} diff --git a/packages/cli-core/src/commands/migrate/transformers/registry.ts b/packages/cli-core/src/commands/migrate/transformers/registry.ts deleted file mode 100644 index ae2f133b8..000000000 --- a/packages/cli-core/src/commands/migrate/transformers/registry.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Transformer registry. - * - * `migrate import` reads this array to resolve `--transformer` and to list the - * valid choices in help output and tab-completion. - * - * To add a platform: create `transformers/.ts` exporting a - * `TransformerRegistryEntry`, then add it to the array below. - */ - -import type { TransformerRegistryEntry } from "../types.ts"; -import auth0Transformer from "./auth0.ts"; -import authjsTransformer from "./authjs.ts"; -import betterAuthTransformer from "./betterauth.ts"; -import clerkTransformer from "./clerk.ts"; -import firebaseTransformer from "./firebase.ts"; -import supabaseTransformer from "./supabase.ts"; -import workosTransformer from "./workos.ts"; - -export const transformers: TransformerRegistryEntry[] = [ - clerkTransformer, - auth0Transformer, - authjsTransformer, - betterAuthTransformer, - firebaseTransformer, - supabaseTransformer, - workosTransformer, -]; - -/** - * Transformers loaded from a user's `--transformer-file` for this invocation. - * - * Kept beside the built-ins rather than pushed into them, so the shipped list - * is never mutated and `--transformer`'s choices stay exactly the built-in - * keys. One CLI invocation loads at most one, so this holding a single entry is - * the normal case; the array shape just avoids a special case in the lookups. - */ -const customTransformers: TransformerRegistryEntry[] = []; - -export function registerCustomTransformer(entry: TransformerRegistryEntry): void { - customTransformers.push(entry); -} - -/** Test-only: drops anything a previous test registered. */ -export function __resetCustomTransformersForTesting(): void { - customTransformers.length = 0; -} - -/** Built-ins plus whatever `--transformer-file` loaded. */ -export function allTransformers(): TransformerRegistryEntry[] { - return [...transformers, ...customTransformers]; -} - -/** - * The built-in keys, for `--transformer`'s choices and tab-completion. - * - * Deliberately excludes custom transformers: they are selected by path via - * `--transformer-file`, and Commander resolves these choices once at - * registration time, before any file could have been loaded. - */ -export function transformerKeys(): string[] { - return transformers.map((entry) => entry.key); -} - -/** - * Looks up a transformer by key, custom ones included. - * - * @throws Error when no transformer is registered under that key. - */ -export function getTransformer(key: string): TransformerRegistryEntry { - const transformer = allTransformers().find((entry) => entry.key === key); - if (!transformer) { - throw new Error(`Transformer not found for key: ${key}`); - } - return transformer; -} diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts index 6ca7c2dda..ced754db1 100644 --- a/packages/cli-core/src/commands/migrate/types.ts +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -40,9 +40,6 @@ export const PASSWORD_HASHERS = [ /** A user that has passed schema validation and is ready to import. */ export type User = z.infer; -/** Union of all registered transformer keys (e.g. `"clerk"`). */ -export type TransformerKey = string; - /** Totals for a completed import run. */ export type ImportSummary = { totalProcessed: number; @@ -90,21 +87,37 @@ export type PreTransformResult = { data?: Record[]; }; +/** How much of one kind of data a source brings across. */ +export type CarryLevel = "yes" | "no" | "partial"; + +export type Carry = { level: CarryLevel; note: string }; + +/** + * What a source brings across, per kind of data that is easy to lose without + * noticing. Social sign-ins are not listed: no source carries them, and every + * source shares one account-linking note instead. + */ +export type SourceCarries = { passwords: Carry; mfa: Carry; metadata: Carry }; + /** - * A platform transformer: how to get from one source export shape to Clerk's - * import shape. + * A source: how to get from one platform's export shape to Clerk's import + * shape. * * @property transformer - Source field path → Clerk field name. + * @property carries - What comes across: passwords, MFA and metadata. + * @property caveats - Anything else worth knowing before importing. * @property defaults - Values merged into every user from this platform. * @property preTransform - Runs before field mapping. * @property postTransform - Mutates a user after field mapping, given the * run's {@link TransformContext}. */ -export type TransformerRegistryEntry = { +export type SourceEntry = { key: string; label: string; description: string; transformer: Record; + carries: SourceCarries; + caveats?: string[]; defaults?: Record; preTransform?: ( filePath: string, diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts index 02f7f3306..26b61b6e6 100644 --- a/packages/cli-core/src/commands/migrate/wizard.test.ts +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -60,7 +60,7 @@ beforeEach(() => { const textCall = (index: number): Prompt | undefined => mockText.mock.calls[index]?.[0]; const selectCall = (index: number): SelectPrompt | undefined => mockSelect.mock.calls[index]?.[0]; -describe("transformer picker", () => { +describe("source picker", () => { test("is built from the registry, so every platform appears", async () => { mockSelect.mockResolvedValue("auth0"); mockText.mockResolvedValue("users.json"); @@ -78,7 +78,7 @@ describe("transformer picker", () => { ]); }); - test("labels each choice with the transformer's display name", async () => { + test("labels each choice with the source's display name", async () => { mockSelect.mockResolvedValue("clerk"); mockText.mockResolvedValue("users.json"); @@ -87,13 +87,13 @@ describe("transformer picker", () => { expect(selectCall(0)?.choices.map((choice) => choice.name)).toContain("Better Auth"); }); - test("is skipped when --transformer was already passed", async () => { + test("is skipped when --source was already passed", async () => { mockText.mockResolvedValue("users.json"); - const result = await runWizard({ transformer: "clerk" }); + const result = await runWizard({ source: "clerk" }); expect(mockSelect).not.toHaveBeenCalled(); - expect(result.transformer).toBe("clerk"); + expect(result.source).toBe("clerk"); }); }); @@ -127,7 +127,7 @@ describe("file prompt validation", () => { }); describe("firebase hash parameters", () => { - test("are asked for when the firebase transformer is picked", async () => { + test("are asked for when the firebase source is picked", async () => { mockSelect.mockResolvedValue("firebase"); mockText .mockResolvedValueOnce("users.json") @@ -175,7 +175,7 @@ describe("firebase hash parameters", () => { expect(textCall(3)?.default).toBeUndefined(); }); - test("are not asked for on a non-firebase transformer", async () => { + test("are not asked for on a non-firebase source", async () => { mockSelect.mockResolvedValue("auth0"); mockText.mockResolvedValue("users.json"); @@ -217,15 +217,15 @@ describe("firebase hash parameters", () => { describe("throwAgentFlagsRequired", () => { test.each([ - [{ transformer: true, file: true }, /--transformer and --file /], - [{ transformer: true, file: false }, /--transformer \./], - [{ transformer: false, file: true }, /--file \./], + [{ source: true, file: true }, /the file \(or an export run ID\) and --source /], + [{ source: true, file: false }, /Pass --source \./], + [{ source: false, file: true }, /Pass the file \(or an export run ID\)\./], ])("names only the flags that are missing (%p)", (missing, expected) => { expect(() => throwAgentFlagsRequired(missing)).toThrow(expected); }); test("says why it cannot prompt", () => { - expect(() => throwAgentFlagsRequired({ transformer: true, file: true })).toThrow( + expect(() => throwAgentFlagsRequired({ source: true, file: true })).toThrow( /cannot prompt in agent mode/, ); }); diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index b2ca866c1..0b49a45e2 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -14,11 +14,11 @@ import { log } from "../../lib/log.ts"; import { text } from "../../lib/prompts.ts"; import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebase-hash.ts"; import { fileExists, getFileType } from "./lib/transform.ts"; -import { transformers } from "./transformers/registry.ts"; +import { sources } from "./sources/registry.ts"; import type { FirebaseHashConfig } from "./types.ts"; export type WizardResult = { - transformer: string; + source: string; file: string; firebaseHashConfig?: FirebaseHashConfig; }; @@ -29,12 +29,12 @@ function hint(description: string): string { return firstSentence.length > 96 ? `${firstSentence.slice(0, 93)}...` : firstSentence; } -async function pickTransformer(): Promise { +async function pickSource(): Promise { // Built from the registry, so a new platform appears here with no second // place to update. return select({ message: "Which platform are you migrating from?", - choices: transformers.map((entry) => ({ + choices: sources.map((entry) => ({ name: entry.label, value: entry.key, description: hint(entry.description), @@ -102,27 +102,27 @@ async function askNumber(label: string): Promise { } /** - * Fills in whichever of transformer and file were not passed as flags. + * Fills in whichever of source and file were not passed. * * @param provided - Flags the caller already supplied; those are not asked for. */ export async function runWizard( provided: { - transformer?: string; + source?: string; file?: string; firebaseHashConfig?: FirebaseHashConfig; } & FirebaseHashFlags, ): Promise { - const transformer = provided.transformer ?? (await pickTransformer()); + const source = provided.source ?? (await pickSource()); const file = provided.file ?? (await askFile()); let firebaseHashConfig = provided.firebaseHashConfig; - if (transformer === "firebase" && !firebaseHashConfig) { + if (source === "firebase" && !firebaseHashConfig) { firebaseHashConfig = resolveFirebaseHashConfig(provided, "firebase") ?? (await askFirebaseHashConfig()); } - return { transformer, file, ...(firebaseHashConfig ? { firebaseHashConfig } : {}) }; + return { source, file, ...(firebaseHashConfig ? { firebaseHashConfig } : {}) }; } /** @@ -131,10 +131,10 @@ export async function runWizard( * Names exactly the flags that are missing, so the caller can retry without * guessing which of the two it forgot. */ -export function throwAgentFlagsRequired(missing: { transformer: boolean; file: boolean }): never { +export function throwAgentFlagsRequired(missing: { source: boolean; file: boolean }): never { const flags = [ - missing.transformer ? "--transformer " : undefined, - missing.file ? "--file " : undefined, + missing.file ? "the file (or an export run ID)" : undefined, + missing.source ? "--source " : undefined, ].filter(Boolean); throwUsageError( @@ -143,7 +143,7 @@ export function throwAgentFlagsRequired(missing: { transformer: boolean; file: b undefined, [ { - command: `clerk migrate import -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, + command: `clerk migrate import users.json --source ${sources[0]?.key ?? "clerk"} -y`, description: "Run non-interactively", }, ], diff --git a/packages/cli-core/src/test/integration/completion.test.ts b/packages/cli-core/src/test/integration/completion.test.ts index e1e10d2e1..35a1152ba 100644 --- a/packages/cli-core/src/test/integration/completion.test.ts +++ b/packages/cli-core/src/test/integration/completion.test.ts @@ -220,6 +220,12 @@ describe("generateCompletions", () => { expect(names).toContain("DELETE"); }); + test("completes --source with the built-in migrate sources", () => { + const names = completionNames("migrate", "import", "--source", ""); + expect(names).toContain("clerk"); + expect(names).toContain("betterauth"); + }); + test("returns empty for options with unknown values (file paths)", () => { const result = complete("config", "pull", "--output", ""); expect(result.completions).toEqual([]); @@ -272,6 +278,12 @@ describe("generateCompletions", () => { expect(names).toContain("settings"); }); + test("migrate sources: suggests the built-in sources", () => { + const names = completionNames("migrate", "sources", ""); + expect(names).toContain("supabase"); + expect(names).toContain("workos"); + }); + test("open dashboard: filters subpaths by prefix", () => { const names = completionNames("open", "dashboard", "u"); expect(names).toContain("users"); From 983b1f38dd2a9c8ea821f642256834e571f33d68 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 29 Sep 2026 18:04:22 -0400 Subject: [PATCH 058/141] fix(migrate): correct what each source carries - Metadata a user could edit on the source now goes to `unsafe_metadata`, which is also user-editable in Clerk. That covers Auth0 `user_metadata`, Supabase `raw_user_meta_data` and WorkOS `metadata`. Public metadata is read-only to the user, so the user would have lost the ability to edit it. - Better Auth passwords: the hasher is detected per user rather than assumed to be bcrypt. Better Auth's own scrypt (`<32 hex>:<128 hex>`) is sent as `scrypt:16384:16:1$salt$key` with hasher `scrypt_werkzeug`. `$2a/b/y$` is sent as bcrypt, and `$argon2id$`/`$argon2i$` as argon2. Any other value is dropped: the user imports without a password, and their run line records `passwordDropped: true`. `sources betterauth` explains that Better Auth NFKC-normalizes passwords and Clerk does not. - Social sign-ins: every export now prints the shared account-linking note. It replaces the Auth.js-only claim that users "will use the same providers". New E2E tests (`test/e2e/migrate.test.ts`) check the two things only a real instance can confirm. A real Better Auth scrypt hash imports and passes `verify_password`. A user whose only email is unverified is refused by an instance that requires email. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 41 +++++- .../src/commands/migrate/export/authjs.ts | 2 +- .../src/commands/migrate/export/shared.ts | 4 + .../src/commands/migrate/import-users.test.ts | 15 ++ .../src/commands/migrate/import-users.ts | 1 + .../src/commands/migrate/sources/auth0.ts | 4 +- .../commands/migrate/sources/betterauth.ts | 65 +++++++- .../commands/migrate/sources/sources.test.ts | 58 +++++++- .../src/commands/migrate/sources/supabase.ts | 8 +- .../src/commands/migrate/sources/workos.ts | 4 +- .../src/commands/migrate/validator.ts | 4 +- test/e2e/migrate.test.ts | 139 ++++++++++++++++++ 12 files changed, 316 insertions(+), 29 deletions(-) create mode 100644 test/e2e/migrate.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index bb694161a..51e0b8fb0 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -550,11 +550,11 @@ project's `.gitignore` first, because run files carry user data. Each run is a folder named for its ID, `YYYYMMDD-HHmmss-xxxx`: -| File | Contents | -| -------------- | -------------------------------------------------------------------------------------------------------- | -| `run.json` | Kind, status, start and finish times, the target, the source, the file and its sha256, and the counts | -| `users.ndjson` | One line per user outcome: `sourceId`, `clerkId`, `status`, and `reason`, `error` or `code` when present | -| `lock` | The PID of the process writing the run, while it runs | +| File | Contents | +| -------------- | --------------------------------------------------------------------------------------------------------------------------- | +| `run.json` | Kind, status, start and finish times, the target, the source, the file and its sha256, and the counts | +| `users.ndjson` | One line per user outcome: `sourceId`, `clerkId`, `status`, and `reason`, `error`, `code` or `passwordDropped` when present | +| `lock` | The PID of the process writing the run, while it runs | A user's status is `created`, `failed`, `skipped`, `deleted` or `exported`. The last line for each `sourceId` wins. A `429` retry, an extra email or phone that did not @@ -680,6 +680,37 @@ The `userId` check is the load-bearing one: without it the import would run to completion and create every user with no `external_id`, which is what makes a migration re-runnable. +### Metadata + +Metadata a user can edit on the source platform — Auth0's `user_metadata`, +Supabase's `raw_user_meta_data`, WorkOS's `metadata` — goes to Clerk's +`unsafe_metadata`, which is the user-editable one. Public metadata is read-only +to the user, so putting it there would take away an edit the user had. Auth0's +`app_metadata` goes to `private_metadata`. + +### Better Auth passwords + +Better Auth hashes with its own scrypt by default and lets an app swap in bcrypt +or argon2, so one database can hold more than one kind. The hasher is detected +per user: + +| Stored value | Sent as | +| --------------------------- | ---------------------------------------------------------- | +| `<32 hex>:<128 hex>` | `scrypt:16384:16:1$$`, hasher `scrypt_werkzeug` | +| `$2a$`, `$2b$` or `$2y$` | `bcrypt` | +| `$argon2id$` or `$argon2i$` | `argon2id` or `argon2i` | +| anything else | dropped | + +Better Auth's scrypt uses the hex salt string as the salt and a 64-byte key, +which is what `scrypt_werkzeug` verifies once N, r and p are written inline. It +also normalizes a password to NFKC before hashing, and Clerk does not, so a +password whose NFKC form differs will not verify and that user resets it. +`clerk migrate sources betterauth` lists that caveat. + +A password Clerk cannot verify is **dropped, not rejected**: the user imports +without it and can sign in another way or reset it. Their run line carries +`passwordDropped: true`. + ### Verified vs unverified identifiers Every platform records verification differently, and each source declares diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 90a36712e..6c14da347 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -140,7 +140,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { if (users.length > 0) { log.warn( - "Auth.js core stores no passwords — its users sign in with OAuth or email links, so they arrive without credentials and will use the same providers in Clerk.", + "Auth.js core stores no passwords — its users sign in with OAuth or email links, so they arrive without credentials.", ); } }); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index a0ac34fd5..09e163614 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -13,6 +13,7 @@ import path from "node:path"; import { dim, green, yellow } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { ENVELOPE_VERSION, type ExportEnvelope } from "../lib/export-file.ts"; +import { ACCOUNT_LINKING_NOTE } from "../sources/registry.ts"; import { resolveRunsDir, sha256File, @@ -183,6 +184,9 @@ export function finishExport(input: FinishExportInput): FinishedExport { log.success(`Exported ${users.length} user${users.length === 1 ? "" : "s"} to ${outputPath}`); log.info(dim(`Run ${record.id}. See each user with \`clerk migrate runs ${record.id}\`.`)); + log.blank(); + log.info(dim(ACCOUNT_LINKING_NOTE)); + log.blank(); for (const line of formatImportCommand(record.id)) log.info(line); diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index e9a7b06ea..517dc7a6a 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -232,6 +232,21 @@ describe("importUsers", () => { expect(requests.filter((r) => r.url.endsWith("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/v1/phone_numbers"))).toHaveLength(1); }); + test("marks a user whose password the source dropped", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [user({ passwordDropped: true })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(lines[0]).toMatchObject({ status: "created", passwordDropped: true }); + // Never sent: it is the CLI's own bookkeeping, not a Clerk field. + expect(JSON.stringify(requests[0]?.body)).not.toContain("passwordDropped"); + }); + test("notes a failed additional identifier without failing the user", async () => { stub((url) => url.endsWith("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/v1/email_addresses") diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index c29ccdad9..1d097ddbc 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -323,6 +323,7 @@ export async function importUsers(options: ImportUsersOptions): Promise:<128 hex key>`, N=16384, r=16, p=1. */ +const BETTER_AUTH_SCRYPT = /^([0-9a-f]{32}):([0-9a-f]{128})$/i; + +/** + * Works out which hasher produced a stored Better Auth password, one user at + * a time. + * + * Better Auth hashes with its own scrypt by default and lets an app swap in + * bcrypt or argon2, so one database can hold more than one kind. Better Auth's + * scrypt uses the hex salt string itself as the salt and a 64-byte key, which + * is exactly what `scrypt_werkzeug` verifies once the parameters are written + * inline. + * + * @returns The digest and hasher to send, or `undefined` when the hash is not + * one Clerk can verify. + */ +export function detectBetterAuthHash( + hash: string, +): + | { password: string; passwordHasher: "scrypt_werkzeug" | "bcrypt" | "argon2id" | "argon2i" } + | undefined { + const scrypt = BETTER_AUTH_SCRYPT.exec(hash); + if (scrypt) { + return { + password: `scrypt:16384:16:1$${scrypt[1]}$${scrypt[2]}`, + passwordHasher: "scrypt_werkzeug", + }; + } + if (/^\$2[aby]\$/.test(hash)) return { password: hash, passwordHasher: "bcrypt" }; + if (hash.startsWith("$argon2id$")) return { password: hash, passwordHasher: "argon2id" }; + if (hash.startsWith("$argon2i$")) return { password: hash, passwordHasher: "argon2i" }; + return undefined; +} + const betterAuthSource = { key: "betterauth", label: "Better Auth", description: - "Works with the Better Auth export. Supports bcrypt passwords and the admin plugin's banned flag.", + "Works with the Better Auth export. Detects scrypt, bcrypt and argon2 passwords per user, and carries the admin plugin's banned flag.", carries: { - passwords: { level: "yes", note: "bcrypt hashes from the credential account come across." }, + passwords: { + level: "yes", + note: "Better Auth's scrypt, bcrypt and argon2 hashes come across, detected per user. Any other hash is dropped, and that user resets their password.", + }, mfa: { level: "no", note: "The two-factor plugin's secrets are not exported. Users enrol again in Clerk.", @@ -28,6 +66,9 @@ const betterAuthSource = { note: "Plugin columns such as `role` have no Clerk equivalent and are left out.", }, }, + caveats: [ + "Better Auth's scrypt normalizes a password to NFKC before hashing it; Clerk hashes it as typed. A password whose NFKC form differs — full-width characters, some ligatures — will not verify, and that user resets it.", + ], transformer: { user_id: "userId", email: "email", @@ -45,13 +86,23 @@ const betterAuthSource = { routeByVerification(user, "phone", "phoneVerified", "boolean"); splitName(user); + if (typeof user.password === "string" && user.password) { + const detected = detectBetterAuthHash(user.password); + if (detected) { + user.password = detected.password; + user.passwordHasher = detected.passwordHasher; + } else { + // Imported without it rather than rejected: the user can still sign + // in another way, or reset it. + delete user.password; + user.passwordDropped = true; + } + } + // Only carry `banned` when it is actually true — Better Auth writes false // for every user that was never banned, and sending that to Clerk is noise. if (user.banned !== true) delete user.banned; }, - defaults: { - passwordHasher: "bcrypt" as const, - }, } satisfies SourceEntry; export default betterAuthSource; diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index e86424eb5..f244d4329 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -167,7 +167,9 @@ describe("auth0", () => { expect("phoneVerified" in (user ?? {})).toBe(false); }); - test("keeps user_metadata public and app_metadata private", async () => { + // `user_metadata` is the user's own to edit in Auth0, which is what Clerk's + // unsafe metadata is; public metadata is read-only to the user. + test("sends user_metadata to unsafe metadata and app_metadata to private", async () => { const { users } = await load("auth0", [ { ...base, @@ -176,7 +178,8 @@ describe("auth0", () => { app_metadata: { plan: "pro" }, }, ]); - expect(users[0]?.publicMetadata).toEqual({ theme: "dark" }); + expect(users[0]?.unsafeMetadata).toEqual({ theme: "dark" }); + expect(users[0]?.publicMetadata).toBeUndefined(); expect(users[0]?.privateMetadata).toEqual({ plan: "pro" }); }); }); @@ -206,11 +209,11 @@ describe("workos", () => { expect(user?.unverifiedEmailAddresses).toBe(unverified); }); - test("keeps metadata public", async () => { + test("sends metadata to unsafe metadata", async () => { const { users } = await load("workos", [ { ...base, email_verified: true, metadata: { plan: "pro" } }, ]); - expect(users[0]?.publicMetadata).toEqual({ plan: "pro" }); + expect(users[0]?.unsafeMetadata).toEqual({ plan: "pro" }); }); // No other transformer omits it. WorkOS never returns a digest, so naming a @@ -271,9 +274,50 @@ describe("authjs", () => { describe("betterauth", () => { const base = { user_id: "ba1", email: "a@x.dev", email_verified: true }; - test("maps the credential hash and defaults the hasher to bcrypt", async () => { - const { users } = await load("betterauth", [{ ...base, password_hash: "$2a$10$hash" }]); - expect(users[0]).toMatchObject({ password: "$2a$10$hash", passwordHasher: "bcrypt" }); + // Better Auth's own scrypt: a 16-byte hex salt, a colon, a 64-byte hex key. + const SALT = "a".repeat(32); + const KEY = "b".repeat(128); + + test.each([ + [`${SALT}:${KEY}`, `scrypt:16384:16:1$${SALT}$${KEY}`, "scrypt_werkzeug"], + ["$2a$10$hash", "$2a$10$hash", "bcrypt"], + ["$2b$10$hash", "$2b$10$hash", "bcrypt"], + ["$2y$10$hash", "$2y$10$hash", "bcrypt"], + [ + "$argon2id$v=19$m=65536,t=3,p=4$c2FsdA$aGFzaA", + "$argon2id$v=19$m=65536,t=3,p=4$c2FsdA$aGFzaA", + "argon2id", + ], + [ + "$argon2i$v=19$m=4096,t=3,p=1$c2FsdA$aGFzaA", + "$argon2i$v=19$m=4096,t=3,p=1$c2FsdA$aGFzaA", + "argon2i", + ], + ])("detects the hasher per user: %s", async (stored, password, passwordHasher) => { + const { users } = await load("betterauth", [{ ...base, password_hash: stored }]); + expect(users[0]).toMatchObject({ password, passwordHasher }); + expect(users[0]?.passwordDropped).toBeUndefined(); + }); + + // Imported without the password rather than rejected: the user can still + // sign in another way, or reset it. + test.each([["plaintext"], ["$pbkdf2$abc"], [`${SALT}:short`]])( + "drops a password it cannot verify (%s) and imports the user", + async (stored) => { + const { users, validationFailed } = await load("betterauth", [ + { ...base, password_hash: stored }, + ]); + expect(validationFailed).toBe(0); + expect(users[0]?.password).toBeUndefined(); + expect(users[0]?.passwordHasher).toBeUndefined(); + expect(users[0]?.passwordDropped).toBe(true); + }, + ); + + test("names no hasher for a user without a password", async () => { + const { users } = await load("betterauth", [base]); + expect(users[0]?.passwordHasher).toBeUndefined(); + expect(users[0]?.passwordDropped).toBeUndefined(); }); test("routes an unverified phone", () => { diff --git a/packages/cli-core/src/commands/migrate/sources/supabase.ts b/packages/cli-core/src/commands/migrate/sources/supabase.ts index 76091518a..a4c202710 100644 --- a/packages/cli-core/src/commands/migrate/sources/supabase.ts +++ b/packages/cli-core/src/commands/migrate/sources/supabase.ts @@ -33,7 +33,7 @@ const supabaseSource = { }, metadata: { level: "partial", - note: "`raw_user_meta_data` → public metadata. `raw_app_meta_data` is not carried.", + note: "`raw_user_meta_data` → unsafe metadata, which users can edit, as in Supabase. `raw_app_meta_data` is not carried.", }, }, transformer: { @@ -45,7 +45,7 @@ const supabaseSource = { encrypted_password: "password", phone: "phone", phone_confirmed_at: "phoneConfirmedAt", - raw_user_meta_data: "publicMetadata", + raw_user_meta_data: "unsafeMetadata", created_at: "createdAt", }, postTransform: (user) => { @@ -55,8 +55,8 @@ const supabaseSource = { // A basic SQL export has no first_name/last_name columns; the name lives in // user metadata instead, under whichever key the provider happened to use. - if (!user.firstName && user.publicMetadata && typeof user.publicMetadata === "object") { - const meta = user.publicMetadata as Record; + if (!user.firstName && user.unsafeMetadata && typeof user.unsafeMetadata === "object") { + const meta = user.unsafeMetadata as Record; const displayName = stripDiscriminator(meta.display_name ?? meta.first_name ?? meta.name); if (displayName) { const parts = displayName.split(/\s+/); diff --git a/packages/cli-core/src/commands/migrate/sources/workos.ts b/packages/cli-core/src/commands/migrate/sources/workos.ts index bfa571b95..5041dacb4 100644 --- a/packages/cli-core/src/commands/migrate/sources/workos.ts +++ b/packages/cli-core/src/commands/migrate/sources/workos.ts @@ -28,7 +28,7 @@ const workosSource = { note: "WorkOS never returns password hashes. Users reset their password, or sign in with SSO.", }, mfa: { level: "no", note: "WorkOS returns TOTP secrets at enrolment only." }, - metadata: { level: "yes", note: "`metadata` → public metadata." }, + metadata: { level: "yes", note: "`metadata` → unsafe metadata." }, }, transformer: { id: "userId", @@ -36,7 +36,7 @@ const workosSource = { email_verified: "emailVerified", first_name: "firstName", last_name: "lastName", - metadata: "publicMetadata", + metadata: "unsafeMetadata", created_at: "createdAt", }, postTransform: (user) => { diff --git a/packages/cli-core/src/commands/migrate/validator.ts b/packages/cli-core/src/commands/migrate/validator.ts index 2ebc31166..80954c526 100644 --- a/packages/cli-core/src/commands/migrate/validator.ts +++ b/packages/cli-core/src/commands/migrate/validator.ts @@ -5,7 +5,7 @@ * * ============================================================================ * ONLY EDIT THIS IF YOU ARE ADDING A NEW FIELD. - * Adding support for a new source platform means adding a transformer, not + * Adding support for a new source platform means adding a source, not * touching the schema. * ============================================================================ */ @@ -50,6 +50,8 @@ export const userSchema = z // Password password: z.string().optional(), passwordHasher: passwordHasherEnum.optional(), + /** Set by a source that found a password Clerk cannot verify, and left it out. */ + passwordDropped: z.boolean().optional(), // 2FA totpSecret: z.string().optional(), backupCodesEnabled: z.boolean().optional(), diff --git a/test/e2e/migrate.test.ts b/test/e2e/migrate.test.ts new file mode 100644 index 000000000..9dbf2c608 --- /dev/null +++ b/test/e2e/migrate.test.ts @@ -0,0 +1,139 @@ +/** + * Live-BAPI tests for `clerk migrate import`, covering what only a real + * instance can answer: + * + * - A Better Auth scrypt hash, sent as `scrypt_werkzeug`, verifies against the + * password it was made from. A unit test can only check the string shape. + * - A user whose only email is unverified, imported into an instance that + * requires an email, is refused by Clerk. The import's checks treat that + * user as a reject on the strength of this test. + * + * Requires `CLERK_PLATFORM_API_KEY` and `CLERK_CLI_TEST_APP_ID`. Locally, run + * via `bun run test:e2e:op` so 1Password resolves both in-memory. + */ + +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { randomBytes, scryptSync } from "node:crypto"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const CLI_PATH = join(import.meta.dir, "../../packages/cli-core/src/cli.ts"); + +let APP_ID: string; +let workDir: string; +let emailRequired = false; +const importRuns: string[] = []; + +async function cli(args: string[]) { + return Bun.$`bun ${CLI_PATH} ${args} --app ${APP_ID}` + .env({ + ...process.env, + CLERK_CONFIG_DIR: join(workDir, "config"), + CLERK_MIGRATE_DIR: join(workDir, "runs"), + CLERK_TELEMETRY_DISABLED: "1", + }) + .cwd(workDir) + .quiet() + .nothrow(); +} + +beforeAll(async () => { + const appId = process.env.CLERK_CLI_TEST_APP_ID; + if (!appId || !process.env.CLERK_PLATFORM_API_KEY) { + throw new Error( + "CLERK_CLI_TEST_APP_ID and CLERK_PLATFORM_API_KEY are required. " + + "Run via `bun run test:e2e:op` for local 1Password injection.", + ); + } + APP_ID = appId; + workDir = mkdtempSync(join(tmpdir(), "clerk-cli-e2e-migrate-")); + + const config = await cli(["config", "pull", "--keys", "auth_email"]); + const authEmail = ( + JSON.parse(config.stdout.toString()) as { + auth_email?: { required_for_sign_up?: boolean }; + } + ).auth_email; + emailRequired = Boolean(authEmail?.required_for_sign_up); +}); + +// Undo every import, so the test app does not fill up with migrated users. +afterAll(async () => { + await Promise.all(importRuns.map(async (runId) => cli(["migrate", "undo", runId, "--yes"]))); + rmSync(workDir, { recursive: true, force: true }); +}, 60_000); + +/** Imports `users` as a Better Auth export and returns each user's run line. */ +async function importBetterAuth(users: Record[]) { + const file = join(workDir, `betterauth-${randomBytes(4).toString("hex")}.json`); + writeFileSync(file, JSON.stringify(users)); + + await cli(["migrate", "import", file, "--source", "betterauth", "--yes"]); + + const runsDir = join(workDir, "runs"); + const [runId] = readdirSync(runsDir) + .filter( + (id) => JSON.parse(readFileSync(join(runsDir, id, "run.json"), "utf-8")).kind === "import", + ) + .filter((id) => !importRuns.includes(id)); + if (!runId) throw new Error("The import recorded no run."); + importRuns.push(runId); + + return readFileSync(join(runsDir, runId, "users.ndjson"), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); +} + +/** A password hashed exactly the way Better Auth's default hasher does it. */ +function betterAuthHash(password: string): string { + const salt = randomBytes(16).toString("hex"); + const key = scryptSync(password.normalize("NFKC"), salt, 64, { + N: 16384, + r: 16, + p: 1, + maxmem: 128 * 16384 * 16 * 2, + }); + return `${salt}:${key.toString("hex")}`; +} + +test("a Better Auth scrypt hash imports and verifies against its password", async () => { + const hex = randomBytes(6).toString("hex"); + const password = `Migrate${hex}!1`; + + const [line] = await importBetterAuth([ + { + user_id: `ba_${hex}`, + email: `${hex}+clerk_test@clerkcookie.com`, + email_verified: true, + password_hash: betterAuthHash(password), + }, + ]); + expect(line).toMatchObject({ status: "created" }); + + const verify = await cli([ + "api", + `/users/${line?.clerkId as string}/verify_password`, + "-X", + "POST", + "-d", + JSON.stringify({ password }), + ]); + expect(JSON.parse(verify.stdout.toString())).toMatchObject({ verified: true }); +}, 60_000); + +test("a user whose only email is unverified is refused where email is required", async () => { + if (!emailRequired) { + // The test app decides this, not the test; say so rather than pass silently. + console.warn("Skipped: the test app's development instance does not require an email."); + return; + } + const hex = randomBytes(6).toString("hex"); + + const [line] = await importBetterAuth([ + { user_id: `ba_${hex}`, email: `${hex}+clerk_test@clerkcookie.com`, email_verified: false }, + ]); + + expect(line).toMatchObject({ status: "failed" }); +}, 60_000); From d56fa31df32b14b0ef329d905c2892988cfa8ceb Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 29 Sep 2026 18:18:16 -0400 Subject: [PATCH 059/141] feat(migrate)!: check every import against the instance, continue re-runs, and ask before writing `clerk migrate import ` now works the same way every time. Arguments. The file is positional. `-f`, `-r/--resume-after`, `--skip-unsupported-providers` and the transformer branch of the agent error are removed. A human at a terminal is asked for the path, and asked for a source only when the file has none. An agent, a non-TTY run, or `--json` without the file exits 2. Re-runs. A run matches when the file's sha256, the source (and a custom source's hash) and the instance ID are all the same. The latest matching import run decides what happens: - An interrupted run continues and skips the users it created. - A partial run continues and retries its failed and skipped users. - A complete run does nothing: "Already imported in run X", exit 0. - An undone run, or no match, starts a new run. `--new-run` skips the lookup. A live lock exits 2. Checks. `checkImport()` runs before every import, and `--dry-run` stops after it. Each rejected user gets the first reason that applies: - schema-invalid - a source ID, email or phone that repeats in the file - a missing required identifier. An unverified email doesn't count where email is required (G20) - a bad hash shape for bcrypt, scrypt_firebase, argon2 or scrypt_werkzeug - a Supabase user whose only provider is disabled - a user already in the instance, found by a batched `GET /v1/users` lookup through the scheduler. Users created by the continued run are ignored. - a development instance's quota; `--allow-partial` imports up to the headroom in file order Any reject stops the import unless `--allow-partial` is passed. With it, each reject is recorded as `skipped` with its reason. Warnings cover fields the instance is not set up to store, fields Clerk won't store, and dropped passwords. Each flagged setting prints a `clerk config patch`. The readiness report's multiselect offer and its config writes are removed. Consent. Writing needs `--yes` or a yes at the prompt. Otherwise the run prints the checks and exits 2 with the exact command. This closes the gap where agent and non-TTY runs imported without consent. Output. Next steps point at `runs ` and `undo `. A complete import names the export and run folders it no longer needs, with the `rm -rf` for each. `--json` returns `{ target, run, resume, checks, result }`. The E2E test for G20 now checks both halves: BAPI refuses a user with no email, and the checks reject an unverified-only user up front. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 417 +++---- .../src/commands/migrate/index.test.ts | 27 +- .../cli-core/src/commands/migrate/index.ts | 47 +- .../src/commands/migrate/lib/checks.test.ts | 287 +++++ .../src/commands/migrate/lib/checks.ts | 476 +++++++ .../migrate/lib/modify-settings.test.ts | 80 +- .../commands/migrate/lib/modify-settings.ts | 60 +- .../commands/migrate/lib/readiness.test.ts | 232 +--- .../src/commands/migrate/lib/readiness.ts | 289 +---- .../src/commands/migrate/lib/transform.ts | 39 +- .../src/commands/migrate/lib/user-lookup.ts | 8 +- .../commands/migrate/run-interactive.test.ts | 444 +------ .../cli-core/src/commands/migrate/run.test.ts | 681 +++++----- packages/cli-core/src/commands/migrate/run.ts | 1102 ++++++++--------- .../src/commands/migrate/sources/supabase.ts | 2 +- .../src/commands/migrate/wizard.test.ts | 161 +-- .../cli-core/src/commands/migrate/wizard.ts | 107 +- test/e2e/migrate.test.ts | 30 +- 18 files changed, 2048 insertions(+), 2441 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/checks.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/checks.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 51e0b8fb0..8c9ab0311 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -29,80 +29,142 @@ and lists the subcommands below. The direction is always spelled out — `migrate import` moves users **into** Clerk, `migrate export` gets them **out** of a source platform — so neither is implied by the group. -### `clerk migrate import` (interactive) - -Bare `clerk migrate import` walks a human through the import instead of -demanding flags. - -```sh -clerk migrate import -``` - -It picks the source from a list built off the registry, asks for the file, -and collects Firebase's hash parameters when they are needed. Anything already -passed as a flag is not asked for. - -Then it prints the [Migration Readiness report](#migration-readiness-report), -offers to [change whatever it flagged](#changing-the-flagged-settings), and -waits for confirmation. Declining writes nothing to Clerk. - -**Agent mode never prompts.** `clerk migrate import` with no flags exits with a -usage error naming exactly what to pass: - -``` -`clerk migrate import` is interactive and cannot prompt in agent mode. -Pass the file (or an export run ID) and --source . -``` - ### `clerk migrate import` -Reads an exported user file, maps it onto Clerk's user schema, validates every -record, and creates the users through the Backend API. +Reads an exported user file, maps it onto Clerk's user schema, checks every +user against the destination instance, and creates them through the Backend +API. ```sh -clerk migrate import 20260929-141502-a1b2 -y # an export run -clerk migrate import users.json --source clerk -y +clerk migrate import 20260929-141502-a1b2 --dry-run # check, write nothing +clerk migrate import 20260929-141502-a1b2 --yes # an export run +clerk migrate import users.json --source clerk --yes # any other file +clerk migrate import # a human is asked ``` -| Flag | Description | -| --------------------------------------- | ---------------------------------------------------------------- | -| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | -| `--source ` | Where the file came from: a [source](#sources), or one you wrote | -| `-f, --file ` | Path to the export. `.json` or `.csv` | -| `-r, --resume-after ` | Skip every user up to and including this **source** ID | -| `--require-password` | Import only users that carry a password digest | -| `--skip-unsupported-providers` | Supabase: skip users whose only social provider is off in Clerk | -| `--firebase-signer-key ` | Firebase base64 signer key | -| `--firebase-salt-separator ` | Firebase base64 salt separator | -| `--firebase-rounds ` | Firebase scrypt rounds | -| `--firebase-mem-cost ` | Firebase scrypt memory cost | -| `-y, --yes` | Skip the confirmation prompt | -| `--runs-dir ` | Where runs are kept (see [Runs](#clerk-migrate-runs)) | +| Flag | Description | +| --------------------------------------- | ------------------------------------------------------------------- | +| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | +| `--source ` | Where the file came from: a [source](#sources), or one you wrote | +| `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | +| `--allow-partial` | Import the users that pass, and record the rest as skipped | +| `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | +| `--require-password` | Import only users that carry a password digest | +| `--firebase-signer-key ` | Firebase base64 signer key (overrides the export file) | +| `--firebase-salt-separator ` | Firebase base64 salt separator | +| `--firebase-rounds ` | Firebase scrypt rounds | +| `--firebase-mem-cost ` | Firebase scrypt memory cost | +| `-y, --yes` | Import without prompting | +| `--json` | Output as JSON. Never prompts, so importing needs `--yes` | +| `--runs-dir ` | Where runs are kept (see [Runs](#clerk-migrate-runs)) | Plus the targeting flags from the table above: `--secret-key`, `--app` and `--instance`. -The file is the positional argument or `--file`, not both. An export run ID -stands for the file that run wrote, and the import records it as `fromExport`. +An export run ID stands for the file that run wrote, and the import records it +as `fromExport`. A file `clerk migrate export` wrote carries its source, so it +needs no `--source`, and a `--source` that contradicts it exits 2. Any other +file — a bare JSON array, a CSV, Firebase's own `{ "users": [...] }` — needs +`--source`. + +**What a human is asked, and what an agent is told.** A human at a terminal who +leaves out the file is asked for its path, and is asked for a source only when +the file does not name one. An agent, a non-TTY run, or `--json` without the +file exits 2 naming what to pass. + +**Nothing is written without consent.** After the checks, a human is asked +`Import N users?`, and declining writes nothing. `--yes` skips the question. +Without either — an agent, a non-TTY run, `--json` — the run prints the checks +and exits 2 with the exact command to run. -A file `clerk migrate export` wrote carries its source, so it needs no -`--source`. A `--source` that contradicts it exits 2. Any other file -— a bare JSON array, a CSV, Firebase's own `{ "users": [...] }` — needs -`--source`, and omitting it fails with a usage error that names the valid -values. +**Every run prints its target first**, then which [case](#re-running) applies, +then the checks. Failures do not stop the run: each user's outcome is written to the -[run](#clerk-migrate-runs) and the import continues. A `429` backs off — honouring `Retry-After` when the response -carries it — and retries up to 5 times before the user is recorded as failed. -The command exits non-zero if any user failed. +[run](#clerk-migrate-runs) and the import continues. A `429` backs off — +honouring `Retry-After` when the response carries it — and retries up to 5 +times before the user is recorded as failed. The command exits 1 if any user +failed. + +An **unrecognized password hasher** aborts the whole run before anything is +sent, because it would import credentials nobody can sign in with. + +`--json` returns `{ target, run, resume, checks, result }`. + +#### Re-running + +Running the same import again continues where it left off. The match is the +file's sha256, the source (and a custom source's content hash), and the +instance ID; the latest matching import run decides what happens: + +| Latest match | Re-running does | +| ----------------------------------------- | ---------------------------------------------------------------------- | +| none | a new run | +| interrupted (dead lock or no finish time) | continues the same run, skipping the users it created | +| `partial` | continues the same run, retrying the users that failed or were skipped | +| `complete` | nothing: prints "Already imported in run …" and exits 0 | +| `undone` | a new run | + +`--new-run` skips the lookup. A run another live process holds exits 2. + +When an import completes, it names the folders it no longer needs: the export it +read, which holds your users' data, and its own run, which only `undo` needs. +Each comes with the `rm -rf` to remove it. + +#### Checks + +Every import runs the checks before writing anything, and `--dry-run` stops +after them. They sort the users three ways: + +- **Rejected** — users Clerk would refuse. Each gets the first reason that + applies: + - it failed schema validation + - its source ID, email or phone repeats an earlier user in the file + - it lacks an identifier the instance requires. An email or phone counts + only when it is verified, because an unverified one is attached after the + user exists + - its password is not the shape its hasher says (`bcrypt`, `scrypt_firebase`, + `argon2i`/`argon2id` and `scrypt_werkzeug` are checked; other hashers are + not) + - Supabase: its only provider is not enabled in Clerk + - the instance already has a user with its source ID, email, phone or + username (a batched `GET /v1/users` lookup, 100 values a request, through + the scheduler). The users a continued run created do not count + - a development instance: it is past the 100-user headroom, counted in file + order +- **Imported, but not everything comes across** — fields the instance is not + set up to store, fields Clerk has no place for (`Clerk won't store: …`), and + passwords a source had to drop. +- **Imported** — everyone else. + +Any reject stops the import, and it exits 2 with the command that adds +`--allow-partial`. With `--allow-partial`, the rest import and each reject is +recorded as `skipped` with its reason. `--dry-run` exits 2 when the real run +would be refused, and 0 otherwise. -Two failures do abort the whole run, because continuing would produce a -corrupt instance: +``` +Checks + 120 users checked + ✗ 12 users rejected + 12: only has an unverified email, and this instance requires an email + u_17, u_22, u_40, u_51, u_88, and 7 more + ⚠ Imported, but not everything comes across + 6 users have a username, which this instance is not set up to store + Clerk won't store: department (120 users) + ✓ 108 users to import + +Or change the instance instead + Make Email optional at sign-up + clerk config patch --app app_… --instance ins_… --json '{"auth_email":{"required_for_sign_up":false}}' + Enable Username + clerk config patch --app app_… --instance ins_… --json '{"auth_username":{"used_for_sign_up":true}}' +``` -- An **unrecognized password hasher**, which would import credentials nobody can - sign in with. -- A **`--resume-after` ID that is not in the file**, which would otherwise - re-import every user the previous run already created. +The fixes are offers, not corrections: an instance that requires an email is +configured as its owner intended, and fixing the export may be the answer. When +the instance settings cannot be read (BAPI `/v1/domains` → the instance's +Frontend API `/v1/environment`), required fields are not checked and the run +says so. #### Additional identifiers @@ -125,17 +187,12 @@ that, assuming ~100ms of API latency. Both are overridable: A non-numeric or non-positive value is ignored in favour of the default. -**Development instances warn when an import may exceed their user limit.** New -development instances are created with a 100-user limit; production instances -have none. Before importing, the run reads the instance's current user count -(`GET /v1/users/count`) and warns when the file would take it past 100. - -The run then stops and asks before going ahead. It is a prompt rather than a -hard refusal because the number checked against may not be this instance's: -Clerk raises a development instance's limit on request, and the raised value -(`max_allowed_users`) is not served by BAPI, DAPI or FAPI — so the CLI can show -the live count but never the live limit. Declining aborts before anything is -written to Clerk; `-y` and agent mode proceed on the warning alone. +A development instance's user limit is checked with the other +[checks](#checks): new development instances are created with a 100-user limit, +production instances have none, and the run reads the live count +(`GET /v1/users/count`). The limit itself is not served by any API, so a +development instance Clerk has raised may accept more than the checks allow; +`--allow-partial` imports up to the headroom. Users that do exceed the limit come back in the error breakdown as `You have reached your limit of N users`, annotated with what a development @@ -341,8 +398,8 @@ unreachable host and a closed port as "Connection closed", so: **`supabase` reads the database rather than the Admin API** because `encrypted_password` exists only there. An API-based export would force every user to reset their password; this one carries the bcrypt digests across. It -also keeps `raw_app_meta_data`, which is what `--skip-unsupported-providers` -reads at import time. +also keeps `raw_app_meta_data`, which is what the import's +[checks](#checks) read for each user's providers. **`authjs` tries `User`, then `user`, then `users`.** Auth.js has no single schema — Prisma capitalizes the table, Drizzle does not, and Postgres treats @@ -749,19 +806,6 @@ Firebase secret. An export with no password hashes needs no parameters at all. -### `--skip-unsupported-providers` (Supabase) - -Reads each user's `raw_app_meta_data.providers` and cross-references it against -the social providers the destination instance has enabled (via BAPI -`/v1/domains` → the instance's Frontend API `/v1/environment`). - -A user is skipped **only when every one of their providers is disabled**. Anyone -who can still sign in another way — email, phone, or an enabled social provider -— is imported. The number skipped is reported, broken down by provider. - -If the instance configuration cannot be read, nobody is skipped and a warning is -printed: a failed lookup must not be mistaken for "no providers are enabled". - ## Schema fields What a source maps _onto_. Every user is validated against this schema @@ -836,172 +880,6 @@ stamping every user with today's. | `skipLegalChecks` | `boolean` | Skip legal acceptance checks | | `skipPasswordChecks` | `boolean` | Skip password requirements on import | -## Migration Readiness report - -Printed immediately before the confirmation prompt, so declining aborts with -nothing written to Clerk. Skipped only for `-y`, which says "don't ask, don't -lecture" and should not pay for the two extra round-trips. Agent runs without -`-y` still get it — an agent can act on it exactly as a human would. - -It cross-references the file against the destination instance's live settings -(BAPI `/v1/domains` → that instance's Frontend API `/v1/environment`) and -answers the two questions worth answering before writing anything: **who won't -be imported**, and **who will arrive incomplete**. - -``` -Migration readiness - 120 users in this file - 3 failed validation and will be skipped - - ✗ 12 users will not be imported - 12 have no email, which this instance requires - If you import them, this applies to them too: - 12 have a phone, which this instance is not set up to store - ⚠ 20 users will be imported, but not everything they carry - 14 have no password, which this instance requires — they will have to reset it to sign in - 6 have a username, which this instance is not set up to store - ✓ 88 users will be imported in full - -Identifiers - ⚠ Email — required in Clerk, and not every user has one — 108/120 users - ⚠ Username — not enabled in Clerk — 6/120 users - -Social connections - ✓ Google — enabled in Clerk — 40/120 users - ⚠ Discord — not enabled in Clerk — 12/120 users - -⚠ 3 settings need attention -``` - -### The two blocks - -**The outcome block** classifies each user **once**, into the worst outcome that -applies to them, so its three totals add up to the file. This matters: per-field -coverage cannot answer "how many won't be imported", because the users missing -an email and the users missing a password overlap by an amount only a per-user -pass knows. A user rejected for their missing email is not also counted under -the missing password they happen to share. - -**"If you import them, this applies to them too"** is the part that stops the -settings interacting invisibly. A user who is not being created cannot lose a -field, so a setting that only affects rejected users costs nothing _today_ and -would otherwise never be mentioned — right up until the operator relaxes the -requirement rejecting them, at which point all of it lands at once. Naming it -up front is what turns - -> make email optional → re-check → discover the phones are being dropped → -> enable phone → re-check - -into a single decision with both offers visible. It is also why a setting can -be flagged in the section rows while contributing nothing to the ✗/⚠/✓ totals. - -**The section rows below** are the other question — per-field coverage against -each setting — and deliberately do not restate user counts, which would read as -contradicting the block above. - -### Which settings cost what - -| Setting | Consequence | -| ------------------------------------------ | --------------------------------------------------------------------------------------------- | -| Identifier (email/phone/username) required | **Not imported.** `POST /v1/users` enforces the sign-up identifier requirements. | -| Password required, user has none | **Imported without a password.** The import sends `skip_password_requirement`, so the user is | -| | created and has to reset their password before they can sign in with one. | -| Attribute disabled in Clerk | **Imported without that field.** The instance has nowhere to put it. | -| Social provider disabled | **Imported**, but that sign-in method is unavailable to them. | - -Social rows are not part of the per-user outcome counts: which providers a user -signed up with lives in the raw export rather than the transformed user, so it -cannot be attributed per user. Their coverage row still names them. - -If the instance settings cannot be read — the secret key is rejected, or FAPI -is unreachable — the report degrades to a coverage-only listing with a note. -Nothing is flagged in that case: "could not read" is not the same as "switched -off", and treating it as such would raise alarms about settings that are -perfectly fine. - -### Changing the flagged settings - -When the report flags anything, a human run offers one selectable change per -flagged row before the import confirmation, so acting on the report does not -mean leaving the CLI for the dashboard: - -``` -Update this instance's settings first? (enter to skip) - ◻ Make Email optional at sign-up - ◻ Enable Discord sign-in - ↑/↓ to navigate • Space: select • a: all • Enter: confirm -``` - -**Nothing is preselected** — relaxing an instance's sign-up requirements is a -real decision, not a default — and selecting nothing continues to the import -prompt with the instance untouched, which is what "enter to skip" is there to -say. - -`a: all` is added to clack's legend in `lib/prompts.ts`: `MultiSelectPrompt` -has always bound `a` to toggle everything (and `i` to invert), but clack's -footer never listed them and takes no override, so the key was undiscoverable. -It applies to every multiselect in the CLI, because it is a property of the -prompt rather than of any one question. - -These are offers, not corrections: **a flagged setting is not a wrong setting.** -An instance that genuinely requires an email address is configured exactly as -its owner intended, and the right answer may well be to fix the export instead. - -Whatever is selected becomes a single `PATCH` of the instance config document, -the same document `clerk config patch` writes. The report is then redrawn so -the confirmation that follows is against the settings the write established. - -**The offer repeats while anything is still flagged.** A redraw is another -decision point, not a receipt: applying one change routinely leaves others -worth making, and each round re-offers only what is left. It ends when the -report has nothing flagged, when the operator selects nothing, or when there is -nothing offerable for the rows that remain — so reaching the second change -never costs a second run of the command. - -The redraw is computed from the write, **not** from a second settings fetch. -Clerk's Frontend API is eventually consistent, so a `/v1/environment` read -issued this soon after the config write routinely still reports the pre-write -settings — which would redraw the report with every row the operator just -cleared still flagged. The Platform API accepting the write is the -authoritative statement of what took, exactly as `clerk config patch` treats -it (see that command's [round-trip verification](../config/README.md#round-trip-verification) -notes for the same reasoning). - -The config leaves each option writes are not shown in the prompt — internal -detail an operator cannot act on — but they are fixed and listed here: - -| Flagged row | Change offered | -| ------------------------------- | --------------------------------------------------------------------------------- | -| Email/Phone/Username — required | `auth_.required_for_sign_up → false` | -| Email — disabled | `auth_email.used_for_sign_up → true` + `verification_strategies → ["email_code"]` | -| Phone — disabled | `auth_phone.used_for_sign_up → true` + `verification_strategies → ["phone_code"]` | -| Username — disabled | `auth_username.used_for_sign_up → true` | -| Password — required / disabled | `auth_password.required → false` / `auth_password.enabled → true` | -| First/Last name | `user_model..required → false` / `user_model..enabled → true` | -| Social provider — disabled | `connection_oauth_.enabled → true` | - -`used_for_sign_up` is the enable field that matters: `POST /v1/users` validates -an import against the instance's sign-up requirements, not its sign-in -strategies. - -**Email and phone take two writes, not one.** They are _verifiable_ attributes, -and Clerk rejects one that is on with no way to verify it: - -``` -422 phone_number: verifiable attributes need to have at least one verification -``` - -Switching the attribute off empties `verification_strategies`, so whatever -turns it back on has to put a strategy back in the same request. Username, -password and the name fields are not verifiable and take one write each. - -The offer is skipped entirely for `-y` and in agent mode, both of which say -"don't prompt". It also stands down, with a warning rather than a failed run, -when the instance to configure cannot be resolved (a bare `--secret-key` in an -unlinked directory) or when it is a **keyless** application — the Backend API -those are reachable through has no route for any of these settings, so -`clerk auth login` is the way in. - ## Artifacts | Path | Contents | @@ -1033,22 +911,23 @@ grep '"sourceId":"user_123"' .clerk/migrate/20260929-141502-a1b2/users.ndjson ## API Endpoints -| Method | Path | Used by | -| ------ | -------------------------- | ------------------------------------------------------------------------------------ | -| `POST` | `/v1/users` | `migrate import` — creates each user | -| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | -| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | -| `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | -| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | -| `GET` | `/v1/domains` | Readiness report and `--skip-unsupported-providers` — resolves the Frontend API host | - -The readiness report also reads the instance's Frontend API -`GET /v1/environment` (bootstrapping a dev browser first on development -instances), and its settings-change offer writes through the Platform API: - -| Method | Path | Used by | -| ------- | ----------------------------------------------------------------- | ------------------------------------------------------ | -| `PATCH` | `/v1/platform/applications/{appID}/instances/{instanceID}/config` | Applying the settings changes selected from the report | +| Method | Path | Used by | +| -------- | -------------------------- | ------------------------------------------------------------------------------ | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | +| `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | +| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | +| `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | +| `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | +| `GET` | `/v1/instance` | `migrate import`, `undo`, `export clerk` — names the instance behind the key | +| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | + +The checks also read the instance's Frontend API `GET /v1/environment` +(bootstrapping a dev browser first on development instances) for its +attributes and enabled social providers. Nothing in `clerk migrate` writes +instance settings: the checks print the `clerk config patch` to run instead. Three exports talk to their own platform rather than to Clerk: @@ -1069,10 +948,6 @@ The two Identity Toolkit paths are on `identitytoolkit.googleapis.com`, or on The three database exports (`supabase`, `authjs`, `betterauth`) make no HTTP calls at all — they connect over `--db-url`. -The readiness report and `--skip-unsupported-providers` additionally read the -instance's Frontend API `GET /v1/environment` for its attributes and enabled -social providers. - ## Notes - `userId` in the source file becomes the Clerk user's `external_id`. That is diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index cd617e397..a0373c82f 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -35,10 +35,11 @@ describe("registerMigrate", () => { test.each([ "--source", - "--file", - "--resume-after", + "--dry-run", + "--allow-partial", + "--new-run", "--require-password", - "--skip-unsupported-providers", + "--json", "--firebase-signer-key", "--firebase-salt-separator", "--firebase-rounds", @@ -62,7 +63,13 @@ describe("registerMigrate", () => { expect(findCommand(["migrate", ...names])).toBeUndefined(); }); - test.each(["--transformer", "--transformer-file"])("migrate import drops %s", (flag) => { + test.each([ + "--transformer", + "--transformer-file", + "--file", + "--resume-after", + "--skip-unsupported-providers", + ])("migrate import drops %s", (flag) => { expect(findCommand(["migrate", "import"])?.options.map((o) => o.long)).not.toContain(flag); }); @@ -182,11 +189,13 @@ describe("registerMigrate", () => { expect(option?.argChoices).toBeUndefined(); }); - test.each([ - ["-f", "--file"], - ["-r", "--resume-after"], - ["-y", "--yes"], - ])("exposes %s as the short form of %s", (short, long) => { + test("takes the file, or the export run that wrote it, as an optional argument", () => { + const [argument] = findCommand(["migrate", "import"])?.registeredArguments ?? []; + expect(argument?.name()).toBe("file|export-run-id"); + expect(argument?.required).toBe(false); + }); + + test.each([["-y", "--yes"]])("exposes %s as the short form of %s", (short, long) => { const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === long); expect(option?.short).toBe(short); }); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index 43ec89376..a55bc9259 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -16,19 +16,17 @@ export function registerMigrate(program: Program): void { .command("migrate") .description("Migrate users into Clerk from another auth provider or another Clerk instance") .setExamples([ - { command: "clerk migrate import", description: "Walk through an import interactively" }, { - command: "clerk migrate import users.json --source clerk -y", - description: "Import users from a Clerk export", + command: "clerk migrate export supabase", + description: "Export users from Supabase into a new run", }, { - command: - "clerk migrate import users.json --source supabase --skip-unsupported-providers -y", - description: "Skip Supabase users whose provider is not enabled", + command: "clerk migrate import 20260929-141502-a1b2 --dry-run", + description: "Check an export against the instance, and write nothing", }, { - command: "clerk migrate export supabase", - description: "Export users from Supabase, ready to import", + command: "clerk migrate import 20260929-141502-a1b2 --yes", + description: "Import it", }, { command: "clerk migrate runs", description: "List every migration run" }, { @@ -68,14 +66,11 @@ export function registerMigrate(program: Program): void { "--source ", "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`", ) - .option("-f, --file ", "Path to the exported user data (JSON or CSV)") - .option("-r, --resume-after ", "Skip every user up to and including this source ID") + .option("--dry-run", "Check the file against the instance, report, and write nothing") + .option("--allow-partial", "Import the users that pass the checks, and skip the rest") + .option("--new-run", "Start a new run instead of continuing an earlier one of this file") .option("--require-password", "Import only users that have a password") - .option( - "--skip-unsupported-providers", - "Supabase: skip users whose only social provider is not enabled in Clerk", - ) - .option("--firebase-signer-key ", "Firebase base64 signer key") + .option("--firebase-signer-key ", "Firebase base64 signer key (overrides the export file)") .option("--firebase-salt-separator ", "Firebase base64 salt separator") .option("--firebase-rounds ", "Firebase scrypt rounds", (value: string) => parseIntegerOption(value, "--firebase-rounds", { min: 1 }), @@ -83,33 +78,29 @@ export function registerMigrate(program: Program): void { .option("--firebase-mem-cost ", "Firebase scrypt memory cost", (value: string) => parseIntegerOption(value, "--firebase-mem-cost", { min: 1 }), ) - .option("-y, --yes", "Skip the confirmation prompt") + .option("-y, --yes", "Import without prompting") + .option("--json", "Output as JSON; never prompts, so pair it with --yes to import") .option("--secret-key ", "Backend API secret key to use") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) .setExamples([ { - command: "clerk migrate import 20260929-141502-a1b2 -y", - description: "Import what an export run wrote", + command: "clerk migrate import 20260929-141502-a1b2 --dry-run", + description: "Check what an export run wrote, and write nothing", }, { - command: "clerk migrate import users.json --source clerk -y", - description: "Import a Clerk Dashboard export", + command: "clerk migrate import 20260929-141502-a1b2 --yes", + description: "Import it. Run it again to continue after a failure", }, { - command: "clerk migrate import users.csv --source clerk --require-password -y", - description: "Import only the users that carry a password digest", + command: "clerk migrate import users.json --source clerk --allow-partial --yes", + description: "Import a Clerk Dashboard export, skipping users that would be rejected", }, { - command: "clerk migrate import users.json --source ./my-source.ts -y", + command: "clerk migrate import users.json --source ./my-source.ts --yes", description: "Import with a source you wrote", }, - { - command: - "clerk migrate import users.json --source supabase --skip-unsupported-providers -y", - description: "Skip Supabase users whose only provider is not enabled in Clerk", - }, ]) .action(async (input, _opts, cmd) => migrate.run({ diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts new file mode 100644 index 000000000..c3737cdd5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -0,0 +1,287 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import type { User } from "../types.ts"; +import { checkImport, hashShapeProblem, type CheckInput } from "./checks.ts"; + +const BCRYPT = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"; + +const settings = (attributes: object, social: object = {}) => + ({ attributes, social }) as unknown as UserSettingsJSON; + +const EMAIL_REQUIRED = settings({ email_address: { enabled: true, required: true } }); + +const user = (userId: string, fields: Partial = {}): User => + ({ userId, email: `${userId}@x.dev`, ...fields }) as User; + +let originalFetch: typeof globalThis.fetch; +/** Users `GET /v1/users` finds, whatever the filter. */ +let existing: Record[]; + +beforeAll(() => { + originalFetch = globalThis.fetch; +}); + +afterAll(() => { + globalThis.fetch = originalFetch; +}); + +beforeEach(() => { + existing = []; + globalThis.fetch = (async (input: string | URL | Request) => { + const url = new URL(input.toString()); + const wanted = new Set(url.searchParams.values()); + return Response.json( + existing.filter((candidate) => + [ + candidate.external_id, + candidate.username, + ...((candidate.email_addresses as { email_address: string }[] | undefined) ?? []).map( + (email) => email.email_address, + ), + ].some((value) => wanted.has(value as string)), + ), + ); + }) as typeof fetch; +}); + +function input(overrides: Partial = {}): CheckInput { + return { + users: [], + failures: [], + settings: null, + instanceType: "prod", + target: { + env: "production", + instanceId: "ins_1", + instanceType: "prod", + keySource: "--secret-key", + }, + secretKey: "sk_live_x", + schedule: async (fn) => fn(), + ...overrides, + }; +} + +const reasonsOf = async (overrides: Partial) => + Object.fromEntries( + (await checkImport(input(overrides))).rejects.map((reject) => [reject.sourceId, reject.reason]), + ); + +describe("rejects", () => { + test("a user that failed validation", async () => { + const checks = await checkImport( + input({ failures: [{ userId: "bad", row: 0, error: "Invalid email", path: ["email"] }] }), + ); + expect(checks.rejects).toEqual([{ sourceId: "bad", reason: "invalid: Invalid email" }]); + expect(checks.total).toBe(1); + }); + + test("a duplicate source ID, email or phone within the file", async () => { + expect( + await reasonsOf({ + users: [ + user("a"), + user("a", { email: "other@x.dev" }), + user("b", { email: "A@x.dev" }), + user("c", { phone: "+15555550100" }), + user("d", { phone: "+15555550100" }), + ], + }), + ).toEqual({ + a: "duplicate source ID in the file", + b: "email is also used by another user in the file", + d: "phone number is also used by another user in the file", + }); + }); + + // G20: an unverified email is attached after the user exists, so it cannot + // satisfy a sign-up requirement. + test("a user with only an unverified email, where email is required", async () => { + expect( + await reasonsOf({ + settings: EMAIL_REQUIRED, + users: [ + user("ok"), + user("unverified", { email: undefined, unverifiedEmailAddresses: ["u@x.dev"] }), + user("none", { email: undefined, username: "none" }), + ], + }), + ).toEqual({ + unverified: "only has an unverified email, and this instance requires an email", + none: "no email, which this instance requires", + }); + }); + + test("a password that is not the shape its hasher says", async () => { + expect( + await reasonsOf({ + users: [ + user("good", { password: BCRYPT, passwordHasher: "bcrypt" }), + user("bad", { password: "not-a-hash", passwordHasher: "bcrypt" }), + ], + }), + ).toEqual({ bad: "password is not a bcrypt hash ($2a$/$2b$/$2y$, 60 characters)" }); + }); + + test("a user already in the instance, by source ID, email or username", async () => { + existing = [ + { id: "user_1", external_id: "by-id" }, + { id: "user_2", email_addresses: [{ email_address: "by-email@x.dev" }] }, + { id: "user_3", username: "taken" }, + ]; + + expect( + await reasonsOf({ + users: [ + user("by-id"), + user("by-email"), + user("by-username", { username: "Taken" }), + user("new"), + ], + }), + ).toEqual({ + "by-id": "already in the instance, with this source ID", + "by-email": "email is already used by a user in the instance", + "by-username": "username is already taken in the instance", + }); + }); + + // Continuing a run finds the users it created: that is expected, not a clash. + test("not a user the run being continued created", async () => { + existing = [{ id: "user_1", external_id: "mine" }]; + + expect( + await reasonsOf({ users: [user("mine")], continuedClerkIds: new Set(["user_1"]) }), + ).toEqual({}); + }); + + test("a supabase user whose only provider is disabled", async () => { + const rows = [ + { id: "only-discord", raw_app_meta_data: { providers: ["discord"] } }, + { id: "has-email", raw_app_meta_data: { providers: ["email", "discord"] } }, + ]; + expect( + await reasonsOf({ + settings: settings( + { email_address: { enabled: true } }, + { oauth_google: { enabled: true } }, + ), + supabaseRows: rows, + users: [user("only-discord"), user("has-email")], + }), + ).toEqual({ "only-discord": "only signs in with Discord, which is not enabled in Clerk" }); + }); + + test("the users past a development instance's headroom, in file order", async () => { + const checks = await checkImport( + input({ + instanceType: "dev", + existingUsers: 98, + users: [user("a"), user("b"), user("c")], + }), + ); + expect(checks.importable.map((entry) => entry.userId)).toEqual(["a", "b"]); + expect(checks.rejects).toEqual([ + { sourceId: "c", reason: "over the development instance's 100-user limit" }, + ]); + expect(checks.quota).toEqual({ existing: 98, limit: 100, headroom: 2, over: 1 }); + }); + + test("groups the rejects by reason", async () => { + const checks = await checkImport( + input({ + settings: EMAIL_REQUIRED, + users: [ + user("a", { email: undefined, username: "a" }), + user("b", { email: undefined, username: "b" }), + ], + }), + ); + expect(checks.rejectReasons).toEqual([ + { reason: "no email, which this instance requires", count: 2 }, + ]); + }); +}); + +describe("warnings", () => { + test("a field the instance is not set up to store", async () => { + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true }, phone_number: { enabled: false } }), + users: [user("a", { phone: "+15555550100" })], + }), + ); + expect(checks.warnings).toContain( + "1 user has a phone, which this instance is not set up to store", + ); + expect(checks.rejects).toEqual([]); + }); + + test("fields Clerk has no place for", async () => { + const checks = await checkImport( + input({ users: [user("a")], unknownFields: { department: 3, role: 1 } }), + ); + expect(checks.warnings).toContain("Clerk won't store: department (3 users), role (1 user)"); + }); + + test("passwords a source had to drop", async () => { + const checks = await checkImport(input({ users: [user("a", { passwordDropped: true })] })); + expect(checks.warnings[0]).toContain("1 password Clerk cannot verify will be dropped"); + }); +}); + +describe("fixes", () => { + test("print a `clerk config patch` per flagged setting, targeting the instance", async () => { + const checks = await checkImport( + input({ + settings: settings({ + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }), + target: { + env: "production", + appId: "app_1", + instanceId: "ins_1", + instanceType: "prod", + keySource: "linked profile", + }, + users: [user("a"), user("b", { email: undefined, username: "b" })], + }), + ); + expect(checks.fixes).toEqual([ + { + label: "Make Email optional at sign-up", + command: `clerk config patch --app app_1 --instance ins_1 --json '{"auth_email":{"required_for_sign_up":false}}'`, + }, + ]); + }); + + test("offer nothing when the settings could not be read", async () => { + const checks = await checkImport(input({ users: [user("a")] })); + expect(checks.fixes).toEqual([]); + expect(checks.settingsUnavailable).toBe(true); + }); +}); + +describe("hashShapeProblem", () => { + test.each([ + [BCRYPT, "bcrypt"], + ["hash$salt$signer$sep$8$14", "scrypt_firebase"], + ["$argon2id$v=19$m=65536,t=3,p=4$c2FsdA$aGFzaA", "argon2id"], + [`scrypt:16384:16:1$${"a".repeat(32)}$${"b".repeat(128)}`, "scrypt_werkzeug"], + // Not checked: a bad digest there fails only at sign-in. + ["anything", "pbkdf2_sha256"], + ])("accepts %s as %s", (password, hasher) => { + expect(hashShapeProblem(password, hasher)).toBeUndefined(); + }); + + test.each([ + ["$2a$10$short", "bcrypt"], + ["hash$salt$signer$sep$eight$14", "scrypt_firebase"], + ["hash$salt", "scrypt_firebase"], + ["argon2id$...", "argon2id"], + ["scrypt:16384:16:1$salt$not-hex!", "scrypt_werkzeug"], + ])("rejects %s as %s", (password, hasher) => { + expect(hashShapeProblem(password, hasher)).toBeDefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts new file mode 100644 index 000000000..391a9065e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -0,0 +1,476 @@ +/** + * `checkImport()`: what the instance will do with each user, worked out before + * anything is written. + * + * Every import runs these checks, and `--dry-run` stops after them. They sort + * the file into three piles: + * + * - **Rejects** — users Clerk would refuse, or that the run would refuse on + * Clerk's behalf. Any reject stops the import unless `--allow-partial` is + * passed, and then each one is recorded as `skipped` with its reason. + * - **Warnings** — users that import, but lose something on the way: a field + * the instance is not set up to store, a field Clerk has no place for, a + * password it cannot verify. + * - **Importable** — everyone else, in file order. + * + * Each rejected user gets the first reason that applies to them, so the counts + * add up to the file. + */ + +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import type { SpinnerControls } from "../../../lib/spinner.ts"; +import { isEnabled, isRequired, type AttributeName } from "../../users/interactive/attributes.ts"; +import { splitIdentifiers } from "../import-users.ts"; +import type { User } from "../types.ts"; +import { analyzeFields, hasValue } from "./analysis.ts"; +import { enabledSocialProviders, providerLabel, toClerkStrategy } from "./clerk-config.ts"; +import { DEV_USER_LIMIT } from "./instance.ts"; +import { buildChangePayload, buildSettingChanges } from "./modify-settings.ts"; +import { buildReadinessReport } from "./readiness.ts"; +import type { ApiScheduler } from "./scheduler.ts"; +import { + countSocialProviders, + findDisabledProviders, + findUsersWithOnlyDisabledProviders, +} from "./supabase-providers.ts"; +import type { ClerkTarget } from "./target.ts"; +import type { ValidationFailure } from "./transform.ts"; +import { lookupUsers, type LookedUpUser } from "./user-lookup.ts"; + +export type Reject = { sourceId: string; reason: string }; + +export type ReasonCount = { reason: string; count: number }; + +/** A `clerk config patch` that would stop a setting costing users. */ +export type Fix = { label: string; command: string }; + +export type Quota = { + /** Users already in the instance, or `null` when the count could not be read. */ + existing: number | null; + limit: number; + headroom: number; + /** Importable users over the headroom. */ + over: number; +}; + +export type ImportChecks = { + /** Users in the file being considered, valid or not. */ + total: number; + /** Users that pass every check, in file order. */ + importable: User[]; + /** Users that do not, each with the first reason that applies to them. */ + rejects: Reject[]; + rejectReasons: ReasonCount[]; + warnings: string[]; + fixes: Fix[]; + quota?: Quota; + /** True when the instance's settings could not be read. */ + settingsUnavailable: boolean; +}; + +export type CheckInput = { + users: User[]; + failures: ValidationFailure[]; + /** Fields the source produced that Clerk has no place for → how many users carry each. */ + unknownFields?: Record; + /** The raw Supabase rows, for the provider checks. Absent for any other source. */ + supabaseRows?: Record[]; + settings: UserSettingsJSON | null; + /** Users already in the instance, for the quota. Development instances only. */ + existingUsers?: number | null; + instanceType: "dev" | "prod"; + target: ClerkTarget; + secretKey: string; + schedule: ApiScheduler; + /** Clerk IDs the run being continued created: finding them in the instance is expected. */ + continuedClerkIds?: Set; + spinner?: SpinnerControls; +}; + +const plural = (count: number, word: string) => `${count} ${word}${count === 1 ? "" : "s"}`; + +// --- Hash shapes ----------------------------------------------------------- + +/** + * Why a password digest cannot be what its hasher says it is, for the hashers + * whose shape is cheap and certain to check. + * + * Every other hasher is "can't verify": its shape is not checked, and a bad + * digest there still fails only at sign-in. + */ +export function hashShapeProblem(password: string, hasher: string): string | undefined { + switch (hasher) { + case "bcrypt": + return /^\$2[aby]\$\d\d\$[./A-Za-z0-9]{53}$/.test(password) + ? undefined + : "password is not a bcrypt hash ($2a$/$2b$/$2y$, 60 characters)"; + case "scrypt_firebase": { + const parts = password.split("$"); + const numeric = (value: string | undefined) => /^\d+$/.test(value ?? ""); + return parts.length === 6 && + parts.slice(0, 4).every(Boolean) && + numeric(parts[4]) && + numeric(parts[5]) + ? undefined + : "password is not a Firebase scrypt digest (hash$salt$signerKey$saltSeparator$rounds$memCost)"; + } + case "argon2i": + case "argon2id": + return password.startsWith("$argon2") ? undefined : "password is not an argon2 hash"; + case "scrypt_werkzeug": + return /^scrypt:\d+:\d+:\d+\$[^$]+\$[0-9a-f]+$/i.test(password) + ? undefined + : "password is not a Werkzeug scrypt hash (scrypt:N:r:p$salt$hex)"; + default: + return undefined; + } +} + +// --- Per-user checks ------------------------------------------------------- + +/** + * Identifiers Clerk creates a user under. A required one the user does not + * carry leaves nothing to create them with. + * + * An email or phone counts only when it is verified: an unverified one is + * attached after the user exists, so it cannot satisfy a sign-up requirement. + */ +function missingRequiredIdentifier( + user: User, + settings: UserSettingsJSON | null, +): string | undefined { + if (!settings) return undefined; + const required = (attribute: AttributeName) => isRequired(settings, attribute); + const identifiers = splitIdentifiers(user); + + if (required("email_address") && !identifiers.primaryEmail) { + return identifiers.unverifiedEmails.length > 0 + ? "only has an unverified email, and this instance requires an email" + : "no email, which this instance requires"; + } + if (required("phone_number") && !identifiers.primaryPhone) { + return identifiers.unverifiedPhones.length > 0 + ? "only has an unverified phone number, and this instance requires one" + : "no phone number, which this instance requires"; + } + if (required("username") && !hasValue(user.username)) { + return "no username, which this instance requires"; + } + return undefined; +} + +/** First user in the file to claim each email, phone and source ID. */ +function findFileDuplicates(users: User[]): Map { + const reasons = new Map(); + const seenIds = new Set(); + const emails = new Map(); + const phones = new Map(); + + for (const user of users) { + if (seenIds.has(user.userId)) { + reasons.set(user.userId, "duplicate source ID in the file"); + continue; + } + seenIds.add(user.userId); + + const identifiers = splitIdentifiers(user); + const ownEmails = [identifiers.primaryEmail, ...identifiers.additionalEmails].filter( + (value): value is string => Boolean(value), + ); + const ownPhones = [identifiers.primaryPhone, ...identifiers.additionalPhones].filter( + (value): value is string => Boolean(value), + ); + + const emailOwner = ownEmails.map((email) => emails.get(email.toLowerCase())).find(Boolean); + const phoneOwner = ownPhones.map((phone) => phones.get(phone)).find(Boolean); + if (emailOwner) { + reasons.set(user.userId, "email is also used by another user in the file"); + continue; + } + if (phoneOwner) { + reasons.set(user.userId, "phone number is also used by another user in the file"); + continue; + } + for (const email of ownEmails) emails.set(email.toLowerCase(), user.userId); + for (const phone of ownPhones) phones.set(phone, user.userId); + } + return reasons; +} + +/** + * Users the instance already holds, found by source ID, primary email, + * primary phone or username. + * + * Only what `POST /v1/users` itself carries is looked up: an extra email that + * collides is attached after the user exists, fails on its own, and is noted + * on the user's line rather than failing them. + */ +async function findInstanceDuplicates( + users: User[], + input: CheckInput, +): Promise> { + const byExternalId = new Map(); + const byEmail = new Map(); + const byPhone = new Map(); + const byUsername = new Map(); + + for (const user of users) { + const identifiers = splitIdentifiers(user); + byExternalId.set(user.userId, user.userId); + if (identifiers.primaryEmail) byEmail.set(identifiers.primaryEmail.toLowerCase(), user.userId); + if (identifiers.primaryPhone) byPhone.set(identifiers.primaryPhone, user.userId); + if (typeof user.username === "string" && user.username) { + byUsername.set(user.username.toLowerCase(), user.userId); + } + } + + const lookup = async ( + filter: "external_id" | "email_address" | "phone_number" | "username", + values: Iterable, + ) => + lookupUsers({ + filter, + values: [...values], + secretKey: input.secretKey, + schedule: input.schedule, + spinner: input.spinner, + label: "Checking for users already in the instance", + }); + + const found: LookedUpUser[] = ( + await Promise.all([ + lookup("external_id", byExternalId.keys()), + lookup("email_address", byEmail.keys()), + lookup("phone_number", byPhone.keys()), + lookup("username", byUsername.keys()), + ]) + ).flat(); + + const reasons = new Map(); + const claim = (sourceId: string | undefined, reason: string) => { + if (sourceId && !reasons.has(sourceId)) reasons.set(sourceId, reason); + }; + + for (const existing of found) { + if (input.continuedClerkIds?.has(existing.id)) continue; + if (existing.external_id) { + claim(byExternalId.get(existing.external_id), "already in the instance, with this source ID"); + } + for (const email of existing.email_addresses ?? []) { + claim( + byEmail.get((email.email_address ?? "").toLowerCase()), + "email is already used by a user in the instance", + ); + } + for (const phone of existing.phone_numbers ?? []) { + claim( + byPhone.get(phone.phone_number ?? ""), + "phone number is already used by a user in the instance", + ); + } + if (existing.username) { + claim( + byUsername.get(existing.username.toLowerCase()), + "username is already taken in the instance", + ); + } + } + return reasons; +} + +/** Supabase users whose every provider is off in Clerk: they could never sign in. */ +function findDisabledProviderRejects(input: CheckInput): Map { + const reasons = new Map(); + if (!input.supabaseRows || !input.settings) return reasons; + + const enabled = enabledSocialProviders(input.settings); + const disabled = findDisabledProviders(input.supabaseRows, enabled, toClerkStrategy); + if (disabled.length === 0) return reasons; + + const { excludedIds } = findUsersWithOnlyDisabledProviders(input.supabaseRows, disabled); + const names = disabled.map(providerLabel).join(", "); + for (const id of excludedIds) { + reasons.set( + id, + `only signs in with ${names}, which ${disabled.length === 1 ? "is" : "are"} not enabled in Clerk`, + ); + } + return reasons; +} + +// --- Warnings and fixes ---------------------------------------------------- + +/** The Supabase rows for just these users, so provider counts describe them alone. */ +function rowsFor(input: CheckInput, users: User[]): Record[] { + if (!input.supabaseRows) return []; + const ids = new Set(users.map((user) => user.userId)); + return input.supabaseRows.filter((row) => ids.has(String(row.id))); +} + +function buildWarnings(input: CheckInput, importable: User[]): string[] { + const warnings: string[] = []; + + if (input.settings && importable.length > 0) { + const report = buildReadinessReport({ + analysis: analyzeFields(importable), + settings: input.settings, + providerCounts: countSocialProviders(rowsFor(input, importable)), + }); + for (const item of report.blocking) { + if (item.consequence !== "drops") continue; + if (item.clerkRequired === true) { + const missing = importable.length - item.userCount; + warnings.push( + item.key === "password" + ? `${plural(missing, "user")} without a password, which this instance requires: they reset it to sign in` + : `${plural(missing, "user")} without a ${item.label.toLowerCase()}, which this instance requires`, + ); + } else { + warnings.push( + item.section === "social" + ? `${plural(item.userCount, "user")} signed in with ${item.label}, which is not enabled in Clerk` + : `${plural(item.userCount, "user")} ${item.userCount === 1 ? "has" : "have"} a ${item.label.toLowerCase()}, which this instance is not set up to store`, + ); + } + } + } + + const dropped = importable.filter((user) => user.passwordDropped).length; + if (dropped > 0) { + warnings.push( + `${plural(dropped, "password")} Clerk cannot verify will be dropped: those users reset it to sign in`, + ); + } + + const unknown = Object.entries(input.unknownFields ?? {}); + if (unknown.length > 0) { + warnings.push( + `Clerk won't store: ${unknown + .sort((a, b) => b[1] - a[1]) + .map(([field, count]) => `${field} (${plural(count, "user")})`) + .join(", ")}`, + ); + } + + return warnings; +} + +/** Shell-quotes a JSON payload for a single-quoted argument. */ +function quoteJson(payload: unknown): string { + return `'${JSON.stringify(payload).replace(/'/g, "'\\''")}'`; +} + +/** + * One `clerk config patch` per flagged setting, built from the same rows the + * warnings and rejects come from. + * + * These are offers, not corrections: an instance that requires an email is + * configured as its owner intended, and fixing the export may be the answer. + */ +function buildFixes(input: CheckInput, users: User[]): Fix[] { + if (!input.settings || users.length === 0) return []; + + const report = buildReadinessReport({ + analysis: analyzeFields(users), + settings: input.settings, + providerCounts: countSocialProviders(rowsFor(input, users)), + }); + + // An unverified email does not satisfy a required one, so a file of only + // unverified addresses flags the requirement even when every user has one. + const unverifiedOnly = users.some((user) => { + const identifiers = splitIdentifiers(user); + return !identifiers.primaryEmail && identifiers.unverifiedEmails.length > 0; + }); + const flagged = report.blocking.slice(); + if ( + unverifiedOnly && + isRequired(input.settings, "email_address") && + isEnabled(input.settings, "email_address") && + !flagged.some((item) => item.key === "email_address") + ) { + const email = report.items.find((item) => item.key === "email_address"); + if (email) + flagged.unshift({ ...email, clerkRequired: true, blocking: true, consequence: "rejects" }); + } + + const flags = input.target.appId + ? ` --app ${input.target.appId} --instance ${input.target.instanceId}` + : ""; + return buildSettingChanges(flagged).map((change) => ({ + label: change.label, + command: `clerk config patch${flags} --json ${quoteJson(buildChangePayload([change]))}`, + })); +} + +// --- The whole check ------------------------------------------------------- + +function countReasons(rejects: Reject[]): ReasonCount[] { + const counts = new Map(); + for (const { reason } of rejects) counts.set(reason, (counts.get(reason) ?? 0) + 1); + return [...counts].map(([reason, count]) => ({ reason, count })); +} + +export async function checkImport(input: CheckInput): Promise { + const rejects: Reject[] = input.failures.map((failure) => ({ + sourceId: failure.userId, + reason: `invalid: ${failure.error}`, + })); + + const fileDuplicates = findFileDuplicates(input.users); + const disabledProviders = findDisabledProviderRejects(input); + + let candidates: User[] = []; + for (const user of input.users) { + const reason = + fileDuplicates.get(user.userId) ?? + missingRequiredIdentifier(user, input.settings) ?? + (user.password && user.passwordHasher + ? hashShapeProblem(user.password, user.passwordHasher) + : undefined) ?? + disabledProviders.get(user.userId); + if (reason) rejects.push({ sourceId: user.userId, reason }); + else candidates.push(user); + } + + const instanceDuplicates = + candidates.length > 0 + ? await findInstanceDuplicates(candidates, input) + : new Map(); + const unique: User[] = []; + for (const user of candidates) { + const reason = instanceDuplicates.get(user.userId); + if (reason) rejects.push({ sourceId: user.userId, reason }); + else unique.push(user); + } + candidates = unique; + + // The limit is a development-instance default, not a number the API serves, + // so it is checked against the live count and the importable users alone. + let quota: Quota | undefined; + if (input.instanceType === "dev") { + const headroom = Math.max(0, DEV_USER_LIMIT - (input.existingUsers ?? 0)); + const over = Math.max(0, candidates.length - headroom); + quota = { existing: input.existingUsers ?? null, limit: DEV_USER_LIMIT, headroom, over }; + if (over > 0) { + for (const user of candidates.slice(headroom)) { + rejects.push({ + sourceId: user.userId, + reason: `over the development instance's ${DEV_USER_LIMIT}-user limit`, + }); + } + candidates = candidates.slice(0, headroom); + } + } + + return { + total: input.users.length + input.failures.length, + importable: candidates, + rejects, + rejectReasons: countReasons(rejects), + warnings: buildWarnings(input, candidates), + fixes: buildFixes(input, input.users), + ...(quota ? { quota } : {}), + settingsUnavailable: input.settings === null, + }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts index e9b55e8e0..db8e84366 100644 --- a/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test"; import type { UserSettingsJSON } from "../../../lib/fapi.ts"; import type { FieldAnalysis } from "./analysis.ts"; import { buildReadinessReport } from "./readiness.ts"; -import { applyChanges, buildChangePayload, buildSettingChanges } from "./modify-settings.ts"; +import { buildChangePayload, buildSettingChanges } from "./modify-settings.ts"; /** Instance settings carrying only the attributes and providers a test names. */ function settings(config: { @@ -223,81 +223,3 @@ describe("the payload", () => { expect(buildChangePayload([])).toEqual({}); }); }); - -/** - * The redraw after a write comes from `applyChanges`, not a second fetch: - * Clerk's Frontend API is eventually consistent, so re-reading straight after - * the patch returns the pre-write settings and redraws every row just cleared. - */ -describe("the settings after a write", () => { - /** The two fields a change touches, as `settings()` above builds them. */ - const attr = (value: { enabled: boolean; required: boolean }) => - value as unknown as UserSettingsJSON["attributes"]["email_address"]; - - test("drops the requirement a relaxation removed", () => { - const before = settings({ attributes: { email_address: { enabled: true, required: true } } }); - const input = { - analysis: analysis({ - totalUsers: 5, - identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, - }), - settings: before, - }; - - const after = applyChanges(before, changesFor(input)); - - expect(after?.attributes.email_address).toEqual(attr({ enabled: true, required: false })); - // The report is rebuilt from this, so the row must stop being flagged. - expect(buildReadinessReport({ ...input, settings: after }).blocking).toEqual([]); - }); - - test("turns on what an enable switched on", () => { - const before = settings({ attributes: { username: { enabled: false } } }); - const changes = changesFor({ - analysis: analysis({ totalUsers: 2, identifiers: { username: 2 } as never }), - settings: before, - }); - - expect(applyChanges(before, changes)?.attributes.username).toEqual( - attr({ enabled: true, required: false }), - ); - }); - - test("enables a provider under Clerk's strategy name, not the source platform's", () => { - const before = settings({ - attributes: { email_address: { enabled: true } }, - social: { oauth_x: { enabled: false } }, - }); - const changes = changesFor({ - analysis: analysis({ - totalUsers: 2, - identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, - }), - settings: before, - providerCounts: { twitter: 2 }, - }); - - expect(applyChanges(before, changes)?.social).toEqual({ - oauth_x: { enabled: true }, - } as unknown as UserSettingsJSON["social"]); - }); - - test("leaves the settings it was given untouched", () => { - const before = settings({ attributes: { email_address: { enabled: true, required: true } } }); - const changes = changesFor({ - analysis: analysis({ - totalUsers: 5, - identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, - }), - settings: before, - }); - - applyChanges(before, changes); - - expect(before.attributes.email_address).toMatchObject({ required: true }); - }); - - test("passes null through — unreadable settings flag nothing to change", () => { - expect(applyChanges(null, [])).toBeNull(); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts index 8136f6ca1..a430200e3 100644 --- a/packages/cli-core/src/commands/migrate/lib/modify-settings.ts +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts @@ -1,11 +1,11 @@ /** - * Turning a flagged Migration Readiness row into the instance-config change - * that would stop it being flagged. + * Turning a flagged readiness row into the instance-config change that would + * stop it being flagged, printed as a `clerk config patch` command. * - * The report already knows which settings will cost users; without this the - * only way to act on it is to leave the CLI, find the setting in the dashboard, - * and come back. Each change is a single leaf in the Platform API's config - * document, so they compose into one `PATCH` however many the operator picks. + * The checks already know which settings will cost users; without this the + * only way to act on them is to leave the CLI, find the setting in the + * dashboard, and come back. Each change is a single leaf in the Platform API's + * config document. * * These are offers, not corrections. A flagged setting is not a wrong setting — * an instance that genuinely requires an email address is configured exactly as @@ -18,7 +18,6 @@ * it and still points at the dashboard. */ -import type { UserSettingsJSON } from "../../../lib/fapi.ts"; import { toClerkStrategy } from "./clerk-config.ts"; import type { ReadinessItem, ReadinessSection } from "./readiness.ts"; @@ -142,50 +141,3 @@ export function buildChangePayload(changes: SettingChange[]): Record; - const strategy = toClerkStrategy(change.id); - social[strategy] = { ...social[strategy], enabled: true }; - continue; - } - - const attributes = next.attributes as Record; - attributes[change.id] = - change.kind === "enable" - ? { - ...attributes[change.id], - enabled: true, - required: attributes[change.id]?.required ?? false, - } - : { - ...attributes[change.id], - enabled: attributes[change.id]?.enabled ?? true, - required: false, - }; - } - - return next; -} diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts index 842e9c4b5..f9824a37e 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import type { UserSettingsJSON } from "../../../lib/fapi.ts"; -import { analyzeFields, type FieldAnalysis } from "./analysis.ts"; -import { buildReadinessReport, formatReadinessReport, type ReadinessItem } from "./readiness.ts"; +import type { FieldAnalysis } from "./analysis.ts"; +import { buildReadinessReport, type ReadinessItem } from "./readiness.ts"; /** Instance settings carrying only the attributes and providers a test names. */ function settings(config: { @@ -230,12 +230,6 @@ describe("when the instance settings cannot be read", () => { test("still reports what the file contains", () => { expect(unreadable().items.map((entry) => entry.label)).toEqual(["Email"]); }); - - test("renders a note explaining the checks are coverage only", () => { - const output = formatReadinessReport(unreadable()).join("\n"); - expect(output).toContain("Could not read this instance's settings"); - expect(output).toContain("dashboard.clerk.com"); - }); }); describe("file-level totals", () => { @@ -265,225 +259,3 @@ describe("file-level totals", () => { * because per-field coverage cannot answer them: the users missing an email and * the users missing a password overlap by an amount only a per-user pass knows. */ -describe("what the settings mean for these users", () => { - const REQUIRE_EMAIL_AND_PASSWORD = settings({ - attributes: { - email_address: { enabled: true, required: true }, - password: { enabled: true, required: true }, - }, - }); - - /** Two with everything, two with no email, one with an email but no password. */ - const USERS = [ - { userId: "a", email: "a@x.dev", password: "hash" }, - { userId: "b", email: "b@x.dev", password: "hash" }, - { userId: "c", username: "c" }, - { userId: "d", username: "d" }, - { userId: "e", email: "e@x.dev" }, - ] as never; - - const outcomes = () => - buildReadinessReport({ - analysis: analyzeFields(USERS), - users: USERS, - settings: REQUIRE_EMAIL_AND_PASSWORD, - }).outcomes; - - test("the three totals account for every user exactly once", () => { - const result = outcomes(); - expect(result).toMatchObject({ rejected: 2, incomplete: 1, complete: 2 }); - expect((result?.rejected ?? 0) + (result?.incomplete ?? 0) + (result?.complete ?? 0)).toBe(5); - }); - - // The file has three users without a password, but two of them are already - // rejected for the email — counting them twice would overstate the damage. - test("a rejected user is not also counted as incomplete", () => { - expect(outcomes()?.incompleteReasons).toEqual([ - { - label: "Password", - count: 1, - detail: expect.stringContaining("1 has no password, which this instance requires"), - }, - ]); - }); - - test("names why the rejected users are rejected", () => { - expect(outcomes()?.rejectedReasons).toEqual([ - { label: "Email", count: 2, detail: "2 have no email, which this instance requires" }, - ]); - }); - - /** - * The rejected users lose nothing today — they are not being created. But the - * moment the operator relaxes the requirement rejecting them (one of the - * changes on offer) every masked setting lands at once. Surfacing it here is - * what saves an apply → re-check → discover → apply → re-check loop. - */ - describe("what is masked behind a rejection", () => { - // b and c have no email, so both are rejected; b also carries a phone the - // instance is not set up to store. Exactly the shape the supabase sample - // hits: every phone belongs to a user who has no email. - const MASKED_USERS = [ - { userId: "a", email: "a@x.dev" }, - { userId: "b", username: "b", phone: "+15551234567" }, - { userId: "c", username: "c" }, - ] as never; - - const report = (attributes: Record) => - buildReadinessReport({ - analysis: analyzeFields(MASKED_USERS), - users: MASKED_USERS, - settings: settings({ attributes }), - }); - - const REQUIRE_EMAIL_PHONE_OFF = { - email_address: { enabled: true, required: true }, - phone_number: { enabled: false }, - username: { enabled: true }, - }; - - test("counts a setting that only bites once the rejected users get in", () => { - const outcomes = report(REQUIRE_EMAIL_PHONE_OFF).outcomes; - - expect(outcomes).toMatchObject({ rejected: 2, incomplete: 0, complete: 1 }); - expect(outcomes?.maskedReasons).toEqual([ - { - label: "Phone", - count: 1, - detail: "1 has a phone, which this instance is not set up to store", - }, - ]); - }); - - test("keeps it out of the incomplete count, which is about users being imported", () => { - expect(report(REQUIRE_EMAIL_PHONE_OFF).outcomes?.incompleteReasons).toEqual([]); - }); - - test("renders it under the rejected group", () => { - const output = formatReadinessReport(report(REQUIRE_EMAIL_PHONE_OFF)).join("\n"); - - expect(output).toContain("If you import them, this applies to them too:"); - expect(output).toContain("1 has a phone, which this instance is not set up to store"); - }); - - // Enabling phone is the other change on offer, and it empties the block — - // which is the check that the two offers really do interact this way. - test("is empty once the masked setting is no longer a problem", () => { - const outcomes = report({ - email_address: { enabled: true, required: true }, - phone_number: { enabled: true }, - username: { enabled: true }, - }).outcomes; - - expect(outcomes).toMatchObject({ rejected: 2 }); - expect(outcomes?.maskedReasons).toEqual([]); - }); - }); - - test("a disabled attribute costs the users who carry it, not the ones who don't", () => { - const users = [ - { userId: "a", email: "a@x.dev", username: "a" }, - { userId: "b", email: "b@x.dev" }, - ] as never; - - const result = buildReadinessReport({ - analysis: analyzeFields(users), - users, - settings: settings({ - attributes: { email_address: { enabled: true }, username: { enabled: false } }, - }), - }).outcomes; - - expect(result).toMatchObject({ rejected: 0, incomplete: 1, complete: 1 }); - expect(result?.incompleteReasons[0]?.detail).toContain("not set up to store"); - }); - - test("is omitted when the caller passes no users", () => { - const report = buildReadinessReport({ - analysis: analyzeFields(USERS), - settings: REQUIRE_EMAIL_AND_PASSWORD, - }); - expect(report.outcomes).toBeUndefined(); - }); - - test("renders each outcome with the reasons behind it", () => { - const output = formatReadinessReport( - buildReadinessReport({ - analysis: analyzeFields(USERS), - users: USERS, - settings: REQUIRE_EMAIL_AND_PASSWORD, - }), - ).join("\n"); - - expect(output).toContain("2 users will not be imported"); - expect(output).toContain("1 user will be imported, but not everything they carry"); - expect(output).toContain("2 users will be imported in full"); - expect(output).toContain("they will have to reset it to sign in"); - }); -}); - -describe("rendering", () => { - const blocked = () => - buildReadinessReport({ - analysis: analysis({ - totalUsers: 10, - identifiers: { verifiedEmails: 7, hasAnyIdentifier: 8, username: 10 } as never, - }), - settings: settings({ - attributes: { - email_address: { enabled: true, required: true }, - username: { enabled: true }, - }, - }), - validationFailed: 2, - }); - - test("leads with the counts an operator needs before confirming", () => { - const output = formatReadinessReport(blocked()).join("\n"); - expect(output).toContain("10 users in this file"); - expect(output).toContain("2 failed validation"); - expect(output).toContain("2 without any identifier"); - }); - - test("names the blocking rows and points at the dashboard", () => { - const output = formatReadinessReport(blocked()).join("\n"); - expect(output).toContain("1 setting needs attention"); - expect(output).toContain("required in Clerk, and not every user has one"); - expect(output).toContain("dashboard.clerk.com"); - }); - - test("confirms a clean report when nothing blocks", () => { - const output = formatReadinessReport( - buildReadinessReport({ - analysis: analysis({ - totalUsers: 2, - identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, - }), - settings: settings({ attributes: { email_address: { enabled: true } } }), - }), - ).join("\n"); - expect(output).toContain("Every field in this file is configured in Clerk"); - }); - - test("does not claim everything is configured when settings were unreadable", () => { - const output = formatReadinessReport( - buildReadinessReport({ analysis: analysis({ totalUsers: 1 }), settings: null }), - ).join("\n"); - expect(output).not.toContain("Every field in this file is configured"); - }); - - test("renders section headings only for sections that have rows", () => { - const output = formatReadinessReport( - buildReadinessReport({ - analysis: analysis({ - totalUsers: 1, - identifiers: { verifiedEmails: 1, hasAnyIdentifier: 1 } as never, - }), - settings: settings({ attributes: { email_address: { enabled: true } } }), - }), - ).join("\n"); - expect(output).toContain("Identifiers"); - expect(output).not.toContain("Social connections"); - expect(output).not.toContain("User model"); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts index e476da4a1..60cd35585 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -1,27 +1,19 @@ /** - * The Migration Readiness report: what the import file contains, cross- - * referenced against what the destination instance actually accepts. + * What the import file contains, cross-referenced against what the + * destination instance accepts, one row per field. * - * Ported from the standalone migration-tool's `displayCrossReference`. Split - * into a pure {@link buildReadinessReport} and a separate renderer so the - * cross-reference decisions are testable without parsing coloured output. - * - * The point of the report is to surface, *before* anything is written to - * Clerk, the two failure modes a migration only discovers halfway through: - * a field Clerk requires that some users lack, and a social provider users - * signed up with that Clerk has not enabled. + * Ported from the standalone migration-tool's `displayCrossReference`. The + * import's checks read the flagged rows for their drop warnings and their + * `clerk config patch` fixes; which users are rejected is decided per user in + * `checks.ts`. */ import type { UserSettingsJSON } from "../../../lib/fapi.ts"; -import { bold, dim, green, red, yellow } from "../../../lib/color.ts"; // Pure attribute lookups, shared with the `users` create wizard. import { isEnabled, isRequired, type AttributeName } from "../../users/interactive/attributes.ts"; -import type { User } from "../types.ts"; -import { hasValue, type FieldAnalysis } from "./analysis.ts"; +import type { FieldAnalysis } from "./analysis.ts"; import { providerLabel, toClerkStrategy } from "./clerk-config.ts"; -export const DASHBOARD_URL = "https://dashboard.clerk.com/~/user-authentication"; - export type ReadinessSection = "identifiers" | "auth" | "social" | "model"; /** @@ -62,39 +54,6 @@ export type ReadinessItem = { detail?: string; }; -/** One reason users are affected, and how many of them it affects. */ -export type OutcomeReason = { label: string; count: number; detail: string }; - -/** - * What the settings mean for the users in the file, counted per user rather - * than per field. - * - * Per-field coverage cannot answer "how many users will not be imported" — - * the users missing an email and the users missing a username overlap by an - * unknown amount. Each user is classified once, into the worst outcome that - * applies to them, so the three totals add up to the file. - */ -export type ImportOutcomes = { - rejected: number; - rejectedReasons: OutcomeReason[]; - /** - * What *else* affects the rejected users — surfaced now rather than after - * they become importable. - * - * A user who is not being created cannot lose a field, so these settings cost - * nothing today and would otherwise go unmentioned. But the moment the - * operator relaxes the requirement rejecting them, every one of these lands. - * Reporting it only afterwards turns one decision into a apply → re-check → - * discover → apply → re-check loop, which is exactly what the report exists - * to prevent. - */ - maskedReasons: OutcomeReason[]; - incomplete: number; - incompleteReasons: OutcomeReason[]; - /** Imported with everything the file carries for them. */ - complete: number; -}; - export type ReadinessReport = { totalUsers: number; /** Users with no identifier at all; they cannot be imported under any settings. */ @@ -105,8 +64,6 @@ export type ReadinessReport = { blocking: ReadinessItem[]; /** True when the instance settings could not be read. */ settingsUnavailable: boolean; - /** Omitted when the caller passed no users to classify. */ - outcomes?: ImportOutcomes; }; type BuildInput = { @@ -116,12 +73,6 @@ type BuildInput = { validationFailed?: number; /** Source-platform provider key → user count. Supabase exports only. */ providerCounts?: Record; - /** - * The users themselves, for the per-user outcome counts. Optional so callers - * that only need the coverage rows (and tests working from a synthetic - * {@link FieldAnalysis}) do not have to supply them. - */ - users?: User[]; }; /** @@ -248,231 +199,5 @@ export function buildReadinessReport(input: BuildInput): ReadinessReport { items, blocking, settingsUnavailable: settings === null, - ...(input.users ? { outcomes: countOutcomes(input.users, blocking) } : {}), }; } - -/** Whether a user carries the field an attribute row is about. */ -const CARRIES: Record) => boolean> = { - email_address: (u) => - hasValue(u.email) || hasValue(u.emailAddresses) || hasValue(u.unverifiedEmailAddresses), - phone_number: (u) => - hasValue(u.phone) || hasValue(u.phoneNumbers) || hasValue(u.unverifiedPhoneNumbers), - username: (u) => hasValue(u.username), - password: (u) => hasValue(u.password), - first_name: (u) => hasValue(u.firstName), - last_name: (u) => hasValue(u.lastName), -}; - -/** Which users a flagged row actually affects: the ones missing it, or carrying it. */ -function affects(item: ReadinessItem, user: Record): boolean { - const carries = CARRIES[item.key]; - if (!carries) return false; - // A required row costs the users without it; a disabled row costs the ones with it. - return item.clerkRequired === true ? !carries(user) : carries(user); -} - -/** - * One reason line: how many users, what they are missing or carrying, and what - * the instance does about it. Count first, because that is what is being - * decided on. - */ -function describe(item: ReadinessItem, count: number): string { - const noun = item.label.toLowerCase(); - const have = count === 1 ? "has" : "have"; - - if (item.clerkRequired === true) { - const consequence = item.key === "password" ? " — they will have to reset it to sign in" : ""; - return `${count} ${have} no ${noun}, which this instance requires${consequence}`; - } - return `${count} ${have} a ${noun}, which this instance is not set up to store`; -} - -function toReasons(counts: Map): OutcomeReason[] { - return [...counts].map(([label, { item, count }]) => ({ - label, - count, - detail: describe(item, count), - })); -} - -/** - * Classifies every user into exactly one outcome, worst first. - * - * Social rows are left out: which providers a user signed up with lives in the - * raw export rather than the transformed `User`, so they cannot be counted per - * user here. Their coverage row still names them. - */ -function countOutcomes(users: User[], blocking: ReadinessItem[]): ImportOutcomes { - const rejecting = blocking.filter((item) => item.consequence === "rejects"); - const dropping = blocking.filter( - (item) => item.consequence === "drops" && item.section !== "social", - ); - - type Tally = Map; - const rejectedBy: Tally = new Map(); - const droppedBy: Tally = new Map(); - const maskedBy: Tally = new Map(); - let rejected = 0; - let incomplete = 0; - let complete = 0; - - const tally = (into: Tally, item: ReadinessItem) => { - const entry = into.get(item.label) ?? { item, count: 0 }; - entry.count++; - into.set(item.label, entry); - }; - - for (const entry of users) { - const user = entry as unknown as Record; - const gaps = dropping.filter((item) => affects(item, user)); - - const refusals = rejecting.filter((item) => affects(item, user)); - if (refusals.length > 0) { - rejected++; - for (const item of refusals) tally(rejectedBy, item); - // Their gaps are still tallied, into a separate bucket. Dropping them - // here is what makes the settings interact invisibly: relaxing the - // requirement that rejects these users lets them in, and only then does - // whatever else affects them show up — a second round trip to learn - // something that was knowable now. - for (const item of gaps) tally(maskedBy, item); - continue; - } - - if (gaps.length === 0) { - complete++; - continue; - } - - incomplete++; - for (const item of gaps) tally(droppedBy, item); - } - - return { - rejected, - rejectedReasons: toReasons(rejectedBy), - maskedReasons: toReasons(maskedBy), - incomplete, - incompleteReasons: toReasons(droppedBy), - complete, - }; -} - -const SECTION_ORDER: ReadinessSection[] = ["identifiers", "auth", "social", "model"]; -const SECTION_LABELS: Record = { - identifiers: "Identifiers", - auth: "Authentication", - social: "Social connections", - model: "User model", -}; - -function renderItem(item: ReadinessItem, total: number): string { - const coverage = item.userCount === total ? "all users" : `${item.userCount}/${total} users`; - - if (item.blocking) { - return ` ${yellow("⚠")} ${item.label} — ${yellow(item.detail ?? "needs attention")} — ${dim(coverage)}`; - } - if (item.clerkEnabled === true) { - return ` ${green("✓")} ${item.label} — ${dim(`enabled in Clerk — ${coverage}`)}`; - } - // Settings unavailable: state coverage without claiming anything about Clerk. - return ` ${yellow("!")} ${item.label} — ${dim(`${coverage} — check it is enabled in Clerk`)}`; -} - -const users = (count: number) => `${count} user${count === 1 ? "" : "s"}`; - -/** - * The three outcomes, each with the reasons behind it. - * - * This is the part of the report that answers "so what": which users the - * instance will refuse, which will arrive with something missing, and why. - * Per-field coverage lives further down and is a different question. - */ -function renderOutcomes(outcomes: ImportOutcomes): string[] { - const lines: string[] = []; - - const group = ( - symbol: string, - colour: (text: string) => string, - headline: string, - reasons: OutcomeReason[], - ) => { - lines.push(` ${colour(symbol)} ${colour(headline)}`); - for (const reason of reasons) lines.push(` ${dim(reason.detail)}`); - }; - - if (outcomes.rejected > 0) { - group("✗", red, `${users(outcomes.rejected)} will not be imported`, outcomes.rejectedReasons); - - // Named here rather than left for a second run of the report: these are the - // settings that start costing something the moment the rejection above is - // lifted, and lifting it is one of the changes on offer. - if (outcomes.maskedReasons.length > 0) { - lines.push(` ${dim("If you import them, this applies to them too:")}`); - for (const reason of outcomes.maskedReasons) lines.push(` ${dim(reason.detail)}`); - } - } - if (outcomes.incomplete > 0) { - group( - "⚠", - yellow, - `${users(outcomes.incomplete)} will be imported, but not everything they carry`, - outcomes.incompleteReasons, - ); - } - if (outcomes.complete > 0) { - lines.push(` ${green("✓")} ${green(`${users(outcomes.complete)} will be imported in full`)}`); - } - - return lines; -} - -/** Renders the report for a human, as lines. */ -export function formatReadinessReport(report: ReadinessReport): string[] { - const lines: string[] = [bold("Migration readiness")]; - - lines.push(` ${users(report.totalUsers)} in this file`); - if (report.validationFailed > 0) { - lines.push(` ${yellow(`${report.validationFailed} failed validation and will be skipped`)}`); - } - if (report.withoutIdentifier > 0) { - lines.push( - ` ${red(`${report.withoutIdentifier} without any identifier — cannot be imported`)}`, - ); - } - - if (report.outcomes) { - const outcomeLines = renderOutcomes(report.outcomes); - if (outcomeLines.length > 0) lines.push("", ...outcomeLines); - } - - if (report.settingsUnavailable) { - lines.push( - "", - ` ${yellow("!")} ${dim("Could not read this instance's settings, so the checks below are coverage only.")}`, - ` ${dim(` Verify your settings at ${DASHBOARD_URL}`)}`, - ); - } - - for (const section of SECTION_ORDER) { - const sectionItems = report.items.filter((item) => item.section === section); - if (sectionItems.length === 0) continue; - - lines.push("", bold(SECTION_LABELS[section])); - for (const item of sectionItems) lines.push(renderItem(item, report.totalUsers)); - } - - lines.push(""); - if (report.blocking.length > 0) { - const count = report.blocking.length; - lines.push( - yellow(`⚠ ${count} setting${count === 1 ? "" : "s"} need${count === 1 ? "s" : ""} attention`), - dim(` ${DASHBOARD_URL}`), - ); - } else if (!report.settingsUnavailable) { - lines.push(green("✓ Every field in this file is configured in Clerk")); - } - - return lines; -} diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index b4e872dba..7ebee5657 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -305,17 +305,28 @@ export function consolidateClerkIdentifiers(user: Record): void * importing those users would store credentials nobody can ever sign in with, * and the fix is a one-word edit to the transformer. */ +/** Every field the import schema declares; anything else is stripped. */ +const SCHEMA_FIELDS: ReadonlySet = new Set(Object.keys(userSchema.shape)); + export function validatePreparedUsers(users: Record[]): { users: User[]; validationFailed: number; failures: ValidationFailure[]; + /** Fields a source produced that Clerk has no place for → how many users carry each. */ + unknownFields: Record; } { const validated: User[] = []; const failures: ValidationFailure[] = []; + const unknownFields: Record = {}; let validationFailed = 0; for (let i = 0; i < users.length; i++) { const user = users[i] as Record; + for (const [field, value] of Object.entries(user)) { + if (!SCHEMA_FIELDS.has(field) && value !== undefined && value !== null && value !== "") { + unknownFields[field] = (unknownFields[field] ?? 0) + 1; + } + } const result = userSchema.safeParse(user); if (result.success) { @@ -350,7 +361,7 @@ export function validatePreparedUsers(users: Record[]): { }); } - return { users: validated, validationFailed, failures }; + return { users: validated, validationFailed, failures, unknownFields }; } function addDefaultFields( @@ -373,7 +384,12 @@ export function transformUsers( users: Record[], key: string, options: TransformOptions = {}, -): { transformedData: User[]; validationFailed: number; failures: ValidationFailure[] } { +): { + transformedData: User[]; + validationFailed: number; + failures: ValidationFailure[]; + unknownFields: Record; +} { const transformer = getSource(key); const context = options.context ?? {}; const transformed: Record[] = []; @@ -390,7 +406,12 @@ export function transformUsers( } if (options.validate === false) { - return { transformedData: transformed as User[], validationFailed: 0, failures: [] }; + return { + transformedData: transformed as User[], + validationFailed: 0, + failures: [], + unknownFields: {}, + }; } const result = validatePreparedUsers(transformed); @@ -398,6 +419,7 @@ export function transformUsers( transformedData: result.users, validationFailed: result.validationFailed, failures: result.failures, + unknownFields: result.unknownFields, }; } @@ -477,14 +499,19 @@ export async function loadUsersFromFile( file: string, key: string, options: TransformOptions = {}, -): Promise<{ users: User[]; validationFailed: number; failures: ValidationFailure[] }> { +): Promise<{ + users: User[]; + validationFailed: number; + failures: ValidationFailure[]; + unknownFields: Record; +}> { const transformer = getSource(key); const raw = await readUsersFromFile(file, transformer); const withDefaults = addDefaultFields(raw, transformer); - const { transformedData, validationFailed, failures } = transformUsers( + const { transformedData, validationFailed, failures, unknownFields } = transformUsers( withDefaults, key, options, ); - return { users: transformedData, validationFailed, failures }; + return { users: transformedData, validationFailed, failures, unknownFields }; } diff --git a/packages/cli-core/src/commands/migrate/lib/user-lookup.ts b/packages/cli-core/src/commands/migrate/lib/user-lookup.ts index 394749541..8c27d8a65 100644 --- a/packages/cli-core/src/commands/migrate/lib/user-lookup.ts +++ b/packages/cli-core/src/commands/migrate/lib/user-lookup.ts @@ -19,12 +19,18 @@ export const LOOKUP_BATCH = 100; export type LookedUpUser = { id: string; external_id?: string | null; + username?: string | null; last_sign_in_at?: number | null; email_addresses?: { email_address?: string }[]; phone_numbers?: { phone_number?: string }[]; }; -export type LookupFilter = "user_id" | "external_id" | "email_address" | "phone_number"; +export type LookupFilter = + | "user_id" + | "external_id" + | "email_address" + | "phone_number" + | "username"; /** Splits `items` into chunks of at most `size`. */ export function batch(items: T[], size: number): T[][] { diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index b6e69e48c..991c64e04 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -1,6 +1,6 @@ /** - * The human-mode half of `migrate import`: the wizard fills in missing flags, the - * readiness report renders, and declining the confirmation writes nothing. + * The human-mode half of `migrate import`: the prompts fill in what was not + * passed, and nothing is written until the operator says yes. * * Kept in its own file because `mock.module` registrations are process-lifetime, * and `bun test --parallel` puts several files in each worker — so a mocked @@ -14,43 +14,20 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { getMode, setMode, type Mode } from "../../mode.ts"; -import { keylessTargetStubs, listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; -import type { InstanceTarget } from "../../lib/keyless-target.ts"; +import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; const mockSelect = mock(async () => "clerk" as unknown); const mockText = mock(async () => "export.json" as unknown); -type MultiselectConfig = { options: { value: string; label: string; hint?: string }[] }; -const mockMultiselect = mock(async (_config: MultiselectConfig) => [] as unknown[]); let confirmAnswer = true; /** Every confirmation the run put up, in order — the wording is the assertion. */ let confirmMessages: string[] = []; let originalMode: Mode; -const ACCOUNT_TARGET: InstanceTarget = { - kind: "account", - ctx: { - appId: "app_1", - appLabel: "Migration Test", - instanceId: "ins_1", - instanceLabel: "development", - }, - label: "Migration Test (development)", -}; -let instanceTarget: InstanceTarget | Error = ACCOUNT_TARGET; - mock.module("../../lib/listage.ts", () => ({ ...listageStubs, select: (...args: unknown[]) => mockSelect(...(args as [])), })); -mock.module("../../lib/keyless-target.ts", () => ({ - ...keylessTargetStubs, - resolveInstanceTarget: async () => { - if (instanceTarget instanceof Error) throw instanceTarget; - return instanceTarget; - }, -})); - // Every export of the real module must appear here — a missing one is a link // error at import time, which takes down the whole file rather than one prompt. mock.module("../../lib/prompts.ts", () => ({ @@ -58,7 +35,7 @@ mock.module("../../lib/prompts.ts", () => ({ confirmMessages.push(message); return confirmAnswer; }, - multiselect: (...args: unknown[]) => mockMultiselect(...(args as [MultiselectConfig])), + multiselect: async () => [], text: (...args: unknown[]) => mockText(...(args as [])), password: async () => "", editor: async () => "{}", @@ -67,6 +44,7 @@ mock.module("../../lib/prompts.ts", () => ({ const { run } = await import("./run.ts"); const { UserAbortError } = await import("../../lib/errors.ts"); const { _setConfigDir } = await import("../../lib/config.ts"); +const { listRuns, startRun } = await import("./lib/run-store.ts"); const captured = useCaptureLog(); @@ -74,26 +52,20 @@ let workDir: string; let configDir: string; let originalCwd: string; let originalFetch: typeof globalThis.fetch; -let requests: { method: string; url: string; body: unknown }[]; +let requests: { method: string; url: string }[]; const EXPORT = [ { id: "u1", primary_email_address: "a@x.dev" }, { id: "u2", primary_email_address: "b@x.dev" }, ]; -const baseOptions = { source: "clerk", file: "export.json", secretKey: "sk_test_x" }; - -let originalPlatformKey: string | undefined; +const runsDir = () => path.join(workDir, ".clerk", "migrate"); beforeAll(() => { originalMode = getMode(); setMode("human"); originalCwd = process.cwd(); originalFetch = globalThis.fetch; - // Pinned rather than inherited: the settings-fix write goes through the - // Platform API, and CI has neither a `.env.local` nor a login session. - originalPlatformKey = process.env.CLERK_PLATFORM_API_KEY; - process.env.CLERK_PLATFORM_API_KEY = "ak_test"; workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-config-")); _setConfigDir(configDir); @@ -103,8 +75,6 @@ beforeAll(() => { afterAll(() => { setMode(originalMode); globalThis.fetch = originalFetch; - if (originalPlatformKey === undefined) delete process.env.CLERK_PLATFORM_API_KEY; - else process.env.CLERK_PLATFORM_API_KEY = originalPlatformKey; _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); @@ -115,397 +85,113 @@ beforeEach(() => { requests = []; confirmAnswer = true; confirmMessages = []; - instanceTarget = ACCOUNT_TARGET; mockSelect.mockReset(); mockText.mockReset(); - mockMultiselect.mockReset(); mockSelect.mockResolvedValue("clerk"); mockText.mockResolvedValue("export.json"); - mockMultiselect.mockResolvedValue([]); fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); - stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input.toString()); + const method = init?.method ?? "GET"; + requests.push({ method, url: url.toString() }); + if (url.pathname === "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/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (url.pathname === "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/v1/users" && method === "GET") return Response.json([]); + if (url.pathname === "/v1/users/count") return Response.json({ total_count: 0 }); + return Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; }); afterEach(() => { process.exitCode = 0; }); -type StubSettings = { attributes?: object; social?: object } | null; - -let currentSettings: StubSettings = null; -/** What the instance reports once a config PATCH lands, when a test sets one. */ -let settingsAfterFix: StubSettings = null; - -/** Stubs BAPI plus the FAPI environment lookup the readiness report needs. */ -function stubInstanceSettings(settings: StubSettings) { - currentSettings = settings; - settingsAfterFix = null; - globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { - const url = input.toString(); - requests.push({ - method: init?.method ?? "GET", - url, - body: init?.body ? JSON.parse(init.body as string) : null, - }); - if (url.endsWith("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/v1/domains")) { - if (!currentSettings) return new Response("nope", { status: 500 }); - return Response.json({ - data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], - }); - } - if (url.includes("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/v1/dev_browser")) return Response.json({ token: "jwt" }); - if (url.includes("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/v1/environment")) return Response.json({ user_settings: currentSettings }); - if (url.endsWith("/instances/ins_1/config")) { - if (settingsAfterFix) currentSettings = settingsAfterFix; - return Response.json({ config_version: "v1_patched" }); - } - return Response.json({ id: "user_created" }); - }) as unknown as typeof fetch; -} +const created = () => + requests.filter((r) => r.method === "POST" && new URL(r.url).pathname === "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/v1/users"); -const created = () => requests.filter((r) => r.url.endsWith("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/v1/users")); +const importOptions = { input: "export.json", source: "clerk", secretKey: "sk_test_x" }; -describe("the wizard fills in missing flags", () => { - test("bare `clerk migrate import` prompts for the source and file, then imports", async () => { +describe("prompts for what was not passed", () => { + test("bare `clerk migrate import` asks for the file, then the source, then imports", async () => { await run({ secretKey: "sk_test_x" }); - expect(mockSelect).toHaveBeenCalledTimes(1); expect(mockText).toHaveBeenCalledTimes(1); + expect(mockSelect).toHaveBeenCalledTimes(1); expect(created()).toHaveLength(2); }); - test("asks only for what the flags did not supply", async () => { - await run({ ...baseOptions, source: "clerk" }); + test("asks only for what was not passed", async () => { + await run(importOptions); - expect(mockSelect).not.toHaveBeenCalled(); expect(mockText).not.toHaveBeenCalled(); - }); - - test("prompts for the file when only the source was passed", async () => { - await run({ source: "clerk", secretKey: "sk_test_x" }); - expect(mockSelect).not.toHaveBeenCalled(); - expect(mockText).toHaveBeenCalledTimes(1); - }); -}); - -describe("the readiness report", () => { - test("renders before the confirmation", async () => { - await run(baseOptions); - expect(captured.err).toContain("Migration readiness"); - expect(captured.err).toContain("2 users in this file"); }); - // The whole point of the report: seeing what will go wrong, then backing out - // before a single user exists in the destination instance. - test("declining afterwards writes nothing to Clerk", async () => { - confirmAnswer = false; - - await expect(run(baseOptions)).rejects.toThrow(UserAbortError); - - expect(captured.err).toContain("Migration readiness"); - expect(created()).toHaveLength(0); - }); - - test("accepting proceeds with the import", async () => { - confirmAnswer = true; - - await run(baseOptions); - - expect(created()).toHaveLength(2); - }); - - test("flags a field Clerk requires that not every user has", async () => { - stubInstanceSettings({ - attributes: { - email_address: { enabled: true, required: true }, - username: { enabled: true }, - }, + // The envelope names the source, so asking would only invite a wrong answer. + test("does not ask for a source when the file names its own", async () => { + const exportRun = startRun(runsDir(), { + kind: "export", + target: { platform: "clerk" }, + source: "clerk", }); + const file = path.join(exportRun.dir, "export.json"); fs.writeFileSync( - path.join(workDir, "export.json"), - JSON.stringify([ - { id: "u1", primary_email_address: "a@x.dev" }, - { id: "u2", username: "bob" }, - ]), + file, + JSON.stringify({ + clerkMigrate: 1, + source: "clerk", + exportedAt: "2026-09-01T00:00:00.000Z", + runId: exportRun.record.id, + users: EXPORT, + }), ); + exportRun.update({ file: { path: file, sha256: "x" } }); + exportRun.finish(); - await run(baseOptions); + await run({ input: exportRun.record.id, secretKey: "sk_test_x" }); - // The outcome block is the point: one user has no email, and an instance - // that requires one will refuse them. - expect(captured.err).toContain("1 user will not be imported"); - expect(captured.err).toContain("1 has no email, which this instance requires"); - expect(captured.err).toContain("1 setting needs attention"); - }); - - test("degrades to a note when the instance settings cannot be read", async () => { - stubInstanceSettings(null); - - await run(baseOptions); - - expect(captured.err).toContain("Could not read this instance's settings"); - expect(created()).toHaveLength(2); - }); - - test("is skipped for a -y run, which pays for no extra round-trips", async () => { - await run({ ...baseOptions, yes: true }); - - expect(requests.some((r) => r.url.endsWith("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/v1/domains"))).toBe(false); - expect(captured.err).not.toContain("Migration readiness"); + expect(mockSelect).not.toHaveBeenCalled(); expect(created()).toHaveLength(2); }); }); -describe("fixing the instance's settings from the report", () => { - /** An export whose second user has no email, against a required-email instance. */ - function blockedOnRequiredEmail() { - stubInstanceSettings({ - attributes: { - email_address: { enabled: true, required: true }, - username: { enabled: true }, - }, - }); - fs.writeFileSync( - path.join(workDir, "export.json"), - JSON.stringify([ - { id: "u1", primary_email_address: "a@x.dev" }, - { id: "u2", username: "bob" }, - ]), - ); - } - - /** Two flagged rows: email required with a user lacking it, username switched off. */ - function blockedOnTwoSettings() { - stubInstanceSettings({ - attributes: { - email_address: { enabled: true, required: true }, - username: { enabled: false }, - }, - }); - fs.writeFileSync( - path.join(workDir, "export.json"), - JSON.stringify([ - { id: "u1", primary_email_address: "a@x.dev", username: "alice" }, - { id: "u2", username: "bob" }, - ]), - ); - } - - const patched = () => requests.filter((r) => r.url.endsWith("/instances/ins_1/config")); - const offered = (round = 0) => mockMultiselect.mock.calls[round]?.[0]?.options ?? []; - - // Plain labels only: the config leaf each one writes is internal detail an - // operator cannot act on and does not need to read. - test("offers one change per blocking row, named in plain terms", async () => { - blockedOnRequiredEmail(); - - await run(baseOptions); - - expect(offered()).toEqual([ - { value: "email_address", label: "Make Email optional at sign-up" }, - ]); - }); - - test("does not ask when nothing is blocking", async () => { - await run(baseOptions); +describe("consent", () => { + test("asks before writing, naming how many users", async () => { + await run(importOptions); - expect(mockMultiselect).not.toHaveBeenCalled(); - expect(created()).toHaveLength(2); - }); - - // Relaxing an instance's sign-up requirements is a real decision, so nothing - // is preselected and an empty answer must leave the instance untouched. - test("selecting nothing changes nothing and continues to the import", async () => { - blockedOnRequiredEmail(); - mockMultiselect.mockResolvedValue([]); - - await run(baseOptions); - - expect(patched()).toHaveLength(0); - expect(created()).toHaveLength(2); - }); - - test("selecting a change patches the instance and re-renders the report", async () => { - blockedOnRequiredEmail(); - mockMultiselect.mockResolvedValue(["email_address"]); - - await run(baseOptions); - - expect(patched()).toHaveLength(1); - expect(patched()[0]).toMatchObject({ - method: "PATCH", - body: { auth_email: { required_for_sign_up: false } }, - }); - expect(captured.err).toContain("Updated 1 setting"); - // The redraw clears the row that was just fixed, so the confirmation that - // follows is against the settings the write established. - expect(captured.err).toContain("Every field in this file is configured in Clerk"); - expect(created()).toHaveLength(2); + expect(confirmMessages).toEqual(["Import 2 users?"]); }); - /** - * The redraw must not re-read the Frontend API. It is eventually consistent, - * so a fetch this soon after the write returns the pre-write settings and - * redraws the report with every row the operator just cleared still flagged. - */ - test("redraws from the write rather than re-reading stale settings", async () => { - blockedOnRequiredEmail(); - // Anything read back now would still say "required" — as it did in practice. - settingsAfterFix = { - attributes: { - email_address: { enabled: true, required: true }, - username: { enabled: true }, - }, - }; - mockMultiselect.mockResolvedValue(["email_address"]); - - await run(baseOptions); - - expect(requests.filter((r) => r.url.includes("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/v1/environment"))).toHaveLength(1); - expect(captured.err).toContain("Every field in this file is configured in Clerk"); - // Flagged in the first report, and only there — the redraw is clean even - // though a re-read at this moment would still have reported it. - expect(captured.err.split("setting needs attention")).toHaveLength(2); - }); + test("prints the checks before the question", async () => { + await run(importOptions); - // A keyless application is only reachable through the Backend API, which has - // no route for any of these settings — saying so beats a confusing rejection. - test("stands down for a keyless application and still imports", async () => { - blockedOnRequiredEmail(); - instanceTarget = { - kind: "keyless", - keyless: { secretKey: "sk_test_x", source: ".env" }, - label: "keyless", - }; - mockMultiselect.mockResolvedValue(["email_address"]); - - await run(baseOptions); - - expect(patched()).toHaveLength(0); - expect(captured.err).toContain("clerk auth login"); - expect(created()).toHaveLength(2); + expect(captured.err).toContain("Checks"); }); - /** - * Each redraw is another decision point, not a receipt. Applying one change - * routinely leaves others still worth making, and an operator should not have - * to re-run the whole command to reach them. - */ - describe("offering again while anything is still flagged", () => { - test("re-offers what is left, without the change already applied", async () => { - blockedOnTwoSettings(); - mockMultiselect.mockResolvedValueOnce(["email_address"]); - mockMultiselect.mockResolvedValueOnce(["username"]); - - await run(baseOptions); - - expect(offered(0).map((option) => option.value)).toEqual(["email_address", "username"]); - expect(offered(1).map((option) => option.value)).toEqual(["username"]); - expect(patched()).toHaveLength(2); - expect(patched()[1]).toMatchObject({ - body: { auth_username: { used_for_sign_up: true } }, - }); - }); - - test("stops once nothing is flagged, rather than asking again", async () => { - blockedOnTwoSettings(); - mockMultiselect.mockResolvedValueOnce(["email_address"]); - mockMultiselect.mockResolvedValueOnce(["username"]); - - await run(baseOptions); - - expect(mockMultiselect).toHaveBeenCalledTimes(2); - expect(captured.err).toContain("Every field in this file is configured in Clerk"); - expect(created()).toHaveLength(2); - }); - - test("stops when the operator skips, leaving the rest flagged", async () => { - blockedOnTwoSettings(); - mockMultiselect.mockResolvedValueOnce(["email_address"]); - mockMultiselect.mockResolvedValueOnce([]); - - await run(baseOptions); - - expect(mockMultiselect).toHaveBeenCalledTimes(2); - expect(patched()).toHaveLength(1); - expect(created()).toHaveLength(2); - }); - - // A selection naming nothing on offer is the same as no selection, and must - // not become an empty PATCH. - test("sends nothing when the selection matches no offered change", async () => { - blockedOnRequiredEmail(); - mockMultiselect.mockResolvedValue(["not_a_real_change"]); + test("declining writes nothing to Clerk, and records no run", async () => { + confirmAnswer = false; - await run(baseOptions); + await expect(run(importOptions)).rejects.toThrow(UserAbortError); - expect(patched()).toHaveLength(0); - expect(created()).toHaveLength(2); - }); + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); }); - test("warns instead of failing the run when the instance cannot be resolved", async () => { - blockedOnRequiredEmail(); - instanceTarget = new Error("not linked"); - mockMultiselect.mockResolvedValue(["email_address"]); - - await run(baseOptions); + test("--yes does not ask", async () => { + await run({ ...importOptions, yes: true }); - expect(patched()).toHaveLength(0); - expect(captured.err).toContain("nothing was changed"); + expect(confirmMessages).toEqual([]); expect(created()).toHaveLength(2); }); -}); - -describe("guards that still apply interactively", () => { - /** Makes `GET /v1/users/count` report an instance with one seat left. */ - function stubNearlyFullInstance(): void { - const inner = globalThis.fetch; - globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { - if (input.toString().includes("/v1/users/count")) { - return Response.json({ object: "total_count", total_count: 99 }); - } - return inner(input, init); - }) as typeof fetch; - } - - test("the dev-instance user limit, which the operator can agree to import past", async () => { - stubNearlyFullInstance(); - confirmAnswer = true; - - await run(baseOptions); - - expect(captured.err).toContain("100-user limit"); - expect(created()).toHaveLength(2); - }); - - // Asking "Import 2 users?" after warning that one of them cannot fit is the - // report and the quota disagreeing in the same run. - test("the final prompt restates the quota split rather than the file size", async () => { - stubNearlyFullInstance(); - - await run(baseOptions); - - expect(confirmMessages).toContain("Import 1 user and expect 1 to fail?"); - }); - - test("the final prompt names the whole file when the quota is not in play", async () => { - await run(baseOptions); - - expect(confirmMessages).toContain("Import 2 users?"); - }); - - test("declining the user-limit prompt writes nothing to Clerk", async () => { - stubNearlyFullInstance(); - confirmAnswer = false; - await expect(run(baseOptions)).rejects.toThrow(UserAbortError); + // `--json` means nobody reads a prompt, even at a terminal. + test("--json never asks, and without --yes refuses", async () => { + await expect(run({ ...importOptions, json: true })).rejects.toThrow(/needs consent/); - // Aborted before the readiness report, so nothing was read from FAPI either. - expect(captured.err).not.toContain("Migration readiness"); + expect(confirmMessages).toEqual([]); expect(created()).toHaveLength(0); }); @@ -522,7 +208,7 @@ describe("guards that still apply interactively", () => { ]), ); - await expect(run(baseOptions)).rejects.toThrow(/Invalid password hasher/); + await expect(run(importOptions)).rejects.toThrow(/Invalid password hasher/); expect(created()).toHaveLength(0); }); }); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index b29b332c6..7c22ae50d 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -3,16 +3,18 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { _setConfigDir } from "../../lib/config.ts"; -import { CliError } from "../../lib/errors.ts"; +import { type CliError, EXIT_CODE } from "../../lib/errors.ts"; import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; // Every test below names its own `--secret-key`, which short-circuits the // signed-in check — except the one that asserts what happens without it. mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); -import { latestUserLines, listRuns, startRun } from "./lib/run-store.ts"; +import { latestUserLines, listRuns, readRun, startRun } from "./lib/run-store.ts"; import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; -import { applyResumeAfter, explainErrors, run, validateRunOptions } from "./run.ts"; -import type { User } from "./types.ts"; +import { explainErrors, run, validateRunOptions } from "./run.ts"; + +/** A real-shaped bcrypt digest: the checks reject anything that is not. */ +const BCRYPT = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"; let workDir: string; let configDir: string; @@ -21,8 +23,6 @@ let originalCwd: string; /** Where runs land for a project rooted at `workDir`. */ const runsDir = () => path.join(workDir, ".clerk", "migrate"); -const users = (...ids: string[]): User[] => ids.map((userId) => ({ userId }) as User); - beforeAll(() => { originalCwd = process.cwd(); workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-"))); @@ -62,23 +62,21 @@ describe("validateRunOptions", () => { }); }); -describe("applyResumeAfter", () => { - test("returns everything when no ID is given", () => { - expect(applyResumeAfter(users("a", "b"), undefined)).toHaveLength(2); - }); - - test("skips up to and including the named user", () => { - expect(applyResumeAfter(users("a", "b", "c"), "b").map((u) => u.userId)).toEqual(["c"]); - }); - - test("returns nothing when the named user is last", () => { - expect(applyResumeAfter(users("a", "b"), "b")).toEqual([]); - }); - - test("throws rather than silently re-importing everyone", () => { - expect(() => applyResumeAfter(users("a"), "zz")).toThrow(CliError); - }); -}); +type Stub = { + /** What `/v1/environment` reports; `null` makes the settings unreadable. */ + settings?: { attributes?: object; social?: object } | null; + /** Users already in the instance, as `GET /v1/users` returns them. */ + existing?: { + id: string; + external_id?: string; + username?: string; + email_addresses?: { email_address: string }[]; + }[]; + /** `GET /v1/users/count`. */ + count?: number; + /** Source IDs whose `POST /v1/users` fails with a 422. */ + failing?: Set; +}; describe("run", () => { const captured = useCaptureLog(); @@ -89,12 +87,63 @@ describe("run", () => { { id: "u1", primary_email_address: "a@x.dev", - password_digest: "d1", + password_digest: BCRYPT, password_hasher: "bcrypt", }, { id: "u2", primary_email_address: "b@x.dev" }, ]; + /** A fake Clerk: the instance, its settings, its users, and the writes. */ + function stubClerk(stub: Stub = {}): void { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input.toString()); + const method = init?.method ?? "GET"; + const body = init?.body ? (JSON.parse(init.body as string) as Record) : null; + requests.push({ method, url: url.toString(), body }); + + if (url.pathname === "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/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (url.pathname === "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/v1/domains") { + if (!stub.settings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.pathname.includes("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/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.pathname.includes("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/v1/environment")) { + return Response.json({ user_settings: stub.settings }); + } + if (url.pathname === "/v1/users/count") { + return Response.json({ object: "total_count", total_count: stub.count ?? 0 }); + } + if (method === "GET" && url.pathname === "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/v1/users") { + const wanted = new Set(url.searchParams.values()); + return Response.json( + (stub.existing ?? []).filter( + (user) => + wanted.has(user.external_id ?? "") || + wanted.has(user.username ?? "") || + (user.email_addresses ?? []).some((email) => wanted.has(email.email_address)), + ), + ); + } + if (method === "POST" && url.pathname === "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/v1/users") { + const externalId = body?.external_id as string; + if (stub.failing?.has(externalId)) { + return Response.json( + { + errors: [{ code: "form_identifier_exists", message: "That email address is taken." }], + }, + { status: 422 }, + ); + } + return Response.json({ id: `user_${externalId}` }); + } + return Response.json({ id: "ok" }); + }) as typeof fetch; + } + beforeAll(() => { originalFetch = globalThis.fetch; }); @@ -105,14 +154,7 @@ describe("run", () => { fs.rmSync(runsDir(), { recursive: true, force: true }); fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); - globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { - requests.push({ - method: init?.method ?? "GET", - url: input.toString(), - body: init?.body ? JSON.parse(init.body as string) : null, - }); - return new Response(JSON.stringify({ id: "user_created" }), { status: 200 }); - }) as typeof fetch; + stubClerk(); }); afterEach(() => { @@ -122,16 +164,24 @@ describe("run", () => { const baseOptions = { source: "clerk", - file: "export.json", + input: "export.json", yes: true, secretKey: "sk_test_x", }; - test("refuses before the wizard when nobody is signed in", async () => { + const created = () => + requests + .filter((r) => r.method === "POST" && r.url.endsWith("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/v1/users")) + .map((r) => (r.body as { external_id: string }).external_id); + + const exitCodeOf = async (promise: Promise) => + ((await promise.catch((caught: unknown) => caught)) as CliError | undefined)?.exitCode; + + test("refuses before anything else when nobody is signed in", async () => { const previous = process.env.CLERK_SECRET_KEY; delete process.env.CLERK_SECRET_KEY; try { - await expect(run({ source: "clerk", file: "export.json", yes: true })).rejects.toThrow( + await expect(run({ source: "clerk", input: "export.json", yes: true })).rejects.toThrow( /Not logged in/, ); expect(requests).toHaveLength(0); @@ -143,16 +193,16 @@ describe("run", () => { test("imports every user in the file end to end", async () => { await run(baseOptions); - const created = requests.filter((r) => r.url.endsWith("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/v1/users")); - expect(created).toHaveLength(2); - expect(created[0]?.method).toBe("POST"); - expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual([ - "u1", - "u2", - ]); + expect(created()).toEqual(["u1", "u2"]); expect(captured.err).toContain("Imported:"); }); + test("prints the target first", async () => { + await run(baseOptions); + expect(captured.err).toContain("Target: instance (development, ins_1)"); + expect(captured.err.indexOf("Target:")).toBeLessThan(captured.err.indexOf("Checks")); + }); + test("records the run in the project's run store", async () => { await run(baseOptions); @@ -163,19 +213,25 @@ describe("run", () => { status: "complete", source: "clerk", counts: { total: 2, created: 2 }, - target: { keySource: "--secret-key", instanceType: "dev" }, + target: { keySource: "--secret-key", instanceType: "dev", instanceId: "ins_1" }, }); - expect(record?.id).toMatch(/^\d{8}-\d{6}-[0-9a-f]{4}$/); expect(record?.file?.path).toBe(path.join(workDir, "export.json")); expect(record?.file?.sha256).toMatch(/^[0-9a-f]{64}$/); const lines = [...latestUserLines(runsDir(), record!.id).values()]; expect(lines.map((line) => [line.sourceId, line.status, line.clerkId])).toEqual([ - ["u1", "created", "user_created"], - ["u2", "created", "user_created"], + ["u1", "created", "user_u1"], + ["u2", "created", "user_u2"], ]); }); + // A complete import leaves the export's user data behind; say how to remove it. + test("names the folders a complete import no longer needs", async () => { + await run(baseOptions); + const [record] = listRuns(runsDir()); + expect(captured.err).toContain(`rm -rf ${path.join(runsDir(), record!.id)}`); + }); + describe("export envelopes", () => { /** An export run whose envelope holds `users`, as `clerk migrate export` writes it. */ function exportRun(source: string, rows: unknown[], extra: Record = {}) { @@ -196,7 +252,7 @@ describe("run", () => { return { record: run.finish(), file }; } - const { source: _source, file: _file, ...noSource } = baseOptions; + const { source: _source, input: _input, ...noSource } = baseOptions; test("imports by export run ID, with the source the envelope names", async () => { const { record } = exportRun("clerk", export2); @@ -211,7 +267,7 @@ describe("run", () => { test("imports an envelope file with no source named", async () => { const { file } = exportRun("clerk", export2); - await run({ ...noSource, file }); + await run({ ...noSource, input: file }); expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); }); @@ -295,66 +351,11 @@ describe("run", () => { expect(listRuns(runsDir())).toHaveLength(0); }); - test("--require-password imports only the users that have one", async () => { + test("--require-password leaves out the users without one", async () => { await run({ ...baseOptions, requirePassword: true }); - const created = requests.filter((r) => r.url.endsWith("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/v1/users")); - expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u1"]); - expect(captured.err).toContain("skipping 1 user without a password"); - }); - - test("--resume-after skips everyone up to and including that ID", async () => { - await run({ ...baseOptions, resumeAfter: "u1" }); - - const created = requests.filter((r) => r.url.endsWith("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/v1/users")); - expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u2"]); - }); - - test("records validation failures in the run and imports the rest", async () => { - fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify([...export2, { id: "u3" }])); - - await run(baseOptions); - - expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); - expect(captured.err).toContain("1 user failed validation"); - - const [record] = listRuns(runsDir()); - expect(record).toMatchObject({ status: "partial", counts: { created: 2, failed: 1 } }); - expect(latestUserLines(runsDir(), record!.id).get("u3")).toMatchObject({ - status: "failed", - code: "validation", - }); - }); - - /** Makes `GET /v1/users/count` report an instance that already holds users. */ - function stubUserCount(total: number): void { - const inner = globalThis.fetch; - globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { - if (input.toString().includes("/v1/users/count")) { - return Response.json({ object: "total_count", total_count: total }); - } - return inner(input, init); - }) as typeof fetch; - } - - // `baseOptions` passes -y, which has nobody to answer the prompt this warning - // otherwise raises — see run-interactive.test.ts for the prompt itself. - test("warns under -y when an import may exceed the development-instance user limit", async () => { - stubUserCount(99); - - await run(baseOptions); - - expect(captured.err).toContain("100-user limit"); - expect(captured.err).toContain("already holds 99"); - expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); - }); - - test("stays quiet when the instance has room for the whole file", async () => { - stubUserCount(10); - - await run(baseOptions); - - expect(captured.err).not.toContain("100-user limit"); + expect(created()).toEqual(["u1"]); + expect(captured.err).toContain("leaving out 1 user without a password"); }); test("aborts before any API call when the hasher is unrecognized", async () => { @@ -371,119 +372,242 @@ describe("run", () => { ); await expect(run(baseOptions)).rejects.toThrow(/Invalid password hasher/); - expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(0); + expect(created()).toHaveLength(0); }); - test("exits non-zero when some users failed", async () => { - globalThis.fetch = (async () => - new Response(JSON.stringify({ errors: [{ code: "e", message: "taken" }] }), { - status: 422, - })) as unknown as typeof fetch; + test("exits 1 when some users failed", async () => { + stubClerk({ failing: new Set(["u2"]) }); await run(baseOptions); + expect(process.exitCode).toBe(1); + expect(listRuns(runsDir())[0]).toMatchObject({ status: "partial", counts: { failed: 1 } }); }); - // Tests run non-TTY, so `isHuman()` is false and the wizard path is never - // reached — the same guard an agent hits. - describe("without --source or a file", () => { - test.each([ - [{}, /the file \(or an export run ID\) and --source /], - [{ source: "clerk" }, /Pass the file \(or an export run ID\)\./], - [{ file: "export.json" }, /Pass --source \./], - ])("names the missing flags rather than prompting (%p)", async (partial, expected) => { - await expect(run({ ...partial, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( - expected, + // Tests run non-TTY, the same signal an agent gives. + describe("without a file or a source", () => { + test("names what to pass rather than prompting for the file", async () => { + await expect(run({ source: "clerk", yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /needs the file to import, or the export run that wrote it, and cannot prompt here/, ); expect(requests).toHaveLength(0); }); - test("explains that it cannot prompt", async () => { - await expect(run({ yes: true, secretKey: "sk_test_x" })).rejects.toThrow( - /cannot prompt in agent mode/, - ); + test("asks for --source when the file does not name its own", async () => { + await expect( + run({ input: "export.json", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/Missing --source/); }); }); - describe("readiness report", () => { - /** Stubs BAPI plus the FAPI environment lookup the report depends on. */ - function stubInstanceSettings( - settings: { attributes?: object; social?: object } | null, - onUsers?: () => Response, - ) { - globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { - const url = input.toString(); - requests.push({ - method: init?.method ?? "GET", - url, - body: init?.body ? JSON.parse(init.body as string) : null, - }); - if (url.endsWith("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/v1/domains")) { - if (!settings) return new Response("nope", { status: 500 }); - return Response.json({ - data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], - }); - } - if (url.includes("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/v1/dev_browser")) return Response.json({ token: "jwt" }); - if (url.includes("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/v1/environment")) return Response.json({ user_settings: settings }); - return onUsers ? onUsers() : Response.json({ id: "user_created" }); - }) as unknown as typeof fetch; - } + describe("continuing an earlier run", () => { + test("a complete run is not imported again", async () => { + await run(baseOptions); + requests = []; - const created = () => requests.filter((r) => r.url.endsWith("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/v1/users")); + await run(baseOptions); - // `-y` means nobody is watching, so the two extra round-trips buy nothing. - test("is skipped for a -y run", async () => { - stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); + expect(created()).toHaveLength(0); + expect(captured.err).toContain("Already imported in run"); + expect(listRuns(runsDir())).toHaveLength(1); + }); + test("--new-run imports it again as a new run", async () => { await run(baseOptions); + requests = []; - expect(requests.some((r) => r.url.endsWith("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/v1/domains"))).toBe(false); - expect(captured.err).not.toContain("Migration readiness"); - expect(created()).toHaveLength(2); + await run({ ...baseOptions, newRun: true }); + + expect(created()).toEqual(["u1", "u2"]); + expect(listRuns(runsDir())).toHaveLength(2); }); - test("renders before any user is created, and flags a required-but-missing field", async () => { - stubInstanceSettings({ - attributes: { - email_address: { enabled: true, required: true }, - username: { enabled: true }, - }, + test("a partial run retries only the users that did not make it, in the same run", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + + requests = []; + process.exitCode = 0; + stubClerk(); + await run(baseOptions); + + expect(created()).toEqual(["u2"]); + expect(captured.err).toContain(`Continuing run ${first!.id}, which finished partial`); + const runs = listRuns(runsDir()); + expect(runs).toHaveLength(1); + expect(runs[0]).toMatchObject({ id: first!.id, status: "complete", counts: { created: 2 } }); + }); + + test("an interrupted run skips the users it already created", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + // A crash never writes a finish time. + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + + requests = []; + process.exitCode = 0; + stubClerk(); + await run(baseOptions); + + expect(created()).toEqual(["u2"]); + expect(captured.err).toContain("which was interrupted"); + }); + + test("an undone run is imported again as a new run", async () => { + await run(baseOptions); + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + fs.writeFileSync( + path.join(runsDir(), first!.id, "run.json"), + JSON.stringify({ ...record, status: "undone" }), + ); + requests = []; + + await run(baseOptions); + + expect(created()).toEqual(["u1", "u2"]); + expect(listRuns(runsDir())).toHaveLength(2); + }); + + test("a run another live process holds refuses with exit 2", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + // PID 1 is always alive, and never this test. + fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); + + expect(await exitCodeOf(run(baseOptions))).toBe(EXIT_CODE.USAGE); + }); + + // An edited file is a different job. + test("a changed file is a new run", async () => { + await run(baseOptions); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3", primary_email_address: "c@x.dev" }]), + ); + requests = []; + + await run(baseOptions); + + expect(listRuns(runsDir())).toHaveLength(2); + }); + }); + + describe("checks", () => { + test("a reject stops the import, with the command that imports the rest", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("1 user would be rejected, so nothing was imported"); + expect(error.examples?.[0]?.command).toContain("--allow-partial --yes"); + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); + }); + + test("--allow-partial imports the rest and records each reject as skipped", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + + await run({ ...baseOptions, allowPartial: true }); + + expect(created()).toEqual(["u1", "u2"]); + const [record] = listRuns(runsDir()); + expect(record).toMatchObject({ status: "partial", counts: { created: 2, skipped: 1 } }); + expect(latestUserLines(runsDir(), record!.id).get("u3")).toMatchObject({ + status: "skipped", + reason: expect.stringContaining("invalid:"), + }); + // The operator accepted the skips; only a failed user exits 1. + expect(process.exitCode).toBe(0); + }); + + test("--dry-run writes nothing, and exits 2 when the import would be refused", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + + await run({ ...baseOptions, dryRun: true }); + + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); + expect(captured.err).toContain("Dry run: nothing was written."); + expect(process.exitCode).toBe(2); + }); + + test("--dry-run exits 0 when nothing would be rejected", async () => { + await run({ ...baseOptions, dryRun: true }); + + expect(created()).toHaveLength(0); + expect(process.exitCode).toBe(0); + }); + + test("a user whose only email is unverified is rejected where email is required, with a fix", async () => { + stubClerk({ + settings: { attributes: { email_address: { enabled: true, required: true } } }, }); - // One user has no email, so a required email address will cost them. fs.writeFileSync( path.join(workDir, "export.json"), JSON.stringify([ { id: "u1", primary_email_address: "a@x.dev" }, - { id: "u2", username: "bob" }, + { id: "u2", unverified_email_addresses: "b@x.dev" }, ]), ); - await run({ ...baseOptions, yes: false }); + await run({ ...baseOptions, dryRun: true }); + + expect(captured.err).toContain( + "only has an unverified email, and this instance requires an email", + ); + expect(captured.err).toContain( + `clerk config patch --json '{"auth_email":{"required_for_sign_up":false}}'`, + ); + }); + + test("a user already in the instance is rejected", async () => { + stubClerk({ + existing: [{ id: "user_old", email_addresses: [{ email_address: "b@x.dev" }] }], + }); - expect(captured.err).toContain("Migration readiness"); - expect(captured.err).toContain("1 user will not be imported"); + await run({ ...baseOptions, dryRun: true }); - // The report was printed before the first POST /v1/users. - const reportIndex = requests.findIndex((r) => r.url.includes("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/v1/environment")); - const firstCreate = requests.findIndex((r) => r.url.endsWith("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/v1/users")); - expect(reportIndex).toBeGreaterThanOrEqual(0); - expect(reportIndex).toBeLessThan(firstCreate); + expect(captured.err).toContain("email is already used by a user in the instance"); + expect(Bun.stripANSI(captured.err)).toContain("u2"); }); - test("degrades to a note when the instance settings cannot be read", async () => { - stubInstanceSettings(null); + test("the dev quota rejects users past the headroom; --allow-partial imports up to it", async () => { + stubClerk({ count: 99 }); - await run({ ...baseOptions, yes: false }); + expect(await exitCodeOf(run(baseOptions))).toBe(EXIT_CODE.USAGE); + expect(created()).toHaveLength(0); - expect(captured.err).toContain("Could not read this instance's settings"); - expect(created()).toHaveLength(2); + await run({ ...baseOptions, allowPartial: true }); + expect(created()).toEqual(["u1"]); + const [record] = listRuns(runsDir()); + expect(latestUserLines(runsDir(), record!.id).get("u2")?.reason).toContain("100-user limit"); }); - test("cross-references supabase providers against the instance", async () => { - stubInstanceSettings({ - attributes: { email_address: { enabled: true } }, - social: { oauth_google: { enabled: true } }, + test("supabase users whose only provider is disabled are rejected", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }, }); fs.writeFileSync( path.join(workDir, "export.json"), @@ -494,42 +618,70 @@ describe("run", () => { email_confirmed_at: "2024-01-01 00:00:00+00", raw_app_meta_data: '{"providers":["discord"]}', }, + { + id: "sb2", + email: "b@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email","discord"]}', + }, ]), ); - await run({ ...baseOptions, source: "supabase", yes: false }); + await run({ ...baseOptions, source: "supabase", dryRun: true }); - expect(captured.err).toContain("Social connections"); - expect(captured.err).toContain("Discord"); - expect(captured.err).toContain("not enabled in Clerk"); + expect(captured.err).toContain("only signs in with Discord, which is not enabled in Clerk"); + expect(captured.err).toContain( + "1 user signed in with Discord, which is not enabled in Clerk", + ); }); - // Supabase lists `email` and `phone` in `providers` alongside real social - // connections, and Clerk has no `oauth_email` to enable — so counting them - // as social flagged every password user as a blocking problem. - test("leaves supabase's email and phone pseudo-providers out of the social section", async () => { - stubInstanceSettings({ - attributes: { email_address: { enabled: true } }, - social: { oauth_google: { enabled: true } }, - }); + test("names the fields Clerk won't store", async () => { fs.writeFileSync( path.join(workDir, "export.json"), - JSON.stringify([ - { - id: "sb1", - email: "a@x.dev", - email_confirmed_at: "2024-01-01 00:00:00+00", - raw_app_meta_data: '{"providers":["email","discord"]}', - }, - ]), + JSON.stringify(export2.map((user) => ({ ...user, department: "Sales" }))), + ); + + await run({ ...baseOptions, dryRun: true }); + + expect(captured.err).toContain("Clerk won't store: department (2 users)"); + }); + }); + + describe("consent", () => { + test("without --yes where nobody can be asked: the preview, then exit 2 with the command", async () => { + const error = (await run({ ...baseOptions, yes: false }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("needs consent. Pass --yes to confirm"); + expect(error.examples?.[0]?.command).toBe( + "clerk migrate import export.json --source clerk --secret-key --yes", ); + expect(captured.err).toContain("Checks"); + expect(created()).toHaveLength(0); + }); - await run({ ...baseOptions, source: "supabase", yes: false }); + test("--json --yes returns { target, run, checks, result }", async () => { + await run({ ...baseOptions, json: true }); - const social = captured.err.slice(captured.err.indexOf("Social connections")); - expect(social).toContain("Discord"); - expect(social).not.toContain("Email"); - expect(social).not.toContain("Phone"); + expect(JSON.parse(captured.out)).toMatchObject({ + target: { instanceId: "ins_1" }, + run: { kind: "import", status: "complete" }, + checks: { total: 2, importable: 2, rejects: [] }, + result: { created: 2, failed: 0, skipped: 0 }, + }); + }); + + test("--json without --yes returns the preview with consent required, and exits 2", async () => { + expect(await exitCodeOf(run({ ...baseOptions, yes: false, json: true }))).toBe( + EXIT_CODE.USAGE, + ); + expect(JSON.parse(captured.out)).toMatchObject({ + consent: "required", + checks: { importable: 2 }, + }); + expect(created()).toHaveLength(0); }); }); @@ -559,8 +711,8 @@ describe("run", () => { fs.writeFileSync( path.join(workDir, "export.json"), JSON.stringify([ - { account_ref: "mp_1", contact_email: "a@x.dev", given: "Ada", pw: "$2b$10$hash" }, - { account_ref: "mp_2", contact_email: "b@x.dev", given: "", pw: "$2b$10$hash" }, + { account_ref: "mp_1", contact_email: "a@x.dev", given: "Ada", pw: BCRYPT }, + { account_ref: "mp_2", contact_email: "b@x.dev", given: "", pw: BCRYPT }, ]), ); }); @@ -573,7 +725,7 @@ describe("run", () => { test("imports through a user-authored source", async () => { await run({ - file: "export.json", + input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x", @@ -589,7 +741,7 @@ describe("run", () => { test("applies the custom source's defaults and postTransform", async () => { await run({ - file: "export.json", + input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x", @@ -603,7 +755,7 @@ describe("run", () => { // An edited source is a different source, so the run records which one. test("records the custom source's content hash on the run", async () => { - await run({ file: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); const [record] = listRuns(runsDir()); expect(record?.source).toBe("myplatform"); @@ -612,7 +764,7 @@ describe("run", () => { test("an unknown built-in key is a usage error listing the valid ones", async () => { await expect( - run({ file: "export.json", source: "okta", yes: true, secretKey: "sk_test_x" }), + run({ input: "export.json", source: "okta", yes: true, secretKey: "sk_test_x" }), ).rejects.toThrow(/Unknown source "okta". Valid sources: clerk, auth0/); expect(created()).toHaveLength(0); }); @@ -620,7 +772,7 @@ describe("run", () => { test("fails before any request when the file is not there", async () => { await expect( run({ - file: "export.json", + input: "export.json", source: "./nope.ts", yes: true, secretKey: "sk_test_x", @@ -637,14 +789,14 @@ describe("run", () => { ); await expect( - run({ file: "export.json", source: bad, yes: true, secretKey: "sk_test_x" }), + run({ input: "export.json", source: bad, yes: true, secretKey: "sk_test_x" }), ).rejects.toThrow(/no source field maps to `userId`/); expect(requests).toHaveLength(0); }); test("still requires a file", async () => { await expect(run({ source: customFile, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( - /Pass the file \(or an export run ID\)/, + /needs the file to import/, ); }); }); @@ -668,7 +820,7 @@ describe("run", () => { ["authjs", [{ id: "aj1", email: "a@x.dev", email_verified: "2024-01-01T00:00:00Z" }], "aj1"], [ "betterauth", - [{ user_id: "ba1", email: "a@x.dev", email_verified: true, password_hash: "$2a$10$h" }], + [{ user_id: "ba1", email: "a@x.dev", email_verified: true, password_hash: BCRYPT }], "ba1", ], [ @@ -678,7 +830,7 @@ describe("run", () => { id: "sb1", email: "a@x.dev", email_confirmed_at: "2024-06-29 20:25:06+00", - encrypted_password: "$2b$10$h", + encrypted_password: BCRYPT, }, ], "sb1", @@ -749,99 +901,6 @@ describe("run", () => { ); }); }); - - describe("--skip-unsupported-providers", () => { - const supabaseExport = [ - { - id: "sb_email", - email: "a@x.dev", - email_confirmed_at: "2024-01-01 00:00:00+00", - raw_app_meta_data: '{"providers":["email"]}', - }, - { - id: "sb_discord", - email: "b@x.dev", - email_confirmed_at: "2024-01-01 00:00:00+00", - raw_app_meta_data: '{"providers":["discord"]}', - }, - { - id: "sb_both", - email: "c@x.dev", - email_confirmed_at: "2024-01-01 00:00:00+00", - raw_app_meta_data: '{"providers":["email","discord"]}', - }, - ]; - - /** Stubs BAPI plus the FAPI environment lookup the check depends on. */ - function stubInstance(enabledSocial: Record | null) { - globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { - const url = input.toString(); - requests.push({ - method: init?.method ?? "GET", - url, - body: init?.body ? JSON.parse(init.body as string) : null, - }); - if (url.endsWith("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/v1/domains")) { - if (!enabledSocial) return new Response("nope", { status: 500 }); - return Response.json({ - data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], - }); - } - if (url.includes("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/v1/dev_browser")) return Response.json({ token: "jwt" }); - if (url.includes("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/v1/environment")) { - return Response.json({ user_settings: { social: enabledSocial } }); - } - return Response.json({ id: "user_created" }); - }) as unknown as typeof fetch; - } - - const created = () => - requests - .filter((r) => r.url.endsWith("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/v1/users")) - .map((r) => (r.body as { external_id: string }).external_id); - - beforeEach(() => { - fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(supabaseExport)); - }); - - test("skips only the user whose sole provider is disabled", async () => { - stubInstance({ oauth_google: { enabled: true }, oauth_discord: { enabled: false } }); - - await run({ ...baseOptions, source: "supabase", skipUnsupportedProviders: true }); - - expect(created()).toEqual(["sb_email", "sb_both"]); - expect(captured.err).toContain("skipping 1 user "); - expect(captured.err).toContain("discord: 1"); - }); - - test("imports everyone when the provider is enabled", async () => { - stubInstance({ oauth_discord: { enabled: true } }); - - await run({ ...baseOptions, source: "supabase", skipUnsupportedProviders: true }); - - expect(created()).toHaveLength(3); - }); - - // A failed lookup must not be read as "nothing is enabled" — that would - // silently drop every social user. - test("imports everyone when the instance config cannot be read", async () => { - stubInstance(null); - - await run({ ...baseOptions, source: "supabase", skipUnsupportedProviders: true }); - - expect(created()).toHaveLength(3); - expect(captured.err).toContain("Could not read the instance's enabled providers"); - }); - - test("is a no-op with a warning on a non-supabase source", async () => { - fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); - - await run({ ...baseOptions, skipUnsupportedProviders: true }); - - expect(created()).toHaveLength(2); - expect(captured.err).toContain("only applies to supabase"); - }); - }); }); describe("explainErrors", () => { diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index a7b1cb2fa..9da82c7ed 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -1,20 +1,24 @@ /** - * `clerk migrate import` — the user import itself. - * - * Ported from the standalone migration-tool's `src/migrate/cli.ts` - * (`runNonInteractive`), with auth moved onto the CLI's standard secret-key - * resolution chain and every failure raised as a `CliError` instead of - * `console.error` + `process.exit`. + * `clerk migrate import ` — the user import itself. * * Registered as the `import` subcommand. The exported handler keeps the name - * `run` because `import` is a reserved word. Whatever the flags did not supply - * is filled in by `wizard.ts` for a human, or raised as a usage error naming - * the missing flags for an agent, which cannot answer a prompt. + * `run` because `import` is a reserved word. + * + * An import goes through the same steps every time: + * + * 1. Settle the file (an export run ID stands for the file that run wrote), + * the source (a file from `clerk migrate export` names its own) and the + * target instance, and print the target. + * 2. Decide whether this continues an earlier run of the same file, source and + * instance, from the run store. + * 3. Run `checkImport()` against the real instance. `--dry-run` stops here. + * 4. Refuse on any reject unless `--allow-partial`, then ask for consent. + * Nothing is written without `--yes` or a yes at the prompt. + * 5. Import, one run line per user. */ import { bold, dim, green, red, yellow } from "../../lib/color.ts"; import { resolveProfile } from "../../lib/config.ts"; -import path from "node:path"; import { hasAccountCredentials } from "../../lib/credential-store.ts"; import { AUTH_ERROR_REASON, @@ -24,55 +28,35 @@ import { throwUsageError, throwUserAbort, } from "../../lib/errors.ts"; -import { - resolveInstanceTarget, - resolveKeylessTarget, - type InstanceTarget, -} from "../../lib/keyless-target.ts"; +import { resolveKeylessTarget } from "../../lib/keyless-target.ts"; import { log } from "../../lib/log.ts"; import { NEXT_STEPS, printAgentNextSteps } from "../../lib/next-steps.ts"; -import { confirm, multiselect } from "../../lib/prompts.ts"; +import { confirm } from "../../lib/prompts.ts"; import { withGutter, withSpinner } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; -import { writeInstanceConfig } from "../config/io.ts"; import { importUsers } from "./import-users.ts"; -import { analyzeFields } from "./lib/analysis.ts"; +import { checkImport, type ImportChecks } from "./lib/checks.ts"; +import { fetchInstanceSettings, fetchUserCount } from "./lib/clerk-config.ts"; +import { readEnvelope, type ExportEnvelope } from "./lib/export-file.ts"; import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebase-hash.ts"; -import { - enabledSocialProviders, - fetchInstanceSettings, - fetchUserCount, - toClerkStrategy, -} from "./lib/clerk-config.ts"; -import { - buildReadinessReport, - DASHBOARD_URL, - formatReadinessReport, - type ReadinessReport, -} from "./lib/readiness.ts"; -import { - applyChanges, - buildChangePayload, - buildSettingChanges, - type SettingChange, -} from "./lib/modify-settings.ts"; import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; -import { readEnvelope, type ExportEnvelope } from "./lib/export-file.ts"; import { + continueRun, + latestUserLines, + listRuns, readRun, resolveRunsDir, RUN_ID_PATTERN, + runDir, + runState, sha256File, startRun, + type Run, type RunRecord, } from "./lib/run-store.ts"; -import { - countSocialProviders, - findDisabledProviders, - findUsersWithOnlyDisabledProviders, - readSupabaseRows, -} from "./lib/supabase-providers.ts"; -import { describeTarget, resolveClerkTarget } from "./lib/target.ts"; +import { createApiScheduler } from "./lib/scheduler.ts"; +import { readSupabaseRows } from "./lib/supabase-providers.ts"; +import { printTarget, resolveClerkTarget } from "./lib/target.ts"; import { fileExists, getFileType, @@ -80,8 +64,8 @@ import { resolveImportFilePath, } from "./lib/transform.ts"; import { resolveSource, sourceKeys } from "./sources/registry.ts"; -import type { ImportSummary, User } from "./types.ts"; -import { runWizard, throwAgentFlagsRequired } from "./wizard.ts"; +import type { ImportSummary } from "./types.ts"; +import { promptForFile, promptForFirebaseHashConfig, promptForSource } from "./wizard.ts"; import { login } from "../auth/login.ts"; import { link } from "../link/index.ts"; @@ -92,21 +76,33 @@ export type MigrateRunOptions = { source?: string; /** Content hash of a custom `--source`, set once it is loaded. */ sourceHash?: string; + /** The file to read, once `input` is resolved. Not a flag. */ file?: string; - resumeAfter?: string; requirePassword?: boolean; + /** Check against the instance, report, and write nothing. */ + dryRun?: boolean; + /** Import the users that pass, and record the rest as skipped. */ + allowPartial?: boolean; + /** Start a fresh run even when an earlier one matches. */ + newRun?: boolean; yes?: boolean; + json?: boolean; secretKey?: string; app?: string; instance?: string; - /** Supabase: drop users whose only social provider is disabled in Clerk. */ - skipUnsupportedProviders?: boolean; /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ runsDir?: string; } & FirebaseHashFlags; +const plural = (count: number, word: string) => `${count} ${word}${count === 1 ? "" : "s"}`; + +/** Nobody can answer a prompt: an agent, a non-TTY run, or `--json`. */ +function canPrompt(options: MigrateRunOptions): boolean { + return !options.json && isHuman() && !isAgent(); +} + /** - * Validates the flags a run needs before anything is read or sent. + * Validates what a run needs before anything is read or sent. * * `--source` has already been resolved to a registered key by the time this * runs, custom sources included. @@ -124,7 +120,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate import users.json --source clerk -y", + command: "clerk migrate import users.json --source clerk --yes", description: "Import a Clerk export", }, ], @@ -137,7 +133,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate import users.json --source clerk -y", + command: "clerk migrate import users.json --source clerk --yes", description: "Import a Clerk export", }, ], @@ -153,24 +149,6 @@ export function validateRunOptions(options: MigrateRunOptions): { return { source: options.source, file: options.file }; } -/** - * Drops every user up to and including `resumeAfter`. - * - * @throws CliError when the ID is not in the file — silently importing the - * whole set would duplicate everything the previous run already created. - */ -export function applyResumeAfter(users: User[], resumeAfter: string | undefined): User[] { - if (!resumeAfter) return users; - - const index = users.findIndex((user) => user.userId === resumeAfter); - if (index === -1) { - throw new CliError(`Could not find user ID "${resumeAfter}" in the import file.`, { - code: ERROR_CODE.USAGE_ERROR, - }); - } - return users.slice(index + 1); -} - /** Where a production instance's operator changes the SMS country blocklist. */ const SMS_SETTINGS_URL = "https://dashboard.clerk.com/~/customization/sms/settings"; @@ -219,352 +197,7 @@ export function explainErrors(errors: Iterable, instanceType: InstanceTy return notes; } -function formatSummary( - summary: ImportSummary, - run: RunRecord, - runFolder: string, - instanceType: InstanceType, -): string { - const inFile = summary.totalProcessed + summary.validationFailed; - const lines = [ - `${bold("Total users in file:")} ${inFile}`, - `${green("Imported:")} ${summary.successful}`, - `${red("Failed:")} ${summary.failed}`, - ]; - - if (summary.validationFailed > 0) { - lines.push(`${yellow("Failed validation:")} ${summary.validationFailed}`); - } - if (summary.errorBreakdown.size > 0) { - lines.push("", bold("Error breakdown:")); - for (const [error, count] of summary.errorBreakdown) { - lines.push(` ${count} user${count === 1 ? "" : "s"}: ${error}`); - } - for (const note of explainErrors(summary.errorBreakdown.keys(), instanceType)) { - lines.push("", note); - } - } - lines.push("", dim(`Run ${run.id}: ${runFolder}`)); - - return lines.join("\n"); -} - -/** - * Stops an import that looks likely to exhaust a development instance's user - * quota, and asks before letting it through anyway. - * - * A prompt rather than a hard refusal, because the number it checks against - * cannot be trusted to be this instance's: {@link DEV_USER_LIMIT} is only what - * an instance is *created* with, Clerk raises it per instance on request, and - * no public endpoint serves the real value. The existing user count is live; - * the limit it is measured against is not. Refusing outright would block - * imports the destination would happily accept, so the operator — who can ask - * Clerk what their limit is — gets the last word. - * - * `-y` and agent mode proceed on the warning alone, matching the import - * confirmation below: neither has anyone to answer the question. - * - * @returns How many of `incoming` the quota is expected to reject, or `0` when - * the whole file fits. The final import prompt reports the same split, so - * that "yes" is never a bigger number than the instance will accept. - * @throws UserAbortError when the operator declines. - */ -async function confirmDevUserLimit( - incoming: number, - secretKey: string, - yes: boolean, -): Promise { - const existing = await withSpinner("Checking the instance's user count...", async () => - fetchUserCount(secretKey), - ); - const headroom = Math.max(0, DEV_USER_LIMIT - (existing ?? 0)); - if (incoming <= headroom) return 0; - - const rejected = incoming - headroom; - const held = existing === null ? "" : `, and this one already holds ${existing}`; - log.warn( - `Development instances default to a ${DEV_USER_LIMIT}-user limit${held}. About ${rejected} of the ` + - `${incoming} user${incoming === 1 ? "" : "s"} in this file will be rejected with a quota error unless ` + - `Clerk has raised this instance's limit — the limit itself is not readable from the API.\n` + - `Import into a production instance to bring everyone across, or contact support to raise the limit.`, - ); - - if (yes || !isHuman() || isAgent()) return rejected; - - const proceed = await confirm({ - message: `Continue anyway, expecting about ${rejected} user${rejected === 1 ? "" : "s"} to be rejected?`, - default: false, - }); - if (!proceed) throwUserAbort(); - - return rejected; -} - -/** - * Drops users whose only way into Clerk is a social provider the destination - * instance has not enabled. - * - * Only meaningful for Supabase exports — it is the one platform whose export - * records per-user providers. If the instance's configuration cannot be read, - * nobody is dropped: a failed lookup must not be mistaken for "no providers - * are enabled". - * - * @returns The source IDs to skip. - */ -async function findDisabledProviderUsers( - file: string, - source: string, - secretKey: string, -): Promise> { - const none = new Set(); - if (source !== "supabase") { - log.warn(`--skip-unsupported-providers only applies to supabase exports; ignoring.`); - return none; - } - - const settings = await withSpinner("Checking enabled providers...", async () => - fetchInstanceSettings(secretKey), - ); - const enabled = settings ? enabledSocialProviders(settings) : null; - if (!enabled) { - log.warn( - "Could not read the instance's enabled providers; importing every user. Re-run with --verbose for details.", - ); - return none; - } - - const rows = await readSupabaseRows(file); - const disabled = findDisabledProviders(rows, enabled, toClerkStrategy); - if (disabled.length === 0) { - log.info("Every provider in this export is enabled in Clerk; no users skipped."); - return none; - } - - const { excludedIds, byProvider } = findUsersWithOnlyDisabledProviders(rows, disabled); - if (excludedIds.size === 0) { - log.info( - `${disabled.join(", ")} not enabled in Clerk, but every user has another way to sign in; none skipped.`, - ); - return none; - } - - const breakdown = Object.entries(byProvider) - .map(([provider, count]) => `${provider}: ${count}`) - .join(", "); - log.warn( - `--skip-unsupported-providers: skipping ${excludedIds.size} user${excludedIds.size === 1 ? "" : "s"} whose only provider is not enabled in Clerk (${breakdown}).`, - ); - - return excludedIds; -} - -type ReportInput = { - users: User[]; - file: string; - source: string; - secretKey: string; - validationFailed: number; -}; - -/** - * Everything the report needs except the instance's settings — the half that - * comes from the file, and so does not change when the instance does. - */ -async function readFileSide(input: ReportInput) { - // Only Supabase exports record per-user providers, so only they can be - // cross-referenced against the instance's social connections. - let providerCounts: Record | undefined; - if (input.source === "supabase") { - try { - providerCounts = countSocialProviders(await readSupabaseRows(input.file)); - } catch (error) { - log.debug(`migrate: could not read providers for the readiness report: ${String(error)}`); - } - } - - return { - analysis: analyzeFields(input.users), - validationFailed: input.validationFailed, - providerCounts, - }; -} - -function printReport(report: ReadinessReport): void { - log.blank(); - for (const line of formatReadinessReport(report)) log.info(line); - log.blank(); -} - -/** - * Offers to change the instance's settings, one selectable change per flagged - * row. - * - * Without this the report names something the operator has to leave the CLI to - * act on. Nothing is preselected and selecting nothing continues to the import - * prompt unchanged: a flagged setting is not a wrong setting, and relaxing an - * instance's sign-up requirements is a real decision rather than a default. - * - * @returns The changes that were written, so the caller can redraw the report. - */ -async function offerSettingChanges( - report: ReadinessReport, - options: MigrateRunOptions, -): Promise { - const changes = buildSettingChanges(report.blocking); - if (changes.length === 0) return []; - - // Navigation keys are in the prompt's own footer; what that footer cannot say - // is that selecting nothing is a valid answer rather than an unfinished one. - const chosen = await multiselect({ - message: "Update this instance's settings first? (enter to skip)", - options: changes.map((change) => ({ value: change.id, label: change.label })), - initialValues: [], - required: false, - }); - // Filtered before anything is resolved or sent: a selection that matches no - // offered change is the same as no selection, and must not become an empty - // PATCH. - const applied = changes.filter((change) => chosen.includes(change.id)); - if (applied.length === 0) return []; - - // Resolved here rather than up front: an operator who selects nothing should - // not pay for a Platform API round-trip, and a target that cannot be resolved - // (a bare `--secret-key` against an unlinked directory) should not fail the - // whole run before the report has even been offered. - let target: InstanceTarget; - try { - target = await resolveInstanceTarget({ app: options.app, instance: options.instance }); - } catch (error) { - log.warn( - "Could not resolve which instance to configure, so nothing was changed. " + - "Link a project with `clerk link`, or pass `--app `.", - ); - log.debug(`migrate: settings change target unresolved: ${String(error)}`); - return []; - } - - // The Backend API a keyless application is reachable through has no route for - // any of these settings — `config patch` rejects the same payload by name. - if (target.kind === "keyless") { - log.warn( - "These settings need an account to change. Run `clerk auth login` to claim this application, " + - `then re-run, or update them at ${DASHBOARD_URL}.`, - ); - return []; - } - - await withSpinner(`Updating settings on ${target.label}...`, async () => - writeInstanceConfig(target, buildChangePayload(applied), { - method: "PATCH", - failureContext: "Failed to update instance settings", - }), - ); - log.success(`Updated ${applied.length} setting${applied.length === 1 ? "" : "s"}.`); - - return applied; -} - -/** - * Prints the Migration Readiness report: what the file contains, cross- - * referenced against what the destination instance accepts. - * - * Rendered immediately before the confirmation prompt, so declining that - * prompt aborts with nothing written to Clerk. - * - * Skipped only for `-y`, which says "don't ask, don't lecture" and should not - * pay for two extra network round-trips. Agent mode still gets it: an agent - * driving a migration can act on "10 users will not be imported, because email - * is required" exactly as a human would — but not the prompt, which needs one. - */ -async function showReadinessReport( - input: ReportInput & { skipReport: boolean; options: MigrateRunOptions }, -): Promise { - if (input.skipReport) return; - - let settings = await withSpinner("Checking instance settings...", async () => - fetchInstanceSettings(input.secretKey), - ); - const fileSide = { ...(await readFileSide(input)), users: input.users }; - - let report = buildReadinessReport({ ...fileSide, settings }); - printReport(report); - - if (!isHuman() || isAgent()) return; - - // Every redraw is another decision point, not a receipt. Applying one change - // routinely leaves others still worth making — and can surface consequences - // that were masked behind the row just cleared — so the offer repeats for as - // long as the report has something to offer. - while (report.blocking.length > 0) { - const applied = await offerSettingChanges(report, input.options); - // Nothing selected, nothing offerable, or nowhere to write it: the operator - // has said their piece and the import prompt is next. - if (applied.length === 0) return; - - // Redrawn from the write, not from a re-read. Clerk's Frontend API is - // eventually consistent, so fetching settings again here routinely returns - // the pre-write ones and redraws every row the operator just cleared. - settings = applyChanges(settings, applied); - report = buildReadinessReport({ ...fileSide, settings }); - printReport(report); - } -} - -/** - * Fills in a missing `--source`/file interactively, or explains what - * to pass. - * - * Agent mode is the CLI's existing non-interactive signal, so an agent that - * runs bare `clerk migrate import` gets a usage error naming the flags rather than a - * prompt it cannot answer. - */ -async function resolveMissingOptions(options: MigrateRunOptions): Promise { - const missing = { source: !options.source, file: !options.file }; - if (!missing.source && !missing.file) return options; - - if (isAgent() || !isHuman()) { - throwAgentFlagsRequired(missing); - } - - // Resolved before the prompt only when `--source firebase` was already - // passed; otherwise the wizard picks the platform first and looks them up - // itself, so a non-Firebase migration never reads them at all. - const firebaseHashConfig = resolveFirebaseHashConfig(options, options.source); - const answers = await runWizard({ ...options, firebaseHashConfig }); - - return { - ...options, - source: answers.source, - file: answers.file, - ...(answers.firebaseHashConfig - ? { - firebaseSignerKey: answers.firebaseHashConfig.base64_signer_key, - firebaseSaltSeparator: answers.firebaseHashConfig.base64_salt_separator, - firebaseRounds: answers.firebaseHashConfig.rounds, - firebaseMemCost: answers.firebaseHashConfig.mem_cost, - } - : {}), - }; -} - -/** - * Resolves `--source` to a registered key, loading a custom source from its - * path so the rest of the run treats it exactly like a built-in. - * - * @throws UsageError for an unknown key. - */ -async function applySource(options: MigrateRunOptions): Promise { - if (!options.source) return options; - const resolved = await resolveSource(options.source); - if (resolved.path) { - log.info(`Loaded the \`${resolved.key}\` source from ${options.source}.`); - } - return { - ...options, - source: resolved.key, - ...(resolved.hash ? { sourceHash: resolved.hash } : {}), - }; -} +// --- Settling the file, source and target ---------------------------------- /** * Makes sure there is somewhere to import *into* before anything else happens. @@ -572,9 +205,7 @@ async function applySource(options: MigrateRunOptions): Promise { // An unclaimed accountless application keeps its only secret key on disk. if (await resolveKeylessTarget({ instance: options.instance })) return; - const interactive = isHuman() && !isAgent(); + const interactive = canPrompt(options); if (!(await hasAccountCredentials())) { if (!interactive) { @@ -598,7 +229,8 @@ async function ensureImportTarget(options: MigrateRunOptions): Promise { examples: [ { command: "clerk auth login", description: "Sign in, then re-run the import" }, { - command: "clerk migrate import users.json --source clerk -y --secret-key sk_test_...", + command: + "clerk migrate import users.json --source clerk --yes --secret-key sk_test_...", description: "Import without signing in", }, ], @@ -617,17 +249,33 @@ async function ensureImportTarget(options: MigrateRunOptions): Promise { } /** - * Settles which file to read: the positional argument or `--file`, where the - * argument may name the export run that wrote the file. + * Settles which file to read. The argument may name the export run that wrote + * the file; a human who gave none is asked for a path. */ async function resolveInput( options: MigrateRunOptions, -): Promise<{ file?: string; fromExport?: string }> { - if (options.input && options.file) { - throwUsageError("Name the file once: either as the argument or with --file, not both."); +): Promise<{ file: string; fromExport?: string }> { + const value = options.input; + if (!value) { + if (!canPrompt(options)) { + throwUsageError( + "`clerk migrate import` needs the file to import, or the export run that wrote it, and cannot prompt here.", + undefined, + undefined, + [ + { + command: "clerk migrate import 20260929-141502-a1b2 --yes", + description: "Import what an export run wrote", + }, + { + command: "clerk migrate import users.json --source clerk --yes", + description: "Import a file", + }, + ], + ); + } + return { file: await promptForFile() }; } - const value = options.input ?? options.file; - if (!value) return {}; if (!RUN_ID_PATTERN.test(value) || fileExists(value)) return { file: value }; const runsDir = await resolveRunsDir(options.runsDir); @@ -643,6 +291,25 @@ async function resolveInput( return { file: record.file.path, fromExport: record.id }; } +/** + * Resolves `--source` to a registered key, loading a custom source from its + * path so the rest of the run treats it exactly like a built-in. + * + * @throws UsageError for an unknown key. + */ +async function applySource(options: MigrateRunOptions): Promise { + if (!options.source) return options; + const resolved = await resolveSource(options.source); + if (resolved.path && !options.json) { + log.info(`Loaded the \`${resolved.key}\` source from ${options.source}.`); + } + return { + ...options, + source: resolved.key, + ...(resolved.hash ? { sourceHash: resolved.hash } : {}), + }; +} + /** * The source an export file names for itself, checked against the one the * flags name. @@ -664,172 +331,467 @@ function applyEnvelope( return { ...options, source: envelope.source }; } -export async function run(rawOptions: MigrateRunOptions): Promise { - await ensureImportTarget(rawOptions); - rawOptions = await applySource(rawOptions); +// --- Continuing an earlier run --------------------------------------------- - const input = await resolveInput(rawOptions); - rawOptions = { ...rawOptions, file: input.file, input: undefined }; - const envelope = - input.file && fileExists(input.file) - ? readEnvelope(resolveImportFilePath(input.file)) - : undefined; - rawOptions = applyEnvelope(rawOptions, envelope); +/** How this import relates to earlier runs of the same file, source and instance. */ +export type ResumeCase = + | { kind: "new" } + | { kind: "continue"; record: RunRecord; because: "interrupted" | "partial" } + | { kind: "complete"; record: RunRecord }; - const options = await resolveMissingOptions(rawOptions); +/** + * Finds the latest import run of this file (by sha256), this source (and a + * custom source's hash) and this instance, and decides what a re-run does: + * + * - none, or undone: a new run + * - interrupted: continue it, skipping the users it created + * - partial: continue it, retrying the users that failed or were skipped + * - complete: nothing; the file is already in + * + * @throws UsageError when that run is still running in another process. + */ +export function findResume( + runsDir: string, + match: { sha256: string; source: string; sourceHash?: string; instanceId: string }, +): ResumeCase { + const latest = listRuns(runsDir).find( + (record) => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + record.source === match.source && + record.sourceHash === match.sourceHash && + record.target.instanceId === match.instanceId, + ); + if (!latest) return { kind: "new" }; - const { source, file } = validateRunOptions(options); - // The flags win, so a rotated key can be passed without re-exporting. - const firebaseHashConfig = - resolveFirebaseHashConfig(options, source) ?? - (source === "firebase" ? envelope?.firebase : undefined); + const state = runState(runsDir, latest); + if (state === "running") { + throwUsageError( + `Run ${latest.id} is importing this file right now in another process. Wait for it to finish.`, + ); + } + if (state === "complete") return { kind: "complete", record: latest }; + if (state === "undone") return { kind: "new" }; + return { + kind: "continue", + record: latest, + because: state === "interrupted" ? "interrupted" : "partial", + }; +} + +// --- Reporting ------------------------------------------------------------- + +/** How many rejected source IDs to name per reason before summing the rest. */ +const REJECT_SAMPLE = 5; + +function printChecks(checks: ImportChecks): void { + log.blank(); + log.info(bold("Checks")); + log.info(` ${plural(checks.total, "user")} checked`); + + if (checks.rejects.length > 0) { + log.info(` ${red("✗")} ${red(`${plural(checks.rejects.length, "user")} rejected`)}`); + for (const { reason, count } of checks.rejectReasons) { + const ids = checks.rejects + .filter((reject) => reject.reason === reason) + .map((reject) => reject.sourceId); + const sample = ids.slice(0, REJECT_SAMPLE).join(", "); + const more = ids.length > REJECT_SAMPLE ? `, and ${ids.length - REJECT_SAMPLE} more` : ""; + log.info(` ${count}: ${reason}`); + log.info(` ${dim(sample + more)}`); + } + } + + if (checks.warnings.length > 0) { + log.info(` ${yellow("⚠")} ${yellow("Imported, but not everything comes across")}`); + for (const warning of checks.warnings) log.info(` ${dim(warning)}`); + } - await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { - const { secretKey, target } = await resolveClerkTarget(options); - const limits = resolveLimits(secretKey); + log.info(` ${green("✓")} ${green(`${plural(checks.importable.length, "user")} to import`)}`); - const { - users: loaded, - validationFailed, - failures, - } = await withSpinner(`Loading users from ${file}...`, async () => - loadUsersFromFile(file, source, { context: { firebaseHashConfig } }), + if (checks.settingsUnavailable) { + log.info( + ` ${yellow("!")} ${dim("Could not read this instance's settings, so required fields were not checked.")}`, ); + } - let users = applyResumeAfter(loaded, options.resumeAfter); - if (options.resumeAfter) { - log.info(`Resuming after ${options.resumeAfter} (${loaded.length - users.length} skipped).`); + if (checks.fixes.length > 0) { + log.blank(); + log.info(bold("Or change the instance instead")); + for (const fix of checks.fixes) { + log.info(` ${fix.label}`); + log.info(dim(` ${fix.command}`)); } + } +} - // Users left out on purpose. Recorded as skipped, so the run says who they - // were rather than only how many. - const skipped: { user: User; reason: string }[] = []; +function checksJson(checks: ImportChecks) { + return { + total: checks.total, + importable: checks.importable.length, + rejects: checks.rejects, + rejectReasons: checks.rejectReasons, + warnings: checks.warnings, + fixes: checks.fixes, + ...(checks.quota ? { quota: checks.quota } : {}), + settingsUnavailable: checks.settingsUnavailable, + }; +} - if (options.skipUnsupportedProviders) { - const excluded = await findDisabledProviderUsers(file, source, secretKey); - for (const user of users.filter((candidate) => excluded.has(candidate.userId))) { - skipped.push({ user, reason: "only provider is not enabled in Clerk" }); - } - users = users.filter((user) => !excluded.has(user.userId)); +function formatSummary( + summary: ImportSummary, + skipped: number, + record: RunRecord, + runFolder: string, + instanceType: InstanceType, +): string[] { + const lines = [ + `${green("Imported:")} ${summary.successful}`, + `${red("Failed:")} ${summary.failed}`, + ]; + if (skipped > 0) lines.push(`${yellow("Skipped:")} ${skipped}`); + + if (summary.errorBreakdown.size > 0) { + lines.push("", bold("Error breakdown:")); + for (const [error, count] of summary.errorBreakdown) { + lines.push(` ${plural(count, "user")}: ${error}`); + } + for (const note of explainErrors(summary.errorBreakdown.keys(), instanceType)) { + lines.push("", note); } + } + lines.push("", dim(`Run ${record.id}: ${runFolder}`)); + return lines; +} + +/** + * Once every user is in, the files that hold them are only a liability: the + * export carries user data and password hashes, and the run is only needed + * for `undo`. + */ +function cleanupLines(runsDir: string, record: RunRecord): string[] { + const lines: string[] = []; + if (record.fromExport) { + lines.push( + `The export in run ${record.fromExport} holds your users' data. Once you have checked the import, delete it:`, + dim(` rm -rf ${runDir(runsDir, record.fromExport)}`), + ); + } + lines.push( + `Keep run ${record.id} while you might still undo it. After that:`, + dim(` rm -rf ${runDir(runsDir, record.id)}`), + ); + return lines; +} + +// --- The import ------------------------------------------------------------ + +/** The exact command that would carry on from here, for the consent and refusal messages. */ +function commandFor(options: MigrateRunOptions, fromExport: string | undefined, extra: string[]) { + const input = fromExport ?? options.input ?? options.file ?? ""; + const parts = ["clerk migrate import", input]; + if (!fromExport && options.source) parts.push("--source", options.source); + if (options.allowPartial) parts.push("--allow-partial"); + if (options.newRun) parts.push("--new-run"); + if (options.requirePassword) parts.push("--require-password"); + if (options.secretKey) parts.push("--secret-key", ""); + if (options.app) parts.push("--app", options.app); + if (options.instance) parts.push("--instance", options.instance); + if (options.runsDir) parts.push("--runs-dir", options.runsDir); + return [...parts, ...extra].join(" "); +} + +/** Records the checks' rejects as skipped users, so the run says who they were. */ +function recordRejects(run: Run, checks: ImportChecks): void { + for (const { sourceId, reason } of checks.rejects) { + run.append({ sourceId, status: "skipped", reason }); + } +} + +export async function run(rawOptions: MigrateRunOptions): Promise { + await ensureImportTarget(rawOptions); + let options = await applySource(rawOptions); + + const input = await resolveInput(options); + options = { ...options, file: input.file }; + const envelope = fileExists(input.file) + ? readEnvelope(resolveImportFilePath(input.file)) + : undefined; + options = applyEnvelope(options, envelope); + + // Asked only when the file does not say where it came from. + if (!options.source && canPrompt(options) && fileExists(input.file)) { + options = { ...options, source: await promptForSource() }; + } + + const { source, file } = validateRunOptions(options); + // The flags win, so a rotated key can be passed without re-exporting. + let firebaseHashConfig = + resolveFirebaseHashConfig(options, source) ?? + (source === "firebase" ? envelope?.firebase : undefined); + if (source === "firebase" && !firebaseHashConfig && canPrompt(options)) { + firebaseHashConfig = await promptForFirebaseHashConfig(); + } + + await withGutter( + "Migrating users to Clerk", + async ({ setNextSteps }) => { + const { secretKey, target } = await resolveClerkTarget(options); + if (!options.json) printTarget(target); + const limits = resolveLimits(secretKey); - if (options.requirePassword) { - const withPassword = users.filter((user) => Boolean(user.password)); - const dropped = users.length - withPassword.length; - if (dropped > 0) { + const filePath = resolveImportFilePath(file); + const sha256 = sha256File(filePath); + const runsDir = await resolveRunsDir(options.runsDir, { write: !options.dryRun }); + + const resume: ResumeCase = options.newRun + ? { kind: "new" } + : findResume(runsDir, { + sha256, + source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), + instanceId: target.instanceId, + }); + + if (resume.kind === "complete") { + if (options.json) { + log.data(JSON.stringify({ target, run: resume.record, alreadyImported: true }, null, 2)); + } else { + log.success( + `Already imported in run ${resume.record.id}. Pass --new-run to import it again.`, + ); + } + return; + } + + // Users the run being continued already created are done: they are not + // checked or sent again, and finding them in the instance is expected. + const continued = resume.kind === "continue" ? resume.record : undefined; + const done = new Map(); + if (continued) { + for (const line of latestUserLines(runsDir, continued.id).values()) { + if (line.status === "created" && line.clerkId) done.set(line.sourceId, line.clerkId); + } + } + + if (!options.json) { log.info( - `--require-password: skipping ${dropped} user${dropped === 1 ? "" : "s"} without a password.`, + resume.kind === "continue" + ? `Continuing run ${resume.record.id}, which ${resume.because === "interrupted" ? "was interrupted" : "finished partial"}: ` + + `${plural(done.size, "user")} already imported ${done.size === 1 ? "is" : "are"} left alone.` + : "Starting a new run.", ); } - for (const user of users.filter((candidate) => !candidate.password)) { - skipped.push({ user, reason: "no password (--require-password)" }); - } - users = withPassword; - } - if (validationFailed > 0) { - log.warn( - `${validationFailed} user${validationFailed === 1 ? "" : "s"} failed validation and will be skipped.`, + const loaded = await withSpinner(`Loading users from ${file}...`, async () => + loadUsersFromFile(file, source, { context: { firebaseHashConfig } }), ); - } - - const runsDir = await resolveRunsDir(options.runsDir, { write: true }); - const beginRun = () => { - const filePath = resolveImportFilePath(file); - const run = startRun(runsDir, { - kind: "import", - target, - source, - ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), - file: { path: filePath, sha256: sha256File(filePath) }, - ...(input.fromExport ? { fromExport: input.fromExport } : {}), - }); - for (const failure of failures) { - run.append({ - sourceId: failure.userId, - status: "failed", - error: `${failure.error} (${failure.path.join(".") || "user"}, row ${failure.row + 1})`, - code: "validation", - }); - } - for (const { user, reason } of skipped) { - run.append({ sourceId: user.userId, status: "skipped", reason }); + let users = loaded.users.filter((user) => !done.has(user.userId)); + const failures = loaded.failures.filter((failure) => !done.has(failure.userId)); + + // An instruction about this import, not a prediction: users without a + // password are left out of the job rather than recorded as skipped. + if (options.requirePassword) { + const withPassword = users.filter((user) => Boolean(user.password)); + const dropped = users.length - withPassword.length; + if (dropped > 0 && !options.json) { + log.info( + `--require-password: leaving out ${plural(dropped, "user")} without a password.`, + ); + } + users = withPassword; } - return run; - }; - - if (users.length === 0) { - log.warn("No users left to import."); - // Still a run: the record of who failed validation, and why, is the one - // thing this attempt produced. - if (failures.length > 0 || skipped.length > 0) { - const record = beginRun().finish(); - log.info(dim(`Run ${record.id}: ${path.join(runsDir, record.id)}`)); - process.exitCode = 1; + + let supabaseRows: Record[] | undefined; + if (source === "supabase") { + try { + supabaseRows = await readSupabaseRows(file); + } catch (error) { + log.debug(`migrate: could not read Supabase providers: ${String(error)}`); + } } - return; - } - const quotaRejections = - limits.instanceType === "dev" - ? await confirmDevUserLimit(users.length, secretKey, Boolean(options.yes)) - : 0; + const [settings, existingUsers] = await withSpinner("Checking the instance...", async () => + Promise.all([ + fetchInstanceSettings(secretKey), + limits.instanceType === "dev" ? fetchUserCount(secretKey) : Promise.resolve(null), + ]), + ); - log.info( - `Importing ${users.length} user${users.length === 1 ? "" : "s"} from ${source} into ` + - `${describeTarget(target)}.`, - ); + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + const checks = await withSpinner("Checking users against the instance...", async (spinner) => + checkImport({ + users, + failures, + unknownFields: loaded.unknownFields, + ...(supabaseRows ? { supabaseRows } : {}), + settings, + existingUsers, + instanceType: limits.instanceType, + target, + secretKey, + schedule, + continuedClerkIds: new Set(done.values()), + spinner, + }), + ); - await showReadinessReport({ - users, - file, - source, - secretKey, - validationFailed, - skipReport: Boolean(options.yes), - options, - }); - - if (!options.yes && isHuman() && !isAgent()) { - // The readiness report counts the whole file, because settings decide - // what Clerk *accepts*. The quota decides how much of it gets in at all, - // so the last prompt — the one that starts writing — restates that split - // rather than asking about a number the instance will not take. - const importable = users.length - quotaRejections; - const proceed = await confirm({ - message: quotaRejections - ? `Import ${importable} user${importable === 1 ? "" : "s"} and expect ${quotaRejections} to fail?` - : `Import ${users.length} user${users.length === 1 ? "" : "s"}?`, - default: false, - }); - if (!proceed) throwUserAbort(); - } + const refused = checks.rejects.length > 0 && !options.allowPartial; + const preview = (extra: Record) => + log.data( + JSON.stringify( + { + target, + run: continued ?? null, + resume: resume.kind, + checks: checksJson(checks), + ...extra, + }, + null, + 2, + ), + ); - const run = beginRun(); - const summary = await withSpinner(`Importing users: [0/${users.length}]...`, async (spinner) => - importUsers({ - users, - secretKey, - limits, - record: run.append, - skipPasswordRequirement: !options.requirePassword, - validationFailed, - spinner, - }), - ); - const record = run.finish(); + if (!options.json) printChecks(checks); - log.info(formatSummary(summary, record, run.dir, limits.instanceType)); + if (options.dryRun) { + if (options.json) preview({ dryRun: true }); + else { + log.blank(); + log.info(dim("Dry run: nothing was written.")); + } + // The exit code says what the real run would do. + if (refused) process.exitCode = 2; + return; + } - // Offered even when some users failed: a partial import is exactly when - // reading the per-user record matters most. - const steps = - summary.failed > 0 - ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(record.id) - : NEXT_STEPS.MIGRATE_DONE(record.id); - setNextSteps(steps); - printAgentNextSteps(steps); + if (refused) { + if (options.json) preview({ refused: true }); + throwUsageError( + `${plural(checks.rejects.length, "user")} would be rejected, so nothing was imported. ` + + "Fix them, or pass --allow-partial to import the rest and record them as skipped.", + undefined, + undefined, + [ + { + command: commandFor(options, input.fromExport, ["--allow-partial", "--yes"]), + description: "Import the users that pass", + }, + ], + ); + } - if (summary.failed > 0) process.exitCode = 1; - }); + if (checks.importable.length === 0 && checks.rejects.length === 0) { + if (options.json) preview({ nothingToImport: true }); + else log.warn("No users left to import."); + return; + } + + // Rule 1: nothing is written without consent — `--yes`, or a yes at a + // prompt. An agent, a non-TTY run and `--json` never prompt. + if (!options.yes) { + if (!canPrompt(options)) { + if (options.json) preview({ consent: "required" }); + throwUsageError( + `\`clerk migrate import\` will create ${plural(checks.importable.length, "user")} and needs consent. Pass --yes to confirm.`, + undefined, + undefined, + [ + { + command: commandFor(options, input.fromExport, ["--yes"]), + description: "Run the import", + }, + ], + ); + } + log.blank(); + const proceed = await confirm({ + message: `Import ${plural(checks.importable.length, "user")}?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + const run = continued + ? continueRun(runsDir, continued) + : startRun(runsDir, { + kind: "import", + target, + source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), + file: { path: filePath, sha256 }, + ...(input.fromExport ? { fromExport: input.fromExport } : {}), + }); + recordRejects(run, checks); + + const summary = + checks.importable.length > 0 + ? await withSpinner( + `Importing users: [0/${checks.importable.length}]...`, + async (spinner) => + importUsers({ + users: checks.importable, + secretKey, + limits, + record: run.append, + skipPasswordRequirement: !options.requirePassword, + spinner, + }), + ) + : { + totalProcessed: 0, + successful: 0, + failed: 0, + validationFailed: 0, + errorBreakdown: new Map(), + }; + const record = run.finish(); + if (summary.failed > 0) process.exitCode = 1; + + if (options.json) { + log.data( + JSON.stringify( + { + target, + run: record, + resume: resume.kind, + checks: checksJson(checks), + result: { + created: summary.successful, + failed: summary.failed, + skipped: checks.rejects.length, + errors: [...summary.errorBreakdown].map(([error, count]) => ({ error, count })), + }, + }, + null, + 2, + ), + ); + return; + } + + log.blank(); + for (const line of formatSummary( + summary, + checks.rejects.length, + record, + run.dir, + limits.instanceType, + )) { + log.info(line); + } + if (record.status === "complete") { + log.blank(); + for (const line of cleanupLines(runsDir, record)) log.info(line); + } + + const steps = + summary.failed > 0 + ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(record.id) + : NEXT_STEPS.MIGRATE_DONE(record.id); + setNextSteps(steps); + printAgentNextSteps(steps); + }, + { skip: Boolean(options.json) }, + ); } diff --git a/packages/cli-core/src/commands/migrate/sources/supabase.ts b/packages/cli-core/src/commands/migrate/sources/supabase.ts index a4c202710..ab67146d8 100644 --- a/packages/cli-core/src/commands/migrate/sources/supabase.ts +++ b/packages/cli-core/src/commands/migrate/sources/supabase.ts @@ -24,7 +24,7 @@ const supabaseSource = { key: "supabase", label: "Supabase", description: - "Works with a Supabase `auth.users` export. Use --skip-unsupported-providers to drop users whose only social provider is not enabled in Clerk.", + "Works with a Supabase `auth.users` export. Users whose only social provider is not enabled in Clerk are rejected by the import's checks.", carries: { passwords: { level: "yes", note: "bcrypt `encrypted_password` hashes come across." }, mfa: { diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts index 26b61b6e6..824e0ebb7 100644 --- a/packages/cli-core/src/commands/migrate/wizard.test.ts +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -26,47 +26,39 @@ mock.module("../../lib/prompts.ts", () => ({ editor: async () => "{}", })); -const { runWizard, throwAgentFlagsRequired } = await import("./wizard.ts"); -const { _setConfigDir } = await import("../../lib/config.ts"); +const { promptForFile, promptForFirebaseHashConfig, promptForSource } = await import("./wizard.ts"); let workDir: string; -let configDir: string; let originalCwd: string; beforeAll(() => { originalCwd = process.cwd(); workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-"))); - configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-config-")); - _setConfigDir(configDir); process.chdir(workDir); fs.writeFileSync(path.join(workDir, "users.json"), "[]"); fs.writeFileSync(path.join(workDir, "other.csv"), ""); + fs.writeFileSync(path.join(workDir, "notes.txt"), ""); }); afterAll(() => { - _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); - fs.rmSync(configDir, { recursive: true, force: true }); }); beforeEach(() => { mockSelect.mockReset(); mockText.mockReset(); - fs.rmSync(path.join(configDir, "config.json"), { force: true }); }); -/** The config object the wizard passed to its Nth `text`/`select` prompt. */ +/** The config object passed to the Nth `text`/`select` prompt. */ const textCall = (index: number): Prompt | undefined => mockText.mock.calls[index]?.[0]; const selectCall = (index: number): SelectPrompt | undefined => mockSelect.mock.calls[index]?.[0]; -describe("source picker", () => { +describe("promptForSource", () => { test("is built from the registry, so every platform appears", async () => { mockSelect.mockResolvedValue("auth0"); - mockText.mockResolvedValue("users.json"); - - await runWizard({}); + expect(await promptForSource()).toBe("auth0"); expect(selectCall(0)?.choices.map((choice) => choice.value)).toEqual([ "clerk", "auth0", @@ -80,65 +72,48 @@ describe("source picker", () => { test("labels each choice with the source's display name", async () => { mockSelect.mockResolvedValue("clerk"); - mockText.mockResolvedValue("users.json"); - - await runWizard({}); - + await promptForSource(); expect(selectCall(0)?.choices.map((choice) => choice.name)).toContain("Better Auth"); }); - - test("is skipped when --source was already passed", async () => { - mockText.mockResolvedValue("users.json"); - - const result = await runWizard({ source: "clerk" }); - - expect(mockSelect).not.toHaveBeenCalled(); - expect(result.source).toBe("clerk"); - }); }); -describe("file prompt validation", () => { +describe("promptForFile", () => { const validate = async () => { - mockSelect.mockResolvedValue("clerk"); mockText.mockResolvedValue("users.json"); - await runWizard({}); - return textCall(0)?.validate; + await promptForFile(); + return textCall(0)?.validate as (value?: string) => string | undefined; }; - test.each([ - ["users.json", undefined], - ["other.csv", undefined], - ])("accepts %s", async (file, expected) => { - expect((await validate())?.(file)).toBe(expected as undefined); - }); - - test("rejects an empty answer", async () => { - expect((await validate())?.("")).toMatch(/required/); + test("returns the trimmed path", async () => { + mockText.mockResolvedValue(" users.json "); + expect(await promptForFile()).toBe("users.json"); }); - test("rejects a file that does not exist", async () => { - expect((await validate())?.("missing.json")).toMatch(/File not found/); + test("accepts an existing JSON or CSV file", async () => { + const check = await validate(); + expect(check("users.json")).toBeUndefined(); + expect(check("other.csv")).toBeUndefined(); }); - test("rejects an unsupported extension", async () => { - fs.writeFileSync(path.join(workDir, "notes.txt"), ""); - expect((await validate())?.("notes.txt")).toMatch(/\.json or \.csv/); + test.each([ + ["", /required/], + ["nope.json", /File not found/], + ["notes.txt", /\.json or \.csv/], + ])("rejects %p", async (value, message) => { + const check = await validate(); + expect(check(value)).toMatch(message); }); }); -describe("firebase hash parameters", () => { - test("are asked for when the firebase source is picked", async () => { - mockSelect.mockResolvedValue("firebase"); +describe("promptForFirebaseHashConfig", () => { + test("collects all four parameters as a set", async () => { mockText - .mockResolvedValueOnce("users.json") .mockResolvedValueOnce("SIGNER") .mockResolvedValueOnce("Bw==") .mockResolvedValueOnce("8") .mockResolvedValueOnce("14"); - const result = await runWizard({}); - - expect(result.firebaseHashConfig).toEqual({ + expect(await promptForFirebaseHashConfig()).toEqual({ base64_signer_key: "SIGNER", base64_salt_separator: "Bw==", rounds: 8, @@ -146,87 +121,11 @@ describe("firebase hash parameters", () => { }); }); - // Pressing enter through the signer key is how a user says "this export has - // no passwords" — the remaining three would be meaningless without it. - test("stop being asked when the signer key is left blank", async () => { - mockSelect.mockResolvedValue("firebase"); - mockText.mockResolvedValueOnce("users.json").mockResolvedValueOnce(" "); - - const result = await runWizard({}); - - expect(result.firebaseHashConfig).toBeUndefined(); - expect(mockText).toHaveBeenCalledTimes(2); - }); - - // The signer key is a Firebase secret, so it is never written to disk and so - // there is nothing to offer back. A repeat run passes it as a flag or env var. - test("are never pre-filled, because they are not saved", async () => { - mockSelect.mockResolvedValue("firebase"); - mockText - .mockResolvedValueOnce("users.json") - .mockResolvedValueOnce("SIGNER") - .mockResolvedValueOnce("Bw==") - .mockResolvedValueOnce("8") - .mockResolvedValueOnce("14"); - - await runWizard({}); - - expect(textCall(1)?.default).toBeUndefined(); - expect(textCall(3)?.default).toBeUndefined(); - }); - - test("are not asked for on a non-firebase source", async () => { - mockSelect.mockResolvedValue("auth0"); - mockText.mockResolvedValue("users.json"); - - await runWizard({}); - - expect(mockText).toHaveBeenCalledTimes(1); - }); - - test("are not asked for when the flags already supplied them", async () => { - mockSelect.mockResolvedValue("firebase"); - mockText.mockResolvedValue("users.json"); - - const config = { - base64_signer_key: "FLAG", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }; - const result = await runWizard({ firebaseHashConfig: config }); + // An export with no password hashes needs none of them. + test("stops when the signer key is left blank", async () => { + mockText.mockResolvedValueOnce(""); + expect(await promptForFirebaseHashConfig()).toBeUndefined(); expect(mockText).toHaveBeenCalledTimes(1); - expect(result.firebaseHashConfig).toEqual(config); - }); - - test.each([["0"], ["-1"], ["1.5"], ["many"]])("rejects %p as a rounds value", async (value) => { - mockSelect.mockResolvedValue("firebase"); - mockText - .mockResolvedValueOnce("users.json") - .mockResolvedValueOnce("SIGNER") - .mockResolvedValueOnce("Bw==") - .mockResolvedValueOnce("8") - .mockResolvedValueOnce("14"); - - await runWizard({}); - - expect(textCall(3)?.validate?.(value)).toMatch(/positive whole number/); - }); -}); - -describe("throwAgentFlagsRequired", () => { - test.each([ - [{ source: true, file: true }, /the file \(or an export run ID\) and --source /], - [{ source: true, file: false }, /Pass --source \./], - [{ source: false, file: true }, /Pass the file \(or an export run ID\)\./], - ])("names only the flags that are missing (%p)", (missing, expected) => { - expect(() => throwAgentFlagsRequired(missing)).toThrow(expected); - }); - - test("says why it cannot prompt", () => { - expect(() => throwAgentFlagsRequired({ source: true, file: true })).toThrow( - /cannot prompt in agent mode/, - ); }); }); diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index 0b49a45e2..39c9fae7a 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -1,39 +1,31 @@ /** - * The interactive path behind a bare `clerk migrate import`. + * The prompts behind an interactive `clerk migrate import`. * - * Ported from the standalone migration-tool's `src/migrate/cli.ts` interactive - * flow. + * Each one fills in exactly one thing the command was not given: the file, + * the source when the file does not name its own, and Firebase's hash + * parameters when neither the flags nor the export carry them. * - * Agent mode never reaches here — `run` raises a usage error naming the flags - * instead, because an agent cannot answer a prompt. + * Nothing here runs for an agent, a non-TTY run or `--json`: `run` raises a + * usage error naming what to pass instead. */ -import { throwUsageError } from "../../lib/errors.ts"; import { select } from "../../lib/listage.ts"; import { log } from "../../lib/log.ts"; import { text } from "../../lib/prompts.ts"; -import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebase-hash.ts"; import { fileExists, getFileType } from "./lib/transform.ts"; import { sources } from "./sources/registry.ts"; import type { FirebaseHashConfig } from "./types.ts"; -export type WizardResult = { - source: string; - file: string; - firebaseHashConfig?: FirebaseHashConfig; -}; - /** Trims a description down to a single readable hint line. */ function hint(description: string): string { const firstSentence = description.split(". ")[0] ?? description; return firstSentence.length > 96 ? `${firstSentence.slice(0, 93)}...` : firstSentence; } -async function pickSource(): Promise { - // Built from the registry, so a new platform appears here with no second - // place to update. +/** Asks which platform the file came from. Built from the registry. */ +export async function promptForSource(): Promise { return select({ - message: "Which platform are you migrating from?", + message: "Which platform did this file come from?", choices: sources.map((entry) => ({ name: entry.label, value: entry.key, @@ -42,8 +34,9 @@ async function pickSource(): Promise { }); } -async function askFile(): Promise { - return text({ +/** Asks for the file to import. */ +export async function promptForFile(): Promise { + const answer = await text({ message: "Path to the exported user file (JSON or CSV)", validate: (value) => { const file = value?.trim(); @@ -53,16 +46,28 @@ async function askFile(): Promise { return undefined; }, }); + return answer.trim(); +} + +async function askNumber(label: string): Promise { + const answer = await text({ + message: label, + validate: (value) => { + const parsed = Number(value?.trim()); + return Number.isInteger(parsed) && parsed > 0 ? undefined : "Enter a positive whole number"; + }, + }); + return Number(answer.trim()); } /** * Collects Firebase's four hash parameters. * * Asked as a set because a partial set produces a digest that verifies against - * nothing. Pressing enter through all four leaves the config unset, which is + * nothing. Pressing enter at the first leaves the config unset, which is * correct for an export with no password hashes. */ -async function askFirebaseHashConfig(): Promise { +export async function promptForFirebaseHashConfig(): Promise { log.info( "Firebase password hashes need the project's hash parameters. Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", ); @@ -89,63 +94,3 @@ async function askFirebaseHashConfig(): Promise mem_cost: await askNumber("mem cost"), }; } - -async function askNumber(label: string): Promise { - const answer = await text({ - message: label, - validate: (value) => { - const parsed = Number(value?.trim()); - return Number.isInteger(parsed) && parsed > 0 ? undefined : "Enter a positive whole number"; - }, - }); - return Number(answer.trim()); -} - -/** - * Fills in whichever of source and file were not passed. - * - * @param provided - Flags the caller already supplied; those are not asked for. - */ -export async function runWizard( - provided: { - source?: string; - file?: string; - firebaseHashConfig?: FirebaseHashConfig; - } & FirebaseHashFlags, -): Promise { - const source = provided.source ?? (await pickSource()); - const file = provided.file ?? (await askFile()); - - let firebaseHashConfig = provided.firebaseHashConfig; - if (source === "firebase" && !firebaseHashConfig) { - firebaseHashConfig = - resolveFirebaseHashConfig(provided, "firebase") ?? (await askFirebaseHashConfig()); - } - - return { source, file, ...(firebaseHashConfig ? { firebaseHashConfig } : {}) }; -} - -/** - * The error an agent gets instead of a prompt. - * - * Names exactly the flags that are missing, so the caller can retry without - * guessing which of the two it forgot. - */ -export function throwAgentFlagsRequired(missing: { source: boolean; file: boolean }): never { - const flags = [ - missing.file ? "the file (or an export run ID)" : undefined, - missing.source ? "--source " : undefined, - ].filter(Boolean); - - throwUsageError( - `\`clerk migrate import\` is interactive and cannot prompt in agent mode. Pass ${flags.join(" and ")}.`, - undefined, - undefined, - [ - { - command: `clerk migrate import users.json --source ${sources[0]?.key ?? "clerk"} -y`, - description: "Run non-interactively", - }, - ], - ); -} diff --git a/test/e2e/migrate.test.ts b/test/e2e/migrate.test.ts index 9dbf2c608..5ee20f66b 100644 --- a/test/e2e/migrate.test.ts +++ b/test/e2e/migrate.test.ts @@ -5,8 +5,8 @@ * - A Better Auth scrypt hash, sent as `scrypt_werkzeug`, verifies against the * password it was made from. A unit test can only check the string shape. * - A user whose only email is unverified, imported into an instance that - * requires an email, is refused by Clerk. The import's checks treat that - * user as a reject on the strength of this test. + * requires an email, is refused by Clerk. The import's checks reject that + * user up front on the strength of this test, so it checks both halves. * * Requires `CLERK_PLATFORM_API_KEY` and `CLERK_CLI_TEST_APP_ID`. Locally, run * via `bun run test:e2e:op` so 1Password resolves both in-memory. @@ -65,11 +65,11 @@ afterAll(async () => { }, 60_000); /** Imports `users` as a Better Auth export and returns each user's run line. */ -async function importBetterAuth(users: Record[]) { +async function importBetterAuth(users: Record[], extra: string[] = []) { const file = join(workDir, `betterauth-${randomBytes(4).toString("hex")}.json`); writeFileSync(file, JSON.stringify(users)); - await cli(["migrate", "import", file, "--source", "betterauth", "--yes"]); + await cli(["migrate", "import", file, "--source", "betterauth", "--yes", ...extra]); const runsDir = join(workDir, "runs"); const [runId] = readdirSync(runsDir) @@ -131,9 +131,23 @@ test("a user whose only email is unverified is refused where email is required", } const hex = randomBytes(6).toString("hex"); - const [line] = await importBetterAuth([ - { user_id: `ba_${hex}`, email: `${hex}+clerk_test@clerkcookie.com`, email_verified: false }, + // The premise: Clerk itself refuses a user created with no email, which is + // what an unverified-only user is at `POST /v1/users`. + const direct = await cli([ + "api", + "/users", + "-d", + JSON.stringify({ external_id: `direct_${hex}`, skip_password_requirement: true }), ]); - - expect(line).toMatchObject({ status: "failed" }); + expect(direct.exitCode).not.toBe(0); + + // And the import's checks reject them before asking Clerk. + const [line] = await importBetterAuth( + [{ user_id: `ba_${hex}`, email: `${hex}+clerk_test@clerkcookie.com`, email_verified: false }], + ["--allow-partial"], + ); + expect(line).toMatchObject({ + status: "skipped", + reason: "only has an unverified email, and this instance requires an email", + }); }, 60_000); From 9fca06c09e7ab4656af30239b18abff177260d06 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 29 Sep 2026 18:21:19 -0400 Subject: [PATCH 060/141] feat(migrate): print the target first on every command, and name the key's real source `printTarget()` now prints one header everywhere: the environment, the app when the key came from one, the instance ID from `GET /v1/instance`, and where the key came from (`--secret-key`, `--app`, the `CLERK_SECRET_KEY` env var, an accountless app's `.env.local`, or the linked profile). `import` and `undo` print it first. Every export prints its source platform. `export clerk` prints, and records on its run, the instance it reads. `runs` names the runs folder, and every `--json` object carries the same facts as `target`. `describeBapiTarget` followed its own precedence instead of `resolveBapiSecretKey`'s. It: - named the linked app when the key actually came from `CLERK_SECRET_KEY`, which may belong to any app at all - named the key's source only for an accountless app - returned nothing for `--secret-key` It now walks the same chain in the same order, and names the source every time ("My App (development) via the linked profile", "the instance behind --secret-key"). `clerk users` dry-runs print the corrected wording too. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 18 +++- .../src/commands/migrate/export/auth0.ts | 2 + .../src/commands/migrate/export/authjs.ts | 2 + .../src/commands/migrate/export/betterauth.ts | 2 + .../src/commands/migrate/export/clerk.test.ts | 21 ++++ .../src/commands/migrate/export/clerk.ts | 12 ++- .../src/commands/migrate/export/firebase.ts | 2 + .../src/commands/migrate/export/supabase.ts | 2 + .../src/commands/migrate/export/workos.ts | 2 + .../src/commands/migrate/lib/target.test.ts | 96 +++++++++++++++++++ .../src/commands/migrate/lib/target.ts | 21 +++- .../cli-core/src/commands/migrate/run.test.ts | 2 +- .../src/commands/migrate/undo.test.ts | 2 +- .../cli-core/src/lib/bapi-command.test.ts | 55 +++++++++-- packages/cli-core/src/lib/bapi-command.ts | 51 +++++----- 15 files changed, 250 insertions(+), 40 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/target.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 8c9ab0311..7b3473d69 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -20,7 +20,23 @@ from `clerk link`. The **instance type is read from the key**: `sk_live_…` is treated as production, anything else as development. That choice drives the throughput -defaults and the hard development-instance cap below. +defaults and the development-instance user limit below. + +**Every command prints its target first.** `import` and `undo` name the +instance — its environment, its app when the key came from one, and its ID from +`GET /v1/instance` — and where the key came from: `--secret-key`, `--app`, the +`CLERK_SECRET_KEY` env var, an accountless app's `.env.local`, or the linked +profile. An export names its source platform instead, and `export clerk` the +instance it reads. `runs` names the runs folder. `--json` carries the same +facts as `target`. + +``` +Target: My App (app_2x9k…), production instance ins_2x9k… +Key from: linked profile +``` + +The instance ID is what a run records, so `undo` and re-runs can tell whether +the key now in use still addresses the same instance. ## Commands diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index cdc33b3da..3c9ce0f9f 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -22,6 +22,7 @@ import { password as passwordPrompt, text } from "../../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { finishExport, startExportRun } from "./shared.ts"; @@ -333,6 +334,7 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { const resolved = await resolveAuth0Credentials(options); await withGutter("Exporting users from Auth0", async () => { + if (!options.json) printTarget({ platform: "auth0" }); // Only Auth0 can say whether these three go together, and whether the // application carries the `read:users` scope, so a rejected set is asked // for again here. diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 6c14da347..73c9d7f86 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -14,6 +14,7 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { log } from "../../../lib/log.ts"; import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { finishExport, startExportRun } from "./shared.ts"; import { @@ -122,6 +123,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { const dbUrl = await resolveDbUrl(options, AUTHJS_DB); await withGutter("Exporting users from Auth.js", async () => { + if (!options.json) printTarget({ platform: "authjs" }); const { value: { rows, table }, } = await withInputRetry( diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 2398f7c58..54d23d9d8 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -18,6 +18,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { finishExport, startExportRun } from "./shared.ts"; import { @@ -172,6 +173,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const dbUrl = await resolveDbUrl(options, BETTERAUTH_DB); await withGutter("Exporting users from Better Auth", async () => { + if (!options.json) printTarget({ platform: "betterauth" }); const { value: { rows, plugins }, } = await withInputRetry( diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 6c54ed80e..3c5803ab4 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -49,6 +49,10 @@ afterEach(() => { function stubPages(pages: unknown[][]) { let call = 0; globalThis.fetch = (async (input: string | URL | Request) => { + // The export names the instance it reads before paging through it. + if (new URL(input.toString()).pathname === "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/v1/instance") { + return Response.json({ object: "instance", id: "ins_src", environment_type: "production" }); + } requests.push(input.toString()); return Response.json(pages[call++] ?? []); }) as unknown as typeof fetch; @@ -269,6 +273,23 @@ describe("exportClerk", () => { expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); }); + test("names the instance it reads from first, and records it on the run", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(Bun.stripANSI(captured.err)).toContain("Source: Clerk, production instance ins_src"); + expect( + JSON.parse(fs.readFileSync(onlyExportFile().replace("export.json", "run.json"), "utf-8")), + ).toMatchObject({ + target: { + platform: "clerk", + instanceId: "ins_src", + keySource: "the instance behind --secret-key", + }, + }); + }); + test("--output controls the destination, relative to the working directory", async () => { stubPages([[user()], []]); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 63873ff43..403cc5ca6 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -19,6 +19,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; import { retryOn429 } from "../lib/retry.ts"; +import { fetchInstanceIdentity, printTarget } from "../lib/target.ts"; import { resolveClerkSource } from "./clerk-source.ts"; import { finishExport, startExportRun } from "./shared.ts"; @@ -240,13 +241,20 @@ export async function exportClerk(options: ExportClerkOptions): Promise { }); await withGutter("Exporting users from Clerk", async () => { - log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); + const identity = await fetchInstanceIdentity(source.secretKey); + const target = { + platform: "clerk", + env: identity.env, + instanceId: identity.instanceId, + ...(source.target ? { keySource: source.target } : {}), + }; + if (!options.json) printTarget(target); const users = await withSpinner("Fetching users from Clerk...", async (spinner) => fetchAllClerkUsers({ secretKey: source.secretKey, spinner }), ); - const run = await startExportRun(options, { platform: "clerk", appLabel: source.target }); + const run = await startExportRun(options, target); const { users: exported, coverage } = buildClerkExport(users, run.append); finishExport({ run, options, users: exported, coverage }); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 4e94d2e9f..a3aaceb22 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -32,6 +32,7 @@ import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; import type { FirebaseHashConfig } from "../types.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { finishExport, startExportRun } from "./shared.ts"; @@ -515,6 +516,7 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { + if (!options.json) printTarget({ platform: "firebase" }); // Only Google can say whether a well-formed key is still a valid one, so a // revoked or deleted key fails here and is asked for again. const { value: token, input: account } = await withInputRetry( diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index bb6ea8481..33b6a79f6 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -13,6 +13,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { finishExport, startExportRun } from "./shared.ts"; import { @@ -123,6 +124,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { const dbUrl = await resolveDbUrl(options, SUPABASE_DB); await withGutter("Exporting users from Supabase", async () => { + if (!options.json) printTarget({ platform: "supabase" }); const { value: rows } = await withInputRetry( dbUrl, async () => promptDbUrl(SUPABASE_DB), diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index 37fc531e0..17f204ef8 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -24,6 +24,7 @@ import { confirm, password as passwordPrompt } from "../../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; import { withInputRetry } from "../lib/input-retry.ts"; import { createApiScheduler } from "../lib/scheduler.ts"; @@ -440,6 +441,7 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise const resolved = await resolveWorkOsApiKey(options); await withGutter("Exporting users from WorkOS", async () => { + if (!options.json) printTarget({ platform: "workos" }); // Only WorkOS can say whether the key is live, for the right environment, // and not revoked — so a rejected key is asked for again here. The page it // fetches is kept and reused, so proving the key costs no extra request. diff --git a/packages/cli-core/src/commands/migrate/lib/target.test.ts b/packages/cli-core/src/commands/migrate/lib/target.test.ts new file mode 100644 index 000000000..b63af9703 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/target.test.ts @@ -0,0 +1,96 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { describeTarget, fetchInstanceIdentity, printTarget } from "./target.ts"; + +const captured = useCaptureLog(); + +describe("printTarget", () => { + test.each([ + [ + "a linked app", + { + env: "production", + appId: "app_1", + appLabel: "My App", + instanceId: "ins_1", + keySource: "linked profile", + }, + "Target: My App (app_1), production instance ins_1", + "Key from: linked profile", + ], + [ + "a bare --secret-key", + { env: "development", instanceId: "ins_2", keySource: "--secret-key" }, + "Target: development instance ins_2", + "Key from: --secret-key", + ], + [ + "an exported CLERK_SECRET_KEY", + { env: "development", instanceId: "ins_3", keySource: "CLERK_SECRET_KEY env var" }, + "Target: development instance ins_3", + "Key from: CLERK_SECRET_KEY env var", + ], + ])("names %s", (_label, target, first, second) => { + printTarget(target); + const lines = Bun.stripANSI(captured.err).split("\n"); + expect(lines).toEqual([first, second]); + }); + + test("names an export's source platform", () => { + printTarget({ platform: "auth0" }); + expect(captured.err).toBe("Source: auth0"); + }); + + test("names the Clerk instance a Clerk export reads", () => { + printTarget({ platform: "clerk", env: "production", instanceId: "ins_9" }); + expect(Bun.stripANSI(captured.err)).toBe("Source: Clerk, production instance ins_9"); + }); +}); + +describe("describeTarget", () => { + test("is one line for a list", () => { + expect(describeTarget({ appLabel: "My App", env: "development", instanceId: "ins_1" })).toBe( + "My App (development, ins_1)", + ); + expect(describeTarget({ platform: "auth0" })).toBe("auth0"); + }); +}); + +describe("fetchInstanceIdentity", () => { + let originalFetch: typeof globalThis.fetch; + + beforeAll(() => { + originalFetch = globalThis.fetch; + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + test("reads the instance behind the key", async () => { + globalThis.fetch = (async () => + Response.json({ + object: "instance", + id: "ins_7", + environment_type: "production", + })) as unknown as typeof fetch; + + expect(await fetchInstanceIdentity("sk_live_x")).toEqual({ + instanceId: "ins_7", + env: "production", + }); + }); + + // The same key always addresses the same instance, so a hash still tells two + // instances apart when the API cannot say. + test("falls back to a stable hash of the key when the instance cannot be read", async () => { + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + + const first = await fetchInstanceIdentity("sk_test_a"); + expect(first.instanceId).toMatch(/^key_[0-9a-f]{16}$/); + expect(first.env).toBe("development"); + expect(await fetchInstanceIdentity("sk_test_a")).toEqual(first); + expect((await fetchInstanceIdentity("sk_test_b")).instanceId).not.toBe(first.instanceId); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/target.ts b/packages/cli-core/src/commands/migrate/lib/target.ts index 2ac8c1576..d39e3381f 100644 --- a/packages/cli-core/src/commands/migrate/lib/target.ts +++ b/packages/cli-core/src/commands/migrate/lib/target.ts @@ -68,7 +68,7 @@ async function describeKeySource( * same key always addresses the same instance, so it still tells two * instances apart, and nothing is sent anywhere. */ -async function fetchInstanceIdentity( +export async function fetchInstanceIdentity( secretKey: string, ): Promise<{ instanceId: string; env: string }> { const fallbackEnv = detectInstanceType(secretKey) === "prod" ? "production" : "development"; @@ -116,8 +116,23 @@ export function describeTarget(target: RunTarget): string { return where ? `${name} (${where})` : name; } -/** The header every command that acts on an instance prints first. */ +/** + * The header every command prints first: which instance it acts on, and where + * the key came from. An export names its source platform instead, plus the + * Clerk instance when that is what it reads. + * + * `--json` carries the same facts as `target`, so this is for humans only. + */ export function printTarget(target: RunTarget): void { - log.info(`Target: ${describeTarget(target)}`); + const heading = target.platform ? "Source" : "Target"; + const instance = target.instanceId + ? `${target.env ?? "unknown"} instance ${target.instanceId}` + : undefined; + const app = target.appLabel + ? `${target.appLabel}${target.appId ? ` (${target.appId})` : ""}` + : undefined; + const platform = target.platform && target.platform !== "clerk" ? target.platform : undefined; + const parts = [platform ?? (target.platform === "clerk" ? "Clerk" : undefined), app, instance]; + log.info(`${heading}: ${parts.filter(Boolean).join(", ")}`); if (target.keySource) log.info(dim(`Key from: ${target.keySource}`)); } diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 7c22ae50d..790a0413a 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -199,7 +199,7 @@ describe("run", () => { test("prints the target first", async () => { await run(baseOptions); - expect(captured.err).toContain("Target: instance (development, ins_1)"); + expect(captured.err).toContain("Target: development instance ins_1"); expect(captured.err.indexOf("Target:")).toBeLessThan(captured.err.indexOf("Checks")); }); diff --git a/packages/cli-core/src/commands/migrate/undo.test.ts b/packages/cli-core/src/commands/migrate/undo.test.ts index b17cf3c92..94bb89a21 100644 --- a/packages/cli-core/src/commands/migrate/undo.test.ts +++ b/packages/cli-core/src/commands/migrate/undo.test.ts @@ -143,7 +143,7 @@ describe("--dry-run", () => { await undo(record.id, withDir({ dryRun: true })); - expect(captured.err).toContain("Target: instance (development, ins_1)"); + expect(captured.err).toContain("Target: development instance ins_1"); expect(captured.err).toContain("Will delete 2 users"); expect(captured.err).toContain("1 of them has signed in since the import"); expect(deletes()).toHaveLength(0); diff --git a/packages/cli-core/src/lib/bapi-command.test.ts b/packages/cli-core/src/lib/bapi-command.test.ts index 6cc0e2800..3b4fb362d 100644 --- a/packages/cli-core/src/lib/bapi-command.test.ts +++ b/packages/cli-core/src/lib/bapi-command.test.ts @@ -297,7 +297,7 @@ describe("bapi-command", () => { expect(fetchApplicationSpy).not.toHaveBeenCalled(); }); - test("describes the resolved app and instance target", async () => { + test("describes the app and instance --app resolves, and says so", async () => { resolveAppContextSpy.mockResolvedValue({ appId: "app_123", appLabel: "My App", @@ -306,7 +306,7 @@ describe("bapi-command", () => { }); await expect(describeBapiTarget({ app: "app_123", instance: "prod" })).resolves.toBe( - "My App (production)", + "My App (production) via --app", ); expect(resolveAppContextSpy).toHaveBeenCalledWith({ @@ -315,14 +315,41 @@ describe("bapi-command", () => { }); }); - test("returns no target description when only a secret key is available", async () => { - resolveAppContextSpy.mockRejectedValue( - new CliError("linked profile missing", { - code: ERROR_CODE.NOT_LINKED, - }), + test("names the linked profile as the key's source", async () => { + resolveAppContextSpy.mockResolvedValue({ + appId: "app_123", + appLabel: "My App", + instanceId: "ins_dev", + instanceLabel: "development", + }); + + await expect(describeBapiTarget({})).resolves.toBe( + "My App (development) via the linked profile", ); + }); + + test("names --secret-key rather than describing nothing", async () => { + await expect(describeBapiTarget({ secretKey: "sk_test_123" })).resolves.toBe( + "the instance behind --secret-key", + ); + expect(resolveAppContextSpy).not.toHaveBeenCalled(); + }); + + // resolveBapiSecretKey takes an exported key before the linked profile, so + // naming the linked app here would point at an instance the key may not be for. + test("an exported CLERK_SECRET_KEY wins over the linked profile, matching resolveBapiSecretKey", async () => { + process.env.CLERK_SECRET_KEY = "sk_test_env"; + resolveAppContextSpy.mockResolvedValue({ + appId: "app_123", + appLabel: "Linked App", + instanceId: "ins_dev", + instanceLabel: "development", + }); - await expect(describeBapiTarget({ secretKey: "sk_test_123" })).resolves.toBeUndefined(); + await expect(describeBapiTarget({})).resolves.toBe( + "the instance behind the CLERK_SECRET_KEY env var", + ); + expect(resolveAppContextSpy).not.toHaveBeenCalled(); }); test("describes an unclaimed keyless target without querying the account", async () => { @@ -338,11 +365,21 @@ describe("bapi-command", () => { test("an explicit --secret-key wins over a keyless target on disk, matching resolveBapiSecretKey", async () => { resolveKeylessTargetSpy.mockResolvedValue({ secretKey: "sk_test_disk", source: ".env.local" }); - await expect(describeBapiTarget({ secretKey: "sk_test_explicit" })).resolves.toBeUndefined(); + await expect(describeBapiTarget({ secretKey: "sk_test_explicit" })).resolves.toBe( + "the instance behind --secret-key", + ); expect(resolveKeylessTargetSpy).not.toHaveBeenCalled(); }); + test("describes nothing when nothing is linked, leaving the error to the key lookup", async () => { + resolveAppContextSpy.mockRejectedValue( + new CliError("linked profile missing", { code: ERROR_CODE.NOT_LINKED }), + ); + + await expect(describeBapiTarget({})).resolves.toBeUndefined(); + }); + test("throws instance-not-found when the resolved instance is missing from the application", async () => { resolveAppContextSpy.mockResolvedValue({ appId: "app_123", diff --git a/packages/cli-core/src/lib/bapi-command.ts b/packages/cli-core/src/lib/bapi-command.ts index 0607325d6..e154a9152 100644 --- a/packages/cli-core/src/lib/bapi-command.ts +++ b/packages/cli-core/src/lib/bapi-command.ts @@ -18,39 +18,44 @@ interface ResolveBapiSecretKeyOptions { cwd?: string; } +/** + * Names the instance {@link resolveBapiSecretKey} will reach, and where its key + * comes from, for prose ("… for My App (development) via the linked profile"). + * + * Walks the same chain in the same order. An exported `CLERK_SECRET_KEY` wins + * over a linked profile there, so it wins here too: naming the linked app for + * a key that may belong to any app at all would point the reader at the wrong + * instance. + */ export async function describeBapiTarget( options: ResolveBapiSecretKeyOptions, ): Promise { - // An explicit --secret-key wins in resolveBapiSecretKey, so it has no - // app/instance context to describe. - if (options.secretKey) return undefined; - - // Mirrors resolveBapiSecretKey's precedence: an unclaimed keyless project has - // no app/instance to describe, only the key's own source. - const keyless = await resolveKeylessTarget({ - app: options.app, - instance: options.instance, - cwd: options.cwd, - }); - if (keyless) { - return `this accountless application (secret key from ${keyless.source})`; - } + if (options.secretKey) return "the instance behind --secret-key"; - try { + if (options.app) { const ctx = await resolveAppContext({ app: options.app, instance: options.instance, cwd: options.cwd, }); - return `${ctx.appLabel} (${ctx.instanceLabel})`; + return `${ctx.appLabel} (${ctx.instanceLabel}) via --app`; + } + + if (process.env.CLERK_SECRET_KEY) return "the instance behind the CLERK_SECRET_KEY env var"; + + // An unclaimed keyless project has no app/instance to describe, only the + // key's own source. + const keyless = await resolveKeylessTarget({ instance: options.instance, cwd: options.cwd }); + if (keyless) { + return `this accountless application (secret key from ${keyless.source})`; + } + + try { + const ctx = await resolveAppContext({ instance: options.instance, cwd: options.cwd }); + return `${ctx.appLabel} (${ctx.instanceLabel}) via the linked profile`; } catch (error) { - if ( - error instanceof CliError && - error.code === ERROR_CODE.NOT_LINKED && - (options.secretKey || process.env.CLERK_SECRET_KEY) - ) { - return undefined; - } + // Nothing to name; resolveBapiSecretKey raises the error worth reading. + if (error instanceof CliError && error.code === ERROR_CODE.NOT_LINKED) return undefined; throw error; } } From 4bf1cd3fb3972b995d64b3ac53b042c2b71b0c37 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 29 Sep 2026 18:24:04 -0400 Subject: [PATCH 061/141] docs(migrate): rewrite the README around the six commands, the five rules and the run store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README now opens with the command surface, a three-step export → dry run → import walkthrough, and the five rules every command follows: 1. Consent before any write. 2. `--dry-run` checks against the real instance. 3. State lives in one place. 4. The target prints first. 5. Every command takes `--json`, with the same exit codes everywhere. The run store gets its own section, and the commands follow in the order a migration uses them: export, import, runs, undo, sources, help. The docs now say that schema-invalid users are rejected by the checks, not logged and skipped. Stale comments and test names from earlier phases are also updated. `readme.test.ts` now checks that every `migrate` subcommand has a section and that the rules are stated. It also skips synopsis grammar (``, `[--json]`) and Commander's implicit `help` when resolving examples. The changeset describes the command set as it ships. Co-Authored-By: Claude Opus 5.5 --- .changeset/migrate-cli.md | 2 +- .../cli-core/src/commands/migrate/README.md | 555 ++++++++++-------- .../src/commands/migrate/export/clerk.test.ts | 5 +- .../migrate/export/db-exports.test.ts | 2 +- .../src/commands/migrate/lib/analysis.ts | 2 +- .../src/commands/migrate/lib/clerk-config.ts | 2 +- .../migrate/lib/firebase-hash.test.ts | 4 +- .../src/commands/migrate/lib/firebase-hash.ts | 10 +- .../src/commands/migrate/readme.test.ts | 47 +- .../src/commands/migrate/sources/workos.ts | 2 +- .../cli-core/src/commands/migrate/types.ts | 2 +- .../src/commands/migrate/validator.ts | 2 +- 12 files changed, 358 insertions(+), 277 deletions(-) diff --git a/.changeset/migrate-cli.md b/.changeset/migrate-cli.md index 6ba3096b0..dcb4b8f29 100644 --- a/.changeset/migrate-cli.md +++ b/.changeset/migrate-cli.md @@ -2,4 +2,4 @@ "clerk": minor --- -Add `clerk migrate` for importing users with `migrate import`, exporting from supported auth providers, reviewing migration logs, undoing a migration, and extending imports with custom transformers. +Add `clerk migrate` for moving users into Clerk. `migrate export ` exports users from Clerk, Auth0, Supabase, Auth.js, Better Auth, Firebase or WorkOS into a self-describing file. `migrate import ` checks every user against the instance before writing (`--dry-run`, `--allow-partial`), asks before it writes, and continues where a previous run stopped. `migrate runs` shows what every run did, `migrate undo ` deletes the users an import created, and `migrate sources` shows what each source carries, including sources you write yourself (`--source ./my-source.ts`). diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 7b3473d69..23ff8a7c7 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -3,6 +3,50 @@ Migrate users into a Clerk instance from another auth provider, or from another Clerk instance. +``` +clerk migrate export [-o ] [--json] +clerk migrate import [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] +clerk migrate runs [run-id] [--json] +clerk migrate undo [--dry-run] [--yes] [--json] +clerk migrate sources [source] [--json] +clerk migrate help +``` + +A migration is usually three steps: + +```sh +clerk migrate export supabase # 1. a run, holding export.json +clerk migrate import 20260929-141502-a1b2 --dry-run # 2. check it against the instance +clerk migrate import 20260929-141502-a1b2 --yes # 3. import it +``` + +`clerk migrate undo ` takes an import back out, and `clerk migrate runs` +shows what every run did. Every subcommand takes `--runs-dir ` (or +`CLERK_MIGRATE_DIR`) to keep its runs somewhere else. `clerk migrate` on its own +is a group name, not a command: it prints its help. + +## The rules + +Every command follows these: + +1. **Nothing writes without consent.** Consent is a yes at a terminal prompt, or + `--yes`. Without either, `import` and `undo` print what they would do and + exit 2 with the command to run. `--json` means non-interactive: it never + prompts. +2. **`--dry-run` checks against the real instance, and writes nothing.** An + import's [checks](#checks) run before anything is written. Predicted + rejects stop the import unless `--allow-partial` is passed; fields that would + be dropped are warnings. +3. **State lives in one place: the [run store](#the-run-store).** Each run + records its target, its file, and every source ID → Clerk ID outcome, + including the error for each user who failed. `runs`, `undo`, re-runs and + exports all read or write it. +4. **Every command prints its target first:** the environment, app and + instance, and where the key came from. +5. **Every subcommand takes `--json`.** Exit codes: `0` all good, `1` some users + failed, `2` a usage error or a refusal. The UI goes to stderr and data to + stdout. + ## Targeting And Auth `clerk migrate import` resolves its Backend API key through the CLI's standard @@ -38,181 +82,70 @@ Key from: linked profile The instance ID is what a run records, so `undo` and re-runs can tell whether the key now in use still addresses the same instance. -## Commands - -`clerk migrate` on its own is a group name, not a command: it prints its help -and lists the subcommands below. The direction is always spelled out — -`migrate import` moves users **into** Clerk, `migrate export` gets them **out** -of a source platform — so neither is implied by the group. - -### `clerk migrate import` - -Reads an exported user file, maps it onto Clerk's user schema, checks every -user against the destination instance, and creates them through the Backend -API. - -```sh -clerk migrate import 20260929-141502-a1b2 --dry-run # check, write nothing -clerk migrate import 20260929-141502-a1b2 --yes # an export run -clerk migrate import users.json --source clerk --yes # any other file -clerk migrate import # a human is asked -``` - -| Flag | Description | -| --------------------------------------- | ------------------------------------------------------------------- | -| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | -| `--source ` | Where the file came from: a [source](#sources), or one you wrote | -| `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | -| `--allow-partial` | Import the users that pass, and record the rest as skipped | -| `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | -| `--require-password` | Import only users that carry a password digest | -| `--firebase-signer-key ` | Firebase base64 signer key (overrides the export file) | -| `--firebase-salt-separator ` | Firebase base64 salt separator | -| `--firebase-rounds ` | Firebase scrypt rounds | -| `--firebase-mem-cost ` | Firebase scrypt memory cost | -| `-y, --yes` | Import without prompting | -| `--json` | Output as JSON. Never prompts, so importing needs `--yes` | -| `--runs-dir ` | Where runs are kept (see [Runs](#clerk-migrate-runs)) | - -Plus the targeting flags from the table above: `--secret-key`, `--app` and -`--instance`. - -An export run ID stands for the file that run wrote, and the import records it -as `fromExport`. A file `clerk migrate export` wrote carries its source, so it -needs no `--source`, and a `--source` that contradicts it exits 2. Any other -file — a bare JSON array, a CSV, Firebase's own `{ "users": [...] }` — needs -`--source`. - -**What a human is asked, and what an agent is told.** A human at a terminal who -leaves out the file is asked for its path, and is asked for a source only when -the file does not name one. An agent, a non-TTY run, or `--json` without the -file exits 2 naming what to pass. +## The run store -**Nothing is written without consent.** After the checks, a human is asked -`Import N users?`, and declining writes nothing. `--yes` skips the question. -Without either — an agent, a non-TTY run, `--json` — the run prints the checks -and exits 2 with the exact command to run. +Every import, export and undo is a **run**, and the run store is the one place +`clerk migrate` keeps state. -**Every run prints its target first**, then which [case](#re-running) applies, -then the checks. +### Where runs are kept -Failures do not stop the run: each user's outcome is written to the -[run](#clerk-migrate-runs) and the import continues. A `429` backs off — -honouring `Retry-After` when the response carries it — and retries up to 5 -times before the user is recorded as failed. The command exits 1 if any user -failed. +The first of these that is set: -An **unrecognized password hasher** aborts the whole run before anything is -sent, because it would import credentials nobody can sign in with. +1. `--runs-dir ` +2. `CLERK_MIGRATE_DIR` +3. `/.clerk/migrate/` -`--json` returns `{ target, run, resume, checks, result }`. +The project root is the linked profile's directory, then the git toplevel, then +the current directory. Writing to the default location adds `.clerk/` to the +project's `.gitignore` first, because run files carry user data. -#### Re-running +### What a run holds -Running the same import again continues where it left off. The match is the -file's sha256, the source (and a custom source's content hash), and the -instance ID; the latest matching import run decides what happens: +Each run is a folder named for its ID, `YYYYMMDD-HHmmss-xxxx`: -| Latest match | Re-running does | -| ----------------------------------------- | ---------------------------------------------------------------------- | -| none | a new run | -| interrupted (dead lock or no finish time) | continues the same run, skipping the users it created | -| `partial` | continues the same run, retrying the users that failed or were skipped | -| `complete` | nothing: prints "Already imported in run …" and exits 0 | -| `undone` | a new run | +| File | Contents | +| -------------- | --------------------------------------------------------------------------------------------------------------------------- | +| `run.json` | Kind, status, start and finish times, the target, the source, the file and its sha256, and the counts | +| `users.ndjson` | One line per user outcome: `sourceId`, `clerkId`, `status`, and `reason`, `error`, `code` or `passwordDropped` when present | +| `lock` | The PID of the process writing the run, while it runs | -`--new-run` skips the lookup. A run another live process holds exits 2. +A user's status is `created`, `failed`, `skipped`, `deleted` or `exported`. The last line +for each `sourceId` wins. A `429` retry, an extra email or phone that did not +attach, and a validation failure all land in `error`. -When an import completes, it names the folders it no longer needs: the export it -read, which holds your users' data, and its own run, which only `undo` needs. -Each comes with the `rm -rf` to remove it. +A run is `partial` when any user failed or was skipped, and `complete` +otherwise. A run whose process died, or that never recorded a finish time, +lists as `interrupted`. A lock held by a live process refuses a second writer +with exit 2. -#### Checks +`users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C +still leaves a complete record of everything already processed. An export's +file lands in its run folder as `export.json` unless `--output` says otherwise. -Every import runs the checks before writing anything, and `--dry-run` stops -after them. They sort the users three ways: +### Why `users.ndjson` is NDJSON -- **Rejected** — users Clerk would refuse. Each gets the first reason that - applies: - - it failed schema validation - - its source ID, email or phone repeats an earlier user in the file - - it lacks an identifier the instance requires. An email or phone counts - only when it is verified, because an unverified one is attached after the - user exists - - its password is not the shape its hasher says (`bcrypt`, `scrypt_firebase`, - `argon2i`/`argon2id` and `scrypt_werkzeug` are checked; other hashers are - not) - - Supabase: its only provider is not enabled in Clerk - - the instance already has a user with its source ID, email, phone or - username (a batched `GET /v1/users` lookup, 100 values a request, through - the scheduler). The users a continued run created do not count - - a development instance: it is past the 100-user headroom, counted in file - order -- **Imported, but not everything comes across** — fields the instance is not - set up to store, fields Clerk has no place for (`Clerk won't store: …`), and - passwords a source had to drop. -- **Imported** — everyone else. +One JSON object per line, rather than one JSON array per file. A migration is a +long append-only stream, and that format is the one that survives it: -Any reject stops the import, and it exits 2 with the command that adds -`--allow-partial`. With `--allow-partial`, the rest import and each reject is -recorded as `skipped` with its reason. `--dry-run` exits 2 when the real run -would be refused, and 0 otherwise. +- **Appendable.** Each entry is written as it happens, without rewriting the + file. A JSON array would have to be re-serialized on every user. +- **Crash-safe.** Kill the process at any point and every line already written + is still valid. A truncated array is not parseable at all. +- **Streamable.** `tail -f` shows a long import progressing live, and analysis + reads line by line instead of loading a million-user record into memory. -``` -Checks - 120 users checked - ✗ 12 users rejected - 12: only has an unverified email, and this instance requires an email - u_17, u_22, u_40, u_51, u_88, and 7 more - ⚠ Imported, but not everything comes across - 6 users have a username, which this instance is not set up to store - Clerk won't store: department (120 users) - ✓ 108 users to import +Which is also why it greps usefully without any tooling: -Or change the instance instead - Make Email optional at sign-up - clerk config patch --app app_… --instance ins_… --json '{"auth_email":{"required_for_sign_up":false}}' - Enable Username - clerk config patch --app app_… --instance ins_… --json '{"auth_username":{"used_for_sign_up":true}}' +```sh +grep '"status":"created"' .clerk/migrate/20260929-141502-a1b2/users.ndjson | wc -l +grep '"sourceId":"user_123"' .clerk/migrate/20260929-141502-a1b2/users.ndjson ``` -The fixes are offers, not corrections: an instance that requires an email is -configured as its owner intended, and fixing the export may be the answer. When -the instance settings cannot be read (BAPI `/v1/domains` → the instance's -Frontend API `/v1/environment`), required fields are not checked and the run -says so. - -#### Additional identifiers - -Only the first verified email and phone go on `POST /v1/users`. Every -additional verified identifier, and every unverified one, is attached -afterwards with its own request. A failure there is logged and the user still -counts as imported — a duplicate secondary email should not undo an otherwise -successful user. - -#### Throughput - -Defaults follow Clerk's documented `POST /v1/users` limits: 100 req/s for -production instances, 10 req/s for development. Concurrency defaults to ~95% of -that, assuming ~100ms of API latency. Both are overridable: - -| Variable | Effect | -| --------------------------------- | ----------------------------- | -| `CLERK_MIGRATE_RATE_LIMIT` | Requests per second | -| `CLERK_MIGRATE_CONCURRENCY_LIMIT` | Concurrent in-flight requests | - -A non-numeric or non-positive value is ignored in favour of the default. - -A development instance's user limit is checked with the other -[checks](#checks): new development instances are created with a 100-user limit, -production instances have none, and the run reads the live count -(`GET /v1/users/count`). The limit itself is not served by any API, so a -development instance Clerk has raised may accept more than the checks allow; -`--allow-partial` imports up to the headroom. +## Commands -Users that do exceed the limit come back in the error breakdown as -`You have reached your limit of N users`, annotated with what a development -instance can do about it. +The direction is always spelled out — `migrate import` moves users **into** +Clerk, `migrate export` gets them **out** of a source platform — so neither is +implied by the group. ### `clerk migrate export` @@ -254,7 +187,7 @@ having been told not to. | `firebase` | Firebase Identity Toolkit | `--source firebase` | | `workos` | WorkOS User Management API | `--source workos` | -Every export is a [run](#clerk-migrate-runs), and the file lands in the run +Every export is a [run](#the-run-store), and the file lands in the run folder as `export.json`. `--output` writes it somewhere else instead, resolved against the **current directory** like every other path flag here; the run still records where. Nothing is asked about where the file goes. @@ -543,6 +476,196 @@ and every 10 pages during the user fetch. `withSpinner` hands a no-op to anything that is not a TTY, so without this an agent exporting a large tenant would see nothing at all until the run finished. +### `clerk migrate import` + +Reads an exported user file, maps it onto Clerk's user schema, checks every +user against the destination instance, and creates them through the Backend +API. + +```sh +clerk migrate import 20260929-141502-a1b2 --dry-run # check, write nothing +clerk migrate import 20260929-141502-a1b2 --yes # an export run +clerk migrate import users.json --source clerk --yes # any other file +clerk migrate import # a human is asked +``` + +| Flag | Description | +| --------------------------------------- | ------------------------------------------------------------------- | +| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | +| `--source ` | Where the file came from: a [source](#sources), or one you wrote | +| `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | +| `--allow-partial` | Import the users that pass, and record the rest as skipped | +| `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | +| `--require-password` | Import only users that carry a password digest | +| `--firebase-signer-key ` | Firebase base64 signer key (overrides the export file) | +| `--firebase-salt-separator ` | Firebase base64 salt separator | +| `--firebase-rounds ` | Firebase scrypt rounds | +| `--firebase-mem-cost ` | Firebase scrypt memory cost | +| `-y, --yes` | Import without prompting | +| `--json` | Output as JSON. Never prompts, so importing needs `--yes` | +| `--runs-dir ` | Where runs are kept (see [the run store](#the-run-store)) | + +Plus the targeting flags from the table above: `--secret-key`, `--app` and +`--instance`. + +An export run ID stands for the file that run wrote, and the import records it +as `fromExport`. A file `clerk migrate export` wrote carries its source, so it +needs no `--source`, and a `--source` that contradicts it exits 2. Any other +file — a bare JSON array, a CSV, Firebase's own `{ "users": [...] }` — needs +`--source`. + +**What a human is asked, and what an agent is told.** A human at a terminal who +leaves out the file is asked for its path, and is asked for a source only when +the file does not name one. An agent, a non-TTY run, or `--json` without the +file exits 2 naming what to pass. + +**Nothing is written without consent.** After the checks, a human is asked +`Import N users?`, and declining writes nothing. `--yes` skips the question. +Without either — an agent, a non-TTY run, `--json` — the run prints the checks +and exits 2 with the exact command to run. + +**Every run prints its target first**, then which [case](#re-running) applies, +then the checks. + +Failures do not stop the run: each user's outcome is written to the +[run](#the-run-store) and the import continues. A `429` backs off — +honouring `Retry-After` when the response carries it — and retries up to 5 +times before the user is recorded as failed. The command exits 1 if any user +failed. + +An **unrecognized password hasher** aborts the whole run before anything is +sent, because it would import credentials nobody can sign in with. + +`--json` returns `{ target, run, resume, checks, result }`. + +#### Re-running + +Running the same import again continues where it left off. The match is the +file's sha256, the source (and a custom source's content hash), and the +instance ID; the latest matching import run decides what happens: + +| Latest match | Re-running does | +| ----------------------------------------- | ---------------------------------------------------------------------- | +| none | a new run | +| interrupted (dead lock or no finish time) | continues the same run, skipping the users it created | +| `partial` | continues the same run, retrying the users that failed or were skipped | +| `complete` | nothing: prints "Already imported in run …" and exits 0 | +| `undone` | a new run | + +`--new-run` skips the lookup. A run another live process holds exits 2. + +When an import completes, it names the folders it no longer needs: the export it +read, which holds your users' data, and its own run, which only `undo` needs. +Each comes with the `rm -rf` to remove it. + +#### Checks + +Every import runs the checks before writing anything, and `--dry-run` stops +after them. They sort the users three ways: + +- **Rejected** — users Clerk would refuse. Each gets the first reason that + applies: + - it failed schema validation + - its source ID, email or phone repeats an earlier user in the file + - it lacks an identifier the instance requires. An email or phone counts + only when it is verified, because an unverified one is attached after the + user exists + - its password is not the shape its hasher says (`bcrypt`, `scrypt_firebase`, + `argon2i`/`argon2id` and `scrypt_werkzeug` are checked; other hashers are + not) + - Supabase: its only provider is not enabled in Clerk + - the instance already has a user with its source ID, email, phone or + username (a batched `GET /v1/users` lookup, 100 values a request, through + the scheduler). The users a continued run created do not count + - a development instance: it is past the 100-user headroom, counted in file + order +- **Imported, but not everything comes across** — fields the instance is not + set up to store, fields Clerk has no place for (`Clerk won't store: …`), and + passwords a source had to drop. +- **Imported** — everyone else. + +Any reject stops the import, and it exits 2 with the command that adds +`--allow-partial`. With `--allow-partial`, the rest import and each reject is +recorded as `skipped` with its reason. `--dry-run` exits 2 when the real run +would be refused, and 0 otherwise. + +``` +Checks + 120 users checked + ✗ 12 users rejected + 12: only has an unverified email, and this instance requires an email + u_17, u_22, u_40, u_51, u_88, and 7 more + ⚠ Imported, but not everything comes across + 6 users have a username, which this instance is not set up to store + Clerk won't store: department (120 users) + ✓ 108 users to import + +Or change the instance instead + Make Email optional at sign-up + clerk config patch --app app_… --instance ins_… --json '{"auth_email":{"required_for_sign_up":false}}' + Enable Username + clerk config patch --app app_… --instance ins_… --json '{"auth_username":{"used_for_sign_up":true}}' +``` + +The fixes are offers, not corrections: an instance that requires an email is +configured as its owner intended, and fixing the export may be the answer. When +the instance settings cannot be read (BAPI `/v1/domains` → the instance's +Frontend API `/v1/environment`), required fields are not checked and the run +says so. + +#### Additional identifiers + +Only the first verified email and phone go on `POST /v1/users`. Every +additional verified identifier, and every unverified one, is attached +afterwards with its own request. A failure there is logged and the user still +counts as imported — a duplicate secondary email should not undo an otherwise +successful user. + +#### Throughput + +Defaults follow Clerk's documented `POST /v1/users` limits: 100 req/s for +production instances, 10 req/s for development. Concurrency defaults to ~95% of +that, assuming ~100ms of API latency. Both are overridable: + +| Variable | Effect | +| --------------------------------- | ----------------------------- | +| `CLERK_MIGRATE_RATE_LIMIT` | Requests per second | +| `CLERK_MIGRATE_CONCURRENCY_LIMIT` | Concurrent in-flight requests | + +A non-numeric or non-positive value is ignored in favour of the default. + +A development instance's user limit is checked with the other +[checks](#checks): new development instances are created with a 100-user limit, +production instances have none, and the run reads the live count +(`GET /v1/users/count`). The limit itself is not served by any API, so a +development instance Clerk has raised may accept more than the checks allow; +`--allow-partial` imports up to the headroom. + +Users that do exceed the limit come back in the error breakdown as +`You have reached your limit of N users`, annotated with what a development +instance can do about it. + +### `clerk migrate runs` + +`runs` reads the [run store](#the-run-store). + +```sh +clerk migrate runs # every run, newest first +clerk migrate runs 20260929-141502-a1b2 # one run in full +clerk migrate runs --json +``` + +| Flag | Description | +| ------------------- | ----------------------------- | +| `[run-id]` | Show one run instead of all | +| `--json` | The same data, on stdout | +| `--runs-dir ` | Read runs from somewhere else | + +It prints the runs folder first. The listing shows each run's ID, date, kind, +status, target, file and counts. `runs ` adds the error breakdown and the +users that failed or were skipped, with the path to the full record. An unknown +ID exits 2. + ### `clerk migrate undo` Deletes the users an import run created. The import run is the whole record of @@ -585,58 +708,31 @@ already gone from the instance counts as deleted. The undo is a run of its own, every user is deleted. A partial undo exits 1, and running `undo` again retries the users that failed, in the same undo run. -### `clerk migrate runs` - -Every import, export and undo is a **run**, and the run store is the one place -`clerk migrate` keeps state. `runs` reads it. +### `clerk migrate sources` ```sh -clerk migrate runs # every run, newest first -clerk migrate runs 20260929-141502-a1b2 # one run in full -clerk migrate runs --json +clerk migrate sources # every source, with what it carries +clerk migrate sources betterauth # one source in full +clerk migrate sources ./my-source.ts # a source you wrote +clerk migrate sources --json ``` -| Flag | Description | -| ------------------- | ----------------------------- | -| `[run-id]` | Show one run instead of all | -| `--json` | The same data, on stdout | -| `--runs-dir ` | Read runs from somewhere else | - -It prints the runs folder first. The listing shows each run's ID, date, kind, -status, target, file and counts. `runs ` adds the error breakdown and the -users that failed or were skipped, with the path to the full record. An unknown -ID exits 2. - -#### Where runs are kept - -The first of these that is set: - -1. `--runs-dir ` -2. `CLERK_MIGRATE_DIR` -3. `/.clerk/migrate/` - -The project root is the linked profile's directory, then the git toplevel, then -the current directory. Writing to the default location adds `.clerk/` to the -project's `.gitignore` first, because run files carry user data. - -#### What a run holds - -Each run is a folder named for its ID, `YYYYMMDD-HHmmss-xxxx`: +| Flag | Description | +| ---------- | ---------------------------------------------------------------- | +| `[source]` | A built-in key, or the path to a source you wrote, to show fully | +| `--json` | The same data, on stdout | -| File | Contents | -| -------------- | --------------------------------------------------------------------------------------------------------------------------- | -| `run.json` | Kind, status, start and finish times, the target, the source, the file and its sha256, and the counts | -| `users.ndjson` | One line per user outcome: `sourceId`, `clerkId`, `status`, and `reason`, `error`, `code` or `passwordDropped` when present | -| `lock` | The PID of the process writing the run, while it runs | +`sources` alone prints the table above. `sources ` shows one source in +full: its export command, what it carries with a note for each, where each +field lands (`encrypted_password → password`), its fixed defaults, and any +caveats. An unknown key exits 2 and lists the valid ones. There is no +intro/outro gutter: this reads a static registry rather than running anything. -A user's status is `created`, `failed`, `skipped`, `deleted` or `exported`. The last line -for each `sourceId` wins. A `429` retry, an extra email or phone that did not -attach, and a validation failure all land in `error`. +### `clerk migrate help` -A run is `partial` when any user failed or was skipped, and `complete` -otherwise. A run whose process died, or that never recorded a finish time, -lists as `interrupted`. A lock held by a live process refuses a second writer -with exit 2. +`clerk migrate help` and `clerk migrate --help` print the help for the +group or one command, with examples. `clerk migrate help ` does the +same. ## Sources @@ -660,26 +756,6 @@ Clerk, and a user who signs in with one is linked to their imported account by verified email. See [account linking](https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking). -### `clerk migrate sources` - -```sh -clerk migrate sources # every source, with what it carries -clerk migrate sources betterauth # one source in full -clerk migrate sources ./my-source.ts # a source you wrote -clerk migrate sources --json -``` - -| Flag | Description | -| ---------- | ---------------------------------------------------------------- | -| `[source]` | A built-in key, or the path to a source you wrote, to show fully | -| `--json` | The same data, on stdout | - -`sources` alone prints the table above. `sources ` shows one source in -full: its export command, what it carries with a note for each, where each -field lands (`encrypted_password → password`), its fixed defaults, and any -caveats. An unknown key exits 2 and lists the valid ones. There is no -intro/outro gutter: this reads a static registry rather than running anything. - ### `--source` `clerk migrate import` takes `--source `: @@ -835,8 +911,8 @@ not editing it. **Required:** `userId` (`string`). It becomes the Clerk user's `external_id`, which is what makes a migration re-runnable. -**Identifiers.** At least one of these must be present, or the user is logged as -a validation failure and skipped. Each accepts a single value or an array. +**Identifiers.** At least one of these must be present, or the import's checks +reject the user as invalid. Each accepts a single value or an array. | Field | Type | Description | | -------------------------- | -------------------- | ---------------------------------- | @@ -896,35 +972,6 @@ stamping every user with today's. | `skipLegalChecks` | `boolean` | Skip legal acceptance checks | | `skipPasswordChecks` | `boolean` | Skip password requirements on import | -## Artifacts - -| Path | Contents | -| --------------------------------- | ------------------------------------------------------- | -| `//` | One [run](#what-a-run-holds) per import, export or undo | -| `//export.json` | An export's envelope, unless `--output` says otherwise | - -`users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C -still leaves a complete record of everything already processed. - -### Why `users.ndjson` is NDJSON - -One JSON object per line, rather than one JSON array per file. A migration is a -long append-only stream, and that format is the one that survives it: - -- **Appendable.** Each entry is written as it happens, without rewriting the - file. A JSON array would have to be re-serialized on every user. -- **Crash-safe.** Kill the process at any point and every line already written - is still valid. A truncated array is not parseable at all. -- **Streamable.** `tail -f` shows a long import progressing live, and analysis - reads line by line instead of loading a million-user record into memory. - -Which is also why it greps usefully without any tooling: - -```sh -grep '"status":"created"' .clerk/migrate/20260929-141502-a1b2/users.ndjson | wc -l -grep '"sourceId":"user_123"' .clerk/migrate/20260929-141502-a1b2/users.ndjson -``` - ## API Endpoints | Method | Path | Used by | @@ -972,4 +1019,4 @@ calls at all — they connect over `--db-url`. both become arrays, `"true"`/`1` become booleans, and JSON metadata columns are parsed. An empty column is dropped rather than sent as null. - A user must end up with at least one identifier (email, phone or username). - Users that do not are logged as validation failures and skipped. + Users that do not are rejected by the checks as invalid. diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 3c5803ab4..e68c09ceb 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -328,7 +328,7 @@ describe("exportClerk", () => { expect(captured.err).toContain("No users found to export"); expect(captured.err).not.toContain("Next steps"); - expect(captured.err).not.toContain("migrate --transformer"); + expect(captured.err).not.toContain("Import them with"); }); test("agent mode suppresses the Next steps block", async () => { @@ -338,6 +338,7 @@ describe("exportClerk", () => { expect(captured.err).toContain("Exported 1 user"); expect(captured.err).not.toContain("Next steps"); - expect(captured.err).not.toContain("migrate --transformer"); + // The import command still prints, where an agent can read it. + expect(captured.err).toContain("Import them with"); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 9fc918236..e8fdf5052 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -346,7 +346,7 @@ describe("supabase export", () => { expect(byLabel["have a password hash"]).toBe(1); }); - test("keeps raw_app_meta_data, which --skip-unsupported-providers reads", () => { + test("keeps raw_app_meta_data, which the import checks read for providers", () => { const { users } = buildSupabaseExport([ { id: "u1", email: "a@x.dev", raw_app_meta_data: { providers: ["discord"] } }, ]); diff --git a/packages/cli-core/src/commands/migrate/lib/analysis.ts b/packages/cli-core/src/commands/migrate/lib/analysis.ts index 848ec1d76..72afc7a05 100644 --- a/packages/cli-core/src/commands/migrate/lib/analysis.ts +++ b/packages/cli-core/src/commands/migrate/lib/analysis.ts @@ -8,7 +8,7 @@ import type { User } from "../types.ts"; -/** Non-identifier fields the readiness report reports coverage for. */ +/** Non-identifier fields the readiness rows report coverage for. */ export const ANALYZED_FIELDS = [ { key: "firstName", label: "First name" }, { key: "lastName", label: "Last name" }, diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts index f69ae7fab..a14fa0fb2 100644 --- a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -77,7 +77,7 @@ async function fetchFapiHost(secretKey: string): Promise { * * @returns The settings, or `null` when they could not be read. Callers must * treat `null` as "unknown" rather than as "nothing is enabled" — the - * readiness report degrades to a note, and provider skipping stands down. + * import checks say so and skip the checks that need them. */ export async function fetchInstanceSettings(secretKey: string): Promise { try { diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts index b9d536f9d..d795cfee3 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts @@ -8,9 +8,9 @@ const ALL_FLAGS = { firebaseMemCost: 14, }; -describe("gating on the transformer", () => { +describe("gating on the source", () => { test.each([["clerk"], ["supabase"], ["auth0"], ["authjs"], ["betterauth"]])( - "ignores even explicit flags for the %s transformer", + "ignores even explicit flags for the %s source", (transformer) => { expect(resolveFirebaseHashConfig(ALL_FLAGS, transformer)).toBeUndefined(); }, diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts index bab30d1a7..5b64ae12e 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts @@ -1,8 +1,8 @@ /** * Firebase's four scrypt parameters, read from the `--firebase-*` flags. * - * **Only read when the transformer is `firebase`.** `migrate import` is one - * command serving every platform, and nothing but the Firebase transformer + * **Only read when the source is `firebase`.** `migrate import` is one + * command serving every platform, and nothing but the Firebase source * reads the config off {@link TransformContext}. */ @@ -31,16 +31,16 @@ export type FirebaseHashFlags = { * well-formed but verifies against nothing, so every migrated user would fail * to sign in with no error at import time. * - * @param transformer - The platform being migrated. Anything but `firebase` + * @param source - The platform being migrated. Anything but `firebase` * returns immediately. * @returns The config, or `undefined` when none was supplied — which is fine * for an export that carries no password hashes. */ export function resolveFirebaseHashConfig( flags: FirebaseHashFlags, - transformer: string | undefined, + source: string | undefined, ): FirebaseHashConfig | undefined { - if (transformer !== "firebase") return undefined; + if (source !== "firebase") return undefined; const provided = FIREBASE_FLAGS.filter(([key]) => flags[key] !== undefined); if (provided.length === 0) return undefined; diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index 76181117b..b64b5a1fb 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -1,8 +1,8 @@ /** * Keeps README.md and the command tree honest about each other. * - * This README documents seven export platforms, seven sources and the run - * store across ~1000 lines. Checking it by eye at review time does not + * This README documents six commands, seven export platforms, seven sources + * and the run store across ~1000 lines. Checking it by eye at review time does not * scale, and a doc that names a flag the binary rejects is worse than no doc: * the reader trusts it and gets a usage error. * @@ -102,11 +102,22 @@ function migrateTree(): { path: string; command: Command }[] { const EXAMPLES = documentedCommands(README); +/** + * The words of an example, minus grammar: a synopsis writes `` and + * `[--json]` to say what goes there, and those are not values to resolve. + */ +function words(example: string): string[] { + return example + .split(/\s+/) + .slice(1) + .filter((token) => !/^[<[]/.test(token) && !/[>\]]$/.test(token)); +} + /** One case per (example, flag) pair, so a failure names the exact flag. */ const FLAG_USES: [string, string][] = EXAMPLES.flatMap((example) => - example - .split(/\s+/) - .filter((token) => token.startsWith("-")) + words(example) + // Every command accepts `--help`; Commander does not list it as an option. + .filter((token) => token.startsWith("-") && token !== "--help") .map((token) => [example, token.split("=")[0]!] as [string, string]), ); @@ -119,7 +130,8 @@ describe("migrate README", () => { }); test.each(EXAMPLES)("`%s` resolves to a real command", (example) => { - const tokens = example.split(/\s+/).slice(1); + // `help` is Commander's own subcommand on every group, not a registered one. + const tokens = words(example).filter((token) => token !== "help"); const { command, rest } = resolve(tokens); const firstFlag = rest.findIndex((token) => token.startsWith("-")); const positionals = firstFlag === -1 ? rest : rest.slice(0, firstFlag); @@ -132,10 +144,31 @@ describe("migrate README", () => { }); test.each(FLAG_USES)("`%s` uses %s, which the command accepts", (example, flag) => { - const { command } = resolve(example.split(/\s+/).slice(1)); + const { command } = resolve(words(example)); expect(flagsOf(command)).toContain(flag); }); + // The README is organised around the commands, so each one gets a heading. + test.each(resolve(["migrate"]).command.commands.map((child) => child.name()))( + "has a section for `clerk migrate %s`", + (name) => { + expect(README).toContain(`### \`clerk migrate ${name}\``); + }, + ); + + test("states the rules every command follows", () => { + expect(README).toContain("## The rules"); + for (const rule of [ + "Nothing writes without consent", + "`--dry-run` checks against the real instance", + "State lives in one place", + "Every command prints its target first", + "Every subcommand takes `--json`", + ]) { + expect(README).toContain(rule); + } + }); + test.each(migrateTree())("$path documents every flag it accepts", ({ command }) => { const undocumented = flagsOf(command).filter( (flag) => flag.startsWith("--") && flag !== "--help" && !README.includes(flag), diff --git a/packages/cli-core/src/commands/migrate/sources/workos.ts b/packages/cli-core/src/commands/migrate/sources/workos.ts index 5041dacb4..13d13cc30 100644 --- a/packages/cli-core/src/commands/migrate/sources/workos.ts +++ b/packages/cli-core/src/commands/migrate/sources/workos.ts @@ -8,7 +8,7 @@ import { routeByVerification } from "./shared.ts"; * carried through as the Clerk user's `external_id`. * * **There is no `passwordHasher` default here, and that is deliberate.** Every - * other transformer names the hasher its platform ships so a digest can be + * other source names the hasher its platform ships so a digest can be * verified; WorkOS returns no digest to verify. It accepts password hashes on * import and never gives them back, and its TOTP secrets are returned on enrol * only — so a WorkOS migration moves identities, not credentials. Naming a diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts index ced754db1..10487df5a 100644 --- a/packages/cli-core/src/commands/migrate/types.ts +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -65,7 +65,7 @@ export type FirebaseHashConfig = { }; /** - * Per-run values a transformer may need but cannot read from the user record. + * Per-run values a source may need but cannot read from the user record. * * Passed to `postTransform` rather than held in module state so two runs in one * process — or two test files — cannot see each other's configuration. diff --git a/packages/cli-core/src/commands/migrate/validator.ts b/packages/cli-core/src/commands/migrate/validator.ts index 80954c526..af9ea8fbc 100644 --- a/packages/cli-core/src/commands/migrate/validator.ts +++ b/packages/cli-core/src/commands/migrate/validator.ts @@ -26,7 +26,7 @@ export const passwordHasherEnum = z.enum(PASSWORD_HASHERS); * Validates user data before sending it to Clerk. * * Everything is optional except: - * - `userId`, required for tracking, logging and `--resume-after` + * - `userId`, required for tracking, re-runs and `undo` * - `passwordHasher`, required whenever `password` is present * - at least one identifier (email, phone or username) * From ea8fc9ddd48f35ffd5241d22d11b062bf1a747af Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 14:32:29 -0400 Subject: [PATCH 062/141] fix(migrate): keep banned users banned from Better Auth, Firebase and Supabase - Better Auth: SQLite, libSQL/Turso and MySQL return `banned` as 1, and CSV as "true". postTransform runs before normalizeUserData, so its `!== true` check deleted every real ban. It now accepts the same truthy forms as the verification flags. - Firebase: `disabled` was neither exported nor mapped. A disabled account now imports as banned. - Supabase: `banned_until` was not selected. A ban still in force imports as banned; an expired one imports as active, as it is in Supabase. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/export/firebase.test.ts | 5 ++++ .../src/commands/migrate/export/firebase.ts | 2 ++ .../src/commands/migrate/export/supabase.ts | 1 + .../commands/migrate/sources/betterauth.ts | 7 ++++-- .../src/commands/migrate/sources/firebase.ts | 8 ++++++- .../commands/migrate/sources/sources.test.ts | 23 +++++++++++++++++++ .../src/commands/migrate/sources/supabase.ts | 8 +++++++ 7 files changed, 51 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 64909dcd5..2b084d0b6 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -355,6 +355,11 @@ describe("mapFirebaseUserToExport", () => { }); }); +test("mapFirebaseUserToExport keeps disabled only when it is true", () => { + expect(mapFirebaseUserToExport(fbUser(0, { disabled: true })).disabled).toBe(true); + expect("disabled" in mapFirebaseUserToExport(fbUser(0, { disabled: false }))).toBe(false); +}); + describe("buildFirebaseExport", () => { test("counts coverage and records each user", () => { const lines: UserLine[] = []; diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index a3aaceb22..2ebf704cc 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -421,6 +421,8 @@ export function mapFirebaseUserToExport(user: FirebaseUser): Record>'last_name' AS last_name, raw_user_meta_data, raw_app_meta_data, + banned_until, created_at FROM auth.users ORDER BY created_at diff --git a/packages/cli-core/src/commands/migrate/sources/betterauth.ts b/packages/cli-core/src/commands/migrate/sources/betterauth.ts index c81e05ed8..22793b8b4 100644 --- a/packages/cli-core/src/commands/migrate/sources/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/sources/betterauth.ts @@ -1,5 +1,5 @@ import type { SourceEntry } from "../types.ts"; -import { routeByVerification, splitName } from "./shared.ts"; +import { isVerified, routeByVerification, splitName } from "./shared.ts"; /** * Better Auth → Clerk source. @@ -101,7 +101,10 @@ const betterAuthSource = { // Only carry `banned` when it is actually true — Better Auth writes false // for every user that was never banned, and sending that to Clerk is noise. - if (user.banned !== true) delete user.banned; + // SQLite, libSQL and MySQL hand back 1/0 and CSV hands back "true", so this + // runs before normalizeUserData and must accept those too. + if (isVerified(user.banned, "boolean")) user.banned = true; + else delete user.banned; }, } satisfies SourceEntry; diff --git a/packages/cli-core/src/commands/migrate/sources/firebase.ts b/packages/cli-core/src/commands/migrate/sources/firebase.ts index bf21a17fb..bd5152ad2 100644 --- a/packages/cli-core/src/commands/migrate/sources/firebase.ts +++ b/packages/cli-core/src/commands/migrate/sources/firebase.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; import type { PreTransformResult, SourceEntry } from "../types.ts"; -import { routeByVerification, splitName, toIsoDate } from "./shared.ts"; +import { isVerified, routeByVerification, splitName, toIsoDate } from "./shared.ts"; /** * Column order of `firebase auth:export --format=csv`, which writes no header @@ -81,6 +81,7 @@ const firebaseSource = { passwordSalt: "salt", phoneNumber: "phone", displayName: "name", + disabled: "banned", }, postTransform: (user, context) => { @@ -120,6 +121,11 @@ const firebaseSource = { // Firebase exports timestamps as Unix milliseconds, often as strings. user.createdAt = toIsoDate(user.createdAt, true); splitName(user); + + // A disabled Firebase account is Clerk's banned. Runs before + // normalizeUserData, so accept CSV's "true" as well. + if (isVerified(user.banned, "boolean")) user.banned = true; + else delete user.banned; }, defaults: { diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index f244d4329..50e5e41dc 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -331,7 +331,10 @@ describe("betterauth", () => { test.each([ [true, true], + [1, true], + ["true", true], [false, undefined], + [0, undefined], [undefined, undefined], ])("banned=%p is carried through as %p", (banned, expected) => { expect(one("betterauth", { ...base, banned })?.banned).toBe(expected as boolean | undefined); @@ -350,6 +353,15 @@ describe("betterauth", () => { describe("firebase", () => { const base = { localId: "fb1", email: "a@x.dev", emailVerified: true }; + + test.each([ + [true, true], + ["true", true], + [false, undefined], + [undefined, undefined], + ])("disabled=%p carries as banned=%p", (disabled, expected) => { + expect(one("firebase", { ...base, disabled })?.banned).toBe(expected as boolean | undefined); + }); const withHash = { ...base, passwordHash: "SGFzaA==", salt: "U2FsdA==" }; test("builds the scrypt digest Clerk expects, parameters inline", async () => { @@ -413,6 +425,17 @@ describe("firebase", () => { describe("supabase", () => { const base = { id: "sb1", email: "a@x.dev", email_confirmed_at: "2024-06-29 20:25:06.126079+00" }; + // An expired ban means the user is active again in Supabase. + test.each([ + ["2999-01-01 00:00:00+00", true], + ["2020-01-01 00:00:00+00", undefined], + [undefined, undefined], + ])("banned_until=%p carries as banned=%p", (bannedUntil, expected) => { + const user = one("supabase", { ...base, banned_until: bannedUntil }); + expect(user?.banned).toBe(expected as boolean | undefined); + expect("bannedUntil" in (user ?? {})).toBe(false); + }); + test("maps the bcrypt password and converts the PostgreSQL timestamp", async () => { const { users } = await load("supabase", [ { ...base, encrypted_password: "$2b$10$hash", created_at: "2024-06-29 20:25:06.126079+00" }, diff --git a/packages/cli-core/src/commands/migrate/sources/supabase.ts b/packages/cli-core/src/commands/migrate/sources/supabase.ts index ab67146d8..2bfd3a592 100644 --- a/packages/cli-core/src/commands/migrate/sources/supabase.ts +++ b/packages/cli-core/src/commands/migrate/sources/supabase.ts @@ -46,10 +46,18 @@ const supabaseSource = { phone: "phone", phone_confirmed_at: "phoneConfirmedAt", raw_user_meta_data: "unsafeMetadata", + banned_until: "bannedUntil", created_at: "createdAt", }, postTransform: (user) => { user.createdAt = toIsoDate(user.createdAt); + + // Supabase bans until a time; a "permanent" ban is just a far-future one. + // Clerk's ban has no end, so only a ban still in force carries, and it then + // stays until someone lifts it in Clerk. + const bannedUntil = Date.parse(String(toIsoDate(user.bannedUntil))); + if (bannedUntil > Date.now()) user.banned = true; + delete user.bannedUntil; routeByVerification(user, "email", "emailConfirmedAt", "timestamp"); routeByVerification(user, "phone", "phoneConfirmedAt", "timestamp"); From 459887a4d820d5a857875a89bde55b5a20184878 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 14:32:40 -0400 Subject: [PATCH 063/141] fix(migrate): carry Auth0 blocked users and display names - `blocked` was neither exported nor mapped, so blocked Auth0 users imported as active. It now imports as banned. - `name` was dropped. It is now split into first and last name when given_name/family_name are missing. A one-word name, which is Auth0's email default for database users, is left alone. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/auth0.test.ts | 9 +++++++ .../src/commands/migrate/export/auth0.ts | 4 ++++ .../src/commands/migrate/sources/auth0.ts | 13 +++++++++- .../commands/migrate/sources/sources.test.ts | 24 +++++++++++++++++++ 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index c4c33f2ea..27186ff37 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -251,6 +251,15 @@ describe("mapAuth0UserToExport", () => { } }); + test("keeps blocked only when it is true", () => { + expect(mapAuth0UserToExport(auth0User(0, { blocked: true })).blocked).toBe(true); + expect("blocked" in mapAuth0UserToExport(auth0User(0, { blocked: false }))).toBe(false); + }); + + test("keeps name", () => { + expect(mapAuth0UserToExport(auth0User(0, { name: "Ada Lovelace" })).name).toBe("Ada Lovelace"); + }); + test("omits empty metadata", () => { const mapped = mapAuth0UserToExport( auth0User(0, { user_metadata: {}, app_metadata: { plan: "pro" } }), diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 3c9ce0f9f..32e706165 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -264,6 +264,7 @@ export function mapAuth0UserToExport(user: Auth0User): Record { "user_id", "email", "username", + "name", "given_name", "family_name", "phone_number", @@ -278,6 +279,9 @@ export function mapAuth0UserToExport(user: Auth0User): Record { if (user[field] !== undefined) exported[field] = user[field]; } + // Only when true: every unblocked user would otherwise carry a `false`. + if (user.blocked === true) exported.blocked = true; + for (const field of ["user_metadata", "app_metadata"] as const) { const value = user[field]; if (value && typeof value === "object" && Object.keys(value).length > 0) { diff --git a/packages/cli-core/src/commands/migrate/sources/auth0.ts b/packages/cli-core/src/commands/migrate/sources/auth0.ts index 75988d8b2..79c389f17 100644 --- a/packages/cli-core/src/commands/migrate/sources/auth0.ts +++ b/packages/cli-core/src/commands/migrate/sources/auth0.ts @@ -1,5 +1,5 @@ import type { SourceEntry } from "../types.ts"; -import { routeByVerification } from "./shared.ts"; +import { isVerified, routeByVerification, splitName } from "./shared.ts"; /** * Auth0 → Clerk transformer. @@ -33,6 +33,8 @@ const auth0Source = { email: "email", email_verified: "emailVerified", username: "username", + name: "name", + blocked: "banned", given_name: "firstName", family_name: "lastName", phone_number: "phone", @@ -45,6 +47,15 @@ const auth0Source = { postTransform: (user) => { routeByVerification(user, "email", "emailVerified", "boolean"); routeByVerification(user, "phone", "phoneVerified", "boolean"); + + // given_name/family_name win when present. Auth0 fills `name` with the + // email for database users; splitName leaves a one-word value alone. + if (user.firstName || user.lastName) delete user.name; + else splitName(user); + + // Runs before normalizeUserData, so accept CSV's "true" as well. + if (isVerified(user.banned, "boolean")) user.banned = true; + else delete user.banned; }, defaults: { passwordHasher: "bcrypt" as const, diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index 50e5e41dc..fc76ad8e5 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -182,6 +182,30 @@ describe("auth0", () => { expect(users[0]?.publicMetadata).toBeUndefined(); expect(users[0]?.privateMetadata).toEqual({ plan: "pro" }); }); + + test.each([ + [true, true], + ["true", true], + [false, undefined], + [undefined, undefined], + ])("blocked=%p is carried as banned=%p", (blocked, expected) => { + expect(one("auth0", { ...base, blocked })?.banned).toBe(expected as boolean | undefined); + }); + + test("splits name when given_name and family_name are absent", () => { + const user = one("auth0", { ...base, name: "Ada King Lovelace" }); + expect(user).toMatchObject({ firstName: "Ada", lastName: "King Lovelace" }); + }); + + test("prefers given_name/family_name over name", () => { + const user = one("auth0", { ...base, name: "a@x.dev", given_name: "Ada", family_name: "L" }); + expect(user).toMatchObject({ firstName: "Ada", lastName: "L" }); + }); + + test("leaves a one-word name (Auth0's email default) unset", () => { + const user = one("auth0", { ...base, name: "a@x.dev" }); + expect(user?.firstName).toBeUndefined(); + }); }); describe("workos", () => { From 63886d431be92e5666f92e786d2fe38a3348c937 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 14:32:41 -0400 Subject: [PATCH 064/141] feat(migrate): keep the WorkOS tenant's external_id in private metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The WorkOS `user_…` id stays the Clerk external_id. WorkOS's own `external_id`, the tenant's app ID, was dropped by the exporter; it now lands in private metadata as `workosExternalId`. Private because users can edit unsafe metadata, and other records key on this ID. Also corrects the comment claiming POST /v1/users does not accept `locale`. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/workos.test.ts | 15 +++++++-------- .../src/commands/migrate/export/workos.ts | 13 ++++++++++--- .../src/commands/migrate/sources/sources.test.ts | 11 +++++++++++ .../src/commands/migrate/sources/workos.ts | 16 ++++++++++++++-- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts index 2ac38570b..349fb11f4 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.test.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -317,20 +317,19 @@ describe("mapWorkOsUserToExport", () => { profile_picture_url: "https://x.dev/a.png", last_sign_in_at: "2026-01-01", updated_at: "2026-01-01", - external_id: "cust_1", }), ); - for (const noise of [ - "locale", - "profile_picture_url", - "last_sign_in_at", - "updated_at", - "external_id", - ]) { + for (const noise of ["locale", "profile_picture_url", "last_sign_in_at", "updated_at"]) { expect(noise in mapped).toBe(false); } }); + test("keeps the tenant's own external_id", () => { + expect(mapWorkOsUserToExport(workosUser(0, { external_id: "cust_1" })).external_id).toBe( + "cust_1", + ); + }); + test("omits empty metadata", () => { expect("metadata" in mapWorkOsUserToExport(workosUser(0, { metadata: {} }))).toBe(false); expect(mapWorkOsUserToExport(workosUser(0, { metadata: { plan: "pro" } })).metadata).toEqual({ diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index 17f204ef8..49e289226 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -364,8 +364,8 @@ export function buildIdentityReport( * they were fetched. * * A copy rather than the raw record: WorkOS also returns `locale`, - * `profile_picture_url`, `last_sign_in_at` and `updated_at`, none of which - * `POST /v1/users` accepts. `identities` has no target field either and is + * `profile_picture_url`, `last_sign_in_at` and `updated_at`, which the import + * does not carry. `identities` has no target field either and is * dropped at validation, so it rides along purely as a record for whoever runs * the migration. */ @@ -375,7 +375,14 @@ export function mapWorkOsUserToExport( ): Record { const exported: Record = {}; - for (const field of ["id", "email", "first_name", "last_name", "created_at"] as const) { + for (const field of [ + "id", + "email", + "first_name", + "last_name", + "external_id", + "created_at", + ] as const) { if (user[field]) exported[field] = user[field]; } diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index fc76ad8e5..cfe8c2775 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -240,6 +240,17 @@ describe("workos", () => { expect(users[0]?.unsafeMetadata).toEqual({ plan: "pro" }); }); + // The WorkOS id stays the Clerk external_id; the tenant's own ID is kept + // where users cannot edit it. + test("puts WorkOS's external_id in private metadata", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, external_id: "cust_1" }, + ]); + expect(users[0]?.userId).toBe("user_01ABC"); + expect(users[0]?.privateMetadata).toEqual({ workosExternalId: "cust_1" }); + expect("workosExternalId" in (users[0] ?? {})).toBe(false); + }); + // No other transformer omits it. WorkOS never returns a digest, so naming a // hasher would imply a password column that cannot exist. test("names no password hasher, because WorkOS returns no hashes", () => { diff --git a/packages/cli-core/src/commands/migrate/sources/workos.ts b/packages/cli-core/src/commands/migrate/sources/workos.ts index 13d13cc30..beab06bd8 100644 --- a/packages/cli-core/src/commands/migrate/sources/workos.ts +++ b/packages/cli-core/src/commands/migrate/sources/workos.ts @@ -5,7 +5,10 @@ import { routeByVerification } from "./shared.ts"; * WorkOS → Clerk transformer. * * Works with WorkOS's User Management API. `id` is a `user_…` string and is - * carried through as the Clerk user's `external_id`. + * carried through as the Clerk user's `external_id`. WorkOS's own `external_id` + * — the tenant's app ID, when they set one — has nowhere else to go, so it lands + * in private metadata as `workosExternalId`: private because users can edit + * unsafe metadata, and an ID other records key on must not be editable. * * **There is no `passwordHasher` default here, and that is deliberate.** Every * other source names the hasher its platform ships so a digest can be @@ -28,7 +31,10 @@ const workosSource = { note: "WorkOS never returns password hashes. Users reset their password, or sign in with SSO.", }, mfa: { level: "no", note: "WorkOS returns TOTP secrets at enrolment only." }, - metadata: { level: "yes", note: "`metadata` → unsafe metadata." }, + metadata: { + level: "yes", + note: "`metadata` → unsafe metadata. WorkOS's `external_id` → private metadata `workosExternalId`; the WorkOS `id` becomes the Clerk external ID.", + }, }, transformer: { id: "userId", @@ -37,10 +43,16 @@ const workosSource = { first_name: "firstName", last_name: "lastName", metadata: "unsafeMetadata", + external_id: "workosExternalId", created_at: "createdAt", }, postTransform: (user) => { routeByVerification(user, "email", "emailVerified", "boolean"); + + if (user.workosExternalId) { + user.privateMetadata = { workosExternalId: String(user.workosExternalId) }; + } + delete user.workosExternalId; }, } satisfies SourceEntry; From f6a3232a88fe6016f15c68febe62bc34934e1025 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 14:32:41 -0400 Subject: [PATCH 065/141] fix(migrate): warn when Firebase returns no hash for a password user Firebase only returns digests it made itself (scrypt). Users uploaded with bcrypt, HMAC or another hasher come back with an empty passwordHash, and the exporter dropped them silently. It now counts them and warns that they will need to reset their password. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/export/firebase.test.ts | 11 ++++++++ .../src/commands/migrate/export/firebase.ts | 28 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 2b084d0b6..cd3725057 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -374,6 +374,17 @@ describe("buildFirebaseExport", () => { expect(byLabel["have a phone number"]).toBe(1); expect(lines).toHaveLength(2); }); + + // Firebase returns an empty passwordHash for users it did not hash itself. + test("counts password users whose hash Firebase did not return", () => { + const { users, unreadablePasswords } = buildFirebaseExport([ + fbUser(0, { providerUserInfo: [{ providerId: "password" }] }), + fbUser(1, { passwordHash: "", providerUserInfo: [{ providerId: "password" }] }), + fbUser(2, { passwordHash: "", providerUserInfo: [{ providerId: "google.com" }] }), + ]); + expect(unreadablePasswords).toBe(1); + expect("passwordHash" in users[1]!).toBe(false); + }); }); describe("fetchHashConfig", () => { diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 2ebf704cc..4c2d884e5 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -433,12 +433,24 @@ export function mapFirebaseUserToExport(user: FirebaseUser): Record (provider as { providerId?: unknown })?.providerId === "password") + ); +} + export function buildFirebaseExport( users: FirebaseUser[], record: (line: UserLine) => void = () => {}, ) { const exported: Record[] = []; const counts = { email: 0, verified: 0, password: 0, name: 0, phone: 0 }; + // Password users Firebase returned no hash for: it only returns digests it + // made itself (scrypt), and an empty string for users uploaded with bcrypt, + // HMAC or any other hasher. + let unreadablePasswords = 0; for (const user of users) { const userId = String(user.localId ?? ""); @@ -449,6 +461,7 @@ export function buildFirebaseExport( if (mapped.email) counts.email++; if (mapped.emailVerified) counts.verified++; if (mapped.passwordHash) counts.password++; + else if (hasPasswordProvider(user)) unreadablePasswords++; if (mapped.displayName) counts.name++; if (mapped.phoneNumber) counts.phone++; @@ -460,6 +473,7 @@ export function buildFirebaseExport( return { users: exported, + unreadablePasswords, coverage: [ { label: "have an email address", count: counts.email }, { label: "have a verified email", count: counts.verified }, @@ -537,7 +551,11 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise 0) { + log.warn( + `${unreadablePasswords} user${unreadablePasswords === 1 ? " has" : "s have"} a password Firebase did not return. ` + + "Firebase only returns hashes it made itself; users imported into Firebase with bcrypt, HMAC or another hasher come back without one. " + + "They will be imported without a password and need to reset it.", + ); + } }); } From 0effeda361f2a01e46c667f9217cef41fa2303a3 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 14:35:59 -0400 Subject: [PATCH 066/141] fix(migrate): add the leading + to Supabase phone numbers Supabase stores E.164 numbers without the + (14165550123), and nothing put it back, so Clerk could reject the number or read the wrong country code. An all-digit phone now gets the +; one that already has it is left alone. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/sources/sources.test.ts | 14 ++++++++++++++ .../src/commands/migrate/sources/supabase.ts | 3 +++ 2 files changed, 17 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index cfe8c2775..530cb6679 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -460,6 +460,20 @@ describe("firebase", () => { describe("supabase", () => { const base = { id: "sb1", email: "a@x.dev", email_confirmed_at: "2024-06-29 20:25:06.126079+00" }; + test.each([ + ["14165550123", "+14165550123"], + ["+14165550123", "+14165550123"], + ])("phone %p imports as %p", (phone, expected) => { + const user = one("supabase", { ...base, phone, phone_confirmed_at: "2024-06-29 20:25:06+00" }); + expect(user?.phone).toBe(expected); + }); + + test("adds the + to an unconfirmed phone too", () => { + expect(one("supabase", { ...base, phone: "14165550123" })?.unverifiedPhoneNumbers).toBe( + "+14165550123", + ); + }); + // An expired ban means the user is active again in Supabase. test.each([ ["2999-01-01 00:00:00+00", true], diff --git a/packages/cli-core/src/commands/migrate/sources/supabase.ts b/packages/cli-core/src/commands/migrate/sources/supabase.ts index 2bfd3a592..90d85cf7d 100644 --- a/packages/cli-core/src/commands/migrate/sources/supabase.ts +++ b/packages/cli-core/src/commands/migrate/sources/supabase.ts @@ -52,6 +52,9 @@ const supabaseSource = { postTransform: (user) => { user.createdAt = toIsoDate(user.createdAt); + // Supabase stores E.164 without the leading + (14165550123); Clerk needs it. + if (typeof user.phone === "string" && /^\d+$/.test(user.phone)) user.phone = `+${user.phone}`; + // Supabase bans until a time; a "permanent" ban is just a far-future one. // Clerk's ban has no end, so only a ban still in force carries, and it then // stays until someone lifts it in Clerk. From ebd8acb03adb8d93ec6fdf1af79b7cf61a2bad18 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 15:47:19 -0400 Subject: [PATCH 067/141] feat(migrate): let CLERK_MIGRATE_DEV_USER_LIMIT override the dev user limit The checks rejected every user past 100 on a development instance, with no way around it. Clerk raises that limit per instance on request, and the real value is not served by any public API, so an instance Clerk had raised was refused users the API would accept. CLERK_MIGRATE_DEV_USER_LIMIT now sets the limit the checks use, following the rate and concurrency overrides: anything not a positive number falls back to 100. The reject reason names the variable. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 15 +++++----- .../src/commands/migrate/lib/checks.test.ts | 19 ++++++++++++- .../src/commands/migrate/lib/checks.ts | 10 ++++--- .../src/commands/migrate/lib/instance.test.ts | 14 ++++++++++ .../src/commands/migrate/lib/instance.ts | 28 +++++++++++++------ 5 files changed, 65 insertions(+), 21 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 23ff8a7c7..ff8281f4d 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -627,19 +627,20 @@ Defaults follow Clerk's documented `POST /v1/users` limits: 100 req/s for production instances, 10 req/s for development. Concurrency defaults to ~95% of that, assuming ~100ms of API latency. Both are overridable: -| Variable | Effect | -| --------------------------------- | ----------------------------- | -| `CLERK_MIGRATE_RATE_LIMIT` | Requests per second | -| `CLERK_MIGRATE_CONCURRENCY_LIMIT` | Concurrent in-flight requests | +| Variable | Effect | +| --------------------------------- | ------------------------------------------------------------ | +| `CLERK_MIGRATE_RATE_LIMIT` | Requests per second | +| `CLERK_MIGRATE_CONCURRENCY_LIMIT` | Concurrent in-flight requests | +| `CLERK_MIGRATE_DEV_USER_LIMIT` | Development-instance user limit the checks use (default 100) | A non-numeric or non-positive value is ignored in favour of the default. A development instance's user limit is checked with the other [checks](#checks): new development instances are created with a 100-user limit, production instances have none, and the run reads the live count -(`GET /v1/users/count`). The limit itself is not served by any API, so a -development instance Clerk has raised may accept more than the checks allow; -`--allow-partial` imports up to the headroom. +(`GET /v1/users/count`). The limit itself is not served by any API, so for a +development instance Clerk has raised, set `CLERK_MIGRATE_DEV_USER_LIMIT` to +the raised limit; `--allow-partial` imports up to the headroom. Users that do exceed the limit come back in the error breakdown as `You have reached your limit of N users`, annotated with what a development diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index c3737cdd5..0d4a24371 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -182,11 +182,28 @@ describe("rejects", () => { ); expect(checks.importable.map((entry) => entry.userId)).toEqual(["a", "b"]); expect(checks.rejects).toEqual([ - { sourceId: "c", reason: "over the development instance's 100-user limit" }, + { + sourceId: "c", + reason: + "over the development instance's 100-user limit (raised by Clerk? set CLERK_MIGRATE_DEV_USER_LIMIT)", + }, ]); expect(checks.quota).toEqual({ existing: 98, limit: 100, headroom: 2, over: 1 }); }); + test("CLERK_MIGRATE_DEV_USER_LIMIT raises the headroom", async () => { + process.env.CLERK_MIGRATE_DEV_USER_LIMIT = "500"; + try { + const checks = await checkImport( + input({ instanceType: "dev", existingUsers: 98, users: [user("a"), user("b"), user("c")] }), + ); + expect(checks.rejects).toEqual([]); + expect(checks.quota).toEqual({ existing: 98, limit: 500, headroom: 402, over: 0 }); + } finally { + delete process.env.CLERK_MIGRATE_DEV_USER_LIMIT; + } + }); + test("groups the rejects by reason", async () => { const checks = await checkImport( input({ diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index 391a9065e..b5a358681 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -24,7 +24,7 @@ import { splitIdentifiers } from "../import-users.ts"; import type { User } from "../types.ts"; import { analyzeFields, hasValue } from "./analysis.ts"; import { enabledSocialProviders, providerLabel, toClerkStrategy } from "./clerk-config.ts"; -import { DEV_USER_LIMIT } from "./instance.ts"; +import { resolveDevUserLimit } from "./instance.ts"; import { buildChangePayload, buildSettingChanges } from "./modify-settings.ts"; import { buildReadinessReport } from "./readiness.ts"; import type { ApiScheduler } from "./scheduler.ts"; @@ -447,16 +447,18 @@ export async function checkImport(input: CheckInput): Promise { // The limit is a development-instance default, not a number the API serves, // so it is checked against the live count and the importable users alone. + // An instance Clerk has raised sets CLERK_MIGRATE_DEV_USER_LIMIT. let quota: Quota | undefined; if (input.instanceType === "dev") { - const headroom = Math.max(0, DEV_USER_LIMIT - (input.existingUsers ?? 0)); + const limit = resolveDevUserLimit(); + const headroom = Math.max(0, limit - (input.existingUsers ?? 0)); const over = Math.max(0, candidates.length - headroom); - quota = { existing: input.existingUsers ?? null, limit: DEV_USER_LIMIT, headroom, over }; + quota = { existing: input.existingUsers ?? null, limit, headroom, over }; if (over > 0) { for (const user of candidates.slice(headroom)) { rejects.push({ sourceId: user.userId, - reason: `over the development instance's ${DEV_USER_LIMIT}-user limit`, + reason: `over the development instance's ${limit}-user limit (raised by Clerk? set CLERK_MIGRATE_DEV_USER_LIMIT)`, }); } candidates = candidates.slice(0, headroom); diff --git a/packages/cli-core/src/commands/migrate/lib/instance.test.ts b/packages/cli-core/src/commands/migrate/lib/instance.test.ts index ef95b0b92..5eac24b94 100644 --- a/packages/cli-core/src/commands/migrate/lib/instance.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/instance.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { DEV_USER_LIMIT, + resolveDevUserLimit, detectInstanceType, getDefaultConcurrencyLimit, getDefaultRateLimit, @@ -40,6 +41,19 @@ describe("default limits", () => { }); }); +describe("resolveDevUserLimit", () => { + test.each([ + [undefined, 100], + ["500", 500], + ["250.7", 250], + ["0", 100], + ["-5", 100], + ["lots", 100], + ])("CLERK_MIGRATE_DEV_USER_LIMIT=%p gives %i", (value, expected) => { + expect(resolveDevUserLimit({ CLERK_MIGRATE_DEV_USER_LIMIT: value })).toBe(expected); + }); +}); + describe("resolveLimits", () => { test("derives both limits from the key when nothing is overridden", () => { expect(resolveLimits("sk_live_x", {})).toEqual({ diff --git a/packages/cli-core/src/commands/migrate/lib/instance.ts b/packages/cli-core/src/commands/migrate/lib/instance.ts index 24f222a9d..afd09c46e 100644 --- a/packages/cli-core/src/commands/migrate/lib/instance.ts +++ b/packages/cli-core/src/commands/migrate/lib/instance.ts @@ -11,12 +11,28 @@ * * Only a default: Clerk raises it per instance on request, and the real value * (`max_allowed_users`) is not served by BAPI, DAPI or FAPI — only by Clerk's - * internal staff API. So this is a number to warn against, never one to refuse - * an import over; the instance in front of you may be allowed far more. - * Production instances have no limit at all. + * internal staff API. The checks reject users past it, so an instance Clerk has + * raised overrides it with `CLERK_MIGRATE_DEV_USER_LIMIT` (see + * {@link resolveDevUserLimit}). Production instances have no limit at all. */ export const DEV_USER_LIMIT = 100; +/** A positive number from the environment, or undefined. */ +function positive(value: string | undefined): number | undefined { + if (!value) return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; +} + +/** + * The development-instance user limit for this run: `CLERK_MIGRATE_DEV_USER_LIMIT` + * when it is a positive number, otherwise {@link DEV_USER_LIMIT}. + */ +export function resolveDevUserLimit(env: Record = process.env): number { + const override = positive(env.CLERK_MIGRATE_DEV_USER_LIMIT); + return override ? Math.floor(override) : DEV_USER_LIMIT; +} + /** How many times a 429 is retried before the user is recorded as failed. */ export const MAX_RETRIES = 5; @@ -72,12 +88,6 @@ export function resolveLimits( ): ResolvedLimits { const instanceType = detectInstanceType(secretKey); - const positive = (value: string | undefined): number | undefined => { - if (!value) return undefined; - const parsed = Number(value); - return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; - }; - const rateLimit = positive(env.CLERK_MIGRATE_RATE_LIMIT) ?? getDefaultRateLimit(instanceType); const concurrencyLimit = positive(env.CLERK_MIGRATE_CONCURRENCY_LIMIT) ?? getDefaultConcurrencyLimit(rateLimit); From 9ab81bcc7709edb7c2c3745616a506e4f3d40481 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 15:47:20 -0400 Subject: [PATCH 068/141] fix(migrate): check usernames against the instance's username rules The dry run never looked at Clerk's username rules, so it counted users as importable whose usernames the API then refused, e.g. a `.` on an instance without extended special characters. The checks now mirror clerk_go's validate.Username using the username_settings FAPI already serves: length, letters-only when numeric usernames are off, the allowed character set, and no E.164-shaped usernames. A username that only needs extended special characters says so. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/checks.test.ts | 41 +++++++++++++++++ .../src/commands/migrate/lib/checks.ts | 46 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index 0d4a24371..23e3a282e 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -204,6 +204,47 @@ describe("rejects", () => { } }); + describe("usernames", () => { + const withUsernames = (rules: object) => + ({ + attributes: { email_address: { enabled: true }, username: { enabled: true } }, + social: {}, + username_settings: { min_length: 4, max_length: 64, ...rules }, + }) as unknown as UserSettingsJSON; + const reasonFor = async (username: string, rules: object = {}) => + ( + await checkImport( + input({ settings: withUsernames(rules), users: [user("a", { username })] }), + ) + ).rejects[0]?.reason; + + test.each([ + ["ada_l-1", {}], + ["ada.l", { allow_extended_special_characters: true }], + ])("%p is accepted", async (username, rules) => { + expect(await reasonFor(username, rules)).toBeUndefined(); + }); + + test("a . needs extended special characters", async () => { + expect(await reasonFor("ada.lovelace")).toContain("turn on extended special characters"); + }); + + test.each([ + ["ada", "4–64 characters"], + ["12345", "no letters"], + ["ada@x", "characters Clerk does not allow"], + ])("%p is rejected: %s", async (username, reason) => { + expect(await reasonFor(username)).toContain(reason); + }); + + test("a username is not checked when usernames are off", async () => { + const checks = await checkImport( + input({ settings: EMAIL_REQUIRED, users: [user("a", { username: "a.b" })] }), + ); + expect(checks.rejects).toEqual([]); + }); + }); + test("groups the rejects by reason", async () => { const checks = await checkImport( input({ diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index b5a358681..e63eb6670 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -159,6 +159,51 @@ function missingRequiredIdentifier( return undefined; } +/** What FAPI serves under `username_settings`; `@clerk/shared` types only the lengths. */ +type UsernameSettings = { + min_length?: number; + max_length?: number; + allow_extended_special_characters?: boolean; + allow_numeric_usernames?: boolean; +}; + +const USERNAME_DEFAULT = /^[a-zA-Z0-9_-]+$/; +const USERNAME_EXTENDED = /^[a-zA-Z0-9!#$'+.^_`~-]+$/; + +/** + * Clerk's username rules, mirrored from `validate.Username` in clerk_go, so a + * username the instance would refuse is a reject here rather than a failed + * create. Skipped when usernames are off: the readiness warnings cover that. + */ +function usernameProblem(user: User, settings: UserSettingsJSON | null): string | undefined { + const username = user.username; + if (!settings || typeof username !== "string" || !username) return undefined; + if (!isEnabled(settings, "username")) return undefined; + + const rules = (settings as { username_settings?: UsernameSettings }).username_settings ?? {}; + const length = [...username].length; + if (rules.min_length !== undefined && rules.max_length !== undefined) { + if (length < rules.min_length || length > rules.max_length) { + return `username is not ${rules.min_length}–${rules.max_length} characters, which this instance requires`; + } + } + if (!rules.allow_numeric_usernames && !/[a-zA-Z]/.test(username)) { + return "username has no letters; turn on numeric usernames to allow it"; + } + if (rules.allow_extended_special_characters) { + if (!USERNAME_EXTENDED.test(username)) return "username has characters Clerk does not allow"; + if (/^\+[1-9]\d{1,14}$/.test(username)) + return "username is a phone number, which Clerk does not allow"; + return undefined; + } + if (!USERNAME_DEFAULT.test(username)) { + return USERNAME_EXTENDED.test(username) + ? "username has special characters this instance does not allow; turn on extended special characters" + : "username has characters Clerk does not allow"; + } + return undefined; +} + /** First user in the file to claim each email, phone and source ID. */ function findFileDuplicates(users: User[]): Map { const reasons = new Map(); @@ -425,6 +470,7 @@ export async function checkImport(input: CheckInput): Promise { const reason = fileDuplicates.get(user.userId) ?? missingRequiredIdentifier(user, input.settings) ?? + usernameProblem(user, input.settings) ?? (user.password && user.passwordHasher ? hashShapeProblem(user.password, user.passwordHasher) : undefined) ?? From 2c3843943b2c83a43c36fc5ca28c8561ca72ba9c Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 17:02:59 -0400 Subject: [PATCH 069/141] fix(migrate): drop emails and phones the instance has off before creating users The checks warned that a phone on an instance with phone off "is not set up to store", which reads as dropped. But the create body still carried `phone_number`, and Clerk refuses the whole create for it (`phone_number is not a valid parameter`), so every user with a phone failed despite having a verified email. The importable users now come back from the checks without the emails or phones the instance has off. Usernames need nothing: the API ignores those. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/checks.test.ts | 12 +++++++++ .../src/commands/migrate/lib/checks.ts | 25 ++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index 23e3a282e..b3e0389b2 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -275,6 +275,18 @@ describe("warnings", () => { expect(checks.rejects).toEqual([]); }); + // Clerk refuses the whole create when it is sent an identifier the instance + // has off, so the dropped phone has to actually be dropped. + test("strips identifiers the instance has off from the importable users", async () => { + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true }, phone_number: { enabled: false } }), + users: [user("a", { phone: "+15555550100", unverifiedPhoneNumbers: ["+15555550101"] })], + }), + ); + expect(checks.importable).toEqual([user("a")]); + }); + test("fields Clerk has no place for", async () => { const checks = await checkImport( input({ users: [user("a")], unknownFields: { department: 3, role: 1 } }), diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index e63eb6670..3c666d45f 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -456,6 +456,29 @@ function countReasons(rejects: Reject[]): ReasonCount[] { return [...counts].map(([reason, count]) => ({ reason, count })); } +/** + * Removes the emails or phones of an instance that has that identifier off. + * + * The warnings already say they are dropped, but Clerk does not drop them: it + * refuses the whole create (`phone_number is not a valid parameter`). Usernames + * need no such handling, because the API ignores those itself. + */ +function dropDisabledIdentifiers(user: User, settings: UserSettingsJSON | null): User { + if (!settings) return user; + const fields = [ + ...(isEnabled(settings, "email_address") + ? [] + : (["email", "emailAddresses", "unverifiedEmailAddresses"] as const)), + ...(isEnabled(settings, "phone_number") + ? [] + : (["phone", "phoneNumbers", "unverifiedPhoneNumbers"] as const)), + ]; + if (!fields.some((field) => field in user)) return user; + const kept = { ...user }; + for (const field of fields) delete kept[field]; + return kept; +} + export async function checkImport(input: CheckInput): Promise { const rejects: Reject[] = input.failures.map((failure) => ({ sourceId: failure.userId, @@ -513,7 +536,7 @@ export async function checkImport(input: CheckInput): Promise { return { total: input.users.length + input.failures.length, - importable: candidates, + importable: candidates.map((user) => dropDisabledIdentifiers(user, input.settings)), rejects, rejectReasons: countReasons(rejects), warnings: buildWarnings(input, candidates), From 63a38bbd53ca8507c31a1f72d03d6677e43c0d02 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 17:02:59 -0400 Subject: [PATCH 070/141] fix(migrate): say passwords are stored, not dropped, on an instance with passwords off Clerk accepts `password_digest` with passwords off and stores it; the password starts working if passwords are turned on. The warning said the instance was "not set up to store" them. It now says they are stored and work only once passwords are turned on. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/lib/checks.test.ts | 12 ++++++++++++ packages/cli-core/src/commands/migrate/lib/checks.ts | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index b3e0389b2..4d561d40c 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -287,6 +287,18 @@ describe("warnings", () => { expect(checks.importable).toEqual([user("a")]); }); + test("says a password is kept, not dropped, when passwords are off", async () => { + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true }, password: { enabled: false } }), + users: [user("a", { password: BCRYPT, passwordHasher: "bcrypt" })], + }), + ); + expect(checks.warnings).toContain( + "1 user has a password, which this instance does not use: it is stored, and works only once passwords are turned on", + ); + }); + test("fields Clerk has no place for", async () => { const checks = await checkImport( input({ users: [user("a")], unknownFields: { department: 3, role: 1 } }), diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index 3c666d45f..da89f60d3 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -370,6 +370,12 @@ function buildWarnings(input: CheckInput, importable: User[]): string[] { ? `${plural(missing, "user")} without a password, which this instance requires: they reset it to sign in` : `${plural(missing, "user")} without a ${item.label.toLowerCase()}, which this instance requires`, ); + } else if (item.key === "password") { + // Clerk stores a digest even with passwords off, so nothing is lost: + // it starts working if passwords are turned on. + warnings.push( + `${plural(item.userCount, "user")} ${item.userCount === 1 ? "has" : "have"} a password, which this instance does not use: it is stored, and works only once passwords are turned on`, + ); } else { warnings.push( item.section === "social" From 069a1db3e0189d6b87133df1541ee77a721157b0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 17:10:26 -0400 Subject: [PATCH 071/141] fix(migrate): create the user without a phone Clerk refuses The first phone goes on POST /v1/users, so a phone Clerk refused failed the whole user, though each one had an email and a password: a country the instance does not support (`unsupported_country_code`, which names no parameter) or a number that is not E.164 (a form error on `phone_number`). When the create fails on the phone and the user has an email, it is retried without the phone, and the refusal is logged on the user's line, the same way a failed additional identifier already was. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 4 ++ .../src/commands/migrate/import-users.test.ts | 64 +++++++++++++++++++ .../src/commands/migrate/import-users.ts | 41 +++++++++--- 3 files changed, 100 insertions(+), 9 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index ff8281f4d..2566f5ad2 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -621,6 +621,10 @@ afterwards with its own request. A failure there is logged and the user still counts as imported — a duplicate secondary email should not undo an otherwise successful user. +The first phone gets the same treatment when Clerk refuses it — a country the +instance does not support, or a number that is not E.164 — and the user has an +email: the create is retried without the phone, and the refusal is logged. + #### Throughput Defaults follow Clerk's documented `POST /v1/users` limits: 100 req/s for diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index 517dc7a6a..0807c3cd7 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -267,6 +267,70 @@ describe("importUsers", () => { expect(lines[0]?.error).toContain("Failed to add additional email b@x.dev"); }); + // Shapes from clerk_go's apierror: the country error carries its own code + // and no param_name; the E.164 error is a form error on phone_number. + test.each([ + [ + 403, + { + code: "unsupported_country_code", + message: "Unsupported country code", + long_message: "Phone numbers from this country (Netherlands) are currently not supported.", + meta: { alpha2: "NL", country_code: "31" }, + }, + ], + [ + 422, + { + code: "form_param_format_invalid", + message: "is invalid", + long_message: + "Phone number must be a valid phone number according to E.164 international standard.", + meta: { param_name: "phone_number" }, + }, + ], + ])("retries without a phone Clerk refuses (%i), and notes it", async (status, clerkErr) => { + stub((url, attempt) => + url.endsWith("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/v1/users") && attempt === 1 + ? new Response(JSON.stringify({ errors: [clerkErr] }), { status }) + : ok("user_created"), + ); + + const summary = await importUsers({ + users: [user({ phone: "+31612345678" })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + expect(requests).toHaveLength(2); + expect(requests[1]?.body).not.toHaveProperty("phone_number"); + expect(lines[0]?.error).toContain(`Failed to add phone +31612345678: ${clerkErr.long_message}`); + }); + + test("does not retry without the phone when it is the only identifier", async () => { + stub( + () => + new Response( + JSON.stringify({ + errors: [{ code: "x", message: "bad phone", meta: { param_name: "phone_number" } }], + }), + { status: 422 }, + ), + ); + + const summary = await importUsers({ + users: [user({ email: undefined, phone: "+31612345678" })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary.failed).toBe(1); + expect(requests).toHaveLength(1); + }); + test("records a failed user and keeps going", async () => { stub((_url, attempt) => attempt === 1 ? clerkError(422, "that email is taken") : ok("user_ok"), diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 1d097ddbc..54618daa7 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -219,15 +219,35 @@ async function createUser( skipPasswordRequirement: boolean, ): Promise<{ clerkUserId: string; notes: string[] }> { const identifiers = splitIdentifiers(user); + const create = async (body: Record) => + ctx.schedule(async () => + bapiRequest({ + method: "POST", + path: "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/v1/users", + secretKey: ctx.secretKey, + body: JSON.stringify(body), + }), + ); - const response = await ctx.schedule(async () => - bapiRequest({ - method: "POST", - path: "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/v1/users", - secretKey: ctx.secretKey, - body: JSON.stringify(buildCreateUserBody(user, identifiers, skipPasswordRequirement)), - }), - ); + const body = buildCreateUserBody(user, identifiers, skipPasswordRequirement); + const phoneNotes: string[] = []; + let response; + try { + response = await create(body); + } catch (error) { + // A phone Clerk refuses (a country it does not support, a number that is + // not E.164) should not cost a user who has an email to be created under. + // The country error names no parameter, only its own code. + const phoneRefused = + error instanceof BapiError && + (error.code === "unsupported_country_code" || error.meta?.param_name === "phone_number"); + if (!phoneRefused || !identifiers.primaryEmail) throw error; + const { phone_number: _dropped, ...withoutPhone } = body; + response = await create(withoutPhone); + phoneNotes.push( + `Failed to add phone ${identifiers.primaryPhone}: ${(error as BapiError).longMessage ?? (error as BapiError).message}`, + ); + } const clerkUserId = (response.body as { id?: string })?.id ?? ""; @@ -248,7 +268,10 @@ async function createUser( ), ]); - return { clerkUserId, notes: notes.filter((note): note is string => note !== undefined) }; + return { + clerkUserId, + notes: [...phoneNotes, ...notes.filter((note): note is string => note !== undefined)], + }; } export type ImportUsersOptions = { From 7407d735b1405882d11bab7bbb769db2a6783fa0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 17:11:16 -0400 Subject: [PATCH 072/141] fix(migrate): skip anonymous Better Auth users MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The anonymous plugin's guests are throwaway accounts with placeholder `anon-…@anonymous.invalid` emails, and they were created in Clerk as real users without the dry run mentioning them. The export now reads `isAnonymous` when the plugin's column exists, and the betterauth source marks those users with a new `skipReason` field. The checks reject any user carrying one, with that reason, so they are counted and visible, and --allow-partial imports everyone else. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/betterauth.ts | 4 +++- .../cli-core/src/commands/migrate/lib/checks.test.ts | 8 ++++++++ packages/cli-core/src/commands/migrate/lib/checks.ts | 1 + .../src/commands/migrate/sources/betterauth.ts | 8 ++++++++ .../src/commands/migrate/sources/sources.test.ts | 11 +++++++++++ packages/cli-core/src/commands/migrate/validator.ts | 3 +++ 6 files changed, 34 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 54d23d9d8..cdcaf09d1 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -6,7 +6,8 @@ * * Better Auth's schema depends on which plugins are enabled, so the columns * are **detected from the schema** rather than asked for: the username plugin - * adds `username`, admin adds `banned`, phone-number adds `phoneNumber`, and + * adds `username`, admin adds `banned`, phone-number adds `phoneNumber`, + * anonymous adds `isAnonymous`, and * so on. Selecting a column that is not there fails the whole query, and * asking the user which plugins they run is a question their database can * already answer. @@ -40,6 +41,7 @@ export const PLUGIN_COLUMNS = [ "banReason", "banExpires", "twoFactorEnabled", + "isAnonymous", ] as const; export type PluginColumn = (typeof PLUGIN_COLUMNS)[number]; diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index 4d561d40c..0ba27492e 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -204,6 +204,14 @@ describe("rejects", () => { } }); + test("a user its source asked to skip, with the source's reason", async () => { + const checks = await checkImport( + input({ users: [user("a", { skipReason: "anonymous Better Auth user" }), user("b")] }), + ); + expect(checks.rejects).toEqual([{ sourceId: "a", reason: "anonymous Better Auth user" }]); + expect(checks.importable.map((entry) => entry.userId)).toEqual(["b"]); + }); + describe("usernames", () => { const withUsernames = (rules: object) => ({ diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index da89f60d3..4ad4b537e 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -497,6 +497,7 @@ export async function checkImport(input: CheckInput): Promise { let candidates: User[] = []; for (const user of input.users) { const reason = + user.skipReason ?? fileDuplicates.get(user.userId) ?? missingRequiredIdentifier(user, input.settings) ?? usernameProblem(user, input.settings) ?? diff --git a/packages/cli-core/src/commands/migrate/sources/betterauth.ts b/packages/cli-core/src/commands/migrate/sources/betterauth.ts index 22793b8b4..a5f12e50c 100644 --- a/packages/cli-core/src/commands/migrate/sources/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/sources/betterauth.ts @@ -105,6 +105,14 @@ const betterAuthSource = { // runs before normalizeUserData and must accept those too. if (isVerified(user.banned, "boolean")) user.banned = true; else delete user.banned; + + // The anonymous plugin's guests are throwaway accounts with placeholder + // emails (anon-…@…), not people to migrate. + if (isVerified(user.isAnonymous ?? user.is_anonymous, "boolean")) { + user.skipReason = "anonymous Better Auth user"; + } + delete user.isAnonymous; + delete user.is_anonymous; }, } satisfies SourceEntry; diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index 530cb6679..398c554d3 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -309,6 +309,17 @@ describe("authjs", () => { describe("betterauth", () => { const base = { user_id: "ba1", email: "a@x.dev", email_verified: true }; + test.each([ + [1, "anonymous Better Auth user"], + [true, "anonymous Better Auth user"], + [0, undefined], + [undefined, undefined], + ])("isAnonymous=%p sets skipReason %p", (isAnonymous, expected) => { + const user = one("betterauth", { ...base, isAnonymous }); + expect(user?.skipReason).toBe(expected); + expect("isAnonymous" in (user ?? {})).toBe(false); + }); + // Better Auth's own scrypt: a 16-byte hex salt, a colon, a 64-byte hex key. const SALT = "a".repeat(32); const KEY = "b".repeat(128); diff --git a/packages/cli-core/src/commands/migrate/validator.ts b/packages/cli-core/src/commands/migrate/validator.ts index af9ea8fbc..ccfbccab2 100644 --- a/packages/cli-core/src/commands/migrate/validator.ts +++ b/packages/cli-core/src/commands/migrate/validator.ts @@ -52,6 +52,9 @@ export const userSchema = z passwordHasher: passwordHasherEnum.optional(), /** Set by a source that found a password Clerk cannot verify, and left it out. */ passwordDropped: z.boolean().optional(), + // Set by a source for a user that should not be created at all; the checks + // reject it with this reason. Never sent to Clerk. + skipReason: z.string().optional(), // 2FA totpSecret: z.string().optional(), backupCodesEnabled: z.boolean().optional(), From 6630765a1052243cf0e49de3144ff29ec0466497 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 17:38:03 -0400 Subject: [PATCH 073/141] fix(migrate): record each user as created as soon as POST /v1/users returns The `created` line was written only after the user's additional identifiers attached. Those attaches wait on the shared rate-limited scheduler, so at 10 req/s hundreds of users could exist in Clerk without a line, and a run stopped in that window lost them: `undo` left them behind, and a re-run rejected them as already in the instance. A stopped 10,000-user run left 806 such users. The line is now written the moment the create returns. When the attaches or 429 retries add anything, a second `created` line carries it, and as the latest line it wins. A create still in flight when the process exits can land unrecorded, but that window is at most the concurrency limit, not the attach backlog. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/import-users.test.ts | 45 ++++++++++++++++--- .../src/commands/migrate/import-users.ts | 25 +++++++---- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index 0807c3cd7..36c9c3ca5 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -262,9 +262,15 @@ describe("importUsers", () => { }); expect(summary).toMatchObject({ successful: 1, failed: 0 }); - expect(lines).toHaveLength(1); - expect(lines[0]?.status).toBe("created"); - expect(lines[0]?.error).toContain("Failed to add additional email b@x.dev"); + // On record once created, then again with what the attach added. + expect(lines).toHaveLength(2); + expect(lines[0]).toEqual({ + sourceId: lines[0]!.sourceId, + clerkId: "user_created", + status: "created", + }); + expect(lines[1]?.status).toBe("created"); + expect(lines[1]?.error).toContain("Failed to add additional email b@x.dev"); }); // Shapes from clerk_go's apierror: the country error carries its own code @@ -306,7 +312,9 @@ describe("importUsers", () => { expect(summary).toMatchObject({ successful: 1, failed: 0 }); expect(requests).toHaveLength(2); expect(requests[1]?.body).not.toHaveProperty("phone_number"); - expect(lines[0]?.error).toContain(`Failed to add phone +31612345678: ${clerkErr.long_message}`); + expect(lines.at(-1)?.error).toContain( + `Failed to add phone +31612345678: ${clerkErr.long_message}`, + ); }); test("does not retry without the phone when it is the only identifier", async () => { @@ -331,6 +339,29 @@ describe("importUsers", () => { expect(requests).toHaveLength(1); }); + // A run stopped while attaches wait on the scheduler must still have the + // user on record, or `undo` leaves it behind. + test("records the user before its additional identifiers attach", async () => { + let recordedBeforeAttach = false; + stub((url) => { + if (url.endsWith("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/v1/email_addresses")) { + recordedBeforeAttach = lines.some( + (line) => line.status === "created" && line.clerkId === "user_created", + ); + } + return ok("user_created"); + }); + + await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(recordedBeforeAttach).toBe(true); + }); + test("records a failed user and keeps going", async () => { stub((_url, attempt) => attempt === 1 ? clerkError(422, "that email is taken") : ok("user_ok"), @@ -365,9 +396,9 @@ describe("importUsers", () => { expect(summary).toMatchObject({ successful: 1, failed: 0 }); expect(performance.now() - started).toBeGreaterThanOrEqual(900); expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); - expect(lines).toHaveLength(1); - expect(lines[0]).toMatchObject({ status: "created", clerkId: "user_ok" }); - expect(lines[0]?.error).toContain("Rate limit hit (429)"); + expect(lines).toHaveLength(2); + expect(lines[1]).toMatchObject({ status: "created", clerkId: "user_ok" }); + expect(lines[1]?.error).toContain("Rate limit hit (429)"); }); test("gives up after the retry ceiling and records the user as failed", async () => { diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 54618daa7..449dca24d 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -211,12 +211,16 @@ async function attachIdentifier( /** * Creates one user, then attaches any additional identifiers it carries. * + * @param onCreated - Called as soon as the user exists, before the attaches: + * those wait their turn on the shared scheduler, and a run stopped in that + * window must still have the user on record for `undo` and re-runs. * @returns The Clerk ID, and a note for each identifier that did not attach. */ async function createUser( ctx: CreateContext, user: User, skipPasswordRequirement: boolean, + onCreated: (clerkUserId: string) => void, ): Promise<{ clerkUserId: string; notes: string[] }> { const identifiers = splitIdentifiers(user); const create = async (body: Record) => @@ -250,6 +254,7 @@ async function createUser( } const clerkUserId = (response.body as { id?: string })?.id ?? ""; + onCreated(clerkUserId); // Extra identifiers are best-effort: a duplicate secondary email should not // undo a user who was otherwise imported successfully. @@ -333,21 +338,25 @@ export async function importUsers(options: ImportUsersOptions): Promise => { const retries: string[] = []; + const created = (clerkId: string, error?: string) => + record({ + sourceId: user.userId, + clerkId, + status: "created", + ...(error ? { error } : {}), + ...(user.passwordDropped ? { passwordDropped: true } : {}), + }); try { const { clerkUserId, notes } = await retryOn429( - async () => createUser(ctx, user, skipPasswordRequirement), + async () => createUser(ctx, user, skipPasswordRequirement, (clerkId) => created(clerkId)), { onRetry: ({ message }) => retries.push(message) }, ); successful++; processed++; + // The user is already on record; a second line, which wins as the + // latest, adds what happened on the way. const error = [...notes, ...retries].join("; "); - record({ - sourceId: user.userId, - clerkId: clerkUserId, - status: "created", - ...(error ? { error } : {}), - ...(user.passwordDropped ? { passwordDropped: true } : {}), - }); + if (error) created(clerkUserId, error); progress(); } catch (error) { if (error instanceof RateLimitExceededError) { From e8291f053861aac742d89bdf1f2c58452fcc3b6f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 22:39:44 -0400 Subject: [PATCH 074/141] fix(migrate): drop usernames too when the instance has them off 2c384394 claimed the API ignores a username on an instance with usernames off. It does not: it stores the username, or refuses the whole create when the username breaks the default rules (`Username can only contain letters, numbers and - or _.`). Usernames are now dropped with the emails and phones the instance has off. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/lib/checks.test.ts | 8 +++++++- packages/cli-core/src/commands/migrate/lib/checks.ts | 11 +++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index 0ba27492e..5ebadfda4 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -289,7 +289,13 @@ describe("warnings", () => { const checks = await checkImport( input({ settings: settings({ email_address: { enabled: true }, phone_number: { enabled: false } }), - users: [user("a", { phone: "+15555550100", unverifiedPhoneNumbers: ["+15555550101"] })], + users: [ + user("a", { + phone: "+15555550100", + unverifiedPhoneNumbers: ["+15555550101"], + username: "ada.l", + }), + ], }), ); expect(checks.importable).toEqual([user("a")]); diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index 4ad4b537e..a9350930d 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -463,11 +463,13 @@ function countReasons(rejects: Reject[]): ReasonCount[] { } /** - * Removes the emails or phones of an instance that has that identifier off. + * Removes the emails, phones or usernames of an instance that has that + * identifier off. * - * The warnings already say they are dropped, but Clerk does not drop them: it - * refuses the whole create (`phone_number is not a valid parameter`). Usernames - * need no such handling, because the API ignores those itself. + * The warnings already say they are dropped, but Clerk does not drop them. It + * refuses the whole create for a phone (`phone_number is not a valid + * parameter`), and for a username it stores it anyway, or refuses the create + * when the username breaks the default rules. */ function dropDisabledIdentifiers(user: User, settings: UserSettingsJSON | null): User { if (!settings) return user; @@ -478,6 +480,7 @@ function dropDisabledIdentifiers(user: User, settings: UserSettingsJSON | null): ...(isEnabled(settings, "phone_number") ? [] : (["phone", "phoneNumbers", "unverifiedPhoneNumbers"] as const)), + ...(isEnabled(settings, "username") ? [] : (["username"] as const)), ]; if (!fields.some((field) => field in user)) return user; const kept = { ...user }; From 8cfda2e1bd2a14acbb6be88966e18ff2aad8f3f7 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 22:40:54 -0400 Subject: [PATCH 075/141] fix(migrate): retry the instance lookup on 429, and never call an unknown instance a different one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Straight after a large import, `GET /v1/instance` can be rate limited. The lookup then fell back to a hash of the key (`key_…`), and `undo` compared that with the run's `ins_…` ID and refused with "the resolved key addresses … a different instance", which was false. The lookup now retries a 429, honouring Retry-After. If it still falls back, `undo` says it could not confirm the instance and to try again, instead of reporting a mismatch. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/target.test.ts | 14 ++++++++++++++ .../cli-core/src/commands/migrate/lib/target.ts | 8 ++++++-- .../cli-core/src/commands/migrate/undo.test.ts | 12 ++++++++++++ packages/cli-core/src/commands/migrate/undo.ts | 12 ++++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/target.test.ts b/packages/cli-core/src/commands/migrate/lib/target.test.ts index b63af9703..e37110f3f 100644 --- a/packages/cli-core/src/commands/migrate/lib/target.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/target.test.ts @@ -81,6 +81,20 @@ describe("fetchInstanceIdentity", () => { }); }); + test("retries a 429 rather than falling back", async () => { + let calls = 0; + globalThis.fetch = (async () => + ++calls === 1 + ? new Response("{}", { status: 429, headers: { "retry-after": "1" } }) + : Response.json({ + id: "ins_7", + environment_type: "development", + })) as unknown as typeof fetch; + + expect((await fetchInstanceIdentity("sk_test_x")).instanceId).toBe("ins_7"); + expect(calls).toBe(2); + }); + // The same key always addresses the same instance, so a hash still tells two // instances apart when the API cannot say. test("falls back to a stable hash of the key when the instance cannot be read", async () => { diff --git a/packages/cli-core/src/commands/migrate/lib/target.ts b/packages/cli-core/src/commands/migrate/lib/target.ts index d39e3381f..cdef790a4 100644 --- a/packages/cli-core/src/commands/migrate/lib/target.ts +++ b/packages/cli-core/src/commands/migrate/lib/target.ts @@ -16,6 +16,7 @@ import { resolveAppContext } from "../../../lib/config.ts"; import { resolveKeylessTarget } from "../../../lib/keyless-target.ts"; import { log } from "../../../lib/log.ts"; import { detectInstanceType } from "./instance.ts"; +import { retryOn429 } from "./retry.ts"; import type { RunTarget } from "./run-store.ts"; export type TargetOptions = { @@ -66,14 +67,17 @@ async function describeKeySource( * * Falls back to a hash of the key when `GET /v1/instance` cannot be read: the * same key always addresses the same instance, so it still tells two - * instances apart, and nothing is sent anywhere. + * instances apart, and nothing is sent anywhere. A 429 is retried first: it is + * most likely straight after a large import, which is when `undo` runs. */ export async function fetchInstanceIdentity( secretKey: string, ): Promise<{ instanceId: string; env: string }> { const fallbackEnv = detectInstanceType(secretKey) === "prod" ? "production" : "development"; try { - const { body } = await bapiRequest({ method: "GET", path: "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/v1/instance", secretKey }); + const { body } = await retryOn429(async () => + bapiRequest({ method: "GET", path: "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/v1/instance", secretKey }), + ); const instance = body as { id?: unknown; environment_type?: unknown }; if (typeof instance.id === "string" && instance.id.startsWith("ins_")) { return { diff --git a/packages/cli-core/src/commands/migrate/undo.test.ts b/packages/cli-core/src/commands/migrate/undo.test.ts index 94bb89a21..0333cd4c2 100644 --- a/packages/cli-core/src/commands/migrate/undo.test.ts +++ b/packages/cli-core/src/commands/migrate/undo.test.ts @@ -125,6 +125,18 @@ describe("refusals, all exit 2 and delete nothing", () => { expect(deletes()).toHaveLength(0); }); + // `key_` is the identity fallback when GET /v1/instance failed, e.g. rate + // limited straight after a large import. That is unknown, not different. + test("an unconfirmed instance is not reported as a different one", async () => { + const record = importRun({ + target: { instanceId: "key_0123456789abcdef", env: "development" }, + }); + const error = (await undo(record.id, withDir({ yes: true })).catch((e: unknown) => e)) as Error; + expect(error.message).toContain("Could not confirm"); + expect(error.message).not.toContain("addresses instance"); + expect(deletes()).toHaveLength(0); + }); + test("no consent where nobody can be asked: the preview, then the command", async () => { const record = importRun(); diff --git a/packages/cli-core/src/commands/migrate/undo.ts b/packages/cli-core/src/commands/migrate/undo.ts index 8b6aa4b7f..32fcba848 100644 --- a/packages/cli-core/src/commands/migrate/undo.ts +++ b/packages/cli-core/src/commands/migrate/undo.ts @@ -94,6 +94,18 @@ function readImportRun(runsDir: string, runId: string): RunRecord { /** Refuses to delete from an instance the import did not write to. */ function assertSameInstance(record: RunRecord, target: ClerkTarget): void { if (record.target.instanceId === target.instanceId) return; + // A `key_` ID is the fallback for an instance lookup that failed. It cannot + // be compared with an `ins_` ID, so this is "unknown", not "different". + const unconfirmed = [record.target.instanceId, target.instanceId].some((id) => + id?.startsWith("key_"), + ); + if (unconfirmed) { + throwUsageError( + `Could not confirm that the resolved key addresses the instance run ${record.id} imported into: ` + + "Clerk did not answer GET /v1/instance (often rate limiting straight after a large import). " + + "Nothing was deleted. Try again in a minute; --verbose shows the response.", + ); + } throwUsageError( `Run ${record.id} imported into ${describeTarget(record.target)}, but the resolved key ` + `addresses ${describeTarget(target)}. Nothing was deleted.\n` + From 1aea960fe880ee8e3b11a4b63b1fe1ae48b924f0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 22:41:43 -0400 Subject: [PATCH 076/141] fix(migrate): import a one-word name as the first name splitName set firstName/lastName only for two or more words but always deleted the name, so `Cher`, `Prince` and every seed user whose name is a username lost their name in every source that splits one. A one-word name is now the first name, with no last name invented. An email-shaped one-word value is still dropped: Auth0 fills `name` with the email for database users. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/sources/auth0.ts | 2 +- .../cli-core/src/commands/migrate/sources/shared.ts | 10 ++++++---- .../src/commands/migrate/sources/sources.test.ts | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/sources/auth0.ts b/packages/cli-core/src/commands/migrate/sources/auth0.ts index 79c389f17..4f6f1fbdd 100644 --- a/packages/cli-core/src/commands/migrate/sources/auth0.ts +++ b/packages/cli-core/src/commands/migrate/sources/auth0.ts @@ -49,7 +49,7 @@ const auth0Source = { routeByVerification(user, "phone", "phoneVerified", "boolean"); // given_name/family_name win when present. Auth0 fills `name` with the - // email for database users; splitName leaves a one-word value alone. + // email for database users; splitName drops an email-shaped value. if (user.firstName || user.lastName) delete user.name; else splitName(user); diff --git a/packages/cli-core/src/commands/migrate/sources/shared.ts b/packages/cli-core/src/commands/migrate/sources/shared.ts index c4040230a..665074e6d 100644 --- a/packages/cli-core/src/commands/migrate/sources/shared.ts +++ b/packages/cli-core/src/commands/migrate/sources/shared.ts @@ -55,11 +55,11 @@ export function routeByVerification( } /** - * Splits a single display name into `firstName` and `lastName`. + * Splits a single display name into `firstName` and `lastName`: the first word, + * then the rest. A one-word name (`Cher`) becomes the first name alone. * - * Only splits when there are at least two words — a one-word name would - * otherwise produce a first name with no last name, which several instance - * configurations reject. + * A one-word value that is an email address is dropped instead: Auth0 fills + * `name` with the email for database users, and that is not anyone's name. */ export function splitName(user: Record, field = "name"): void { const name = user[field]; @@ -69,6 +69,8 @@ export function splitName(user: Record, field = "name"): void { if (parts.length > 1) { user.firstName = parts[0]; user.lastName = parts.slice(1).join(" "); + } else if (parts[0] && !parts[0].includes("@")) { + user.firstName = parts[0]; } delete user[field]; } diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index 398c554d3..e7691f37d 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -202,7 +202,7 @@ describe("auth0", () => { expect(user).toMatchObject({ firstName: "Ada", lastName: "L" }); }); - test("leaves a one-word name (Auth0's email default) unset", () => { + test("leaves Auth0's email-default name unset", () => { const user = one("auth0", { ...base, name: "a@x.dev" }); expect(user?.firstName).toBeUndefined(); }); @@ -292,9 +292,9 @@ describe("authjs", () => { expect(user?.lastName).toBe(lastName); }); - test("leaves a single-word name unsplit rather than inventing a last name", () => { + test("keeps a single-word name as the first name, without inventing a last name", () => { const user = one("authjs", { ...base, name: "Prince" }); - expect(user?.firstName).toBeUndefined(); + expect(user?.firstName).toBe("Prince"); expect(user?.lastName).toBeUndefined(); expect("name" in (user ?? {})).toBe(false); }); From e0960b10f31f891f22f94267b3c9f7e0da7d26e9 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 22:43:50 -0400 Subject: [PATCH 077/141] fix(migrate): name the columns for a schema the export can't read, and exit 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A missing column (a Prisma `@map("email_verified")`, for one) fell through to "Check the connection string, that the server is running…", which sent the operator the wrong way, and database errors exited 1, which means "some users failed". A missing column now gets its own hint; for Auth.js it names the columns the export reads and how to export a renamed schema by hand. Connection and query errors exit 2, as a refusal. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/db.test.ts | 20 ++++++++++++++++++- .../cli-core/src/commands/migrate/lib/db.ts | 20 ++++++++++++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index 05ae45928..72344f117 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -3,7 +3,7 @@ import { Database } from "bun:sqlite"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { CliError } from "../../../lib/errors.ts"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; import { createDbClient, describeDbError, @@ -240,6 +240,24 @@ describe("withDbClient", () => { ).rejects.toThrow(/expected table was not found/); }); + // A schema the export cannot read is a refusal (exit 2), and the advice is + // about the columns, not the connection. + test("names the columns for a missing one, and exits 2", async () => { + const db = new Database(dbPath); + db.run( + `CREATE TABLE IF NOT EXISTS "User" (id TEXT, name TEXT, email TEXT, email_verified TEXT)`, + ); + db.close(); + + const error = (await withDbClient(dbPath, "authjs", (client) => + client.query(`SELECT no_such_column FROM "User"`), + ).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("reads `id`, `name`, `email` and `emailVerified`"); + expect(error.message).not.toContain("Check the connection string"); + }); + test("passes a CliError through unchanged", async () => { await expect( withDbClient(dbPath, undefined, async () => { diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index 0f2dd4e8b..6c90808a7 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -17,7 +17,7 @@ import { Database } from "bun:sqlite"; import { SQL } from "bun"; -import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, EXIT_CODE } from "../../../lib/errors.ts"; export type DbType = "postgres" | "mysql" | "sqlite"; @@ -316,6 +316,20 @@ export function describeDbError(error: unknown, platform?: DbPlatform): string { return "The database rejected those credentials. Check the user and password in the connection string."; } + // Before the table check: Postgres words a missing column "does not exist" too. + if (/no such column|column .* does not exist|unknown column/i.test(message)) { + const needs: Partial> = { + authjs: + "The Auth.js export reads `id`, `name`, `email` and `emailVerified`. For a schema that renames them " + + '(Prisma `@map("email_verified")`, for one), export the users with your own query, ' + + "`SELECT id, name, email, email_verified FROM …`, and import that file with the authjs source.", + }; + return ( + needs[platform as DbPlatform] ?? + "The user table is missing a column the export reads. Check the schema matches the platform's default." + ); + } + if (/does not exist|unknown database|no such table|permission denied/i.test(message)) { if (platform === "supabase") { return ( @@ -341,7 +355,7 @@ function connectionError( const message = error instanceof Error ? error.message : String(error); return new CliError( `Could not connect to ${redactConnectionString(connectionString)}: ${message}\n\n${describeDbError(error, platform)}`, - { code: ERROR_CODE.USAGE_ERROR }, + { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }, ); } @@ -365,7 +379,7 @@ export async function withDbClient( if (error instanceof CliError) throw error; throw new CliError( `${error instanceof Error ? error.message : String(error)}\n\n${describeDbError(error, platform)}`, - { code: ERROR_CODE.USAGE_ERROR }, + { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }, ); } finally { await client.close().catch(() => {}); From a33767f617554822de3a8ec61ded7c7ad022dcbc Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 22:45:27 -0400 Subject: [PATCH 078/141] fix(migrate): drop placeholder emails Clerk refuses, and reject users left with none Clerk refuses any email on a non-routable TLD (.local, .invalid, .test, .example, .arpa), which is where placeholder addresses live: Auth.js phone-OTP users carry a verified `@phone.local`. As the first identifier on POST /v1/users it failed the whole create, and the dry run had predicted the user would import. The checks now drop those emails, warning how many users had one, and reject a user left with no identifier, naming the email. Clerk's own `*.clerk.test` addresses are kept. The public-suffix check Clerk also runs is not mirrored: it would take the list as a dependency. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/checks.test.ts | 36 ++++++++++ .../src/commands/migrate/lib/checks.ts | 66 +++++++++++++++++-- 2 files changed, 98 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index 5ebadfda4..5b3189e46 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -212,6 +212,42 @@ describe("rejects", () => { expect(checks.importable.map((entry) => entry.userId)).toEqual(["b"]); }); + describe("emails Clerk refuses", () => { + test("a user whose only identifier is a placeholder email is rejected", async () => { + const checks = await checkImport( + input({ users: [user("a", { email: "15551234@phone.local" })] }), + ); + expect(checks.rejects).toEqual([ + { sourceId: "a", reason: "only has an email Clerk refuses (15551234@phone.local)" }, + ]); + }); + + test("a placeholder email is dropped, and the user imports on what is left", async () => { + const checks = await checkImport( + input({ + users: [ + user("a", { + email: "a@x.dev", + unverifiedEmailAddresses: ["anon-1@anonymous.invalid"], + }), + ], + }), + ); + expect(checks.rejects).toEqual([]); + expect(checks.importable).toEqual([user("a", { email: "a@x.dev" })]); + expect(checks.warnings).toContain( + "1 user has an email Clerk refuses (.local, .invalid, .test, .example, .arpa), which is dropped", + ); + }); + + test("Clerk's own .clerk.test addresses are kept", async () => { + const checks = await checkImport( + input({ users: [user("a", { email: "a@dev.clerk.test" })] }), + ); + expect(checks.importable).toEqual([user("a", { email: "a@dev.clerk.test" })]); + }); + }); + describe("usernames", () => { const withUsernames = (rules: object) => ({ diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index a9350930d..fbcbae249 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -204,6 +204,46 @@ function usernameProblem(user: User, settings: UserSettingsJSON | null): string return undefined; } +/** + * TLDs Clerk refuses for any email, from clerk_go's + * `emailaddress.nonRoutableTLDs`. This is where placeholder addresses live + * (`…@phone.local`, `anon-…@anonymous.invalid`). Clerk also refuses a TLD not + * on the public suffix list; that would take the list as a dependency. + */ +const NON_ROUTABLE_TLDS = new Set(["arpa", "local", "invalid", "example", "test"]); +const EMAIL_FIELDS = ["email", "emailAddresses", "unverifiedEmailAddresses"] as const; + +function isRefusedEmail(email: string): boolean { + const host = email.slice(email.lastIndexOf("@") + 1).toLowerCase(); + // Clerk's own dev domains sit under `.test` and are accepted. + if (host.endsWith(".clerk.test")) return false; + return NON_ROUTABLE_TLDS.has(host.slice(host.lastIndexOf(".") + 1)); +} + +/** The user without the emails Clerk would refuse, and those emails. */ +function dropRefusedEmails(user: User): { user: User; refused: string[] } { + const refused: string[] = []; + let kept: User | undefined; + for (const field of EMAIL_FIELDS) { + const value = user[field]; + if (value === undefined) continue; + const list = Array.isArray(value) ? value : [value]; + const bad = list.filter(isRefusedEmail); + if (bad.length === 0) continue; + refused.push(...bad); + kept ??= { ...user }; + const good = list.filter((email) => !isRefusedEmail(email)); + if (good.length > 0) kept[field] = good; + else delete kept[field]; + } + return { user: kept ?? user, refused }; +} + +const hasAnyIdentifier = (user: User) => + [...EMAIL_FIELDS, "phone", "phoneNumbers", "unverifiedPhoneNumbers", "username"].some((field) => + hasValue(user[field as keyof User]), + ); + /** First user in the file to claim each email, phone and source ID. */ function findFileDuplicates(users: User[]): Map { const reasons = new Map(); @@ -488,6 +528,13 @@ function dropDisabledIdentifiers(user: User, settings: UserSettingsJSON | null): return kept; } +function placeholderWarning(count: number): string[] { + if (count === 0) return []; + return [ + `${plural(count, "user")} ${count === 1 ? "has" : "have"} an email Clerk refuses (.local, .invalid, .test, .example, .arpa), which is dropped`, + ]; +} + export async function checkImport(input: CheckInput): Promise { const rejects: Reject[] = input.failures.map((failure) => ({ sourceId: failure.userId, @@ -498,9 +545,14 @@ export async function checkImport(input: CheckInput): Promise { const disabledProviders = findDisabledProviderRejects(input); let candidates: User[] = []; - for (const user of input.users) { + const placeholderEmails = new Set(); + for (const original of input.users) { + const { user, refused } = dropRefusedEmails(original); const reason = - user.skipReason ?? + original.skipReason ?? + (refused.length > 0 && !hasAnyIdentifier(user) + ? `only has an email Clerk refuses (${refused[0]})` + : undefined) ?? fileDuplicates.get(user.userId) ?? missingRequiredIdentifier(user, input.settings) ?? usernameProblem(user, input.settings) ?? @@ -509,7 +561,10 @@ export async function checkImport(input: CheckInput): Promise { : undefined) ?? disabledProviders.get(user.userId); if (reason) rejects.push({ sourceId: user.userId, reason }); - else candidates.push(user); + else { + candidates.push(user); + if (refused.length > 0) placeholderEmails.add(user.userId); + } } const instanceDuplicates = @@ -549,7 +604,10 @@ export async function checkImport(input: CheckInput): Promise { importable: candidates.map((user) => dropDisabledIdentifiers(user, input.settings)), rejects, rejectReasons: countReasons(rejects), - warnings: buildWarnings(input, candidates), + warnings: [ + ...buildWarnings(input, candidates), + ...placeholderWarning(candidates.filter((user) => placeholderEmails.has(user.userId)).length), + ], fixes: buildFixes(input, input.users), ...(quota ? { quota } : {}), settingsUnavailable: input.settings === null, From 2ff5ae7e46d3d8f8a708a505019cc3d949888d3d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 22:46:46 -0400 Subject: [PATCH 079/141] fix(migrate): name the record kept when duplicates in the file collide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When two records share an email or phone, the first in the file is kept and the rest are rejected. The source's export order decides which, so the choice could throw away the richer record without saying so. The behaviour stays, and the dry run now says it: the reason reads "…an earlier user in the file, which is kept", each reject carries `keptSourceId` (in --json too), the sample shows `id (kept: other)`, and the skipped line in the run record names it. The README's list of rejects also catches up with the checks added since: source skips, refused emails, username rules and the dev limit override. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 13 +++++- .../src/commands/migrate/lib/checks.test.ts | 20 +++++++++- .../src/commands/migrate/lib/checks.ts | 40 ++++++++++++++----- packages/cli-core/src/commands/migrate/run.ts | 14 +++++-- 4 files changed, 71 insertions(+), 16 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 2566f5ad2..81759303a 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -566,10 +566,18 @@ after them. They sort the users three ways: - **Rejected** — users Clerk would refuse. Each gets the first reason that applies: - it failed schema validation - - its source ID, email or phone repeats an earlier user in the file + - its source requested a skip (Better Auth: an anonymous guest) + - its only email is one Clerk refuses (`.local`, `.invalid`, `.test`, + `.example`, `.arpa`). Such an email is dropped from any other user, with a + warning + - its source ID, email or phone repeats an earlier user in the file. The + first record in the file is kept, whatever either holds, and the reject + names it (`kept: …`) - it lacks an identifier the instance requires. An email or phone counts only when it is verified, because an unverified one is attached after the user exists + - its username breaks the instance's username rules (length, letters, + the allowed special characters) - its password is not the shape its hasher says (`bcrypt`, `scrypt_firebase`, `argon2i`/`argon2id` and `scrypt_werkzeug` are checked; other hashers are not) @@ -577,7 +585,8 @@ after them. They sort the users three ways: - the instance already has a user with its source ID, email, phone or username (a batched `GET /v1/users` lookup, 100 values a request, through the scheduler). The users a continued run created do not count - - a development instance: it is past the 100-user headroom, counted in file + - a development instance: it is past the 100-user headroom + (`CLERK_MIGRATE_DEV_USER_LIMIT` when Clerk raised it), counted in file order - **Imported, but not everything comes across** — fields the instance is not set up to store, fields Clerk has no place for (`Clerk won't store: …`), and diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index 5b3189e46..9d57a7d50 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -89,11 +89,27 @@ describe("rejects", () => { }), ).toEqual({ a: "duplicate source ID in the file", - b: "email is also used by another user in the file", - d: "phone number is also used by another user in the file", + b: "email is also used by an earlier user in the file, which is kept", + d: "phone number is also used by an earlier user in the file, which is kept", }); }); + // The source's order decides which duplicate survives, so the dry run says. + test("a duplicate names the earlier user kept in its place", async () => { + const checks = await checkImport( + input({ + users: [user("email|1", { email: "a@x.dev" }), user("auth0|1", { email: "a@x.dev" })], + }), + ); + expect(checks.rejects).toEqual([ + { + sourceId: "auth0|1", + reason: "email is also used by an earlier user in the file, which is kept", + keptSourceId: "email|1", + }, + ]); + }); + // G20: an unverified email is attached after the user exists, so it cannot // satisfy a sign-up requirement. test("a user with only an unverified email, where email is required", async () => { diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index fbcbae249..ba6c6ef18 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -37,7 +37,12 @@ import type { ClerkTarget } from "./target.ts"; import type { ValidationFailure } from "./transform.ts"; import { lookupUsers, type LookedUpUser } from "./user-lookup.ts"; -export type Reject = { sourceId: string; reason: string }; +export type Reject = { + sourceId: string; + reason: string; + /** For a duplicate: the earlier user in the file that is kept instead. */ + keptSourceId?: string; +}; export type ReasonCount = { reason: string; count: number }; @@ -244,9 +249,17 @@ const hasAnyIdentifier = (user: User) => hasValue(user[field as keyof User]), ); -/** First user in the file to claim each email, phone and source ID. */ -function findFileDuplicates(users: User[]): Map { +/** + * First user in the file to claim each email, phone and source ID. + * + * @returns Each duplicate's reason, and the earlier user kept in its place. + */ +function findFileDuplicates(users: User[]): { + reasons: Map; + keptBy: Map; +} { const reasons = new Map(); + const keptBy = new Map(); const seenIds = new Set(); const emails = new Map(); const phones = new Map(); @@ -268,18 +281,25 @@ function findFileDuplicates(users: User[]): Map { const emailOwner = ownEmails.map((email) => emails.get(email.toLowerCase())).find(Boolean); const phoneOwner = ownPhones.map((phone) => phones.get(phone)).find(Boolean); + // The first record in the file wins, whatever either holds: the source's + // order decides, so the kept ID is named alongside the reject. if (emailOwner) { - reasons.set(user.userId, "email is also used by another user in the file"); + reasons.set(user.userId, "email is also used by an earlier user in the file, which is kept"); + keptBy.set(user.userId, emailOwner); continue; } if (phoneOwner) { - reasons.set(user.userId, "phone number is also used by another user in the file"); + reasons.set( + user.userId, + "phone number is also used by an earlier user in the file, which is kept", + ); + keptBy.set(user.userId, phoneOwner); continue; } for (const email of ownEmails) emails.set(email.toLowerCase(), user.userId); for (const phone of ownPhones) phones.set(phone, user.userId); } - return reasons; + return { reasons, keptBy }; } /** @@ -541,7 +561,7 @@ export async function checkImport(input: CheckInput): Promise { reason: `invalid: ${failure.error}`, })); - const fileDuplicates = findFileDuplicates(input.users); + const { reasons: fileDuplicates, keptBy } = findFileDuplicates(input.users); const disabledProviders = findDisabledProviderRejects(input); let candidates: User[] = []; @@ -560,8 +580,10 @@ export async function checkImport(input: CheckInput): Promise { ? hashShapeProblem(user.password, user.passwordHasher) : undefined) ?? disabledProviders.get(user.userId); - if (reason) rejects.push({ sourceId: user.userId, reason }); - else { + if (reason) { + const kept = reason === fileDuplicates.get(user.userId) ? keptBy.get(user.userId) : undefined; + rejects.push({ sourceId: user.userId, reason, ...(kept ? { keptSourceId: kept } : {}) }); + } else { candidates.push(user); if (refused.length > 0) placeholderEmails.add(user.userId); } diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 9da82c7ed..49cd5fc72 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -394,7 +394,11 @@ function printChecks(checks: ImportChecks): void { for (const { reason, count } of checks.rejectReasons) { const ids = checks.rejects .filter((reject) => reject.reason === reason) - .map((reject) => reject.sourceId); + .map((reject) => + reject.keptSourceId + ? `${reject.sourceId} (kept: ${reject.keptSourceId})` + : reject.sourceId, + ); const sample = ids.slice(0, REJECT_SAMPLE).join(", "); const more = ids.length > REJECT_SAMPLE ? `, and ${ids.length - REJECT_SAMPLE} more` : ""; log.info(` ${count}: ${reason}`); @@ -503,8 +507,12 @@ function commandFor(options: MigrateRunOptions, fromExport: string | undefined, /** Records the checks' rejects as skipped users, so the run says who they were. */ function recordRejects(run: Run, checks: ImportChecks): void { - for (const { sourceId, reason } of checks.rejects) { - run.append({ sourceId, status: "skipped", reason }); + for (const { sourceId, reason, keptSourceId } of checks.rejects) { + run.append({ + sourceId, + status: "skipped", + reason: keptSourceId ? `${reason} (kept: ${keptSourceId})` : reason, + }); } } From f108dc0a99dc08e753008fbf5205dd595561620e Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 22:48:49 -0400 Subject: [PATCH 080/141] fix(migrate): let undo find users whose create was in flight when the import stopped 6630765a records each user as soon as POST /v1/users returns, but a create still in flight when the process dies can land in Clerk with its ID never recorded. A deliberately killed 1,000-user import left exactly one such user behind after undo. Each user now gets a `creating` line just before its POST. When that is a user's latest line, undo looks it up by `external_id` and deletes it with the rest. Matching on external_id is safe there: the import's checks refused any source ID the instance already held, so the user was created by this run. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 34 ++++++++------ .../src/commands/migrate/import-users.test.ts | 26 ++++++++++- .../src/commands/migrate/import-users.ts | 1 + .../src/commands/migrate/lib/run-store.ts | 7 ++- .../src/commands/migrate/undo.test.ts | 35 ++++++++++++++ .../cli-core/src/commands/migrate/undo.ts | 46 +++++++++++++++++-- 6 files changed, 128 insertions(+), 21 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 81759303a..b7c1e18d3 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -684,8 +684,13 @@ ID exits 2. Deletes the users an import run created. The import run is the whole record of what to delete: every source ID whose latest line is `created`, by the Clerk ID -recorded beside it. Nothing is matched by searching the instance, so a user the -import did not create is never in scope. +recorded beside it. + +The one search is for a source ID whose latest line is `creating`: the run +stopped with that user's `POST /v1/users` in flight, so Clerk may hold the user +without its ID on record. Those are looked up by `external_id`. That match is +safe because the import's checks refused any source ID the instance already +held, so a user the import did not create is never in scope. ```sh clerk migrate undo 20260929-141502-a1b2 --dry-run # preview, delete nothing @@ -988,18 +993,19 @@ stamping every user with today's. ## API Endpoints -| Method | Path | Used by | -| -------- | -------------------------- | ------------------------------------------------------------------------------ | -| `POST` | `/v1/users` | `migrate import` — creates each user | -| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | -| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | -| `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | -| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | -| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | -| `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | -| `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | -| `GET` | `/v1/instance` | `migrate import`, `undo`, `export clerk` — names the instance behind the key | -| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | +| Method | Path | Used by | +| -------- | -------------------------- | ------------------------------------------------------------------------------- | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | +| `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | +| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | +| `GET` | `/v1/users?external_id=…` | `migrate undo` — finds users whose create was in flight when the import stopped | +| `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | +| `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | +| `GET` | `/v1/instance` | `migrate import`, `undo`, `export clerk` — names the instance behind the key | +| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | The checks also read the instance's Frontend API `GET /v1/environment` (bootstrapping a dev browser first on development instances) for its diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index 36c9c3ca5..83d5a17be 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -150,7 +150,13 @@ describe("importUsers", () => { let originalFetch: typeof globalThis.fetch; let requests: { method: string; url: string; body: unknown }[]; let lines: UserLine[]; - const record = (line: UserLine) => lines.push(line); + let allLines: UserLine[]; + // `lines` leaves out the `creating` marker every user gets first; one test + // below covers it through `allLines`. + const record = (line: UserLine) => { + allLines.push(line); + if (line.status !== "creating") lines.push(line); + }; beforeAll(() => { originalFetch = globalThis.fetch; @@ -163,6 +169,7 @@ describe("importUsers", () => { beforeEach(() => { requests = []; lines = []; + allLines = []; }); afterEach(() => { @@ -339,6 +346,23 @@ describe("importUsers", () => { expect(requests).toHaveLength(1); }); + // The marker `undo` uses to find a user whose create was in flight when the + // run stopped. + test("marks each user as creating before its POST /v1/users", async () => { + let recordedBeforePost = false; + stub((url) => { + if (url.endsWith("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/v1/users")) { + recordedBeforePost = allLines.some((line) => line.status === "creating"); + } + return ok("user_created"); + }); + + await importUsers({ users: [user()], secretKey: "sk_test_x", limits: LIMITS, record }); + + expect(recordedBeforePost).toBe(true); + expect(allLines.map((line) => line.status)).toEqual(["creating", "created"]); + }); + // A run stopped while attaches wait on the scheduler must still have the // user on record, or `undo` leaves it behind. test("records the user before its additional identifiers attach", async () => { diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 449dca24d..74a9c87ea 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -337,6 +337,7 @@ export async function importUsers(options: ImportUsersOptions): Promise => { + record({ sourceId: user.userId, status: "creating" }); const retries: string[] = []; const created = (clerkId: string, error?: string) => record({ diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index e5b897871..74cb293ff 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -35,7 +35,12 @@ export const RUNS_DIR_DESCRIPTION = `Where migration runs are kept (default: .cl export type RunKind = "import" | "undo" | "export"; export type RunStatus = "running" | "complete" | "partial" | "undone"; -export type UserStatus = "created" | "failed" | "skipped" | "deleted" | "exported"; +/** + * `creating` is written just before `POST /v1/users`. As a user's latest line + * it means the run stopped with that create in flight: the user may exist in + * Clerk without its ID on record, so `undo` looks it up by `external_id`. + */ +export type UserStatus = "creating" | "created" | "failed" | "skipped" | "deleted" | "exported"; /** * What a run acted on. For an import or undo, the Clerk instance. For an diff --git a/packages/cli-core/src/commands/migrate/undo.test.ts b/packages/cli-core/src/commands/migrate/undo.test.ts index 0333cd4c2..7b9f23ea7 100644 --- a/packages/cli-core/src/commands/migrate/undo.test.ts +++ b/packages/cli-core/src/commands/migrate/undo.test.ts @@ -16,6 +16,8 @@ let requests: { method: string; url: string }[]; let instanceUsers: Map; /** Clerk IDs whose DELETE fails with a 500. */ let failing: Set; +/** external_id → Clerk ID, for users whose create was in flight. */ +let inFlight: Map; const IMPORT_STARTED = "2026-09-01T00:00:00.000Z"; const AFTER_IMPORT = Date.parse("2026-09-02T00:00:00.000Z"); @@ -32,6 +34,7 @@ beforeEach(() => { runsDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-undo-"))); requests = []; failing = new Set(); + inFlight = new Map(); instanceUsers = new Map([ ["user_a", null], ["user_b", AFTER_IMPORT], @@ -45,6 +48,14 @@ beforeEach(() => { if (url.pathname === "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/v1/instance") { return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); } + if (method === "GET" && url.pathname === "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/v1/users" && url.searchParams.has("external_id")) { + return Response.json( + url.searchParams + .getAll("external_id") + .filter((externalId) => inFlight.has(externalId)) + .map((externalId) => ({ id: inFlight.get(externalId), external_id: externalId })), + ); + } if (method === "GET" && url.pathname === "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/v1/users") { const ids = url.searchParams.getAll("user_id"); return Response.json( @@ -177,6 +188,30 @@ describe("--dry-run", () => { }); describe("deleting", () => { + // The run stopped with this user's POST /v1/users in flight: Clerk created + // it, but its ID never reached the run record. + test("finds a user whose create was in flight by external_id, and deletes it", async () => { + const run = startRun(runsDir, { + kind: "import", + target: { instanceId: "ins_1", env: "development" }, + source: "clerk", + }); + run.update({ startedAt: IMPORT_STARTED }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + run.append({ sourceId: "d", status: "creating" }); + run.append({ sourceId: "e", status: "creating" }); + const record = run.finish(); + inFlight.set("d", "user_d"); + instanceUsers.set("user_d", null); + + await undo(record.id, withDir({ yes: true })); + + expect(deletes().map((request) => request.url.split("/").pop())).toEqual( + expect.arrayContaining(["user_a", "user_d"]), + ); + expect(deletes()).toHaveLength(2); + }); + test("deletes what the import created, records an undo run, and marks the import undone", async () => { const record = importRun(); diff --git a/packages/cli-core/src/commands/migrate/undo.ts b/packages/cli-core/src/commands/migrate/undo.ts index 32fcba848..0a95a4c01 100644 --- a/packages/cli-core/src/commands/migrate/undo.ts +++ b/packages/cli-core/src/commands/migrate/undo.ts @@ -123,12 +123,17 @@ function findOpenUndo(runsDir: string, importId: string): RunRecord | undefined ); } -/** Every user the import created, minus those an earlier undo already deleted. */ +/** + * Every user the import created, minus those an earlier undo already deleted. + * + * `unconfirmed` holds the source IDs whose create was in flight when the run + * stopped: they may exist in Clerk with no ID on record. + */ function usersToDelete( runsDir: string, record: RunRecord, openUndo: RunRecord | undefined, -): { users: UndoUser[]; alreadyDeleted: number } { +): { users: UndoUser[]; unconfirmed: string[]; alreadyDeleted: number } { const deleted = new Set(); if (openUndo) { for (const line of latestUserLines(runsDir, openUndo.id).values()) { @@ -137,12 +142,38 @@ function usersToDelete( } const users: UndoUser[] = []; + const unconfirmed: string[] = []; for (const line of latestUserLines(runsDir, record.id).values()) { - if (line.status !== "created" || !line.clerkId) continue; if (deleted.has(line.sourceId)) continue; + if (line.status === "creating") unconfirmed.push(line.sourceId); + if (line.status !== "created" || !line.clerkId) continue; users.push({ sourceId: line.sourceId, clerkId: line.clerkId }); } - return { users, alreadyDeleted: deleted.size }; + return { users, unconfirmed, alreadyDeleted: deleted.size }; +} + +/** + * Finds the users behind creates that were in flight when the run stopped. + * + * Matching on `external_id` alone is safe here: the import's checks refused + * any source ID the instance already held, so a user carrying one of these + * was created by this run. + */ +async function findUnconfirmed( + sourceIds: string[], + secretKey: string, + schedule: ApiScheduler, +): Promise { + if (sourceIds.length === 0) return []; + const found = await lookupUsers({ + filter: "external_id", + values: sourceIds, + secretKey, + schedule, + }); + return found + .filter((user) => user.external_id && sourceIds.includes(user.external_id)) + .map((user) => ({ sourceId: user.external_id as string, clerkId: user.id })); } /** @@ -311,9 +342,14 @@ export async function undo(runId: string, options: UndoOptions = {}): Promise 0 ? await withSpinner("Checking the imported users...", async (spinner) => From 3e01d66a1dc57f35fea612c09781d146facbe960 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 22:49:50 -0400 Subject: [PATCH 081/141] fix(migrate): drop a first or last name Clerk refuses, instead of failing the user Clerk refuses a name containing a phone number, an email, a URL with a scheme or path, or HTML (`The first name "+447836887904" is invalid: contains a phone number`). Better Auth's phone sign-up stores the number as the user's name, and since 1aea960f a one-word name reaches Clerk, so every phone-only Better Auth user failed, though the dry run predicted otherwise. The checks now drop such a name, approximating clerk_go's nameForAbusePrevention, and warn how many users lost one. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 6 ++- .../src/commands/migrate/lib/checks.test.ts | 26 +++++++++++ .../src/commands/migrate/lib/checks.ts | 44 ++++++++++++++++++- 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index b7c1e18d3..054d3c045 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -589,8 +589,10 @@ after them. They sort the users three ways: (`CLERK_MIGRATE_DEV_USER_LIMIT` when Clerk raised it), counted in file order - **Imported, but not everything comes across** — fields the instance is not - set up to store, fields Clerk has no place for (`Clerk won't store: …`), and - passwords a source had to drop. + set up to store, fields Clerk has no place for (`Clerk won't store: …`), + passwords a source had to drop, emails Clerk refuses, and names Clerk refuses + (a phone number, email, URL or HTML: Better Auth's phone sign-up stores the + number as the name). - **Imported** — everyone else. Any reject stops the import, and it exits 2 with the command that adds diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index 9d57a7d50..ced8728e9 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -264,6 +264,32 @@ describe("rejects", () => { }); }); + describe("names Clerk refuses", () => { + test.each([ + ["+447836887904"], + ["4165550123"], + ["ada@x.dev"], + ["https://spam.example"], + ["see x.com/win"], + ["Ada"], + ])("%p is dropped, and the user still imports", async (firstName) => { + const checks = await checkImport(input({ users: [user("a", { firstName, lastName: "L" })] })); + expect(checks.rejects).toEqual([]); + expect(checks.importable).toEqual([user("a", { lastName: "L" })]); + expect(checks.warnings).toContain( + "1 user has a name Clerk refuses (a phone number, email, URL or HTML), which is dropped", + ); + }); + + test.each([["Ada"], ["Mary-Jane O'Neil"], ["Louis XIV"], ["Agent 007"], ["redacted.io"]])( + "%p is kept", + async (firstName) => { + const checks = await checkImport(input({ users: [user("a", { firstName })] })); + expect(checks.importable).toEqual([user("a", { firstName })]); + }, + ); + }); + describe("usernames", () => { const withUsernames = (rules: object) => ({ diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index ba6c6ef18..434946656 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -244,6 +244,37 @@ function dropRefusedEmails(user: User): { user: User; refused: string[] } { return { user: kept ?? user, refused }; } +/** + * A name Clerk refuses, approximating clerk_go's `nameForAbusePrevention`: a + * phone number (10–15 digits, or fewer behind a `+`/`00`), an email, a URL with + * a scheme or path, or an HTML tag. Better Auth's phone sign-up stores the + * number as the name, so this is common, not exotic. + */ +function nameProblem(name: string): string | undefined { + for (const candidate of name.match(/(?:\+|00)?\d[\d\s().-]{5,}\d/g) ?? []) { + const digits = candidate.replace(/\D/g, "").length; + const international = /^(\+|00)/.test(candidate.trim()); + if (digits <= 15 && (digits >= 10 || (international && digits >= 7))) return "a phone number"; + } + if (/\S+@\S+\.\S+/.test(name)) return "an email address"; + if (/:\/\/|\b[\w-]+(\.[\w-]+)+[/?#]/.test(name)) return "a URL"; + if (/<\/?[a-z!][^>]*>/i.test(name)) return "HTML"; + return undefined; +} + +/** The user without a first or last name Clerk would refuse. */ +function dropRefusedNames(user: User): { user: User; dropped: boolean } { + let kept: User | undefined; + for (const field of ["firstName", "lastName"] as const) { + const value = user[field]; + if (typeof value === "string" && nameProblem(value)) { + kept ??= { ...user }; + delete kept[field]; + } + } + return { user: kept ?? user, dropped: kept !== undefined }; +} + const hasAnyIdentifier = (user: User) => [...EMAIL_FIELDS, "phone", "phoneNumbers", "unverifiedPhoneNumbers", "username"].some((field) => hasValue(user[field as keyof User]), @@ -548,6 +579,13 @@ function dropDisabledIdentifiers(user: User, settings: UserSettingsJSON | null): return kept; } +function refusedNameWarning(count: number): string[] { + if (count === 0) return []; + return [ + `${plural(count, "user")} ${count === 1 ? "has" : "have"} a name Clerk refuses (a phone number, email, URL or HTML), which is dropped`, + ]; +} + function placeholderWarning(count: number): string[] { if (count === 0) return []; return [ @@ -566,8 +604,11 @@ export async function checkImport(input: CheckInput): Promise { let candidates: User[] = []; const placeholderEmails = new Set(); + const refusedNames = new Set(); for (const original of input.users) { - const { user, refused } = dropRefusedEmails(original); + const named = dropRefusedNames(original); + if (named.dropped) refusedNames.add(original.userId); + const { user, refused } = dropRefusedEmails(named.user); const reason = original.skipReason ?? (refused.length > 0 && !hasAnyIdentifier(user) @@ -629,6 +670,7 @@ export async function checkImport(input: CheckInput): Promise { warnings: [ ...buildWarnings(input, candidates), ...placeholderWarning(candidates.filter((user) => placeholderEmails.has(user.userId)).length), + ...refusedNameWarning(candidates.filter((user) => refusedNames.has(user.userId)).length), ], fixes: buildFixes(input, input.users), ...(quota ? { quota } : {}), From 4b7a604f57b2f3e52167fa09d12c91b9f7e02ff8 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 23:16:42 -0400 Subject: [PATCH 082/141] fix(migrate): detect Supabase password hashers per user, and skip soft-deleted users The Supabase source assumed bcrypt for every row, but Supabase accepts argon2 hashes on import, so those users were rejected outright ("password is not a bcrypt hash"). The hasher is now read from each hash's prefix (bcrypt, argon2id, argon2i), with the detection shared with Better Auth. A hash no hasher fits is dropped, and the user imports and resets their password. Soft-deleted users (`deleted_at` set) were exported and only rejected by accident, because Supabase scrambles their email. The export now reads `deleted_at` (through to_jsonb, so an auth.users without the column still reads) and the source skips those users as "deleted in Supabase". Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 3 ++- .../src/commands/migrate/export/supabase.ts | 4 +++- .../commands/migrate/sources/betterauth.ts | 8 +++---- .../src/commands/migrate/sources/shared.ts | 12 ++++++++++ .../commands/migrate/sources/sources.test.ts | 23 ++++++++++++++++++ .../src/commands/migrate/sources/supabase.ts | 24 +++++++++++++++++-- 6 files changed, 65 insertions(+), 9 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 054d3c045..c8cc8a05c 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -566,7 +566,8 @@ after them. They sort the users three ways: - **Rejected** — users Clerk would refuse. Each gets the first reason that applies: - it failed schema validation - - its source requested a skip (Better Auth: an anonymous guest) + - its source requested a skip (Better Auth: an anonymous guest; Supabase: a + soft-deleted user) - its only email is one Clerk refuses (`.local`, `.invalid`, `.test`, `.example`, `.arpa`). Such an email is dropped from any other user, with a warning diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index e9626f50c..abea84f7e 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -46,8 +46,10 @@ const EXPORT_QUERY = ` raw_user_meta_data, raw_app_meta_data, banned_until, + -- Through to_jsonb so an older auth.users without the column still reads. + to_jsonb(u)->>'deleted_at' AS deleted_at, created_at - FROM auth.users + FROM auth.users u ORDER BY created_at `; diff --git a/packages/cli-core/src/commands/migrate/sources/betterauth.ts b/packages/cli-core/src/commands/migrate/sources/betterauth.ts index a5f12e50c..5d7840ed1 100644 --- a/packages/cli-core/src/commands/migrate/sources/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/sources/betterauth.ts @@ -1,5 +1,5 @@ import type { SourceEntry } from "../types.ts"; -import { isVerified, routeByVerification, splitName } from "./shared.ts"; +import { detectStandardHasher, isVerified, routeByVerification, splitName } from "./shared.ts"; /** * Better Auth → Clerk source. @@ -41,10 +41,8 @@ export function detectBetterAuthHash( passwordHasher: "scrypt_werkzeug", }; } - if (/^\$2[aby]\$/.test(hash)) return { password: hash, passwordHasher: "bcrypt" }; - if (hash.startsWith("$argon2id$")) return { password: hash, passwordHasher: "argon2id" }; - if (hash.startsWith("$argon2i$")) return { password: hash, passwordHasher: "argon2i" }; - return undefined; + const passwordHasher = detectStandardHasher(hash); + return passwordHasher ? { password: hash, passwordHasher } : undefined; } const betterAuthSource = { diff --git a/packages/cli-core/src/commands/migrate/sources/shared.ts b/packages/cli-core/src/commands/migrate/sources/shared.ts index 665074e6d..163c5b459 100644 --- a/packages/cli-core/src/commands/migrate/sources/shared.ts +++ b/packages/cli-core/src/commands/migrate/sources/shared.ts @@ -93,3 +93,15 @@ export function toIsoDate(value: unknown, epochMillis = false): unknown { return Number.isNaN(parsed.getTime()) ? value : parsed.toISOString(); } + +/** + * The hasher for a self-describing bcrypt or argon2 digest, or undefined. + * Platforms that accept imported hashes (Supabase, Better Auth) can hold + * either, so the hasher is read per user rather than assumed per source. + */ +export function detectStandardHasher(hash: string): "bcrypt" | "argon2id" | "argon2i" | undefined { + if (/^\$2[aby]\$/.test(hash)) return "bcrypt"; + if (hash.startsWith("$argon2id$")) return "argon2id"; + if (hash.startsWith("$argon2i$")) return "argon2i"; + return undefined; +} diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index e7691f37d..87b88018e 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -471,6 +471,29 @@ describe("firebase", () => { describe("supabase", () => { const base = { id: "sb1", email: "a@x.dev", email_confirmed_at: "2024-06-29 20:25:06.126079+00" }; + // Supabase accepts argon2 hashes on import, so the hasher is read per user. + test.each([ + ["$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", "bcrypt"], + ["$argon2id$v=19$m=19456,t=2,p=1$c2FsdHNhbHQ$aGFzaGhhc2hoYXNo", "argon2id"], + ["$argon2i$v=19$m=4096,t=3,p=1$c2FsdHNhbHQ$aGFzaGhhc2hoYXNo", "argon2i"], + ])("detects the hasher for %p", (encrypted_password, hasher) => { + const user = one("supabase", { ...base, encrypted_password }); + expect(user?.passwordHasher).toBe(hasher); + expect(user?.password).toBe(encrypted_password); + }); + + test("drops a hash no hasher fits, and imports the user", () => { + const user = one("supabase", { ...base, encrypted_password: "md5:abc" }); + expect(user?.password).toBeUndefined(); + expect(user?.passwordDropped).toBe(true); + }); + + test("skips a soft-deleted user", () => { + const user = one("supabase", { ...base, deleted_at: "2026-01-01 00:00:00+00" }); + expect(user?.skipReason).toBe("deleted in Supabase"); + expect("deletedAt" in (user ?? {})).toBe(false); + }); + test.each([ ["14165550123", "+14165550123"], ["+14165550123", "+14165550123"], diff --git a/packages/cli-core/src/commands/migrate/sources/supabase.ts b/packages/cli-core/src/commands/migrate/sources/supabase.ts index 90d85cf7d..0b7501d5c 100644 --- a/packages/cli-core/src/commands/migrate/sources/supabase.ts +++ b/packages/cli-core/src/commands/migrate/sources/supabase.ts @@ -1,5 +1,5 @@ import type { SourceEntry } from "../types.ts"; -import { routeByVerification, toIsoDate } from "./shared.ts"; +import { detectStandardHasher, routeByVerification, toIsoDate } from "./shared.ts"; /** * Supabase Auth → Clerk transformer. @@ -26,7 +26,10 @@ const supabaseSource = { description: "Works with a Supabase `auth.users` export. Users whose only social provider is not enabled in Clerk are rejected by the import's checks.", carries: { - passwords: { level: "yes", note: "bcrypt `encrypted_password` hashes come across." }, + passwords: { + level: "yes", + note: "bcrypt and argon2 `encrypted_password` hashes come across, detected per user. Any other hash is dropped, and that user resets their password.", + }, mfa: { level: "no", note: "Supabase MFA factors are not exported. Users enrol again in Clerk.", @@ -47,11 +50,28 @@ const supabaseSource = { phone_confirmed_at: "phoneConfirmedAt", raw_user_meta_data: "unsafeMetadata", banned_until: "bannedUntil", + deleted_at: "deletedAt", created_at: "createdAt", }, postTransform: (user) => { user.createdAt = toIsoDate(user.createdAt); + // Supabase accepts bcrypt and argon2 hashes on import, so `bcrypt` (the + // default it hashes with) is only right for most users, not all. + if (typeof user.password === "string" && user.password) { + const hasher = detectStandardHasher(user.password); + if (hasher) { + user.passwordHasher = hasher; + } else { + delete user.password; + user.passwordDropped = true; + } + } + + // A soft-deleted user is gone from the app; Supabase scrambles its email. + if (user.deletedAt) user.skipReason = "deleted in Supabase"; + delete user.deletedAt; + // Supabase stores E.164 without the leading + (14165550123); Clerk needs it. if (typeof user.phone === "string" && /^\d+$/.test(user.phone)) user.phone = `+${user.phone}`; From fdfe548984db32c8c50ed6ea44376122e24d1aab Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 23:18:00 -0400 Subject: [PATCH 083/141] docs(migrate): document the read-only Firebase roles, and name a missing hash permission The README named only "Firebase Authentication Admin", which Google's IAM reference says lacks `firebaseauth.configs.getHashConfig`. It now documents the read-only combination verified live (Firebase Authentication Viewer plus a custom role with configs.get, configs.getHashConfig and users.get), with the gcloud commands. A 403 reading the hash parameters now warns which permission is missing, instead of silently falling back to "pass --firebase-*". The token-exchange error no longer blames roles: a role cannot fail that step, only a revoked or deleted key. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 24 ++++++++++++++++++- .../commands/migrate/export/firebase.test.ts | 4 +++- .../src/commands/migrate/export/firebase.ts | 12 ++++++++-- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index c8cc8a05c..642f46e0d 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -369,7 +369,29 @@ clerk migrate export firebase --service-account ./service-account.json ``` Needs a service account key from **Project settings → Service accounts → -Generate new private key**, with the Firebase Authentication Admin role. +Generate new private key**. The service account needs to read users and the +project's password hash parameters. A read-only account verified to work has: + +- **Firebase Authentication Viewer** (`roles/firebaseauth.viewer`), to read + users; +- a custom role with `firebaseauth.configs.get`, + `firebaseauth.configs.getHashConfig` and `firebaseauth.users.get`. The hash + parameters need `getHashConfig`, which no predefined Firebase Auth role is + documented to include. + +```sh +gcloud iam roles create clerkMigrateHashExport --project=PROJECT_ID \ + --title="Clerk migrate hash export" \ + --permissions=firebaseauth.configs.get,firebaseauth.configs.getHashConfig,firebaseauth.users.get +gcloud projects add-iam-policy-binding PROJECT_ID \ + --member=serviceAccount:SA_EMAIL --role=roles/firebaseauth.viewer +gcloud projects add-iam-policy-binding PROJECT_ID \ + --member=serviceAccount:SA_EMAIL --role=projects/PROJECT_ID/roles/clerkMigrateHashExport +``` + +Without `getHashConfig` the users still export, with their hashes, but the +export says which permission is missing and the import needs the +`--firebase-*` flags instead. Without `--service-account` you are prompted for it, the way `export supabase` prompts for its connection string. The answer can be a path to the downloaded diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index cd3725057..0ec3469cd 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -249,7 +249,7 @@ describe("fetchAccessToken", () => { test("names the role the service account usually lacks", async () => { globalThis.fetch = (async () => new Response("{}", { status: 403 })) as unknown as typeof fetch; - await expect(fetchAccessToken(account)).rejects.toThrow(/Firebase Authentication Admin/); + await expect(fetchAccessToken(account)).rejects.toThrow(/revoked or deleted/); }); // The emulator has no token endpoint; `firebase-admin` uses the same bearer. @@ -408,6 +408,8 @@ describe("fetchHashConfig", () => { test("returns null rather than failing when the call is not permitted", async () => { stubFirebase([[]]); expect(await fetchHashConfig(account, "tok")).toBeNull(); + // Names the permission, so the operator can grant it rather than guess. + expect(captured.err).toContain("firebaseauth.configs.getHashConfig"); }); test("returns null when the response carries no hash config", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 4c2d884e5..7c7d100c5 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -295,7 +295,7 @@ export async function fetchAccessToken(account: ServiceAccount): Promise if (!response.ok || !body.access_token) { throw new CliError( `Google rejected the service account (${response.status}): ${body.error_description ?? body.error ?? "no access token returned"}\n` + - "Check the key has not been revoked, and that the service account has the Firebase Authentication Admin role.", + "Check the key has not been revoked or deleted, in the Google Cloud console under IAM → Service accounts.", { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, ); } @@ -385,7 +385,15 @@ export async function fetchHashConfig( headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, }); if (!response.ok) { - log.debug(`firebase: ${response.status} reading the project config`); + if (response.status === 403) { + log.warn( + "The service account cannot read the project's password hash parameters: it needs the " + + "`firebaseauth.configs.getHashConfig` permission. Grant it and export again, or pass the " + + "parameters to the import with --firebase-*.", + ); + } else { + log.debug(`firebase: ${response.status} reading the project config`); + } return null; } From cea116725987c050e0f39c758e8c8b9ab37258c2 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 1 Oct 2026 23:33:39 -0400 Subject: [PATCH 084/141] docs(migrate): document both verified Firebase role setups fdfe5489 said no predefined Firebase Auth role is documented to include `getHashConfig`. Live runs show Firebase Authentication Admin alone exports the hash parameters, and every native scrypt password then verifies in Clerk. The README now gives both verified setups: Firebase Authentication Admin (simplest), or Viewer plus a custom getHashConfig role (read-only), and names each --firebase-* flag for the fallback. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 642f46e0d..d5d33c2fc 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -368,30 +368,30 @@ only ever signed in with OAuth is still exported. clerk migrate export firebase --service-account ./service-account.json ``` -Needs a service account key from **Project settings → Service accounts → -Generate new private key**. The service account needs to read users and the -project's password hash parameters. A read-only account verified to work has: - -- **Firebase Authentication Viewer** (`roles/firebaseauth.viewer`), to read - users; -- a custom role with `firebaseauth.configs.get`, - `firebaseauth.configs.getHashConfig` and `firebaseauth.users.get`. The hash - parameters need `getHashConfig`, which no predefined Firebase Auth role is - documented to include. - -```sh -gcloud iam roles create clerkMigrateHashExport --project=PROJECT_ID \ - --title="Clerk migrate hash export" \ - --permissions=firebaseauth.configs.get,firebaseauth.configs.getHashConfig,firebaseauth.users.get -gcloud projects add-iam-policy-binding PROJECT_ID \ - --member=serviceAccount:SA_EMAIL --role=roles/firebaseauth.viewer -gcloud projects add-iam-policy-binding PROJECT_ID \ - --member=serviceAccount:SA_EMAIL --role=projects/PROJECT_ID/roles/clerkMigrateHashExport -``` - -Without `getHashConfig` the users still export, with their hashes, but the -export says which permission is missing and the import needs the -`--firebase-*` flags instead. +Create a key at **Project settings → Service accounts → Generate new private +key**. The account needs to read users and the project's password hash +parameters (`signIn.hashConfig`). Either of these works, both verified live: + +- **Firebase Authentication Admin** (`roles/firebaseauth.admin`). The simplest + option, but it can also modify users and auth settings. +- **Read-only:** Firebase Authentication Viewer (`roles/firebaseauth.viewer`) + plus a custom role that adds `firebaseauth.configs.getHashConfig`: + + ```sh + gcloud iam roles create clerkMigrateHashExport --project=PROJECT_ID \ + --title="Clerk migrate hash export" \ + --permissions=firebaseauth.configs.get,firebaseauth.configs.getHashConfig,firebaseauth.users.get + gcloud projects add-iam-policy-binding PROJECT_ID \ + --member=serviceAccount:SA_EMAIL --role=roles/firebaseauth.viewer + gcloud projects add-iam-policy-binding PROJECT_ID \ + --member=serviceAccount:SA_EMAIL --role=projects/PROJECT_ID/roles/clerkMigrateHashExport + ``` + +Without `firebaseauth.configs.getHashConfig`, users still export, but the +export can't read the hash parameters. It says which permission is missing, and +the import then needs `--firebase-signer-key`, `--firebase-salt-separator`, +`--firebase-rounds` and `--firebase-mem-cost` (from **Authentication → Users → +⋮ → Password hash parameters** in the Firebase console). Without `--service-account` you are prompted for it, the way `export supabase` prompts for its connection string. The answer can be a path to the downloaded From 3eec00bde62444d27e71a1c2e4d3890d2b5461b6 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 15:01:49 -0400 Subject: [PATCH 085/141] fix(migrate): make resume and undo safe after an interrupted import - Write `creating` as each POST /v1/users goes out, not when the user is queued, so undo no longer looks up (and deletes) users a run never sent. - Keep `creating` when a create gets no answer (abort, network error, 5xx) instead of recording `failed`. A continued run looks those users up by external_id, adopts the ones Clerk holds, and creates the rest. - Record a created user's extra emails and phones as `pending` until they attach, attach them ahead of queued creates, retry an attach on a 429, and let a continued run finish the ones still pending. - An interrupted run that was undone reads as undone, and a re-import refuses while an undo of the matching run is unfinished. - Undo and adoption leave out a user another import run records as created. The created_at window the review suggested can't work: the import sends the source's created_at. - Look up external_id with a `+` prefix, so BAPI doesn't read a leading `-` as an exclusion. - A failed run-store write now throws, a torn last line is ended before a continued run appends, `creating` makes a run partial, --require-password records the users it leaves out, and a continued run with nothing left is finished. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 58 ++-- .../src/commands/migrate/import-users.test.ts | 162 ++++++++++- .../src/commands/migrate/import-users.ts | 261 ++++++++++++------ .../src/commands/migrate/lib/checks.test.ts | 9 +- .../src/commands/migrate/lib/checks.ts | 9 +- .../commands/migrate/lib/run-store.test.ts | 42 ++- .../src/commands/migrate/lib/run-store.ts | 55 +++- .../src/commands/migrate/lib/scheduler.ts | 33 ++- .../src/commands/migrate/lib/user-lookup.ts | 35 ++- .../cli-core/src/commands/migrate/run.test.ts | 113 +++++++- packages/cli-core/src/commands/migrate/run.ts | 102 +++++-- .../src/commands/migrate/undo.test.ts | 27 ++ .../cli-core/src/commands/migrate/undo.ts | 34 +-- 13 files changed, 746 insertions(+), 194 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index d5d33c2fc..5a191b43c 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -103,23 +103,30 @@ project's `.gitignore` first, because run files carry user data. Each run is a folder named for its ID, `YYYYMMDD-HHmmss-xxxx`: -| File | Contents | -| -------------- | --------------------------------------------------------------------------------------------------------------------------- | -| `run.json` | Kind, status, start and finish times, the target, the source, the file and its sha256, and the counts | -| `users.ndjson` | One line per user outcome: `sourceId`, `clerkId`, `status`, and `reason`, `error`, `code` or `passwordDropped` when present | -| `lock` | The PID of the process writing the run, while it runs | - -A user's status is `created`, `failed`, `skipped`, `deleted` or `exported`. The last line -for each `sourceId` wins. A `429` retry, an extra email or phone that did not -attach, and a validation failure all land in `error`. - -A run is `partial` when any user failed or was skipped, and `complete` -otherwise. A run whose process died, or that never recorded a finish time, +| File | Contents | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `run.json` | Kind, status, start and finish times, the target, the source, the file and its sha256, and the counts | +| `users.ndjson` | One line per user outcome: `sourceId`, `clerkId`, `status`, and `reason`, `error`, `code`, `pending` or `passwordDropped` when present | +| `lock` | The PID of the process writing the run, while it runs | + +A user's status is `creating`, `created`, `failed`, `skipped`, `deleted` or +`exported`. The last line for each `sourceId` wins. A `429` retry, an extra +email or phone that did not attach, and a validation failure all land in +`error`. + +`creating` is written as a user's `POST /v1/users` goes out. It stays the +latest line when no answer says whether the create landed: an abort, a +network error, or a 5xx. A `created` line with `pending` lists the extra emails +and phones not yet attached. + +A run is `partial` when any user failed, was skipped or is still `creating`, +and `complete` otherwise. A run whose process died, or that never recorded a finish time, lists as `interrupted`. A lock held by a live process refuses a second writer with exit 2. `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C -still leaves a complete record of everything already processed. An export's +still leaves a complete record of everything already processed. A line that +cannot be written stops that user's create from going out. An export's file lands in its run folder as `export.json` unless `--output` says otherwise. ### Why `users.ndjson` is NDJSON @@ -573,9 +580,20 @@ instance ID; the latest matching import run decides what happens: | `partial` | continues the same run, retrying the users that failed or were skipped | | `complete` | nothing: prints "Already imported in run …" and exits 0 | | `undone` | a new run | +| has an undo that did not finish | exits 2, naming the `clerk migrate undo` that finishes it | `--new-run` skips the lookup. A run another live process holds exits 2. +A continued run also finishes what the last one left open: + +- A user still `creating` is looked up by `external_id`. One Clerk holds is + adopted as `created`, and not created again; one it doesn't is created. +- A user whose `created` line has `pending` identifiers gets just those + attaches. + +`--require-password` records each user it leaves out as `skipped`, so the run +ends `partial`. + When an import completes, it names the folders it no longer needs: the export it read, which holds your users' data, and its own run, which only `undo` needs. Each comes with the `rm -rf` to remove it. @@ -651,9 +669,10 @@ says so. Only the first verified email and phone go on `POST /v1/users`. Every additional verified identifier, and every unverified one, is attached -afterwards with its own request. A failure there is logged and the user still +afterwards with its own request, ahead of any create still queued, and backs +off on a `429` like the create. A refusal there is logged and the user still counts as imported — a duplicate secondary email should not undo an otherwise -successful user. +successful user. An attach with no answer stays `pending` for a re-run. The first phone gets the same treatment when Clerk refuses it — a country the instance does not support, or a number that is not E.164 — and the user has an @@ -712,10 +731,11 @@ what to delete: every source ID whose latest line is `created`, by the Clerk ID recorded beside it. The one search is for a source ID whose latest line is `creating`: the run -stopped with that user's `POST /v1/users` in flight, so Clerk may hold the user -without its ID on record. Those are looked up by `external_id`. That match is -safe because the import's checks refused any source ID the instance already -held, so a user the import did not create is never in scope. +stopped with that user's `POST /v1/users` sent and unanswered, so Clerk may +hold the user without its ID on record. Those are looked up by `external_id`. +The import's checks refused any source ID the instance already held, but a +later import of the same source IDs could have created one since. So a user +that another import run in the runs folder records as created is left out. ```sh clerk migrate undo 20260929-141502-a1b2 --dry-run # preview, delete nothing diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index 83d5a17be..94fada144 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -269,15 +269,117 @@ describe("importUsers", () => { }); expect(summary).toMatchObject({ successful: 1, failed: 0 }); - // On record once created, then again with what the attach added. - expect(lines).toHaveLength(2); - expect(lines[0]).toEqual({ - sourceId: lines[0]!.sourceId, - clerkId: "user_created", - status: "created", + // On record once created, then with its attach pending, then with what + // the attach added. A refused attach is not retried, so nothing is pending. + expect(lines).toHaveLength(3); + expect(lines[0]).toEqual({ sourceId: "u1", clerkId: "user_created", status: "created" }); + expect(lines[1]?.pending).toEqual([{ kind: "email", value: "b@x.dev", verified: true }]); + expect(lines[2]).toMatchObject({ status: "created", clerkId: "user_created" }); + expect(lines[2]?.error).toContain("Failed to add additional email b@x.dev"); + expect(lines[2]).not.toHaveProperty("pending"); + }); + + test("retries an attach that hits a 429", async () => { + stub((url, attempt) => + url.endsWith("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/v1/email_addresses") && attempt === 1 + ? clerkError(429, "slow down", { "retry-after": "1" }) + : ok("user_created"), + ); + + await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, }); - expect(lines[1]?.status).toBe("created"); - expect(lines[1]?.error).toContain("Failed to add additional email b@x.dev"); + + expect(requests.filter((r) => r.url.endsWith("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/v1/email_addresses"))).toHaveLength(2); + expect(lines.at(-1)).not.toHaveProperty("error"); + expect(lines.at(-1)).not.toHaveProperty("pending"); + }); + + test("keeps an attach with no answer pending, for a continued run", async () => { + stub((url) => + url.endsWith("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/v1/email_addresses") ? clerkError(503, "unavailable") : ok("user_created"), + ); + + await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(lines.at(-1)?.pending).toEqual([{ kind: "email", value: "b@x.dev", verified: true }]); + }); + + // One slot: a user's attaches go ahead of the next queued create, so a run + // stopped midway leaves few users without their extra identifiers. + test("attaches a user's identifiers before the next queued create", async () => { + stub((url) => ok(url.endsWith("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/v1/users") ? "user_created" : "idn_1")); + + await importUsers({ + users: [ + user({ userId: "u1", email: ["a@x.dev", "b@x.dev"] }), + user({ userId: "u2", email: ["c@x.dev", "d@x.dev"] }), + user({ userId: "u3", email: ["e@x.dev", "f@x.dev"] }), + ], + secretKey: "sk_test_x", + limits: { ...LIMITS, concurrencyLimit: 1 }, + record, + }); + + expect(requests.map((r) => new URL(r.url).pathname)).toEqual([ + "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/v1/users", + "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/v1/email_addresses", + "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/v1/users", + "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/v1/email_addresses", + "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/v1/users", + "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/v1/email_addresses", + ]); + }); + + test("attachOnly sends just the pending attaches, and clears them", async () => { + stub(() => ok("idn_1")); + + await importUsers({ + users: [], + attachOnly: [ + { + sourceId: "u1", + clerkId: "user_1", + status: "created", + pending: [{ kind: "phone", value: "+15555550100", verified: false }], + }, + ], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(requests.map((r) => [new URL(r.url).pathname, r.body])).toEqual([ + [ + "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/v1/phone_numbers", + { user_id: "user_1", phone_number: "+15555550100", primary: false, verified: false }, + ], + ]); + expect(lines.at(-1)).toEqual({ sourceId: "u1", clerkId: "user_1", status: "created" }); + }); + + test("an adopted user is not created again; only its extras attach", async () => { + stub(() => ok("idn_1")); + + const summary = await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + adopted: new Map([["u1", "user_found"]]), + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(requests.map((r) => new URL(r.url).pathname)).toEqual(["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/v1/email_addresses"]); + expect(summary.successful).toBe(1); + expect(lines.at(-1)).toMatchObject({ clerkId: "user_found", status: "created" }); }); // Shapes from clerk_go's apierror: the country error carries its own code @@ -386,6 +488,50 @@ describe("importUsers", () => { expect(recordedBeforeAttach).toBe(true); }); + // A user is on record only once its create may land, so `undo` never + // looks up users that were still queued when the run stopped. + test("writes creating only as each POST /v1/users goes out", async () => { + const creatingAtFirstPost: number[] = []; + stub((url) => { + if (url.endsWith("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/v1/users") && creatingAtFirstPost.length === 0) { + creatingAtFirstPost.push(allLines.filter((line) => line.status === "creating").length); + } + return ok("user_created"); + }); + + await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2" }), user({ userId: "u3" })], + secretKey: "sk_test_x", + limits: { ...LIMITS, concurrencyLimit: 1 }, + record, + }); + + expect(creatingAtFirstPost).toEqual([1]); + }); + + test.each([ + ["a 5xx", () => clerkError(502, "bad gateway")], + [ + "a network error", + () => { + throw new TypeError("fetch failed"); + }, + ], + ])("a create that gets %s keeps creating, not failed", async (_label, respond) => { + stub(respond); + + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary.failed).toBe(1); + expect(allLines.map((line) => line.status)).toEqual(["creating"]); + expect([...summary.errorBreakdown.keys()][0]).toContain("a re-run checks"); + }); + test("records a failed user and keeps going", async () => { stub((_url, attempt) => attempt === 1 ? clerkError(422, "that email is taken") : ok("user_ok"), diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 74a9c87ea..ccfd5789e 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -18,10 +18,11 @@ import { bapiRequest } from "../../lib/bapi.ts"; import { BapiError } from "../../lib/errors.ts"; +import { interruptSignal } from "../../lib/signals.ts"; import type { SpinnerControls } from "../../lib/spinner.ts"; import type { ResolvedLimits } from "./lib/instance.ts"; import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; -import type { UserLine } from "./lib/run-store.ts"; +import type { PendingIdentifier, UserLine } from "./lib/run-store.ts"; import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; import type { ImportSummary, User } from "./types.ts"; @@ -174,18 +175,53 @@ type CreateContext = { }; /** - * Attaches one extra identifier. + * True when no answer says whether a request landed: an abort, a network + * error, or a 5xx after which Clerk may still have committed it. A 4xx, and a + * 429 that ran out of retries, are definite refusals. + */ +export function outcomeUnknown(error: unknown): boolean { + if (error instanceof RateLimitExceededError) return false; + if (error instanceof BapiError) return error.status >= 500; + return true; +} + +/** The extra identifiers a user carries, in the order they are attached. */ +export function pendingIdentifiers(identifiers: Identifiers): PendingIdentifier[] { + return [ + ...identifiers.additionalEmails.map((value) => ({ + kind: "email" as const, + value, + verified: true, + })), + ...identifiers.unverifiedEmails.map((value) => ({ + kind: "email" as const, + value, + verified: false, + })), + ...identifiers.additionalPhones.map((value) => ({ + kind: "phone" as const, + value, + verified: true, + })), + ...identifiers.unverifiedPhones.map((value) => ({ + kind: "phone" as const, + value, + verified: false, + })), + ]; +} + +/** + * Attaches one extra identifier, backing off on a 429. * - * @returns A note describing the failure, or `undefined` when it attached. - * Never throws: the user itself was already created. + * @returns A note when Clerk refused it, `pending` when nothing says whether + * it attached. Never throws: the user itself was already created. */ async function attachIdentifier( ctx: CreateContext, clerkUserId: string, - kind: "email" | "phone", - value: string, - verified: boolean, -): Promise { + { kind, value, verified }: PendingIdentifier, +): Promise<{ note?: string; pending?: boolean }> { const path = kind === "email" ? "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/v1/email_addresses" : "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/v1/phone_numbers"; const body = kind === "email" @@ -193,48 +229,75 @@ async function attachIdentifier( : { user_id: clerkUserId, phone_number: value, primary: false, verified }; try { - await ctx.schedule(async () => - bapiRequest({ - method: "POST", - path, - secretKey: ctx.secretKey, - body: JSON.stringify(body), - }), + await retryOn429(async () => + ctx.schedule( + async () => + bapiRequest({ + method: "POST", + path, + secretKey: ctx.secretKey, + body: JSON.stringify(body), + }), + { first: true }, + ), ); - return undefined; + return {}; } catch (error) { + if (outcomeUnknown(error)) return { pending: true }; const label = `${verified ? "additional" : "unverified"} ${kind} ${value}`; - return `Failed to add ${label}: ${(error as Error).message}`; + return { note: `Failed to add ${label}: ${(error as Error).message}` }; } } /** - * Creates one user, then attaches any additional identifiers it carries. + * Attaches each identifier. Extra identifiers are best-effort: a duplicate + * secondary email should not undo a user who was otherwise imported. * - * @param onCreated - Called as soon as the user exists, before the attaches: - * those wait their turn on the shared scheduler, and a run stopped in that - * window must still have the user on record for `undo` and re-runs. - * @returns The Clerk ID, and a note for each identifier that did not attach. + * @returns A note per identifier Clerk refused, and those still pending. + */ +async function attachAll( + ctx: CreateContext, + clerkUserId: string, + identifiers: PendingIdentifier[], +): Promise<{ notes: string[]; pending: PendingIdentifier[] }> { + const results = await Promise.all( + identifiers.map(async (identifier) => attachIdentifier(ctx, clerkUserId, identifier)), + ); + return { + notes: results.flatMap((result) => (result.note ? [result.note] : [])), + pending: identifiers.filter((_, index) => results[index]?.pending), + }; +} + +/** + * Creates one user, retrying without a phone Clerk refuses. + * + * @param sending - Called as each `POST /v1/users` goes out, so the run + * records the user only once a create may actually land. + * @returns The Clerk ID, and a note when the phone was dropped. */ async function createUser( ctx: CreateContext, user: User, + identifiers: Identifiers, skipPasswordRequirement: boolean, - onCreated: (clerkUserId: string) => void, + sending: () => void, ): Promise<{ clerkUserId: string; notes: string[] }> { - const identifiers = splitIdentifiers(user); const create = async (body: Record) => - ctx.schedule(async () => - bapiRequest({ + ctx.schedule(async () => { + // A Ctrl-C hands the slot on to queued creates; none of them was sent. + interruptSignal().throwIfAborted(); + sending(); + return bapiRequest({ method: "POST", path: "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/v1/users", secretKey: ctx.secretKey, body: JSON.stringify(body), - }), - ); + }); + }); const body = buildCreateUserBody(user, identifiers, skipPasswordRequirement); - const phoneNotes: string[] = []; + const notes: string[] = []; let response; try { response = await create(body); @@ -248,43 +311,31 @@ async function createUser( if (!phoneRefused || !identifiers.primaryEmail) throw error; const { phone_number: _dropped, ...withoutPhone } = body; response = await create(withoutPhone); - phoneNotes.push( + notes.push( `Failed to add phone ${identifiers.primaryPhone}: ${(error as BapiError).longMessage ?? (error as BapiError).message}`, ); } - const clerkUserId = (response.body as { id?: string })?.id ?? ""; - onCreated(clerkUserId); - - // Extra identifiers are best-effort: a duplicate secondary email should not - // undo a user who was otherwise imported successfully. - const notes = await Promise.all([ - ...identifiers.additionalEmails.map(async (email) => - attachIdentifier(ctx, clerkUserId, "email", email, true), - ), - ...identifiers.unverifiedEmails.map(async (email) => - attachIdentifier(ctx, clerkUserId, "email", email, false), - ), - ...identifiers.additionalPhones.map(async (phone) => - attachIdentifier(ctx, clerkUserId, "phone", phone, true), - ), - ...identifiers.unverifiedPhones.map(async (phone) => - attachIdentifier(ctx, clerkUserId, "phone", phone, false), - ), - ]); - - return { - clerkUserId, - notes: [...phoneNotes, ...notes.filter((note): note is string => note !== undefined)], - }; + return { clerkUserId: (response.body as { id?: string })?.id ?? "", notes }; } export type ImportUsersOptions = { users: User[]; secretKey: string; limits: ResolvedLimits; - /** Receives one line per user, as each one finishes. */ + /** Receives each user's lines as they happen. */ record: (line: UserLine) => void; + /** + * Users a continued run created whose extra identifiers never attached: their + * latest `created` line, with `pending`. Only the attaches are sent. + */ + attachOnly?: UserLine[]; + /** + * Source ID → Clerk ID for users whose create a stopped run sent with no + * answer, and which a continued run then found in the instance. They are + * not created again; only their extra identifiers are sent. + */ + adopted?: Map; /** Allow users that carry no password. */ skipPasswordRequirement?: boolean; /** Carried into the summary so the report covers the whole file. */ @@ -296,7 +347,8 @@ export type ImportUsersOptions = { * Imports every user, concurrently and within the instance's rate limit. * * A failed user is recorded and the run continues; a 429 backs off (honouring - * `Retry-After`) and retries up to {@link MAX_RETRIES} times. + * `Retry-After`) and retries up to {@link MAX_RETRIES} times. A create with no + * answer keeps its `creating` line, for a continued run or `undo` to resolve. */ export async function importUsers(options: ImportUsersOptions): Promise { const { @@ -304,6 +356,8 @@ export async function importUsers(options: ImportUsersOptions): Promise(), skipPasswordRequirement = true, validationFailed = 0, spinner, @@ -325,54 +379,95 @@ export async function importUsers(options: ImportUsersOptions): Promise { + const recordFailure = ( + userId: string, + message: string, + code: string, + notes: string[], + unknown: boolean, + ) => { failed++; processed++; const normalized = normalizeErrorMessage(message); errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); - record({ sourceId: userId, status: "failed", error: [message, ...notes].join("; "), code }); + // With no answer, the `creating` line stays the latest: Clerk may hold + // the user, and a re-run looks it up before creating it again. + if (!unknown) { + record({ sourceId: userId, status: "failed", error: [message, ...notes].join("; "), code }); + } progress(); }; - const processUser = async (user: User): Promise => { - record({ sourceId: user.userId, status: "creating" }); - const retries: string[] = []; - const created = (clerkId: string, error?: string) => + /** + * Attaches a created user's extra identifiers. The user goes on record with + * them `pending` first, so a run stopped before they attach can finish them. + */ + const finishUser = async (line: UserLine, toAttach: PendingIdentifier[], notes: string[]) => { + const { error: _error, pending: _pending, ...base } = line; + if (toAttach.length > 0) record({ ...base, pending: toAttach }); + const attached = await attachAll(ctx, base.clerkId ?? "", toAttach); + const error = [...notes, ...attached.notes].join("; "); + // A second line, which wins as the latest, adds what happened on the way. + if (toAttach.length > 0 || error) { record({ - sourceId: user.userId, - clerkId, - status: "created", + ...base, ...(error ? { error } : {}), - ...(user.passwordDropped ? { passwordDropped: true } : {}), + ...(attached.pending.length > 0 ? { pending: attached.pending } : {}), }); + } + }; + + const processUser = async (user: User): Promise => { + const retries: string[] = []; + const identifiers = splitIdentifiers(user); + let created: { clerkUserId: string; notes: string[] }; + const adoptedId = adopted.get(user.userId); try { - const { clerkUserId, notes } = await retryOn429( - async () => createUser(ctx, user, skipPasswordRequirement, (clerkId) => created(clerkId)), - { onRetry: ({ message }) => retries.push(message) }, - ); - successful++; - processed++; - // The user is already on record; a second line, which wins as the - // latest, adds what happened on the way. - const error = [...notes, ...retries].join("; "); - if (error) created(clerkUserId, error); - progress(); + created = adoptedId + ? { clerkUserId: adoptedId, notes: [] } + : await retryOn429( + async () => + createUser(ctx, user, identifiers, skipPasswordRequirement, () => + record({ sourceId: user.userId, status: "creating" }), + ), + { onRetry: ({ message }) => retries.push(message) }, + ); } catch (error) { if (error instanceof RateLimitExceededError) { - recordFailure(user.userId, error.message, "429", retries); + recordFailure(user.userId, error.message, "429", retries, false); return; } - const apiError = error as BapiError; + const unknown = outcomeUnknown(error); const message = apiError.longMessage ?? apiError.message ?? "Unknown error"; - recordFailure(user.userId, message, String(apiError.status ?? "unknown"), retries); + recordFailure( + user.userId, + unknown ? `${message} (Clerk may have created the user; a re-run checks)` : message, + String(apiError.status ?? "unknown"), + retries, + unknown, + ); + return; } + + const line: UserLine = { + sourceId: user.userId, + clerkId: created.clerkUserId, + status: "created", + ...(user.passwordDropped ? { passwordDropped: true } : {}), + }; + record(line); + await finishUser(line, pendingIdentifiers(identifiers), [...created.notes, ...retries]); + successful++; + processed++; + progress(); }; progress(); - await Promise.all(users.map(async (user) => processUser(user))); + await Promise.all([ + ...users.map(async (user) => processUser(user)), + ...attachOnly.map(async (line) => finishUser(line, line.pending ?? [], [])), + ]); return { totalProcessed: total, successful, failed, validationFailed, errorBreakdown }; } diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index ced8728e9..999eca548 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -29,7 +29,8 @@ beforeEach(() => { existing = []; globalThis.fetch = (async (input: string | URL | Request) => { const url = new URL(input.toString()); - const wanted = new Set(url.searchParams.values()); + // BAPI strips the `+` the lookup puts on each external_id. + const wanted = new Set([...url.searchParams.values()].map((value) => value.replace(/^\+/, ""))); return Response.json( existing.filter((candidate) => [ @@ -162,12 +163,12 @@ describe("rejects", () => { }); }); - // Continuing a run finds the users it created: that is expected, not a clash. - test("not a user the run being continued created", async () => { + // A continued run found this user behind its own in-flight create. + test("not a user the continued run adopted", async () => { existing = [{ id: "user_1", external_id: "mine" }]; expect( - await reasonsOf({ users: [user("mine")], continuedClerkIds: new Set(["user_1"]) }), + await reasonsOf({ users: [user("mine")], adoptedClerkIds: new Set(["user_1"]) }), ).toEqual({}); }); diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index 434946656..23fed65a9 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -87,8 +87,11 @@ export type CheckInput = { target: ClerkTarget; secretKey: string; schedule: ApiScheduler; - /** Clerk IDs the run being continued created: finding them in the instance is expected. */ - continuedClerkIds?: Set; + /** + * Clerk IDs a continued run found behind its own in-flight creates: finding + * them in the instance is expected. + */ + adoptedClerkIds?: Set; spinner?: SpinnerControls; }; @@ -388,7 +391,7 @@ async function findInstanceDuplicates( }; for (const existing of found) { - if (input.continuedClerkIds?.has(existing.id)) continue; + if (input.adoptedClerkIds?.has(existing.id)) continue; if (existing.external_id) { claim(byExternalId.get(existing.external_id), "already in the instance, with this source ID"); } diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 3f9ac5a3b..cabe2e4f2 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -89,12 +89,15 @@ describe("a run's life", () => { expect(fs.existsSync(path.join(run.dir, "lock"))).toBe(false); }); - test.each([["failed"], ["skipped"]] as const)("finishes partial when a user was %s", (status) => { - const run = startRun(runsDir, init); - run.append({ sourceId: "a", status: "created" }); - run.append({ sourceId: "b", status }); - expect(run.finish().status).toBe("partial"); - }); + test.each([["failed"], ["skipped"], ["creating"]] as const)( + "finishes partial when a user was %s", + (status) => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created" }); + run.append({ sourceId: "b", status }); + expect(run.finish().status).toBe("partial"); + }, + ); test("counts each source ID by its last line", () => { const run = startRun(runsDir, init); @@ -112,6 +115,26 @@ describe("a run's life", () => { expect([...latestUserLines(runsDir, run.record.id).keys()]).toEqual(["a"]); }); + + test("a continued run starts a fresh line after one a crash cut short", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created" }); + fs.appendFileSync(path.join(run.dir, "users.ndjson"), '{"sourceId":"b","sta'); + fs.rmSync(path.join(run.dir, "lock")); + + continueRun(runsDir, run.record).append({ sourceId: "c", status: "created" }); + + expect([...latestUserLines(runsDir, run.record.id).keys()]).toEqual(["a", "c"]); + }); + + // A user created with no line is beyond both undo and a re-run, so the + // create that would follow must not go out. + test("append throws when the line cannot be written", () => { + const run = startRun(runsDir, init); + fs.mkdirSync(path.join(run.dir, "users.ndjson")); + + expect(() => run.append({ sourceId: "a", status: "creating" })).toThrow(); + }); }); describe("locks and interruptions", () => { @@ -130,6 +153,13 @@ describe("locks and interruptions", () => { expect(runState(runsDir, run.record)).toBe("interrupted"); }); + // Undo marks an interrupted import undone, but leaves it with no finish time. + test("an interrupted run that was undone reads as undone", () => { + const run = startRun(runsDir, init); + fs.rmSync(path.join(run.dir, "lock")); + expect(runState(runsDir, { ...run.record, status: "undone" })).toBe("undone"); + }); + test("continuing takes the lock from a dead process and reopens the run", () => { const run = startRun(runsDir, init); run.append({ sourceId: "a", status: "created" }); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 74cb293ff..acb8abaf3 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -36,9 +36,10 @@ export const RUNS_DIR_DESCRIPTION = `Where migration runs are kept (default: .cl export type RunKind = "import" | "undo" | "export"; export type RunStatus = "running" | "complete" | "partial" | "undone"; /** - * `creating` is written just before `POST /v1/users`. As a user's latest line - * it means the run stopped with that create in flight: the user may exist in - * Clerk without its ID on record, so `undo` looks it up by `external_id`. + * `creating` is written as `POST /v1/users` goes out, and stays the latest line + * when no answer says whether the create landed (an abort, a network error, a + * 5xx). The user may then exist in Clerk without its ID on record, so `undo` + * and a continued run look it up by `external_id`. */ export type UserStatus = "creating" | "created" | "failed" | "skipped" | "deleted" | "exported"; @@ -81,10 +82,18 @@ export type RunRecord = { counts: RunCounts; }; +/** An extra email or phone still to attach to a created user. */ +export type PendingIdentifier = { kind: "email" | "phone"; value: string; verified: boolean }; + export type UserLine = { sourceId: string; clerkId?: string; status: UserStatus; + /** + * On a `created` line: the extra identifiers not yet attached. A continued + * run attaches them; a later `created` line without it means they are done. + */ + pending?: PendingIdentifier[]; /** Why a user was skipped. */ reason?: string; error?: string; @@ -244,10 +253,28 @@ export function latestUserLines(runsDir: string, id: string): Map { + const ids = new Set(); + for (const record of listRuns(runsDir)) { + if (record.kind !== "import" || record.id === exceptId) continue; + for (const line of latestUserLines(runsDir, record.id).values()) { + if (line.status === "created" && line.clerkId) ids.add(line.clerkId); + } + } + return ids; +} + /** Every readable run, newest first. */ export function listRuns(runsDir: string): RunRecord[] { let entries: fs.Dirent[]; @@ -295,7 +322,8 @@ export type Run = { /** * Counts the outcomes, settles the status and releases the lock. * - * `partial` when any user failed or was skipped, `complete` otherwise. + * `partial` when any user failed, was skipped or may not have been created, + * `complete` otherwise. */ finish(): RunRecord; }; @@ -308,14 +336,10 @@ function openRun(runsDir: string, record: RunRecord): Run { runsDir, dir, record, + // Throws when the line cannot be written: a user created with no record is + // beyond both `undo` and a re-run, so the create it precedes must not go out. append(line) { - try { - fs.appendFileSync(usersFile, `${JSON.stringify(line)}\n`); - } catch (error) { - // A broken destination must not abort an in-flight migration; the run - // is still making real progress against the API. - log.warn(`Could not write to ${usersFile}: ${(error as Error).message}`); - } + fs.appendFileSync(usersFile, `${JSON.stringify(line)}\n`); }, update(patch) { run.record = { ...run.record, ...patch }; @@ -323,7 +347,7 @@ function openRun(runsDir: string, record: RunRecord): Run { }, finish() { const counts = countLines(latestUserLines(runsDir, run.record.id).values()); - const unfinished = (counts.failed ?? 0) + (counts.skipped ?? 0); + const unfinished = (counts.failed ?? 0) + (counts.skipped ?? 0) + (counts.creating ?? 0); run.update({ counts, status: unfinished > 0 ? "partial" : "complete", @@ -363,6 +387,11 @@ export function startRun(runsDir: string, init: StartRunInit): Run { */ export function continueRun(runsDir: string, record: RunRecord): Run { acquireLock(runsDir, record.id); + // A crash mid-write leaves a torn last line; end it so the next append + // starts a line of its own instead of fusing with it. + const usersFile = path.join(runDir(runsDir, record.id), USERS_FILE); + const written = fs.existsSync(usersFile) ? fs.readFileSync(usersFile, "utf-8") : ""; + if (written && !written.endsWith("\n")) fs.appendFileSync(usersFile, "\n"); const run = openRun(runsDir, record); run.record = { ...record, status: "running" }; delete run.record.finishedAt; diff --git a/packages/cli-core/src/commands/migrate/lib/scheduler.ts b/packages/cli-core/src/commands/migrate/lib/scheduler.ts index 1a7e65992..0971bf98e 100644 --- a/packages/cli-core/src/commands/migrate/lib/scheduler.ts +++ b/packages/cli-core/src/commands/migrate/lib/scheduler.ts @@ -8,34 +8,45 @@ * with ten extra email addresses cannot burst past the instance's rate limit. */ -/** Runs `fn` once a slot is free and the pacing interval has elapsed. */ -export type ApiScheduler = (fn: () => Promise) => Promise; +/** + * Runs `fn` once a slot is free and the pacing interval has elapsed. + * + * `first` puts `fn` ahead of every queued call without it: a user's extra + * identifiers attach before the next user is created, so a run stopped midway + * leaves few users waiting on attaches. + */ +export type ApiScheduler = (fn: () => Promise, options?: { first?: boolean }) => Promise; export function createApiScheduler(concurrencyLimit: number, rateLimit: number): ApiScheduler { const maxConcurrent = Math.max(1, Math.floor(concurrencyLimit)); const intervalMs = Math.ceil(1000 / Math.max(1, rateLimit)); const waiting: (() => void)[] = []; + const waitingFirst: (() => void)[] = []; let active = 0; let nextRequestAt = 0; - async function acquire(): Promise { + async function acquire(first: boolean): Promise { if (active < maxConcurrent) { active++; return Promise.resolve(); } - return new Promise((resolve) => waiting.push(resolve)); + return new Promise((resolve) => (first ? waitingFirst : waiting).push(resolve)); } function release(): void { - const next = waiting.shift(); - // Hand the slot straight to the next waiter; `active` is unchanged because - // the slot never actually frees up. - if (next) next(); - else active--; + // One macrotask later, so the finished call's follow-up (a user's attaches) + // is queued before the slot is handed on. + setImmediate(() => { + const next = waitingFirst.shift() ?? waiting.shift(); + // Hand the slot straight to the next waiter; `active` is unchanged + // because the slot never actually frees up. + if (next) next(); + else active--; + }); } - return async (fn) => { - await acquire(); + return async (fn, options) => { + await acquire(options?.first ?? false); try { const now = Date.now(); const waitMs = Math.max(0, nextRequestAt - now); diff --git a/packages/cli-core/src/commands/migrate/lib/user-lookup.ts b/packages/cli-core/src/commands/migrate/lib/user-lookup.ts index 8c27d8a65..dca4a8673 100644 --- a/packages/cli-core/src/commands/migrate/lib/user-lookup.ts +++ b/packages/cli-core/src/commands/migrate/lib/user-lookup.ts @@ -10,6 +10,7 @@ import { bapiRequest } from "../../../lib/bapi.ts"; import type { SpinnerControls } from "../../../lib/spinner.ts"; import { retryOn429 } from "./retry.ts"; +import { clerkIdsCreatedByOtherRuns } from "./run-store.ts"; import type { ApiScheduler } from "./scheduler.ts"; /** BAPI accepts at most 100 values per filter on `GET /v1/users`. */ @@ -55,7 +56,11 @@ export async function lookupUsers(options: { batches.map(async (values) => { const params = new URLSearchParams(); params.set("limit", String(LOOKUP_BATCH)); - for (const value of values) params.append(options.filter, value); + // BAPI reads a leading `-` on an `external_id` as "exclude" and strips a + // leading `+`, so an explicit `+` keeps a source ID like `-abc` literal. + for (const value of values) { + params.append(options.filter, options.filter === "external_id" ? `+${value}` : value); + } const response = await retryOn429(async () => options.schedule(async () => @@ -75,3 +80,31 @@ export async function lookupUsers(options: { return pages.flat().filter((user) => typeof user.id === "string"); } + +/** + * The users behind creates that were in flight when run `runId` stopped. + * + * Found by `external_id`, which the import's checks refused to reuse, but a + * later run of the same source IDs can still have created one after this run + * stopped. So any Clerk ID another import run records as created is left out. + */ +export async function findInFlight(options: { + runsDir: string; + runId: string; + sourceIds: string[]; + secretKey: string; + schedule: ApiScheduler; +}): Promise<{ sourceId: string; clerkId: string }[]> { + if (options.sourceIds.length === 0) return []; + const found = await lookupUsers({ + filter: "external_id", + values: options.sourceIds, + secretKey: options.secretKey, + schedule: options.schedule, + }); + const otherRuns = clerkIdsCreatedByOtherRuns(options.runsDir, options.runId); + const wanted = new Set(options.sourceIds); + return found + .filter((user) => user.external_id && wanted.has(user.external_id) && !otherRuns.has(user.id)) + .map((user) => ({ sourceId: user.external_id as string, clerkId: user.id })); +} diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 790a0413a..e12921ee6 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -118,7 +118,10 @@ describe("run", () => { return Response.json({ object: "total_count", total_count: stub.count ?? 0 }); } if (method === "GET" && url.pathname === "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/v1/users") { - const wanted = new Set(url.searchParams.values()); + // BAPI strips the `+` the lookup puts on each external_id. + const wanted = new Set( + [...url.searchParams.values()].map((value) => value.replace(/^\+(?!\d)/, "")), + ); return Response.json( (stub.existing ?? []).filter( (user) => @@ -356,6 +359,13 @@ describe("run", () => { expect(created()).toEqual(["u1"]); expect(captured.err).toContain("leaving out 1 user without a password"); + // On record, so the run is partial rather than "Already imported". + const [record] = listRuns(runsDir()); + expect(latestUserLines(runsDir(), record!.id).get("u2")).toMatchObject({ + status: "skipped", + reason: "no password (--require-password)", + }); + expect(record?.status).toBe("partial"); }); test("aborts before any API call when the hasher is unrecognized", async () => { @@ -457,6 +467,107 @@ describe("run", () => { expect(captured.err).toContain("which was interrupted"); }); + /** Rewrites a finished run as one a crash stopped: no finish time. */ + const interrupt = (id: string, patch: Record = {}) => { + const record = readRun(runsDir(), id)!; + delete record.finishedAt; + fs.writeFileSync( + path.join(runsDir(), id, "run.json"), + JSON.stringify({ ...record, ...patch }), + ); + }; + + // The create went out and the run stopped before the answer came back. + test("an interrupted run adopts a user Clerk created with no ID on record", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + fs.appendFileSync( + path.join(runsDir(), first!.id, "users.ndjson"), + `${JSON.stringify({ sourceId: "u2", status: "creating" })}\n`, + ); + interrupt(first!.id); + + requests = []; + process.exitCode = 0; + stubClerk({ existing: [{ id: "user_found", external_id: "u2" }] }); + await run(baseOptions); + + expect(created()).toEqual([]); + expect(captured.err).toContain("1 user whose create was cut off is already in the instance"); + expect(latestUserLines(runsDir(), first!.id).get("u2")).toMatchObject({ + status: "created", + clerkId: "user_found", + }); + expect(readRun(runsDir(), first!.id)?.status).toBe("complete"); + }); + + test("an interrupted run creates a user whose in-flight create never landed", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + fs.appendFileSync( + path.join(runsDir(), first!.id, "users.ndjson"), + `${JSON.stringify({ sourceId: "u2", status: "creating" })}\n`, + ); + interrupt(first!.id); + + requests = []; + process.exitCode = 0; + stubClerk(); + await run(baseOptions); + + expect(created()).toEqual(["u2"]); + }); + + test("a continued run with nothing left to do is finished", async () => { + await run(baseOptions); + const [first] = listRuns(runsDir()); + interrupt(first!.id); + requests = []; + + await run(baseOptions); + + expect(created()).toEqual([]); + expect(captured.err).toContain("No users left to import"); + expect(readRun(runsDir(), first!.id)).toMatchObject({ status: "complete" }); + expect(readRun(runsDir(), first!.id)?.finishedAt).toBeDefined(); + }); + + // "Interrupted, so undo it and start over": the undo marks the run undone + // but leaves it with no finish time. + test("an interrupted run that was then undone is imported again as a new run", async () => { + await run(baseOptions); + const [first] = listRuns(runsDir()); + interrupt(first!.id, { status: "undone" }); + requests = []; + + await run(baseOptions); + + expect(created()).toEqual(["u1", "u2"]); + expect(listRuns(runsDir()).filter((record) => record.kind === "import")).toHaveLength(2); + }); + + test("a run with an undo that did not finish refuses with exit 2", async () => { + await run(baseOptions); + const [first] = listRuns(runsDir()); + const undoRun = startRun(runsDir(), { + kind: "undo", + target: { instanceId: "ins_1" }, + undoes: first!.id, + }); + undoRun.append({ sourceId: "u1", status: "deleted", clerkId: "user_u1" }); + undoRun.append({ sourceId: "u2", status: "failed", clerkId: "user_u2" }); + undoRun.finish(); + requests = []; + + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain(`clerk migrate undo ${first!.id}`); + expect(created()).toEqual([]); + }); + test("an undone run is imported again as a new run", async () => { await run(baseOptions); const [first] = listRuns(runsDir()); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 49cd5fc72..194e2e1cd 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -53,10 +53,12 @@ import { startRun, type Run, type RunRecord, + type UserLine, } from "./lib/run-store.ts"; import { createApiScheduler } from "./lib/scheduler.ts"; import { readSupabaseRows } from "./lib/supabase-providers.ts"; import { printTarget, resolveClerkTarget } from "./lib/target.ts"; +import { findInFlight } from "./lib/user-lookup.ts"; import { fileExists, getFileType, @@ -64,7 +66,7 @@ import { resolveImportFilePath, } from "./lib/transform.ts"; import { resolveSource, sourceKeys } from "./sources/registry.ts"; -import type { ImportSummary } from "./types.ts"; +import type { ImportSummary, User } from "./types.ts"; import { promptForFile, promptForFirebaseHashConfig, promptForSource } from "./wizard.ts"; import { login } from "../auth/login.ts"; import { link } from "../link/index.ts"; @@ -348,7 +350,9 @@ export type ResumeCase = * - partial: continue it, retrying the users that failed or were skipped * - complete: nothing; the file is already in * - * @throws UsageError when that run is still running in another process. + * @throws UsageError when that run is still running in another process, or + * has an undo that never finished: some of its users are gone and some are + * not, so neither continuing it nor starting over is safe. */ export function findResume( runsDir: string, @@ -370,8 +374,21 @@ export function findResume( `Run ${latest.id} is importing this file right now in another process. Wait for it to finish.`, ); } - if (state === "complete") return { kind: "complete", record: latest }; if (state === "undone") return { kind: "new" }; + + const undoRun = listRuns(runsDir).find( + (record) => record.kind === "undo" && record.undoes === latest.id, + ); + if (undoRun && runState(runsDir, undoRun) !== "complete") { + throwUsageError( + `Run ${latest.id} has an undo that did not finish (run ${undoRun.id}). ` + + `Finish it with \`clerk migrate undo ${latest.id}\`, or pass --new-run to import into a new run.`, + ); + } + // Undone in full, though the import run was never marked so. + if (undoRun) return { kind: "new" }; + + if (state === "complete") return { kind: "complete", record: latest }; return { kind: "continue", record: latest, @@ -505,9 +522,15 @@ function commandFor(options: MigrateRunOptions, fromExport: string | undefined, return [...parts, ...extra].join(" "); } -/** Records the checks' rejects as skipped users, so the run says who they were. */ -function recordRejects(run: Run, checks: ImportChecks): void { +/** + * Records the checks' rejects as skipped users, so the run says who they were. + * + * An adopted user keeps its `creating` line: it exists in Clerk, and `undo` + * finds it only through that line. + */ +function recordRejects(run: Run, checks: ImportChecks, adopted: Map): void { for (const { sourceId, reason, keptSourceId } of checks.rejects) { + if (adopted.has(sourceId)) continue; run.append({ sourceId, status: "skipped", @@ -573,15 +596,36 @@ export async function run(rawOptions: MigrateRunOptions): Promise { } // Users the run being continued already created are done: they are not - // checked or sent again, and finding them in the instance is expected. + // checked or sent again. Those whose extra identifiers never attached + // get just the attaches. const continued = resume.kind === "continue" ? resume.record : undefined; const done = new Map(); + const attachOnly: UserLine[] = []; + const inFlight: string[] = []; if (continued) { for (const line of latestUserLines(runsDir, continued.id).values()) { - if (line.status === "created" && line.clerkId) done.set(line.sourceId, line.clerkId); + if (line.status === "creating") inFlight.push(line.sourceId); + if (line.status !== "created" || !line.clerkId) continue; + done.set(line.sourceId, line.clerkId); + if (line.pending?.length) attachOnly.push(line); } } + // Creates the run stopped with no answer to. Those Clerk holds are + // adopted: checked as usual, but never created again. + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + const adopted = new Map(); + if (continued) { + const found = await findInFlight({ + runsDir, + runId: continued.id, + sourceIds: inFlight, + secretKey, + schedule, + }); + for (const user of found) adopted.set(user.sourceId, user.clerkId); + } + if (!options.json) { log.info( resume.kind === "continue" @@ -589,6 +633,11 @@ export async function run(rawOptions: MigrateRunOptions): Promise { `${plural(done.size, "user")} already imported ${done.size === 1 ? "is" : "are"} left alone.` : "Starting a new run.", ); + if (adopted.size > 0) { + log.info( + `${plural(adopted.size, "user")} whose create was cut off ${adopted.size === 1 ? "is" : "are"} already in the instance, and won't be created again.`, + ); + } } const loaded = await withSpinner(`Loading users from ${file}...`, async () => @@ -599,15 +648,15 @@ export async function run(rawOptions: MigrateRunOptions): Promise { // An instruction about this import, not a prediction: users without a // password are left out of the job rather than recorded as skipped. + let withoutPassword: User[] = []; if (options.requirePassword) { - const withPassword = users.filter((user) => Boolean(user.password)); - const dropped = users.length - withPassword.length; - if (dropped > 0 && !options.json) { + withoutPassword = users.filter((user) => !user.password && !adopted.has(user.userId)); + if (withoutPassword.length > 0 && !options.json) { log.info( - `--require-password: leaving out ${plural(dropped, "user")} without a password.`, + `--require-password: leaving out ${plural(withoutPassword.length, "user")} without a password.`, ); } - users = withPassword; + users = users.filter((user) => !withoutPassword.includes(user)); } let supabaseRows: Record[] | undefined; @@ -626,7 +675,6 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ]), ); - const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); const checks = await withSpinner("Checking users against the instance...", async (spinner) => checkImport({ users, @@ -639,7 +687,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { target, secretKey, schedule, - continuedClerkIds: new Set(done.values()), + adoptedClerkIds: new Set(adopted.values()), spinner, }), ); @@ -689,7 +737,14 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ); } - if (checks.importable.length === 0 && checks.rejects.length === 0) { + if ( + checks.importable.length === 0 && + checks.rejects.length === 0 && + withoutPassword.length === 0 && + attachOnly.length === 0 + ) { + // Settled, so a continued run is finished rather than left interrupted. + if (continued) continueRun(runsDir, continued).finish(); if (options.json) preview({ nothingToImport: true }); else log.warn("No users left to import."); return; @@ -730,10 +785,17 @@ export async function run(rawOptions: MigrateRunOptions): Promise { file: { path: filePath, sha256 }, ...(input.fromExport ? { fromExport: input.fromExport } : {}), }); - recordRejects(run, checks); + recordRejects(run, checks, adopted); + for (const user of withoutPassword) { + run.append({ + sourceId: user.userId, + status: "skipped", + reason: "no password (--require-password)", + }); + } const summary = - checks.importable.length > 0 + checks.importable.length > 0 || attachOnly.length > 0 ? await withSpinner( `Importing users: [0/${checks.importable.length}]...`, async (spinner) => @@ -742,6 +804,8 @@ export async function run(rawOptions: MigrateRunOptions): Promise { secretKey, limits, record: run.append, + attachOnly, + adopted, skipPasswordRequirement: !options.requirePassword, spinner, }), @@ -767,7 +831,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { result: { created: summary.successful, failed: summary.failed, - skipped: checks.rejects.length, + skipped: checks.rejects.length + withoutPassword.length, errors: [...summary.errorBreakdown].map(([error, count]) => ({ error, count })), }, }, @@ -781,7 +845,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { log.blank(); for (const line of formatSummary( summary, - checks.rejects.length, + checks.rejects.length + withoutPassword.length, record, run.dir, limits.instanceType, diff --git a/packages/cli-core/src/commands/migrate/undo.test.ts b/packages/cli-core/src/commands/migrate/undo.test.ts index 7b9f23ea7..f7e54f059 100644 --- a/packages/cli-core/src/commands/migrate/undo.test.ts +++ b/packages/cli-core/src/commands/migrate/undo.test.ts @@ -52,6 +52,7 @@ beforeEach(() => { return Response.json( url.searchParams .getAll("external_id") + .map((externalId) => externalId.replace(/^\+/, "")) .filter((externalId) => inFlight.has(externalId)) .map((externalId) => ({ id: inFlight.get(externalId), external_id: externalId })), ); @@ -212,6 +213,32 @@ describe("deleting", () => { expect(deletes()).toHaveLength(2); }); + // Run A stopped with d's create in flight; a later run B then created d. + // The user Clerk holds is B's, so undoing A leaves it alone. + test("leaves an in-flight user that another import run records as created", async () => { + const runA = startRun(runsDir, { + kind: "import", + target: { instanceId: "ins_1", env: "development" }, + source: "clerk", + }); + runA.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + runA.append({ sourceId: "d", status: "creating" }); + const recordA = runA.finish(); + const runB = startRun(runsDir, { + kind: "import", + target: { instanceId: "ins_1", env: "development" }, + source: "clerk", + }); + runB.append({ sourceId: "d", status: "created", clerkId: "user_d" }); + runB.finish(); + inFlight.set("d", "user_d"); + instanceUsers.set("user_d", null); + + await undo(recordA.id, withDir({ yes: true })); + + expect(deletes().map((request) => request.url.split("/").pop())).toEqual(["user_a"]); + }); + test("deletes what the import created, records an undo run, and marks the import undone", async () => { const record = importRun(); diff --git a/packages/cli-core/src/commands/migrate/undo.ts b/packages/cli-core/src/commands/migrate/undo.ts index 0a95a4c01..bf09b388c 100644 --- a/packages/cli-core/src/commands/migrate/undo.ts +++ b/packages/cli-core/src/commands/migrate/undo.ts @@ -40,7 +40,7 @@ import { } from "./lib/run-store.ts"; import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; import { describeTarget, printTarget, resolveClerkTarget, type ClerkTarget } from "./lib/target.ts"; -import { lookupUsers } from "./lib/user-lookup.ts"; +import { findInFlight, lookupUsers } from "./lib/user-lookup.ts"; export type UndoOptions = { dryRun?: boolean; @@ -152,30 +152,6 @@ function usersToDelete( return { users, unconfirmed, alreadyDeleted: deleted.size }; } -/** - * Finds the users behind creates that were in flight when the run stopped. - * - * Matching on `external_id` alone is safe here: the import's checks refused - * any source ID the instance already held, so a user carrying one of these - * was created by this run. - */ -async function findUnconfirmed( - sourceIds: string[], - secretKey: string, - schedule: ApiScheduler, -): Promise { - if (sourceIds.length === 0) return []; - const found = await lookupUsers({ - filter: "external_id", - values: sourceIds, - secretKey, - schedule, - }); - return found - .filter((user) => user.external_id && sourceIds.includes(user.external_id)) - .map((user) => ({ sourceId: user.external_id as string, clerkId: user.id })); -} - /** * Reads each user back from the instance, for the preview. * @@ -348,7 +324,13 @@ export async function undo(runId: string, options: UndoOptions = {}): Promise 0 From 6930163088425477e193b3975e42ee200efc381b Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 15:04:05 -0400 Subject: [PATCH 086/141] docs: add stacked pr plan --- stacked-prs.md | 176 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 stacked-prs.md diff --git a/stacked-prs.md b/stacked-prs.md new file mode 100644 index 000000000..2e8a5cc68 --- /dev/null +++ b/stacked-prs.md @@ -0,0 +1,176 @@ +# Stacked PRs for `clerk migrate` + +Split `ra/integrate-migration-tool-into-cli` (clerk/cli#479: 88 commits, 112 files, +20,421 / -122) into 14 stacked PRs. Each PR carves existing files out of the branch. Nothing gets rewritten. #479 stays open as the reference until the top of the stack merges. + +**#479 stays untouched.** Every PR below is a new branch and a new PR. The stack only reads from #479: + +- Nothing is committed or pushed to `ra/integrate-migration-tool-into-cli`, and it's never rebased or force-pushed. +- #479 itself is never edited, retargeted, marked ready, merged, or closed while the stack is in flight. Decide what to do with it after PR 10 merges. +- Files come from commit `cea11672` (the tip of #479 when this plan was written), never by branching from, cherry-picking onto, or merging the PR branch. Pinning the commit means a later push to #479 can't change what the stack carves. + +## The stack at a glance + +Line counts come from `git diff --numstat origin/main...HEAD`, grouped by file. Counts marked `~` include a test file shared across PRs, so they move a little once you split it. + +| PR | Scope | Src | Tests | Total | +| --- | ------------------------------------------------------------------- | ----: | -----: | -----: | +| 0a | Build and CI chores (off `main`, independent) | 82 | 0 | 82 | +| 0b | Shared lib prep (off `main`, independent) | 93 | 101 | 194 | +| 1a | Import pipeline: map, validate, throttle, run store, target | 1,899 | ~1,500 | ~3,400 | +| 1b | Preflight checks and readiness report | 1,394 | 1,276 | 2,670 | +| 1c | `clerk migrate import` command, behind `CLERK_EXPERIMENTAL=migrate` | 1,505 | ~2,000 | ~3,500 | +| 2 | `migrate runs` and `migrate undo` | 771 | 376 | 1,147 | +| 3 | Export framework and `export clerk` | 1,115 | 951 | 2,066 | +| 4 | `export supabase` and the DB layer | 703 | 707 | 1,410 | +| 5 | Firebase: source, export, hash flags | 795 | 576 | 1,371 | +| 6 | Auth0: source and export | 434 | 346 | 780 | +| 7 | WorkOS: source and export | 559 | 478 | 1,037 | +| 8 | Better Auth and Auth.js: sources and exports | 519 | ~150 | ~670 | +| 9 | `migrate sources` command and custom sources (`--source ./file.ts`) | 394 | 417 | 811 | +| 10 | Un-gate, changeset, root README | ~10 | ~180 | ~190 | + +The 1,067-line `commands/migrate/README.md` doesn't get its own PR. `readme.test.ts` checks the README against the command tree in both directions, so each PR adds the README section for what it ships. + +## Ground rules + +1. **Carve, don't rewrite.** Each PR takes files from #479 with `git checkout cea11672 -- `, run on the new PR's own branch, then trims imports and registry entries so it compiles alone. +2. **Every PR passes CI alone.** Run `bun run format && bun run lint && bun run typecheck && bun run test` before you push. +3. **Gate from 1c to 10.** PR 1c adds `lib/experimental.ts` (about 40 lines, the same design as Task 1 of the slice-1 proposal). Without `CLERK_EXPERIMENTAL=migrate`, `migrate` stays hidden from help and completion and exits 2. Each PR can merge to `main` and reach `@canary` without shipping half a feature. PR 10 removes the gate. +4. **Merge bottom-up.** Open each one as a new PR with `gh pr create --draft --base `. Use `git rebase --update-refs` (or Graphite's `gt`) so review fixes low in the stack ripple up in one command. `--update-refs` runs on the stack's branches only, never on `ra/integrate-migration-tool-into-cli`. +5. **0a and 0b branch off `main`**, so they can merge in any order, today. 1a also starts from `origin/main`, and each PR above it starts from the branch below it. No branch starts from `ra/integrate-migration-tool-into-cli`. +6. **Leave #479 alone.** Never build on, push to, or rebase its branch, and never run `gh pr edit`, `gh pr ready`, `gh pr merge`, or `gh pr close` on #479. Do the work in new worktrees, not in a checkout of the PR branch. + +## PR details + +### 0a: Build and CI chores + +- `.github/workflows/ci.yml`: `bun run build` becomes `bun run build:compile` +- `package.json`: `build` alias, Playwright pinned to `1.60.0` +- `packages/cli-core/package.json`: drop the unused `build` script (leave the `csv-parser` and `zod` additions for 1a) +- `scripts/check-bun-version.ts`, `bun.lock` `configVersion` +- `.gitignore`: `exports/` and `*service-account*.json` + +Review focus: none of this touches migrate. Check the Playwright pin still matches main's CI image before merging. + +### 0b: Shared lib prep + +- `lib/fetch.ts` and `lib/errors.ts`: a connection failure becomes a `CliError` with code `network_unreachable` that names the host +- `lib/git.ts` and `lib/keyless.ts`: move `ensureGitignoreEntry` into `git.ts` and export it. Add it to `test/lib/stubs.ts` and the `harness.ts` git mock. +- `lib/spinner.ts`: ignore an empty `setNextSteps([])` +- `lib/bapi-command.ts` and its tests: `describeBapiTarget` names the key's source (`via --app`, `via the linked profile`, `CLERK_SECRET_KEY`) +- `lib/prompts.ts` and `prompts-instructions.test.ts`: advertise `a: all` in the multiselect footer + +Review focus: `describeBapiTarget` changes wording for every command that prints a target, not only migrate. Call that out in the PR description. + +### 1a: Import pipeline (library only, no command) + +Files: + +- `types.ts`, `validator.ts` (+ tests) +- `lib/transform.ts`, `lib/scheduler.ts`, `lib/retry.ts`, `lib/instance.ts`, `lib/run-store.ts`, `lib/target.ts` (+ tests) +- `sources/shared.ts`, `sources/clerk.ts`, `sources/supabase.ts`, `sources/registry.ts` +- `sources/sources.test.ts`: keep only the Clerk, Supabase, and shared cases +- `packages/cli-core/package.json`: add `csv-parser` and `zod` + +Edits to make it stand alone: + +- `sources/registry.ts`: register only `clerk` and `supabase`. Strip `loadCustomSource` and `registerCustomSource` (they return in PR 9). +- `types.ts`: drop `FirebaseHashConfig` if nothing in this PR uses it, or leave the type in place. A type with no caller costs nothing. +- `lib/export-file.ts`: `transform.ts` imports it to read envelopes. Bring it along (51 lines) so `transform.ts` stays untouched. + +Review focus: the run store layout (`run.json`, `users.ndjson`, `lock`), the field mapping for Clerk and Supabase, and the 429 handling. + +### 1b: Preflight checks + +Files: `lib/checks.ts`, `lib/readiness.ts`, `lib/modify-settings.ts`, `lib/analysis.ts`, `lib/clerk-config.ts`, `lib/supabase-providers.ts`, `lib/user-lookup.ts`, plus tests. + +Edits: none expected. `checks.ts` imports `import-users.ts` for one type. Move that type into `types.ts` in this PR, or pull `import-users.ts` down from 1c. + +Review focus: what blocks a user and what drops a field. This is where the behavior that writes to production lives: disabled identifiers, refused emails and names, username rules, duplicates, and the dev user limit. + +### 1c: `clerk migrate import` + +Files: + +- `index.ts`: register only `import` +- `run.ts`, `import-users.ts`, `wizard.ts`, plus `run.test.ts`, `run-interactive.test.ts`, `import-users.test.ts`, `wizard.test.ts`, `index.test.ts` +- `lib/assume-yes.ts`, `lib/input-retry.ts` (the `-y` hook and the credential retry loop) +- `cli-program.ts`, `__complete.ts` (`--source` values), `completion.test.ts` +- `lib/next-steps.ts`: `printAgentNextSteps` +- New: `lib/experimental.ts` and its test (the gate) +- `README.md` (import sections only), `readme.test.ts` +- `test/e2e/migrate.test.ts`: only the "unverified email is refused" case + +Edits to make it stand alone: + +- `run.ts`: remove the Firebase hash flags, `resolveFirebaseHashConfig`, and `promptForFirebaseHashConfig` (about 10 lines; they return in PR 5). +- `run.ts`: `resolveInput` accepts an export run ID. Keep the code, since the run store already exists, and drop its README example until PR 3. +- `next-steps.ts`: `MIGRATE_DONE` points at `migrate runs` and `migrate undo`, which don't exist yet. Point at the run folder path instead; PR 2 restores the original text. +- `run.ts` `cleanupLines`: same fix for its "undo" mention. +- `run.test.ts` and `index.test.ts`: drop the `per-platform imports`, `--source `, and non-Clerk/Supabase cases. + +Continuing a stopped run stays in this PR. It lives inside `run.ts`, and pulling it out means rewriting code that already works. + +Review focus: consent (`--yes`, the TTY prompt, `--json` without `--yes`), the target line, exit codes, and the gate. + +### 2: `migrate runs` and `migrate undo` + +Files: `runs.ts`, `undo.ts`, and their tests. Register both in `index.ts`. Restore `MIGRATE_DONE` and `cleanupLines`. Add the README sections. + +Review focus: `undo` is the one command that deletes users. Check the in-flight case (f108dc0a) and the `--dry-run` path. + +### 3: Export framework and `export clerk` + +Files: `export/index.ts`, `export/shared.ts`, `export/registry.ts` (Clerk only), `export/clerk.ts`, `export/clerk-source.ts`, plus tests. Add the README sections for export and for importing by export run ID. + +Review focus: the envelope format (`ENVELOPE_VERSION = 1`), since every later export writes it. + +### 4: `export supabase` and the DB layer + +Files: `lib/db.ts`, `export/db-options.ts`, `export/supabase.ts`, `lib/db.test.ts`, the Supabase part of `export/db-exports.test.ts`. Add the `Bun.sql` MySQL note to `CLAUDE.md`. + +Review focus: connection-string handling (URL-encoding, libsql/Turso) and the Bun version floor for MySQL binary columns. + +### 5 to 8: One PR per provider + +Each PR adds the source mapping, the export, their tests, the registry entries, the README sections, and the `sources.test.ts` cases for that provider. + +- **5 Firebase:** `sources/firebase.ts`, `export/firebase.ts`, `lib/firebase-hash.ts`. Restore the four `--firebase-*` flags and the hash-config prompt in `run.ts` and `wizard.ts`. +- **6 Auth0:** `sources/auth0.ts`, `export/auth0.ts` +- **7 WorkOS:** `sources/workos.ts`, `export/workos.ts`, the WorkOS entry in `init/scan.ts` +- **8 Better Auth and Auth.js:** `sources/betterauth.ts`, `sources/authjs.ts`, `export/betterauth.ts`, `export/authjs.ts`, the rest of `db-exports.test.ts`, and the "Better Auth scrypt hash" e2e case + +Reorder these freely. They depend only on PR 3 (and PR 4 for the DB-backed two). + +### 9: `migrate sources` and custom sources + +Files: `sources/list.ts`, `sources/load-custom.ts`, plus tests. Restore `registerCustomSource` and `loadCustomSource` in the registry, the `migrate sources` positional completion, and `--source ` handling in `run.ts`. + +It goes after the providers because `sources/list.ts` imports `export/registry.ts` to show how to export from each platform. + +### 10: Un-gate + +Remove `lib/experimental.ts` and the stub branch in `index.ts`. Add `migrate` to the root `README.md`. Add `.changeset/migrate-cli.md`. Re-run the agent baseline against `@canary` before merging. + +## Effort + +- **Carving:** about 1 hour each for PRs 0a, 0b, 2, and 5 to 10, and 2 to 3 hours each for 1a, 1b, 1c, 3, and 4 (the test-file splits take the time). About 2 to 3 days in total. +- **Review:** 14 PRs at 0.1k to 3.5k lines. The three PRs in slice 1 carry the review weight. + +## Risks + +- **1a and 1b merge code that nothing calls** until 1c lands. Reviewers have to read them as libraries. If that's a blocker, merge 1a to 1c together as one reviewed stack, or collapse them into one ~9.6k PR. +- **Rebase churn.** A review fix in 1a ripples through 13 branches. `--update-refs` handles the mechanics, but every PR above it gets re-pushed. +- **Split test files.** `sources.test.ts`, `run.test.ts`, and `db-exports.test.ts` each cover several PRs. Splitting them by `describe` block is the slowest part of the carve. + +## First step + +Create PR 0a in a new worktree off `main`: + +```bash +git worktree add -b migrate/0a-chores ../cli-0a origin/main +cd ../cli-0a +git checkout cea11672 -- .github/workflows/ci.yml scripts/check-bun-version.ts +``` + +Then open it as a new PR with `gh pr create --draft --base main`. From e6421409dac12720bf468fd8c2bf3d8bcbc70b72 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 15:09:50 -0400 Subject: [PATCH 087/141] fix(migrate): stop creating users with more trust than the source gave them - Clerk -> Clerk: an unverified primary email or phone stays unverified, in both `export clerk` and a Dashboard CSV, instead of going on POST /v1/users (which creates it verified). - Auth.js on SQLite: qualify every column, so a missing one is an error rather than a string literal that read as verified, and fall back to a legacy NextAuth `email_verified` column. - The checks reject what POST /v1/users would refuse mid-import: a missing required first or last name, a TOTP secret or backup codes with that feature off (with a fix to turn it on), no password where password is the only sign-in method, and no legal acceptance where legal consent is on. New --skip-legal-checks (or a yes at the prompt) imports those users with skip_legal_checks. - Send skip_restriction_checks: the allowlist and blocklist police sign-ups, and these users already signed up. - Reject a user left with no identifier once the ones the instance has off are stripped. - Read booleans in source hooks case-insensitively, plus t/f, so a spreadsheet's TRUE or psql's t no longer drops a ban or a verification. - An expired Better Auth ban is no longer carried as a permanent one. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 28 +++- .../src/commands/migrate/export/authjs.ts | 54 ++++++-- .../src/commands/migrate/export/clerk.test.ts | 15 +++ .../src/commands/migrate/export/clerk.ts | 11 +- .../migrate/export/db-exports.test.ts | 30 ++++- .../src/commands/migrate/import-users.test.ts | 7 + .../src/commands/migrate/import-users.ts | 4 +- .../cli-core/src/commands/migrate/index.ts | 4 + .../src/commands/migrate/lib/checks.test.ts | 118 ++++++++++++++++- .../src/commands/migrate/lib/checks.ts | 122 +++++++++++++++++- .../cli-core/src/commands/migrate/lib/db.ts | 2 +- .../commands/migrate/lib/modify-settings.ts | 4 +- .../src/commands/migrate/lib/readiness.ts | 14 +- .../commands/migrate/lib/transform.test.ts | 16 ++- .../src/commands/migrate/lib/transform.ts | 21 +-- .../cli-core/src/commands/migrate/run.test.ts | 25 +++- packages/cli-core/src/commands/migrate/run.ts | 20 +++ .../commands/migrate/sources/betterauth.ts | 20 ++- .../src/commands/migrate/sources/shared.ts | 24 +++- .../commands/migrate/sources/sources.test.ts | 19 +++ 20 files changed, 504 insertions(+), 54 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 5a191b43c..92fcb6d12 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -360,11 +360,13 @@ also keeps `raw_app_meta_data`, which is what the import's **`authjs` tries `User`, then `user`, then `users`.** Auth.js has no single schema — Prisma capitalizes the table, Drizzle does not, and Postgres treats the difference as significant once quoted. The run reports which one it found. -Auth.js core stores no passwords, so its users arrive without credentials. +The verified column is read as `emailVerified`, or `email_verified` on a legacy +NextAuth table. Auth.js core stores no passwords, so its users arrive without +credentials. **`betterauth` detects its plugin columns from the schema.** The username -plugin adds `username`, admin adds `banned`, phone-number adds `phoneNumber`, -and so on; selecting a column that is not there fails the whole query, and the +plugin adds `username`, admin adds `banned` (carried only while `banExpires` is +unset or in the future), phone-number adds `phoneNumber`, and so on; selecting a column that is not there fails the whole query, and the database answers the question better than the user can. Passwords come from a `LEFT JOIN` onto the credential `account` row — left, not inner, so a user who only ever signed in with OAuth is still exported. @@ -526,6 +528,7 @@ clerk migrate import # a human is asked | `--allow-partial` | Import the users that pass, and record the rest as skipped | | `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | | `--require-password` | Import only users that carry a password digest | +| `--skip-legal-checks` | Import users with no legal acceptance into an instance requiring it | | `--firebase-signer-key ` | Firebase base64 signer key (overrides the export file) | | `--firebase-salt-separator ` | Firebase base64 salt separator | | `--firebase-rounds ` | Firebase scrypt rounds | @@ -617,6 +620,16 @@ after them. They sort the users three ways: - it lacks an identifier the instance requires. An email or phone counts only when it is verified, because an unverified one is attached after the user exists + - it has no identifier left once those the instance has turned off are + stripped + - it lacks a first or last name the instance requires + - it has an authenticator app secret or backup codes, and the instance has + that turned off. Importing it without them would take away its second + factor, so the checks offer to turn the setting on instead + - it has no password, and password is the instance's only way to sign in + - it has no legal acceptance on record, and the instance requires legal + consent. `--skip-legal-checks`, or a yes at the prompt, imports these users + without it (`skip_legal_checks`), with a warning - its username breaks the instance's username rules (length, letters, the allowed special characters) - its password is not the shape its hasher says (`bcrypt`, `scrypt_firebase`, @@ -625,7 +638,8 @@ after them. They sort the users three ways: - Supabase: its only provider is not enabled in Clerk - the instance already has a user with its source ID, email, phone or username (a batched `GET /v1/users` lookup, 100 values a request, through - the scheduler). The users a continued run created do not count + the scheduler). A user a continued run found behind its own interrupted + create does not count - a development instance: it is past the 100-user headroom (`CLERK_MIGRATE_DEV_USER_LIMIT` when Clerk raised it), counted in file order @@ -665,6 +679,12 @@ the instance settings cannot be read (BAPI `/v1/domains` → the instance's Frontend API `/v1/environment`), required fields are not checked and the run says so. +#### Sign-up restrictions + +Every create sends `skip_restriction_checks: true`. The instance's allowlist, +blocklist, disposable-email and subaddress rules police new sign-ups, and these +users already signed up on the source platform. + #### Additional identifiers Only the first verified email and phone go on `POST /v1/users`. Every diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 73c9d7f86..34c2c5706 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -35,18 +35,40 @@ type AuthJsRow = Record & { email_verified?: unknown; }; -export function buildAuthJsQuery(client: DbClient, table: string): string { +/** + * The verified-email column, in the order tried: the current adapters' name, + * then the legacy NextAuth `users` table's. + */ +const VERIFIED_COLUMNS = ["emailVerified", "email_verified"] as const; + +/** + * Every column is qualified with the table alias: SQLite reads an unqualified + * double-quoted name that matches no column as a string literal, so a missing + * `"emailVerified"` would come back as the text "emailVerified" on every row, + * which reads as verified. + */ +export function buildAuthJsQuery( + client: DbClient, + table: string, + verifiedColumn: (typeof VERIFIED_COLUMNS)[number] = "emailVerified", +): string { const q = (identifier: string) => client.quote(identifier); return ( - `SELECT ${q("id")}, ${q("name")}, ${q("email")}, ${q("emailVerified")} AS ${q("email_verified")} ` + - `FROM ${q(table)} ORDER BY ${q("id")} ASC` + `SELECT u.${q("id")}, u.${q("name")}, u.${q("email")}, u.${q(verifiedColumn)} AS ${q("email_verified")} ` + + `FROM ${q(table)} u ORDER BY u.${q("id")} ASC` ); } +const messageOf = (error: unknown) => (error instanceof Error ? error.message : String(error)); + +/** True for an error that means "no such column". Postgres says "does not exist" for both. */ +function isMissingColumn(error: unknown): boolean { + return /no such column|column .* does not exist|unknown column/i.test(messageOf(error)); +} + /** True for an error that means "wrong table name", not "broken connection". */ function isMissingTable(error: unknown): boolean { - const message = error instanceof Error ? error.message : String(error); - return /does not exist|no such table|doesn't exist|unknown table/i.test(message); + return /does not exist|no such table|doesn't exist|unknown table/i.test(messageOf(error)); } /** @@ -60,12 +82,24 @@ export async function fetchAuthJsUsers( let lastError: unknown; for (const table of TABLE_CANDIDATES) { - try { - return { rows: await client.query(buildAuthJsQuery(client, table)), table }; - } catch (error) { - if (!isMissingTable(error)) throw error; - lastError = error; + let columnError: unknown; + for (const column of VERIFIED_COLUMNS) { + try { + const rows = await client.query(buildAuthJsQuery(client, table, column)); + return { rows, table }; + } catch (error) { + // The table is there: try the other name for the verified column, and + // report the first failure if neither reads. + if (isMissingColumn(error)) { + columnError ??= error; + continue; + } + if (!isMissingTable(error)) throw error; + lastError = error; + break; + } } + if (columnError) throw columnError; } throw lastError instanceof Error diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index e68c09ceb..9d0b3197e 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -109,6 +109,21 @@ describe("mapClerkUserToExport", () => { expect(mapped.verified_email_addresses).toBeUndefined(); }); + // Verify-at-sign-up off lets a primary stay unverified. Exported as primary, + // the import would create it verified. + test("keeps an unverified primary unverified, and promotes a verified one", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "unverified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("b@x.dev"); + expect(mapped.unverified_email_addresses).toEqual(["a@x.dev"]); + }); + test("promotes the first verified address when none is flagged primary", () => { const mapped = mapClerkUserToExport( user({ diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 403cc5ca6..84ab47025 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -88,16 +88,19 @@ function splitIdentifiers( const value = read(entry); if (!value) continue; - if (entry.id && entry.id === primaryId) { + const isVerified = entry.verification?.status === "verified"; + // An unverified primary stays unverified: the import puts a primary on + // POST /v1/users, which creates it verified. + if (entry.id && entry.id === primaryId && isVerified) { primary = value; continue; } - if (entry.verification?.status === "verified") verified.push(value); + if (isVerified) verified.push(value); else unverified.push(value); } - // No primary flagged: promote the first verified one so the export still has - // an identifier the import can lead with. + // No verified primary: promote the first verified one so the export still + // has an identifier the import can lead with. if (!primary && verified.length > 0) primary = verified.shift(); return { primary, verified, unverified }; diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index e8fdf5052..b8dc8e0c1 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -187,8 +187,8 @@ describe("authjs export", () => { test("quotes identifiers for the dialect", async () => { await withClient(authJsDb("User"), async (client) => { - expect(buildAuthJsQuery(client, "User")).toContain('"User"'); - expect(buildAuthJsQuery(client, "User")).toContain('"emailVerified" AS "email_verified"'); + expect(buildAuthJsQuery(client, "User")).toContain('"User" u'); + expect(buildAuthJsQuery(client, "User")).toContain('u."emailVerified" AS "email_verified"'); }); }); @@ -199,6 +199,32 @@ describe("authjs export", () => { expect(rows).toHaveLength(2); }); + // SQLite reads an unqualified "emailVerified" that matches no column as the + // string "emailVerified", which would mark every email verified. + test("reads a legacy NextAuth table's email_verified, and keeps null unverified", async () => { + const file = makeDb((db) => { + db.run( + `CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT, email TEXT, email_verified TEXT)`, + ); + db.run(`INSERT INTO users VALUES (?,?,?,?)`, ["n1", "Nv", "nv@x.dev", null]); + db.run(`INSERT INTO users VALUES (?,?,?,?)`, ["n2", "V", "v@x.dev", "2024-01-15"]); + }); + + const { rows, table } = await withClient(file, fetchAuthJsUsers); + + expect(table).toBe("users"); + expect(rows.map((row) => row.email_verified)).toEqual([null, "2024-01-15"]); + }); + + test("a missing column is an error, not a literal", async () => { + const file = makeDb((db) => { + db.run(`CREATE TABLE "User" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" TEXT)`); + db.run(`INSERT INTO "User" VALUES (?,?,?)`, ["a", "a@x.dev", null]); + }); + + await expect(withClient(file, fetchAuthJsUsers)).rejects.toThrow(/no such column/); + }); + test("fails clearly when no candidate table exists", async () => { const file = makeDb((db) => db.run(`CREATE TABLE unrelated (id TEXT)`)); await expect(withClient(file, fetchAuthJsUsers)).rejects.toThrow( diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index 94fada144..47751cc73 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -82,6 +82,13 @@ describe("buildCreateUserBody", () => { ]); }); + // Allowlists and blocklists police sign-ups; these users already signed up. + test("skips the instance's sign-up restrictions", () => { + expect(buildCreateUserBody(user(), splitIdentifiers(user()), true)).toMatchObject({ + skip_restriction_checks: true, + }); + }); + test("omits fields the source platform never recorded", () => { const body = buildCreateUserBody(user(), splitIdentifiers(user()), true); expect("first_name" in body).toBe(false); diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index ccfd5789e..2cb87b67a 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -131,7 +131,9 @@ export function buildCreateUserBody( identifiers: Identifiers, skipPasswordRequirement: boolean, ): Record { - const body: Record = { external_id: user.userId }; + // The instance's allowlist, blocklist, disposable-email and subaddress rules + // police sign-ups. These users already signed up, on the source platform. + const body: Record = { external_id: user.userId, skip_restriction_checks: true }; if (identifiers.primaryEmail) body.email_address = [identifiers.primaryEmail]; if (identifiers.primaryPhone) body.phone_number = [identifiers.primaryPhone]; diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index a55bc9259..71b5e066e 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -70,6 +70,10 @@ export function registerMigrate(program: Program): void { .option("--allow-partial", "Import the users that pass the checks, and skip the rest") .option("--new-run", "Start a new run instead of continuing an earlier one of this file") .option("--require-password", "Import only users that have a password") + .option( + "--skip-legal-checks", + "Import users with no legal acceptance on record into an instance that requires it", + ) .option("--firebase-signer-key ", "Firebase base64 signer key (overrides the export file)") .option("--firebase-salt-separator ", "Firebase base64 salt separator") .option("--firebase-rounds ", "Firebase scrypt rounds", (value: string) => diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index 999eca548..47f30c9f3 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -1,7 +1,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; import type { UserSettingsJSON } from "../../../lib/fapi.ts"; import type { User } from "../types.ts"; -import { checkImport, hashShapeProblem, type CheckInput } from "./checks.ts"; +import { checkImport, hashShapeProblem, passwordIsOnlySignIn, type CheckInput } from "./checks.ts"; const BCRYPT = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"; @@ -69,6 +69,88 @@ const reasonsOf = async (overrides: Partial) => ); describe("rejects", () => { + // Each one is a user POST /v1/users refuses (clerk_go create_service.go). + describe("what the create refuses", () => { + const EMAIL_ON = { email_address: { enabled: true } }; + + test.each([ + ["first_name", { lastName: "L" }, "no first name, which this instance requires"], + ["last_name", { firstName: "F" }, "no last name, which this instance requires"], + ] as const)("a missing required %s", async (attribute, fields, reason) => { + const reasons = await reasonsOf({ + users: [user("a", fields), user("b", { firstName: "F", lastName: "L" })], + settings: settings({ ...EMAIL_ON, [attribute]: { enabled: true, required: true } }), + }); + expect(reasons).toEqual({ a: reason }); + }); + + test.each([ + [ + "totpSecret", + "authenticator_app", + "has an authenticator app (TOTP) secret, and this instance has authenticator apps off", + ], + ["backupCodes", "backup_code", "has backup codes, and this instance has backup codes off"], + ] as const)("%s with %s off, offering to turn it on", async (field, attribute, reason) => { + const value = field === "backupCodes" ? ["code1"] : "SECRET"; + const checks = await checkImport( + input({ + users: [user("a", { [field]: value })], + settings: settings({ ...EMAIL_ON, [attribute]: { enabled: false } }), + }), + ); + expect(checks.rejects).toEqual([{ sourceId: "a", reason }]); + expect(checks.fixes.map((fix) => fix.command).join("\n")).toContain( + `"auth_multi_factor":{"${attribute}":{"enabled":true}}`, + ); + }); + + test("no password where password is the only way to sign in", async () => { + const passwordOnly = settings({ + email_address: { enabled: true, used_for_first_factor: false, first_factors: [] }, + password: { enabled: true, used_for_first_factor: true, first_factors: ["password"] }, + }); + const reasons = await reasonsOf({ + users: [user("a"), user("b", { password: BCRYPT, passwordHasher: "bcrypt" })], + settings: passwordOnly, + }); + expect(reasons).toEqual({ + a: "no password, and password is this instance's only way to sign in", + }); + }); + + // Firebase or Supabase phone-auth users, into an instance with phone off. + test("a user left with no identifier once disabled ones are stripped", async () => { + const reasons = await reasonsOf({ + users: [user("phone-only", { email: undefined, phone: "+15555550100" }), user("b")], + settings: settings({ email_address: { enabled: true }, phone_number: { enabled: false } }), + }); + expect(reasons).toEqual({ + "phone-only": + "has no identifier this instance accepts (its email, phone or username is turned off)", + }); + }); + + describe("legal consent", () => { + const LEGAL = { + ...settings(EMAIL_ON), + sign_up: { legal_consent_enabled: true }, + } as unknown as UserSettingsJSON; + const users = [user("a"), user("b", { legalAcceptedAt: "2024-01-01T00:00:00.000Z" })]; + + test("rejects a user with no acceptance on record, naming the flag", async () => { + expect((await reasonsOf({ users, settings: LEGAL })).a).toContain("--skip-legal-checks"); + }); + + test("with skipLegalChecks, imports them with skip_legal_checks and a warning", async () => { + const checks = await checkImport(input({ users, settings: LEGAL, skipLegalChecks: true })); + expect(checks.rejects).toEqual([]); + expect(checks.importable.map((u) => u.skipLegalChecks)).toEqual([true, undefined]); + expect(checks.warnings.join("\n")).toContain("1 user has no legal acceptance on record"); + }); + }); + }); + test("a user that failed validation", async () => { const checks = await checkImport( input({ failures: [{ userId: "bad", row: 0, error: "Invalid email", path: ["email"] }] }), @@ -438,6 +520,40 @@ describe("fixes", () => { }); }); +describe("passwordIsOnlySignIn", () => { + const withFactors = (factors: Record, social: object = {}) => + settings( + Object.fromEntries( + Object.entries(factors).map(([name, first_factors]) => [ + name, + { enabled: true, used_for_first_factor: first_factors.length > 0, first_factors }, + ]), + ), + social, + ); + + test.each([ + ["password alone", { password: ["password"] }, {}, true], + // Mirrors clerk_go: a passkey is not counted as a way in without a password. + ["password and passkey", { password: ["password"], passkey: ["passkey"] }, {}, true], + [ + "password and email codes", + { password: ["password"], email_address: ["email_code"] }, + {}, + false, + ], + [ + "password and Google", + { password: ["password"] }, + { oauth_google: { enabled: true, authenticatable: true } }, + false, + ], + ["no password factor", { email_address: ["email_code"] }, {}, false], + ])("%s -> %p", (_label, factors, social, expected) => { + expect(passwordIsOnlySignIn(withFactors(factors, social))).toBe(expected); + }); +}); + describe("hashShapeProblem", () => { test.each([ [BCRYPT, "bcrypt"], diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index 23fed65a9..eb36890fa 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -87,6 +87,11 @@ export type CheckInput = { target: ClerkTarget; secretKey: string; schedule: ApiScheduler; + /** + * Import users with no legal acceptance into an instance that requires it, + * sending `skip_legal_checks`. Without it they are rejected. + */ + skipLegalChecks?: boolean; /** * Clerk IDs a continued run found behind its own in-flight creates: finding * them in the instance is expected. @@ -167,6 +172,87 @@ function missingRequiredIdentifier( return undefined; } +/** A required first or last name the user lacks: `POST /v1/users` refuses it. */ +function missingRequiredName(user: User, settings: UserSettingsJSON | null): string | undefined { + if (!settings) return undefined; + if (isRequired(settings, "first_name") && !hasValue(user.firstName)) { + return "no first name, which this instance requires"; + } + if (isRequired(settings, "last_name") && !hasValue(user.lastName)) { + return "no last name, which this instance requires"; + } + return undefined; +} + +/** MFA a user can carry, and the setting `POST /v1/users` refuses it without. */ +const MFA_SETTINGS = [ + { + field: "totpSecret", + attribute: "authenticator_app", + reason: "has an authenticator app (TOTP) secret, and this instance has authenticator apps off", + label: "Enable authenticator apps", + path: ["auth_multi_factor", "authenticator_app", "enabled"], + }, + { + field: "backupCodes", + attribute: "backup_code", + reason: "has backup codes, and this instance has backup codes off", + label: "Enable backup codes", + path: ["auth_multi_factor", "backup_code", "enabled"], + }, +] as const; + +/** + * MFA the instance has off. Rejected rather than dropped: importing the user + * without it would quietly take away their second factor. + */ +function mfaProblem(user: User, settings: UserSettingsJSON | null): string | undefined { + if (!settings) return undefined; + return MFA_SETTINGS.find( + ({ field, attribute }) => hasValue(user[field]) && !isEnabled(settings, attribute), + )?.reason; +} + +/** Sign-in strategies that don't count as a way in without a password, per clerk_go. */ +const NOT_ALTERNATIVE_SIGN_IN = new Set([ + "password", + "passkey", + "ticket", + "reset_password_email_code", + "reset_password_phone_code", +]); + +/** + * True when the instance has no sign-in strategy but a password. Clerk then + * refuses `skip_password_requirement`, so a user without a digest can't be + * created. Mirrors `create_service.go` and `UserSettings.FirstFactors()`. + */ +export function passwordIsOnlySignIn(settings: UserSettingsJSON): boolean { + const strategies = new Set(); + for (const attribute of Object.values(settings.attributes ?? {})) { + if (attribute?.used_for_first_factor) { + for (const strategy of attribute.first_factors ?? []) strategies.add(strategy); + } + } + for (const [strategy, social] of Object.entries(settings.social ?? {})) { + if (social?.enabled && social.authenticatable) strategies.add(strategy); + } + if (settings.enterprise_sso?.enabled) strategies.add("enterprise_sso"); + return ( + strategies.has("password") && + [...strategies].every((strategy) => NOT_ALTERNATIVE_SIGN_IN.has(strategy)) + ); +} + +/** True when the instance needs legal acceptance this user has no record of. */ +function lacksLegalAcceptance(user: User, settings: UserSettingsJSON | null): boolean { + return ( + Boolean(settings?.sign_up?.legal_consent_enabled) && + !user.legalAcceptedAt && + !user.skipLegalChecks + ); +} + /** What FAPI serves under `username_settings`; `@clerk/shared` types only the lengths. */ type UsernameSettings = { min_length?: number; @@ -542,7 +628,13 @@ function buildFixes(input: CheckInput, users: User[]): Fix[] { const flags = input.target.appId ? ` --app ${input.target.appId} --instance ${input.target.instanceId}` : ""; - return buildSettingChanges(flagged).map((change) => ({ + const settings = input.settings; + const mfa = MFA_SETTINGS.filter( + ({ field, attribute }) => + !isEnabled(settings, attribute) && users.some((user) => hasValue(user[field])), + ).map(({ label, path }) => ({ label, writes: [{ path: [...path], value: true }] })); + + return [...buildSettingChanges(flagged), ...mfa].map((change) => ({ label: change.label, command: `clerk config patch${flags} --json ${quoteJson(buildChangePayload([change]))}`, })); @@ -589,6 +681,13 @@ function refusedNameWarning(count: number): string[] { ]; } +function legalWarning(count: number): string[] { + if (count === 0) return []; + return [ + `${plural(count, "user")} ${count === 1 ? "has" : "have"} no legal acceptance on record, and ${count === 1 ? "is" : "are"} created without it (--skip-legal-checks)`, + ]; +} + function placeholderWarning(count: number): string[] { if (count === 0) return []; return [ @@ -619,6 +718,19 @@ export async function checkImport(input: CheckInput): Promise { : undefined) ?? fileDuplicates.get(user.userId) ?? missingRequiredIdentifier(user, input.settings) ?? + // Stripping the identifiers the instance has off can leave nothing to + // sign in with; Clerk would still create the user. + (!hasAnyIdentifier(dropDisabledIdentifiers(user, input.settings)) + ? "has no identifier this instance accepts (its email, phone or username is turned off)" + : undefined) ?? + missingRequiredName(user, input.settings) ?? + mfaProblem(user, input.settings) ?? + (!user.password && input.settings && passwordIsOnlySignIn(input.settings) + ? "no password, and password is this instance's only way to sign in" + : undefined) ?? + (!input.skipLegalChecks && lacksLegalAcceptance(user, input.settings) + ? "no legal acceptance on record, which this instance requires (--skip-legal-checks imports them without it)" + : undefined) ?? usernameProblem(user, input.settings) ?? (user.password && user.passwordHasher ? hashShapeProblem(user.password, user.passwordHasher) @@ -667,13 +779,19 @@ export async function checkImport(input: CheckInput): Promise { return { total: input.users.length + input.failures.length, - importable: candidates.map((user) => dropDisabledIdentifiers(user, input.settings)), + importable: candidates.map((user) => { + const kept = dropDisabledIdentifiers(user, input.settings); + return lacksLegalAcceptance(kept, input.settings) ? { ...kept, skipLegalChecks: true } : kept; + }), rejects, rejectReasons: countReasons(rejects), warnings: [ ...buildWarnings(input, candidates), ...placeholderWarning(candidates.filter((user) => placeholderEmails.has(user.userId)).length), ...refusedNameWarning(candidates.filter((user) => refusedNames.has(user.userId)).length), + ...legalWarning( + candidates.filter((user) => lacksLegalAcceptance(user, input.settings)).length, + ), ], fixes: buildFixes(input, input.users), ...(quota ? { quota } : {}), diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index 6c90808a7..9e3b0e129 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -320,7 +320,7 @@ export function describeDbError(error: unknown, platform?: DbPlatform): string { if (/no such column|column .* does not exist|unknown column/i.test(message)) { const needs: Partial> = { authjs: - "The Auth.js export reads `id`, `name`, `email` and `emailVerified`. For a schema that renames them " + + "The Auth.js export reads `id`, `name`, `email` and `emailVerified` (or `email_verified`). For a schema that renames them " + '(Prisma `@map("email_verified")`, for one), export the users with your own query, ' + "`SELECT id, name, email, email_verified FROM …`, and import that file with the authjs source.", }; diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts index a430200e3..0ad4f1915 100644 --- a/packages/cli-core/src/commands/migrate/lib/modify-settings.ts +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts @@ -128,7 +128,9 @@ export function buildSettingChanges(flagged: ReadinessItem[]): SettingChange[] { * Changes share parents — `first_name` and `last_name` both write `user_model` * — so leaves are written into a shared tree rather than merged after the fact. */ -export function buildChangePayload(changes: SettingChange[]): Record { +export function buildChangePayload( + changes: Pick[], +): Record { const payload: Record = {}; for (const write of changes.flatMap((change) => change.writes)) { diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts index 60cd35585..02b00c05f 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -80,7 +80,14 @@ type BuildInput = { * carry leaves nothing to create them with, so the API refuses them — which is * why these are the only attributes whose consequence is `rejects`. */ -const IDENTIFIER_ATTRIBUTES = new Set(["email_address", "phone_number", "username"]); +/** Attributes `POST /v1/users` refuses a user without, when they are required. */ +const REJECTING_ATTRIBUTES = new Set([ + "email_address", + "phone_number", + "username", + "first_name", + "last_name", +]); /** An identifier or user-model row, with its blocking verdict. */ function buildAttributeItem( @@ -106,7 +113,7 @@ function buildAttributeItem( clerkEnabled: enabled, clerkRequired: required, blocking: true, - consequence: IDENTIFIER_ATTRIBUTES.has(attribute) ? "rejects" : "drops", + consequence: REJECTING_ATTRIBUTES.has(attribute) ? "rejects" : "drops", // How many users this costs is the outcome block's job. Restating it here // reads as a contradiction, because that block counts each user once and // this row counts the field — a user missing both an email and a password @@ -165,7 +172,8 @@ export function buildReadinessReport(input: BuildInput): ReadinessReport { ]; for (const [label, section, attribute, count] of attributeRows) { - if (count > 0) { + // A required field nobody has still costs every user, so it gets a row. + if (count > 0 || (settings && isRequired(settings, attribute))) { items.push(buildAttributeItem(label, section, attribute, count, settings, total)); } } diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 2f117da56..179b31789 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -124,12 +124,24 @@ describe("consolidateClerkIdentifiers", () => { test("drops the unverified list when every entry is already verified", () => { const user: Record = { phone: "+15555550100", - unverifiedPhoneNumbers: ["+15555550100"], + phoneNumbers: ["+15555550101"], + unverifiedPhoneNumbers: ["+15555550101"], }; consolidateClerkIdentifiers(user); - expect(user.phone).toEqual(["+15555550100"]); + expect(user.phone).toEqual(["+15555550100", "+15555550101"]); expect("unverifiedPhoneNumbers" in user).toBe(false); }); + + // The Dashboard lists an unverified primary under unverified_email_addresses. + test.each([ + ["email", "unverifiedEmailAddresses", "a@x.dev"], + ["phone", "unverifiedPhoneNumbers", "+15555550100"], + ])("keeps an unverified primary %s unverified", (primaryKey, unverifiedKey, value) => { + const user: Record = { [primaryKey]: value, [unverifiedKey]: [value] }; + consolidateClerkIdentifiers(user); + expect(primaryKey in user).toBe(false); + expect(user[unverifiedKey]).toEqual([value]); + }); }); describe("validatePreparedUsers", () => { diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 7ebee5657..56e4040bc 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -12,6 +12,7 @@ import path from "node:path"; import csvParser from "csv-parser"; import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; import { getSource } from "../sources/registry.ts"; +import { normalizeBooleanField } from "../sources/shared.ts"; import { PASSWORD_HASHERS, type TransformContext, type SourceEntry, type User } from "../types.ts"; import { userSchema } from "../validator.ts"; import { isEnvelope } from "./export-file.ts"; @@ -161,21 +162,6 @@ function normalizeStringArrayField(value: unknown): unknown { return parsed; } -function normalizeBooleanField(value: unknown): unknown { - if (typeof value === "boolean") return value; - if (typeof value === "number") { - if (value === 1) return true; - if (value === 0) return false; - return value; - } - if (typeof value !== "string") return value; - - const normalized = value.trim().toLowerCase(); - if (["true", "1", "yes", "y"].includes(normalized)) return true; - if (["false", "0", "no", "n"].includes(normalized)) return false; - return value; -} - function normalizeNumberField(value: unknown): unknown { if (typeof value === "number") return value; if (typeof value !== "string") return value; @@ -278,12 +264,15 @@ export function consolidateClerkIdentifiers(user: Record): void const verified = parseDelimitedStrings(user[verifiedKey]); const unverified = parseDelimitedStrings(user[unverifiedKey]); + // The Dashboard lists an unverified primary under the unverified field. + // Leading the verified list would put it on POST /v1/users, verified. const all: string[] = []; - if (primary) all.push(primary); + if (primary && !unverified.includes(primary)) all.push(primary); for (const value of verified) { if (!all.includes(value)) all.push(value); } if (all.length > 0) user[primaryKey] = all; + else delete user[primaryKey]; delete user[verifiedKey]; const extraUnverified = unverified.filter((value) => !all.includes(value)); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index e12921ee6..67af1558c 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -64,7 +64,7 @@ describe("validateRunOptions", () => { type Stub = { /** What `/v1/environment` reports; `null` makes the settings unreadable. */ - settings?: { attributes?: object; social?: object } | null; + settings?: { attributes?: object; social?: object; sign_up?: object } | null; /** Users already in the instance, as `GET /v1/users` returns them. */ existing?: { id: string; @@ -690,6 +690,29 @@ describe("run", () => { ); }); + test("legal consent: refused without --skip-legal-checks, sent with skip_legal_checks with it", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true } }, + sign_up: { legal_consent_enabled: true }, + }, + }); + + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(captured.err).toContain("no legal acceptance on record"); + expect(created()).toEqual([]); + + await run({ ...baseOptions, skipLegalChecks: true }); + const bodies = requests + .filter((r) => r.method === "POST" && r.url.endsWith("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/v1/users")) + .map((r) => r.body); + expect(bodies).toEqual([ + expect.objectContaining({ external_id: "u1", skip_legal_checks: true }), + expect.objectContaining({ external_id: "u2", skip_legal_checks: true }), + ]); + }); + test("a user already in the instance is rejected", async () => { stubClerk({ existing: [{ id: "user_old", email_addresses: [{ email_address: "b@x.dev" }] }], diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 194e2e1cd..58d58f0ad 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -81,6 +81,11 @@ export type MigrateRunOptions = { /** The file to read, once `input` is resolved. Not a flag. */ file?: string; requirePassword?: boolean; + /** + * Import users with no legal acceptance into an instance that requires it. + * Without it, a prompt asks; where nobody can be asked, they are rejected. + */ + skipLegalChecks?: boolean; /** Check against the instance, report, and write nothing. */ dryRun?: boolean; /** Import the users that pass, and record the rest as skipped. */ @@ -515,6 +520,7 @@ function commandFor(options: MigrateRunOptions, fromExport: string | undefined, if (options.allowPartial) parts.push("--allow-partial"); if (options.newRun) parts.push("--new-run"); if (options.requirePassword) parts.push("--require-password"); + if (options.skipLegalChecks) parts.push("--skip-legal-checks"); if (options.secretKey) parts.push("--secret-key", ""); if (options.app) parts.push("--app", options.app); if (options.instance) parts.push("--instance", options.instance); @@ -675,9 +681,23 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ]), ); + // Creating a user without legal acceptance needs consent of its own: + // the flag, or a yes at a prompt. Otherwise the checks reject them. + let skipLegalChecks = options.skipLegalChecks ?? false; + const withoutLegal = settings?.sign_up?.legal_consent_enabled + ? users.filter((user) => !user.legalAcceptedAt && !user.skipLegalChecks).length + : 0; + if (!skipLegalChecks && withoutLegal > 0 && !options.dryRun && canPrompt(options)) { + skipLegalChecks = await confirm({ + message: `${plural(withoutLegal, "user")} ${withoutLegal === 1 ? "has" : "have"} no legal acceptance on record, which this instance requires. Import them without it?`, + default: false, + }); + } + const checks = await withSpinner("Checking users against the instance...", async (spinner) => checkImport({ users, + skipLegalChecks, failures, unknownFields: loaded.unknownFields, ...(supabaseRows ? { supabaseRows } : {}), diff --git a/packages/cli-core/src/commands/migrate/sources/betterauth.ts b/packages/cli-core/src/commands/migrate/sources/betterauth.ts index 5d7840ed1..fff46cc3a 100644 --- a/packages/cli-core/src/commands/migrate/sources/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/sources/betterauth.ts @@ -1,5 +1,11 @@ import type { SourceEntry } from "../types.ts"; -import { detectStandardHasher, isVerified, routeByVerification, splitName } from "./shared.ts"; +import { + detectStandardHasher, + isVerified, + routeByVerification, + splitName, + toIsoDate, +} from "./shared.ts"; /** * Better Auth → Clerk source. @@ -49,7 +55,7 @@ const betterAuthSource = { key: "betterauth", label: "Better Auth", description: - "Works with the Better Auth export. Detects scrypt, bcrypt and argon2 passwords per user, and carries the admin plugin's banned flag.", + "Works with the Better Auth export. Detects scrypt, bcrypt and argon2 passwords per user, and carries the admin plugin's banned flag while its ban has not expired.", carries: { passwords: { level: "yes", @@ -101,8 +107,16 @@ const betterAuthSource = { // for every user that was never banned, and sending that to Clerk is noise. // SQLite, libSQL and MySQL hand back 1/0 and CSV hands back "true", so this // runs before normalizeUserData and must accept those too. - if (isVerified(user.banned, "boolean")) user.banned = true; + // A ban whose `banExpires` has passed is over: Better Auth lifts it only + // at the user's next sign-in, so the column still says banned. An expiry + // that can't be read keeps the ban. + const rawExpiry = user.banExpires ?? user.ban_expires; + const expiry = typeof rawExpiry === "number" && rawExpiry < 1e11 ? rawExpiry * 1000 : rawExpiry; + const expired = Date.parse(String(toIsoDate(expiry, true))) <= Date.now(); + if (isVerified(user.banned, "boolean") && !expired) user.banned = true; else delete user.banned; + delete user.banExpires; + delete user.ban_expires; // The anonymous plugin's guests are throwaway accounts with placeholder // emails (anon-…@…), not people to migrate. diff --git a/packages/cli-core/src/commands/migrate/sources/shared.ts b/packages/cli-core/src/commands/migrate/sources/shared.ts index 163c5b459..08383e2ec 100644 --- a/packages/cli-core/src/commands/migrate/sources/shared.ts +++ b/packages/cli-core/src/commands/migrate/sources/shared.ts @@ -20,12 +20,30 @@ export type VerificationStyle = "boolean" | "timestamp"; /** CSV exports write SQL NULL as one of these rather than an empty cell. */ const NULLISH_STRINGS = new Set(["", "null", "nil", "undefined", "\\n"]); +/** + * A boolean as a CSV or a database writes it: `TRUE` from a spreadsheet, `t` + * and `f` from psql, `yes`/`no`. Anything else is returned unchanged, so the + * schema can reject it. + */ +export function normalizeBooleanField(value: unknown): unknown { + if (typeof value === "boolean") return value; + if (typeof value === "number") { + if (value === 1) return true; + if (value === 0) return false; + return value; + } + if (typeof value !== "string") return value; + + const normalized = value.trim().toLowerCase(); + if (["true", "1", "yes", "y", "t"].includes(normalized)) return true; + if (["false", "0", "no", "n", "f"].includes(normalized)) return false; + return value; +} + export function isVerified(value: unknown, style: VerificationStyle): boolean { if (value === null || value === undefined) return false; - if (style === "boolean") { - return value === true || value === 1 || value === "true" || value === "1"; - } + if (style === "boolean") return normalizeBooleanField(value) === true; if (value instanceof Date) return !Number.isNaN(value.getTime()); if (typeof value === "number") return true; diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index 87b88018e..b7ed776de 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -107,6 +107,12 @@ describe("isVerified", () => { ["", false], [null, false], [undefined, false], + // A CSV re-saved by a spreadsheet, or written by psql. + ["TRUE", true], + ["FALSE", false], + ["t", true], + ["f", false], + ["Yes", true], ])("boolean style: %p -> %p", (value, expected) => { expect(isVerified(value, "boolean")).toBe(expected); }); @@ -386,6 +392,19 @@ describe("betterauth", () => { expect(one("betterauth", { ...base, banned })?.banned).toBe(expected as boolean | undefined); }); + // Better Auth lifts an expired ban only at the next sign-in, so the column + // can still say banned long after the ban ended. + test.each([ + ["an expiry in the past", "2020-01-01T00:00:00.000Z", undefined], + ["an expiry in the future", "2999-01-01T00:00:00.000Z", true], + ["epoch seconds in the past", 1_577_836_800, undefined], + ["epoch milliseconds in the future", 32_472_144_000_000, true], + ["an unreadable expiry", "soon", true], + ])("a ban with %s -> banned %p", (_label, banExpires, expected) => { + const user = one("betterauth", { ...base, banned: true, banExpires }); + expect(user?.banned).toBe(expected as boolean | undefined); + }); + test("drops plugin-only columns during validation", async () => { const { users } = await load("betterauth", [ { ...base, role: "admin", display_username: "ADA", two_factor_enabled: true }, From 0365fb8430a16a01020a903109d3a4aa475d9ea5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 15:40:35 -0400 Subject: [PATCH 088/141] fix(migrate): fix the Medium review findings and the Standards behavior items - Fixes name their instance with --instance, or point at the Dashboard when Clerk could not name it, so a --secret-key import's fix no longer changes the linked profile's development instance. - import and undo refuse an --instance that the key from --secret-key or CLERK_SECRET_KEY doesn't address. - Run folders are 0700 and export files 0600. - export clerk carries external_id into private metadata as clerkExternalId, pages oldest first and drops repeats. The checks key file duplicates by record, so the kept copy is no longer rejected with its repeat, and only users that pass the other checks claim identifiers. - Add the phpass hasher. A numeric user ID is read as a string. - Read a BOM-prefixed CSV or JSON file, and NDJSON (Auth0's bulk export). A file that isn't valid JSON is named in the error. - Printed commands shell-quote their paths, keep a custom --source's path, and keep --json and the --firebase-* flags (as placeholders). - A lock holding this process's own PID is stale, and refusals name the lock file. `runs` counts an interrupted run from its user lines. - A Firebase hash that came back redacted is treated as missing, with a warning naming the permission. - Usage errors exit 2 instead of 1. An unreadable dev-instance user count is called out instead of checked silently as zero. `clerk update` reports registry_unreachable again when offline. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 51 ++++++++---- .../src/commands/migrate/export/auth0.ts | 11 +-- .../src/commands/migrate/export/clerk.test.ts | 23 ++++++ .../src/commands/migrate/export/clerk.ts | 23 ++++-- .../commands/migrate/export/firebase.test.ts | 11 +++ .../src/commands/migrate/export/firebase.ts | 46 +++++++---- .../commands/migrate/export/shared.test.ts | 10 +++ .../src/commands/migrate/export/shared.ts | 5 +- .../src/commands/migrate/export/workos.ts | 10 +-- .../src/commands/migrate/lib/checks.test.ts | 58 +++++++++++++ .../src/commands/migrate/lib/checks.ts | 81 +++++++++++++------ .../src/commands/migrate/lib/export-file.ts | 40 ++++++++- .../commands/migrate/lib/run-store.test.ts | 23 +++++- .../src/commands/migrate/lib/run-store.ts | 28 +++++-- .../src/commands/migrate/lib/target.test.ts | 47 ++++++++++- .../src/commands/migrate/lib/target.ts | 29 ++++++- .../commands/migrate/lib/transform.test.ts | 38 +++++++++ .../src/commands/migrate/lib/transform.ts | 28 ++++--- .../cli-core/src/commands/migrate/run.test.ts | 62 ++++++++++++-- packages/cli-core/src/commands/migrate/run.ts | 36 ++++++--- .../src/commands/migrate/runs.test.ts | 17 ++++ .../cli-core/src/commands/migrate/runs.ts | 17 +++- .../src/commands/migrate/sources/clerk.ts | 5 +- .../src/commands/migrate/sources/firebase.ts | 12 ++- .../commands/migrate/sources/load-custom.ts | 20 ++--- .../cli-core/src/commands/migrate/types.ts | 1 + .../cli-core/src/commands/migrate/undo.ts | 7 +- .../src/commands/update/index.test.ts | 20 +++++ .../cli-core/src/commands/update/index.ts | 27 +++++-- packages/cli-core/src/lib/config.ts | 2 +- packages/cli-core/src/lib/json-body.ts | 2 +- 31 files changed, 638 insertions(+), 152 deletions(-) create mode 100644 packages/cli-core/src/commands/update/index.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 92fcb6d12..89c734bf1 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -62,6 +62,11 @@ Resolution order: `--secret-key` → `--app` + Platform API lookup → `CLERK_SECRET_KEY` → the keyless project's own key → a linked project profile from `clerk link`. +With a key from `--secret-key` or `CLERK_SECRET_KEY`, the key alone picks the +instance, so `import` and `undo` refuse (exit 2) an `--instance` that names a +different one. `--instance dev` next to an exported `sk_live_…` key would +otherwise write to production. + The **instance type is read from the key**: `sk_live_…` is treated as production, anything else as development. That choice drives the throughput defaults and the development-instance user limit below. @@ -121,8 +126,13 @@ and phones not yet attached. A run is `partial` when any user failed, was skipped or is still `creating`, and `complete` otherwise. A run whose process died, or that never recorded a finish time, -lists as `interrupted`. A lock held by a live process refuses a second writer -with exit 2. +lists as `interrupted`. A lock held by another live process refuses a second +writer with exit 2, and names the lock file to delete if that process is not a +migrate run. A lock holding this process's own PID is stale: in a container the +CLI often gets the same PID every run. + +Run folders are created owner-only (`0700`), and export files `0600`: they hold +password hashes and user data. `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. A line that @@ -544,7 +554,9 @@ An export run ID stands for the file that run wrote, and the import records it as `fromExport`. A file `clerk migrate export` wrote carries its source, so it needs no `--source`, and a `--source` that contradicts it exits 2. Any other file — a bare JSON array, a CSV, Firebase's own `{ "users": [...] }` — needs -`--source`. +`--source`. NDJSON, one user per line (what Auth0's bulk export job writes), is +read too: always for `.ndjson` and `.jsonl`, and for a `.json` file that +doesn't parse whole. A leading BOM is ignored in JSON and CSV. **What a human is asked, and what an agent is told.** A human at a terminal who leaves out the file is asked for its path, and is asked for a source only when @@ -673,6 +685,11 @@ Or change the instance instead clerk config patch --app app_… --instance ins_… --json '{"auth_username":{"used_for_sign_up":true}}' ``` +Each fix names its instance with `--instance`, so it changes the instance the +import targets, whatever the key's source. When Clerk could not name the +instance (a `key_…` fallback ID), the fix points at the Dashboard instead: in +`--json` it carries `url` in place of `command`. + The fixes are offers, not corrections: an instance that requires an email is configured as its owner intended, and fixing the export may be the answer. When the instance settings cannot be read (BAPI `/v1/domains` → the instance's @@ -1025,7 +1042,7 @@ source actually used: `argon2i`, `argon2id`, `awscognito`, `bcrypt`, `bcrypt_peppered`, `bcrypt_sha256_django`, `hmac_sha256_utf16_b64`, `ldap_ssha`, `md5`, -`md5_phpass`, `md5_salted`, `pbkdf2_sha1`, `pbkdf2_sha256`, +`md5_phpass`, `md5_salted`, `phpass`, `pbkdf2_sha1`, `pbkdf2_sha256`, `pbkdf2_sha256_django`, `pbkdf2_sha512`, `pbkdf2_sha512_hex`, `scrypt_firebase`, `scrypt_werkzeug`, `sha256`, `sha256_salted`, `sha512_symfony` @@ -1058,19 +1075,19 @@ stamping every user with today's. ## API Endpoints -| Method | Path | Used by | -| -------- | -------------------------- | ------------------------------------------------------------------------------- | -| `POST` | `/v1/users` | `migrate import` — creates each user | -| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | -| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | -| `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | -| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | -| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | -| `GET` | `/v1/users?external_id=…` | `migrate undo` — finds users whose create was in flight when the import stopped | -| `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | -| `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | -| `GET` | `/v1/instance` | `migrate import`, `undo`, `export clerk` — names the instance behind the key | -| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | +| Method | Path | Used by | +| -------- | ----------------------------------------------- | ------------------------------------------------------------------------------- | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | +| `GET` | `/v1/users?limit=&offset=&order_by=+created_at` | `migrate export clerk` — pages the whole instance, oldest first, 500 at a time | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | +| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | +| `GET` | `/v1/users?external_id=…` | `migrate undo` — finds users whose create was in flight when the import stopped | +| `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | +| `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | +| `GET` | `/v1/instance` | `migrate import`, `undo`, `export clerk` — names the instance behind the key | +| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | The checks also read the instance's Frontend API `GET /v1/environment` (bootstrapping a dev browser first on development instances) for its diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 32e706165..8f773666e 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -14,7 +14,7 @@ * leaving it to be discovered when nobody can sign in. */ -import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { throwUsageError } from "../../../lib/errors.ts"; import { loggedFetch } from "../../../lib/fetch.ts"; import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; @@ -176,10 +176,10 @@ export async function fetchAuth0Token(credentials: Auth0Credentials): Promise { expect(mapped.unverified_email_addresses).toEqual(["a@x.dev"]); }); + // The import puts the old Clerk ID in external_id, so the app's own value + // would otherwise be lost. + test("moves external_id into private metadata, keeping what is there", () => { + const mapped = mapClerkUserToExport( + user({ external_id: "acct_9", private_metadata: { tier: "gold" } }), + ); + expect(mapped.private_metadata).toEqual({ tier: "gold", clerkExternalId: "acct_9" }); + }); + test("promotes the first verified address when none is flagged primary", () => { const mapped = mapClerkUserToExport( user({ @@ -197,6 +206,20 @@ describe("fetchAllClerkUsers", () => { expect(requests[1]).toContain("offset=500"); }); + // Oldest first, so a sign-up mid-export lands at the end; a row a deletion + // shifts onto the next page is dropped as a repeat. + test("pages oldest first and drops a user a shifted page repeats", async () => { + stubPages([ + Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), + [user({ id: "u499" }), user({ id: "v0" })], + ]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(501); + expect(requests[0]).toContain("order_by=%2Bcreated_at"); + }); + // A full final page must still trigger one more request, or an instance whose // size is an exact multiple of the page size would look short by one page. test("makes one more request when the last page is exactly full", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 84ab47025..4183bc926 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -141,6 +141,15 @@ export function mapClerkUserToExport(user: BapiUser): Record { if (value && Object.keys(value).length > 0) exported[target] = value; } + // The import sets external_id to the old Clerk ID, so an app's own + // external_id moves to private metadata rather than being lost. + if (user.external_id) { + exported.private_metadata = { + ...(exported.private_metadata as Record | undefined), + clerkExternalId: user.external_id, + }; + } + if (user.banned) exported.banned = true; if (user.create_organization_enabled !== undefined) { exported.create_organization_enabled = user.create_organization_enabled; @@ -166,27 +175,31 @@ export async function fetchAllClerkUsers(options: { secretKey: string; spinner?: SpinnerControls; }): Promise { - const all: BapiUser[] = []; + const all = new Map(); + // Oldest first, so a sign-up during the export lands at the end instead of + // shifting every later page by one (BAPI's default is newest first). A + // deletion can still shift a page; the Map drops the repeat that causes. + // ponytail: offset paging; /v1/users has no cursor to page by instead. for (let offset = 0; ; offset += PAGE_SIZE) { const response = await retryOn429(async () => bapiRequest({ method: "GET", - path: `/v1/users?limit=${PAGE_SIZE}&offset=${offset}`, + path: `/v1/users?limit=${PAGE_SIZE}&offset=${offset}&order_by=%2Bcreated_at`, secretKey: options.secretKey, }), ); const page = Array.isArray(response.body) ? (response.body as BapiUser[]) : []; - all.push(...page); - options.spinner?.update(`Fetching users from Clerk: ${all.length} so far...`); + for (const user of page) all.set(user.id, user); + options.spinner?.update(`Fetching users from Clerk: ${all.size} so far...`); // A short page means the end; anything else would loop forever on an // instance whose size happens to be a multiple of the page size. if (page.length < PAGE_SIZE) break; } - return all; + return [...all.values()]; } export type ClerkExportResult = { diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 0ec3469cd..cf86218f9 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -385,6 +385,17 @@ describe("buildFirebaseExport", () => { expect(unreadablePasswords).toBe(1); expect("passwordHash" in users[1]!).toBe(false); }); + + // Firebase sends base64 "REDACTED" when the caller may not read hashes: a + // digest that could never verify, which must not be exported as one. + test("treats a redacted hash as none, and counts it", () => { + const { users, redactedPasswords } = buildFirebaseExport([ + fbUser(0, { passwordHash: "UkVEQUNURUQ=", providerUserInfo: [{ providerId: "password" }] }), + ]); + expect(redactedPasswords).toBe(1); + expect("passwordHash" in users[0]!).toBe(false); + expect("salt" in users[0]!).toBe(false); + }); }); describe("fetchHashConfig", () => { diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 7c7d100c5..cad2161d0 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -76,10 +76,7 @@ export type ServiceAccount = { function validateServiceAccount(parsed: unknown, label: string): ServiceAccount { const account = parsed as Partial & { type?: string }; const invalid = (problem: string): never => { - throw new CliError(`${label} is not a usable service account key: ${problem}`, { - code: ERROR_CODE.USAGE_ERROR, - docsUrl: DOCS_URL, - }); + throwUsageError(`${label} is not a usable service account key: ${problem}`, DOCS_URL); }; if (account.type && account.type !== "service_account") { @@ -214,10 +211,7 @@ async function importPrivateKey(pem: string): Promise { try { der = Uint8Array.from(atob(body), (character) => character.charCodeAt(0)); } catch { - throw new CliError("The service account's private_key is not valid base64.", { - code: ERROR_CODE.USAGE_ERROR, - docsUrl: DOCS_URL, - }); + throwUsageError("The service account's private_key is not valid base64.", DOCS_URL); } try { @@ -229,9 +223,9 @@ async function importPrivateKey(pem: string): Promise { ["sign"], ); } catch (error) { - throw new CliError( + throwUsageError( `The service account's private_key could not be read: ${(error as Error).message}`, - { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + DOCS_URL, ); } } @@ -293,10 +287,10 @@ export async function fetchAccessToken(account: ServiceAccount): Promise }; if (!response.ok || !body.access_token) { - throw new CliError( + throwUsageError( `Google rejected the service account (${response.status}): ${body.error_description ?? body.error ?? "no access token returned"}\n` + "Check the key has not been revoked or deleted, in the Google Cloud console under IAM → Service accounts.", - { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + DOCS_URL, ); } @@ -341,9 +335,9 @@ export async function fetchAllFirebaseUsers(options: { }); if (!response.ok) { - throw new CliError( + throwUsageError( `Firebase returned ${response.status} listing users: ${await response.text()}`, - { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + DOCS_URL, ); } @@ -432,8 +426,9 @@ export function mapFirebaseUserToExport(user: FirebaseUser): Record 0) { + log.warn( + `${redactedPasswords} user${redactedPasswords === 1 ? "'s" : "s'"} password hash came back redacted: ` + + "the service account can't read hashes. Grant it `firebaseauth.configs.getHashConfig` and export again, " + + "or those users are imported without a password and need to reset it.", + ); + } + if (unreadablePasswords > 0) { log.warn( `${unreadablePasswords} user${unreadablePasswords === 1 ? " has" : "s have"} a password Firebase did not return. ` + diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts index f1e7f7912..843bdcb03 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.test.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -48,6 +48,16 @@ describe("finishExport", () => { expect(latestUserLines(runsDir, record.id).get("u1")?.status).toBe("exported"); }); + // The file holds password hashes and PII, so no other local account reads it. + test("writes the file owner-only, in an owner-only run folder", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + + const { outputPath } = finishExport({ run, options: {}, users, coverage }); + + expect(fs.statSync(outputPath).mode & 0o777).toBe(0o600); + expect(fs.statSync(path.dirname(outputPath)).mode & 0o777).toBe(0o700); + }); + test("--output writes somewhere else, and the run still records where", async () => { const run = await startExportRun({ runsDir }, { platform: "auth0" }); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index 09e163614..233e00b10 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -58,8 +58,9 @@ export function exportPath(run: Run, output: string | undefined): string { * @returns The absolute path written. */ export function writeExportFile(file: string, envelope: ExportEnvelope): string { - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(file, JSON.stringify(envelope, null, 2)); + // Password hashes, PII and a Firebase signer key: owner-only. + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + fs.writeFileSync(file, JSON.stringify(envelope, null, 2), { mode: 0o600 }); return file; } diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index 49e289226..79ff27f26 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -16,7 +16,7 @@ * discovered when nobody can sign in. */ -import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { throwUsageError } from "../../../lib/errors.ts"; import { loggedFetch } from "../../../lib/fetch.ts"; import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; @@ -166,10 +166,10 @@ export async function fetchWorkOsPage(apiKey: string, after?: string): Promise { }); }); + // A repeated record (an export that paged past a sign-up) must not take the + // kept copy down with it. + test("a repeated source ID rejects only the later copy", async () => { + const checks = await checkImport(input({ users: [user("a"), user("a")] })); + expect(checks.importable.map((u) => u.userId)).toEqual(["a"]); + expect(checks.rejects).toEqual([{ sourceId: "a", reason: "duplicate source ID in the file" }]); + }); + + test("a rejected user does not claim its email from a later one", async () => { + const checks = await checkImport( + input({ + users: [ + user("skipped", { email: "same@x.dev", skipReason: "anonymous Better Auth user" }), + user("kept", { email: "same@x.dev" }), + ], + }), + ); + expect(checks.importable.map((u) => u.userId)).toEqual(["kept"]); + }); + // A continued run found this user behind its own in-flight create. test("not a user the continued run adopted", async () => { existing = [{ id: "user_1", external_id: "mine" }]; @@ -290,6 +310,13 @@ describe("rejects", () => { expect(checks.quota).toEqual({ existing: 98, limit: 100, headroom: 2, over: 1 }); }); + test("warns that an unreadable user count was checked as empty", async () => { + const checks = await checkImport( + input({ instanceType: "dev", existingUsers: null, users: [user("a")] }), + ); + expect(checks.warnings.join("\n")).toContain("Could not read how many users"); + }); + test("CLERK_MIGRATE_DEV_USER_LIMIT raises the headroom", async () => { process.env.CLERK_MIGRATE_DEV_USER_LIMIT = "500"; try { @@ -513,6 +540,37 @@ describe("fixes", () => { ]); }); + // Without --instance, `clerk config patch` changes the linked profile's + // development instance, not the one a --secret-key import targets. + test("name the instance even when the key came from --secret-key", async () => { + const checks = await checkImport( + input({ + settings: EMAIL_REQUIRED, + users: [user("b", { email: undefined, username: "b" })], + }), + ); + expect(checks.fixes[0]?.command).toStartWith("clerk config patch --instance ins_1 --json"); + }); + + test("point at the Dashboard when the instance could not be named", async () => { + const checks = await checkImport( + input({ + settings: EMAIL_REQUIRED, + target: { + env: "production", + instanceId: "key_0123", + instanceType: "prod", + keySource: "--secret-key", + }, + users: [user("b", { email: undefined, username: "b" })], + }), + ); + expect(checks.fixes[0]).toEqual({ + label: "Make Email optional at sign-up", + url: "https://dashboard.clerk.com", + }); + }); + test("offer nothing when the settings could not be read", async () => { const checks = await checkImport(input({ users: [user("a")] })); expect(checks.fixes).toEqual([]); diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index eb36890fa..9fd7880ce 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -47,7 +47,13 @@ export type Reject = { export type ReasonCount = { reason: string; count: number }; /** A `clerk config patch` that would stop a setting costing users. */ -export type Fix = { label: string; command: string }; +/** + * A setting change that would stop users being flagged: a `clerk config patch` + * command, or a Dashboard link when the instance can't be named for one. + */ +export type Fix = { label: string; command?: string; url?: string }; + +const DASHBOARD_URL = "https://dashboard.clerk.com"; export type Quota = { /** Users already in the instance, or `null` when the count could not be read. */ @@ -372,21 +378,24 @@ const hasAnyIdentifier = (user: User) => /** * First user in the file to claim each email, phone and source ID. * + * Keyed by record, not source ID: two records with one source ID must not + * share a verdict, or the one kept would be rejected along with its copy. + * * @returns Each duplicate's reason, and the earlier user kept in its place. */ function findFileDuplicates(users: User[]): { - reasons: Map; - keptBy: Map; + reasons: Map; + keptBy: Map; } { - const reasons = new Map(); - const keptBy = new Map(); + const reasons = new Map(); + const keptBy = new Map(); const seenIds = new Set(); const emails = new Map(); const phones = new Map(); for (const user of users) { if (seenIds.has(user.userId)) { - reasons.set(user.userId, "duplicate source ID in the file"); + reasons.set(user, "duplicate source ID in the file"); continue; } seenIds.add(user.userId); @@ -404,16 +413,13 @@ function findFileDuplicates(users: User[]): { // The first record in the file wins, whatever either holds: the source's // order decides, so the kept ID is named alongside the reject. if (emailOwner) { - reasons.set(user.userId, "email is also used by an earlier user in the file, which is kept"); - keptBy.set(user.userId, emailOwner); + reasons.set(user, "email is also used by an earlier user in the file, which is kept"); + keptBy.set(user, emailOwner); continue; } if (phoneOwner) { - reasons.set( - user.userId, - "phone number is also used by an earlier user in the file, which is kept", - ); - keptBy.set(user.userId, phoneOwner); + reasons.set(user, "phone number is also used by an earlier user in the file, which is kept"); + keptBy.set(user, phoneOwner); continue; } for (const email of ownEmails) emails.set(email.toLowerCase(), user.userId); @@ -625,19 +631,27 @@ function buildFixes(input: CheckInput, users: User[]): Fix[] { flagged.unshift({ ...email, clerkRequired: true, blocking: true, consequence: "rejects" }); } - const flags = input.target.appId - ? ` --app ${input.target.appId} --instance ${input.target.instanceId}` - : ""; + // Always name the instance: without it, `clerk config patch` acts on the + // linked profile's development instance, whatever key this import used. A + // `key_` ID is a stand-in for an instance Clerk didn't name, so there is + // nothing to pass; point at the Dashboard instead. + const { appId, instanceId } = input.target; + const named = instanceId.startsWith("ins_"); + const flags = `${appId ? ` --app ${appId}` : ""} --instance ${instanceId}`; const settings = input.settings; const mfa = MFA_SETTINGS.filter( ({ field, attribute }) => !isEnabled(settings, attribute) && users.some((user) => hasValue(user[field])), ).map(({ label, path }) => ({ label, writes: [{ path: [...path], value: true }] })); - return [...buildSettingChanges(flagged), ...mfa].map((change) => ({ - label: change.label, - command: `clerk config patch${flags} --json ${quoteJson(buildChangePayload([change]))}`, - })); + return [...buildSettingChanges(flagged), ...mfa].map((change) => + named + ? { + label: change.label, + command: `clerk config patch${flags} --json ${quoteJson(buildChangePayload([change]))}`, + } + : { label: change.label, url: DASHBOARD_URL }, + ); } // --- The whole check ------------------------------------------------------- @@ -701,10 +715,11 @@ export async function checkImport(input: CheckInput): Promise { reason: `invalid: ${failure.error}`, })); - const { reasons: fileDuplicates, keptBy } = findFileDuplicates(input.users); const disabledProviders = findDisabledProviderRejects(input); - let candidates: User[] = []; + // Users that pass every per-user check. Only these claim identifiers in + // the file: a rejected record must not cost a later one its email. + const passed: User[] = []; const placeholderEmails = new Set(); const refusedNames = new Set(); for (const original of input.users) { @@ -716,7 +731,6 @@ export async function checkImport(input: CheckInput): Promise { (refused.length > 0 && !hasAnyIdentifier(user) ? `only has an email Clerk refuses (${refused[0]})` : undefined) ?? - fileDuplicates.get(user.userId) ?? missingRequiredIdentifier(user, input.settings) ?? // Stripping the identifiers the instance has off can leave nothing to // sign in with; Clerk would still create the user. @@ -737,14 +751,23 @@ export async function checkImport(input: CheckInput): Promise { : undefined) ?? disabledProviders.get(user.userId); if (reason) { - const kept = reason === fileDuplicates.get(user.userId) ? keptBy.get(user.userId) : undefined; - rejects.push({ sourceId: user.userId, reason, ...(kept ? { keptSourceId: kept } : {}) }); + rejects.push({ sourceId: user.userId, reason }); } else { - candidates.push(user); + passed.push(user); if (refused.length > 0) placeholderEmails.add(user.userId); } } + const { reasons: fileDuplicates, keptBy } = findFileDuplicates(passed); + let candidates: User[] = []; + for (const user of passed) { + const reason = fileDuplicates.get(user); + const kept = keptBy.get(user); + if (reason) + rejects.push({ sourceId: user.userId, reason, ...(kept ? { keptSourceId: kept } : {}) }); + else candidates.push(user); + } + const instanceDuplicates = candidates.length > 0 ? await findInstanceDuplicates(candidates, input) @@ -792,6 +815,12 @@ export async function checkImport(input: CheckInput): Promise { ...legalWarning( candidates.filter((user) => lacksLegalAcceptance(user, input.settings)).length, ), + // An unknown count is checked as zero; say so rather than imply it fit. + ...(quota && quota.existing === null + ? [ + `Could not read how many users this development instance holds, so the ${quota.limit}-user limit was checked as if it were empty`, + ] + : []), ], fixes: buildFixes(input, input.users), ...(quota ? { quota } : {}), diff --git a/packages/cli-core/src/commands/migrate/lib/export-file.ts b/packages/cli-core/src/commands/migrate/lib/export-file.ts index 5d153db5c..c9ebb499d 100644 --- a/packages/cli-core/src/commands/migrate/lib/export-file.ts +++ b/packages/cli-core/src/commands/migrate/lib/export-file.ts @@ -9,6 +9,7 @@ */ import fs from "node:fs"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; import type { FirebaseHashConfig } from "../types.ts"; export const ENVELOPE_VERSION = 1; @@ -43,9 +44,46 @@ export function isEnvelope(value: unknown): value is ExportEnvelope { export function readEnvelope(file: string): ExportEnvelope | undefined { if (!file.toLowerCase().endsWith(".json")) return undefined; try { - const parsed: unknown = JSON.parse(fs.readFileSync(file, "utf-8")); + const parsed = readJsonFile(file); return isEnvelope(parsed) ? parsed : undefined; } catch { return undefined; } } + +/** + * A JSON file's contents, as an import reads it. + * + * A UTF-8 BOM (Excel's "CSV UTF-8", some editors) is stripped. NDJSON, one + * object per line, reads as an array: it is what Auth0's bulk export job + * writes. `.ndjson` and `.jsonl` are always read that way; a `.json` file + * falls back to it only when it doesn't parse whole. + * + * @throws CliError naming the file when it is neither. + */ +export function readJsonFile(file: string): unknown { + const text = fs.readFileSync(file, "utf-8").replace(/^\uFEFF/, ""); + const lines = () => text.split("\n").filter((line) => line.trim()); + const invalid = (error: unknown) => + new CliError(`${file} is not valid JSON: ${(error as Error).message}`, { + code: ERROR_CODE.INVALID_JSON, + }); + + if (/\.(ndjson|jsonl)$/i.test(file)) { + try { + return lines().map((line) => JSON.parse(line) as unknown); + } catch (error) { + throw invalid(error); + } + } + try { + return JSON.parse(text); + } catch (error) { + try { + if (lines().length > 1) return lines().map((line) => JSON.parse(line) as unknown); + } catch { + // Not NDJSON either: report the whole-file error, which names the spot. + } + throw invalid(error); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index cabe2e4f2..aa1b10866 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -75,7 +75,7 @@ describe("a run's life", () => { test("starts running, holding a lock", () => { const run = startRun(runsDir, init); expect(readRun(runsDir, run.record.id)).toMatchObject({ status: "running", kind: "import" }); - expect(runState(runsDir, run.record)).toBe("running"); + expect(fs.readFileSync(path.join(run.dir, "lock"), "utf-8")).toBe(String(process.pid)); }); test("finishes complete when every user made it", () => { @@ -172,12 +172,31 @@ describe("locks and interruptions", () => { expect(again.finish().counts.total).toBe(2); }); + // In a container the CLI often gets the same PID every run, so a killed + // run's lock can hold this process's own PID. + test("a lock holding this process's own PID is stale", () => { + const run = startRun(runsDir, init); + expect(runState(runsDir, run.record)).toBe("interrupted"); + expect(() => continueRun(runsDir, run.record)).not.toThrow(); + }); + + test("reads as running while another live process holds the lock", () => { + const run = startRun(runsDir, init); + fs.writeFileSync(path.join(run.dir, "lock"), "1"); + expect(runState(runsDir, run.record)).toBe("running"); + }); + test("refuses a run another live process holds, with exit 2", () => { const run = startRun(runsDir, init); // PID 1 is always alive, and never this test. fs.writeFileSync(path.join(run.dir, "lock"), "1"); - expect(() => continueRun(runsDir, run.record)).toThrow(/in use by another process \(PID 1\)/); + expect(() => continueRun(runsDir, run.record)).toThrow( + new RegExp( + `in use by another process \\(PID 1\\).*delete ${path.join(run.dir, "lock")}`, + "s", + ), + ); }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index acb8abaf3..347775dcd 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -185,26 +185,39 @@ function isPidAlive(pid: number): boolean { } } -/** The PID holding the run's lock, when that process is still alive. */ +/** + * The PID holding the run's lock, when that process is still alive. + * + * This process's own PID counts as stale: in a container the CLI often gets + * the same PID every run, so a killed run's lock would otherwise read as live. + */ export function liveLockPid(runsDir: string, id: string): number | undefined { let raw: string; try { - raw = fs.readFileSync(path.join(runDir(runsDir, id), LOCK_FILE), "utf-8"); + raw = fs.readFileSync(lockFile(runsDir, id), "utf-8"); } catch { return undefined; } const pid = Number(raw.trim()); - return Number.isInteger(pid) && pid > 0 && isPidAlive(pid) ? pid : undefined; + return Number.isInteger(pid) && pid > 0 && pid !== process.pid && isPidAlive(pid) + ? pid + : undefined; +} + +/** The run's lock file. */ +export function lockFile(runsDir: string, id: string): string { + return path.join(runDir(runsDir, id), LOCK_FILE); } function acquireLock(runsDir: string, id: string): void { const holder = liveLockPid(runsDir, id); - if (holder !== undefined && holder !== process.pid) { + if (holder !== undefined) { throwUsageError( - `Run ${id} is in use by another process (PID ${holder}). Wait for it to finish, then try again.`, + `Run ${id} is in use by another process (PID ${holder}). Wait for it to finish, then try again. ` + + `If that process is not a migrate run, delete ${lockFile(runsDir, id)}.`, ); } - fs.writeFileSync(path.join(runDir(runsDir, id), LOCK_FILE), String(process.pid)); + fs.writeFileSync(lockFile(runsDir, id), String(process.pid)); } // --- Reading --------------------------------------------------------------- @@ -365,7 +378,8 @@ export type StartRunInit = Omit { expect((await fetchInstanceIdentity("sk_test_b")).instanceId).not.toBe(first.instanceId); }); }); + +describe("resolveClerkTarget --instance", () => { + let originalFetch: typeof globalThis.fetch; + let originalKey: string | undefined; + + beforeAll(() => { + originalFetch = globalThis.fetch; + originalKey = process.env.CLERK_SECRET_KEY; + globalThis.fetch = (async () => + Response.json({ id: "ins_prod", environment_type: "production" })) as unknown as typeof fetch; + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + if (originalKey === undefined) delete process.env.CLERK_SECRET_KEY; + else process.env.CLERK_SECRET_KEY = originalKey; + }); + + // An exported production key would otherwise win over `--instance dev` + // without a word, and the import would write to production. + test.each([ + ["an exported key", { instance: "dev" }, "sk_live_x"], + ["--secret-key", { instance: "dev", secretKey: "sk_live_x" }, undefined], + ["--secret-key, with a literal ID", { instance: "ins_dev", secretKey: "sk_live_x" }, undefined], + ])("refuses %s that addresses another instance", async (_label, options, envKey) => { + if (envKey) process.env.CLERK_SECRET_KEY = envKey; + else delete process.env.CLERK_SECRET_KEY; + + const error = (await resolveClerkTarget(options).catch((e: unknown) => e)) as CliError; + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("does not match the key"); + }); + + test.each([["prod"], ["production"], ["ins_prod"]])("accepts --instance %s", async (instance) => { + delete process.env.CLERK_SECRET_KEY; + const { target } = await resolveClerkTarget({ instance, secretKey: "sk_live_x" }); + expect(target.instanceId).toBe("ins_prod"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/target.ts b/packages/cli-core/src/commands/migrate/lib/target.ts index cdef790a4..ca4314791 100644 --- a/packages/cli-core/src/commands/migrate/lib/target.ts +++ b/packages/cli-core/src/commands/migrate/lib/target.ts @@ -12,7 +12,8 @@ import { createHash } from "node:crypto"; import { bapiRequest } from "../../../lib/bapi.ts"; import { dim } from "../../../lib/color.ts"; import { resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; -import { resolveAppContext } from "../../../lib/config.ts"; +import { INSTANCE_ALIASES, resolveAppContext } from "../../../lib/config.ts"; +import { throwUsageError } from "../../../lib/errors.ts"; import { resolveKeylessTarget } from "../../../lib/keyless-target.ts"; import { log } from "../../../lib/log.ts"; import { detectInstanceType } from "./instance.ts"; @@ -93,6 +94,31 @@ export async function fetchInstanceIdentity( return { instanceId: `key_${digest}`, env: fallbackEnv }; } +/** + * Refuses an `--instance` the key does not address. + * + * With a key from `--secret-key` or `CLERK_SECRET_KEY`, the key alone picks + * the instance and `--instance` would be ignored without a word. Migrate + * writes and deletes in bulk, so `--instance dev` next to an exported + * `sk_live_` key must not reach production. + */ +function assertInstanceFlagMatches( + options: TargetOptions, + keySource: string, + identity: { instanceId: string; env: string }, +): void { + const flag = options.instance; + if (!flag || (keySource !== "--secret-key" && !keySource.startsWith("CLERK_SECRET_KEY"))) return; + const wanted = INSTANCE_ALIASES[flag]; + const matches = wanted ? identity.env === wanted : identity.instanceId === flag; + if (matches) return; + throwUsageError( + `--instance ${flag} does not match the key from ${keySource}, which addresses the ` + + `${identity.env} instance ${identity.instanceId}. Nothing was changed.\n` + + "Pass the key for that instance with --secret-key, or drop --instance.", + ); +} + /** Resolves the key, then names the instance it addresses. */ export async function resolveClerkTarget( options: TargetOptions, @@ -100,6 +126,7 @@ export async function resolveClerkTarget( const secretKey = await resolveBapiSecretKey(options); const source = await describeKeySource(options); const { instanceId, env } = await fetchInstanceIdentity(secretKey); + assertInstanceFlagMatches(options, source.keySource, { instanceId, env }); return { secretKey, diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 179b31789..d3a8f265f 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -33,6 +33,8 @@ describe("getFileType", () => { test.each([ ["users.json", "application/json"], ["users.CSV", "text/csv"], + ["users.ndjson", "application/json"], + ["users.jsonl", "application/json"], ["users.txt", undefined], ["users", undefined], ])("%s -> %p", (file, expected) => { @@ -91,6 +93,7 @@ describe("normalizeUserData", () => { ["numeric string limit", { createOrganizationsLimit: "5" }, { createOrganizationsLimit: 5 }], ["JSON metadata", { publicMetadata: '{"plan":"pro"}' }, { publicMetadata: { plan: "pro" } }], ["date string", { createdAt: "2024-01-01" }, { createdAt: "2024-01-01T00:00:00.000Z" }], + ["numeric user ID", { userId: 42 }, { userId: "42" }], ])("normalizes %s", (_label, input, expected) => { expect(normalizeUserData(input)).toMatchObject(expected); }); @@ -216,6 +219,41 @@ describe("loadUsersFromFile", () => { expect(users[0]?.email).toEqual(["a@x.dev", "b@x.dev"]); }); + // Excel's "CSV UTF-8" starts with a BOM, which hid the first column. + test("reads a CSV that starts with a BOM", async () => { + fs.writeFileSync(path.join(workDir, "bom.csv"), "\uFEFFid,primary_email_address\nu3,a@x.dev\n"); + const { users } = await loadUsersFromFile("bom.csv", "clerk"); + expect(users[0]?.userId).toBe("u3"); + }); + + test("reads a JSON file that starts with a BOM", async () => { + fs.writeFileSync( + path.join(workDir, "bom.json"), + `\uFEFF${JSON.stringify([{ id: "u4", primary_email_address: "a@x.dev" }])}`, + ); + const { users } = await loadUsersFromFile("bom.json", "clerk"); + expect(users[0]?.userId).toBe("u4"); + }); + + // What Auth0's bulk export job writes, for tenants over 1,000 users. + test.each([["users-bulk.json"], ["users-bulk.ndjson"]])("reads NDJSON from %s", async (file) => { + fs.writeFileSync( + path.join(workDir, file), + '{"id":"u5","primary_email_address":"a@x.dev"}\n{"id":"u6","primary_email_address":"b@x.dev"}\n', + ); + const { users } = await loadUsersFromFile(file, "clerk"); + expect(users.map((user) => user.userId)).toEqual(["u5", "u6"]); + }); + + test("names the file when it is not valid JSON", async () => { + fs.writeFileSync(path.join(workDir, "broken.json"), "[{"); + const error = (await loadUsersFromFile("broken.json", "clerk").catch( + (e: unknown) => e, + )) as CliError; + expect(error).toBeInstanceOf(CliError); + expect(error.message).toContain("broken.json is not valid JSON"); + }); + test("rejects a JSON file that is not an array of users", async () => { fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 56e4040bc..3ddd2a0b5 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -10,12 +10,12 @@ import fs from "node:fs"; import path from "node:path"; import csvParser from "csv-parser"; -import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; import { getSource } from "../sources/registry.ts"; import { normalizeBooleanField } from "../sources/shared.ts"; import { PASSWORD_HASHERS, type TransformContext, type SourceEntry, type User } from "../types.ts"; import { userSchema } from "../validator.ts"; -import { isEnvelope } from "./export-file.ts"; +import { isEnvelope, readJsonFile } from "./export-file.ts"; export type FileType = "application/json" | "text/csv"; @@ -51,7 +51,7 @@ export function fileExists(file: string): boolean { */ export function getFileType(file: string): FileType | undefined { const ext = path.extname(resolveImportFilePath(file)).toLowerCase(); - if (ext === ".json") return "application/json"; + if (ext === ".json" || ext === ".ndjson" || ext === ".jsonl") return "application/json"; if (ext === ".csv") return "text/csv"; return undefined; } @@ -228,6 +228,8 @@ const DATE_FIELDS = ["createdAt", "legalAcceptedAt"] as const; */ export function normalizeUserData(user: Record): Record { const normalized = { ...user }; + // Integer keys (Better Auth, a custom JSON file) are still IDs. + if (typeof normalized.userId === "number") normalized.userId = String(normalized.userId); const setOrDelete = (field: string, value: unknown) => { if (value === undefined) delete normalized[field]; @@ -332,13 +334,10 @@ export function validatePreparedUsers(users: Record[]): { typeof user.passwordHasher === "string" ? user.passwordHasher : JSON.stringify(user.passwordHasher); - throw new CliError( + throwUsageError( `Invalid password hasher "${invalidHasher}" on user ${String(user.userId)} (row ${i + 1}).\n` + `Expected one of: ${PASSWORD_HASHERS.join(", ")}`, - { - code: ERROR_CODE.USAGE_ERROR, - docsUrl: "https://clerk.com/docs/guides/development/migrating/overview", - }, + "https://clerk.com/docs/guides/development/migrating/overview", ); } @@ -418,7 +417,14 @@ async function readCsv(filePath: string): Promise[]> { return new Promise((resolve, reject) => { const users: Record[] = []; fs.createReadStream(filePath) - .pipe(csvParser({ skipComments: true })) + // Excel's "CSV UTF-8" starts with a BOM, which would otherwise become + // part of the first header and hide that column from every row. + .pipe( + csvParser({ + skipComments: true, + mapHeaders: ({ header }) => header.replace(/^\uFEFF/, ""), + }), + ) .on("data", (row: Record) => users.push(row)) .on("error", reject) .on("end", () => resolve(users)); @@ -436,7 +442,7 @@ async function readUsersFromFile( // An export's envelope already holds the users in the source's own shape, // so there is nothing left for a pre-transform to unwrap. if (type === "application/json") { - const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + const parsed = readJsonFile(filePath); if (isEnvelope(parsed)) return parsed.users; if (!transformer.preTransform) { if (!Array.isArray(parsed)) { @@ -458,7 +464,7 @@ async function readUsersFromFile( if (type === "text/csv") return readCsv(filePath); if (preExtracted) return preExtracted; - const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + const parsed = readJsonFile(filePath); if (!Array.isArray(parsed)) { throw new CliError(`Expected ${file} to contain a JSON array of users, got ${typeof parsed}.`, { code: ERROR_CODE.INVALID_JSON, diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 67af1558c..e7f52daf2 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -235,6 +235,15 @@ describe("run", () => { expect(captured.err).toContain(`rm -rf ${path.join(runsDir(), record!.id)}`); }); + // Pasted unquoted, `rm -rf …/app copy/…` deletes `…/app`. + test("quotes the cleanup paths", async () => { + const spaced = path.join(workDir, "app copy"); + await run({ ...baseOptions, runsDir: spaced }); + + const [record] = listRuns(spaced); + expect(captured.err).toContain(`rm -rf '${path.join(spaced, record!.id)}'`); + }); + describe("export envelopes", () => { /** An export run whose envelope holds `users`, as `clerk migrate export` writes it. */ function exportRun(source: string, rows: unknown[], extra: Record = {}) { @@ -381,7 +390,10 @@ describe("run", () => { ]), ); - await expect(run(baseOptions)).rejects.toThrow(/Invalid password hasher/); + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + expect(error.message).toContain("Invalid password hasher"); + // A usage error, not "some users failed" (exit 1). + expect(error.exitCode).toBe(EXIT_CODE.USAGE); expect(created()).toHaveLength(0); }); @@ -686,7 +698,7 @@ describe("run", () => { "only has an unverified email, and this instance requires an email", ); expect(captured.err).toContain( - `clerk config patch --json '{"auth_email":{"required_for_sign_up":false}}'`, + `clerk config patch --instance ins_1 --json '{"auth_email":{"required_for_sign_up":false}}'`, ); }); @@ -857,6 +869,20 @@ describe("run", () => { const created = () => requests.filter((r) => r.url.endsWith("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/v1/users")); + // The registered key isn't something --source accepts; the path is. + test("the printed command names the source's path, not its key", async () => { + const error = (await run({ + input: "export.json", + source: customFile, + secretKey: "sk_test_x", + json: true, + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.examples?.[0]?.command).toBe( + `clerk migrate import export.json --source ${customFile} --secret-key --json --yes`, + ); + }); + test("imports through a user-authored source", async () => { await run({ input: "export.json", @@ -922,9 +948,14 @@ describe("run", () => { `export default { key: "x", label: "X", transformer: {} };`, ); - await expect( - run({ input: "export.json", source: bad, yes: true, secretKey: "sk_test_x" }), - ).rejects.toThrow(/no source field maps to `userId`/); + const error = (await run({ + input: "export.json", + source: bad, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + expect(error.message).toMatch(/no source field maps to `userId`/); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); expect(requests).toHaveLength(0); }); @@ -1022,6 +1053,27 @@ describe("run", () => { }); }); + test("the printed command keeps the --firebase-* flags, as placeholders", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify({ users: [{ localId: "fb1", email: "a@x.dev", emailVerified: true }] }), + ); + + const error = (await run({ + ...baseOptions, + yes: false, + source: "firebase", + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.examples?.[0]?.command).toContain( + "--firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost ", + ); + }); + test("a partial firebase flag set fails before anything is read", async () => { await expect( run({ ...baseOptions, source: "firebase", firebaseSignerKey: "SIGNER" }), diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 58d58f0ad..f3d838400 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -29,6 +29,7 @@ import { throwUserAbort, } from "../../lib/errors.ts"; import { resolveKeylessTarget } from "../../lib/keyless-target.ts"; +import { quoteArg } from "../../lib/json-body.ts"; import { log } from "../../lib/log.ts"; import { NEXT_STEPS, printAgentNextSteps } from "../../lib/next-steps.ts"; import { confirm } from "../../lib/prompts.ts"; @@ -44,6 +45,8 @@ import { continueRun, latestUserLines, listRuns, + liveLockPid, + lockFile, readRun, resolveRunsDir, RUN_ID_PATTERN, @@ -78,6 +81,8 @@ export type MigrateRunOptions = { source?: string; /** Content hash of a custom `--source`, set once it is loaded. */ sourceHash?: string; + /** `--source` as typed, for printed commands: a custom source's path. Not a flag. */ + sourceArg?: string; /** The file to read, once `input` is resolved. Not a flag. */ file?: string; requirePassword?: boolean; @@ -313,6 +318,7 @@ async function applySource(options: MigrateRunOptions): Promise"; - const parts = ["clerk migrate import", input]; - if (!fromExport && options.source) parts.push("--source", options.source); + const parts = ["clerk migrate import", quoteArg(input)]; + const source = options.sourceArg ?? options.source; + if (!fromExport && source) parts.push("--source", quoteArg(source)); if (options.allowPartial) parts.push("--allow-partial"); if (options.newRun) parts.push("--new-run"); if (options.requirePassword) parts.push("--require-password"); if (options.skipLegalChecks) parts.push("--skip-legal-checks"); + if (options.firebaseSignerKey) parts.push("--firebase-signer-key", ""); + if (options.firebaseSaltSeparator) parts.push("--firebase-salt-separator", ""); + if (options.firebaseRounds) parts.push("--firebase-rounds", ""); + if (options.firebaseMemCost) parts.push("--firebase-mem-cost", ""); if (options.secretKey) parts.push("--secret-key", ""); - if (options.app) parts.push("--app", options.app); - if (options.instance) parts.push("--instance", options.instance); - if (options.runsDir) parts.push("--runs-dir", options.runsDir); + if (options.app) parts.push("--app", quoteArg(options.app)); + if (options.instance) parts.push("--instance", quoteArg(options.instance)); + if (options.runsDir) parts.push("--runs-dir", quoteArg(options.runsDir)); + if (options.json) parts.push("--json"); return [...parts, ...extra].join(" "); } diff --git a/packages/cli-core/src/commands/migrate/runs.test.ts b/packages/cli-core/src/commands/migrate/runs.test.ts index f5e5b0de6..5a751b970 100644 --- a/packages/cli-core/src/commands/migrate/runs.test.ts +++ b/packages/cli-core/src/commands/migrate/runs.test.ts @@ -44,6 +44,23 @@ describe("runs", () => { expect(captured.err).toContain("1 created, 2 failed, 1 skipped"); }); + // Counts are written at the finish, which an interrupted run never reaches. + test("counts an interrupted run from its user lines", async () => { + const run = startRun(runsDir, { + kind: "import", + target: { env: "development", instanceId: "ins_1" }, + source: "clerk", + }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + run.append({ sourceId: "b", status: "created", clerkId: "user_b" }); + fs.rmSync(path.join(run.dir, "lock")); + + await runs(undefined, { runsDir }); + + expect(captured.err).toContain("interrupted"); + expect(captured.err).toContain("2 created"); + }); + test("says so when there are no runs", async () => { await runs(undefined, { runsDir }); expect(captured.err).toContain("No migration runs yet."); diff --git a/packages/cli-core/src/commands/migrate/runs.ts b/packages/cli-core/src/commands/migrate/runs.ts index 5dd2fe6e4..5c9a70d9f 100644 --- a/packages/cli-core/src/commands/migrate/runs.ts +++ b/packages/cli-core/src/commands/migrate/runs.ts @@ -12,6 +12,7 @@ import { throwUsageError } from "../../lib/errors.ts"; import { log } from "../../lib/log.ts"; import { normalizeErrorMessage } from "./import-users.ts"; import { + countLines, latestUserLines, listRuns, readRun, @@ -215,20 +216,30 @@ function printRun(runsDir: string, record: RunRecord): void { for (const step of nextCommands(record, state)) log.info(` → ${step}`); } +/** + * Counts are written when a run finishes, so an interrupted one still holds + * its start's zeros. Those are counted from its user lines instead. + */ +function withLiveCounts(runsDir: string, record: RunRecord): RunRecord { + if (record.finishedAt) return record; + return { ...record, counts: countLines(latestUserLines(runsDir, record.id).values()) }; +} + export async function runs(id: string | undefined, options: RunsOptions = {}): Promise { const runsDir = await resolveRunsDir(options.runsDir); if (id === undefined) { - const records = listRuns(runsDir); + const records = listRuns(runsDir).map((record) => withLiveCounts(runsDir, record)); if (options.json) log.data(JSON.stringify(listJson(runsDir, records), null, 2)); else printList(runsDir, records); return; } - const record = readRun(runsDir, id); - if (!record) { + const stored = readRun(runsDir, id); + if (!stored) { throwUsageError(`No run \`${id}\` in ${runsDir}. Run \`clerk migrate runs\` to list them.`); } + const record = withLiveCounts(runsDir, stored); if (options.json) log.data(JSON.stringify(showJson(runsDir, record), null, 2)); else printRun(runsDir, record); diff --git a/packages/cli-core/src/commands/migrate/sources/clerk.ts b/packages/cli-core/src/commands/migrate/sources/clerk.ts index 20e6fe4c2..e97cba21f 100644 --- a/packages/cli-core/src/commands/migrate/sources/clerk.ts +++ b/packages/cli-core/src/commands/migrate/sources/clerk.ts @@ -20,7 +20,10 @@ const clerkSource = { level: "partial", note: "TOTP secrets and backup codes come across from a Dashboard export only.", }, - metadata: { level: "yes", note: "Public, private and unsafe metadata keep their places." }, + metadata: { + level: "yes", + note: "Public, private and unsafe metadata keep their places. `export clerk` moves each user's external_id to private metadata as `clerkExternalId`, because the import uses external_id for the old Clerk ID.", + }, }, transformer: { id: "userId", diff --git a/packages/cli-core/src/commands/migrate/sources/firebase.ts b/packages/cli-core/src/commands/migrate/sources/firebase.ts index bd5152ad2..d70e755b6 100644 --- a/packages/cli-core/src/commands/migrate/sources/firebase.ts +++ b/packages/cli-core/src/commands/migrate/sources/firebase.ts @@ -1,7 +1,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { readJsonFile } from "../lib/export-file.ts"; import type { PreTransformResult, SourceEntry } from "../types.ts"; import { isVerified, routeByVerification, splitName, toIsoDate } from "./shared.ts"; @@ -50,7 +51,7 @@ const firebaseSource = { } if (fileType === "application/json") { - const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + const parsed = readJsonFile(filePath); if (Array.isArray(parsed)) return { filePath, data: parsed as Record[] }; const users = (parsed as { users?: unknown })?.users; @@ -91,14 +92,11 @@ const firebaseSource = { if (passwordHash && salt) { const config = context.firebaseHashConfig; if (!config) { - throw new CliError( + throwUsageError( "This export contains Firebase password hashes, which need the project's hash parameters to import.\n" + "Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters, then pass:\n" + " --firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost", - { - code: ERROR_CODE.USAGE_ERROR, - docsUrl: "https://clerk.com/docs/guides/development/migrating/firebase", - }, + "https://clerk.com/docs/guides/development/migrating/firebase", ); } diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts index b7ac3e085..03aacebbb 100644 --- a/packages/cli-core/src/commands/migrate/sources/load-custom.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -20,7 +20,7 @@ import fs from "node:fs"; import path from "node:path"; -import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; import type { CarryLevel, SourceEntry } from "../types.ts"; const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; @@ -28,10 +28,7 @@ const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; const LEVELS: readonly CarryLevel[] = ["yes", "no", "partial"]; function invalid(problem: string, file: string): never { - throw new CliError(`${file} is not a valid source: ${problem}`, { - code: ERROR_CODE.USAGE_ERROR, - docsUrl: DOCS_URL, - }); + throwUsageError(`${file} is not a valid source: ${problem}`, DOCS_URL); } /** @@ -157,9 +154,7 @@ export async function loadCustomSource( }); } if (fs.statSync(resolved).isDirectory()) { - throw new CliError(`${resolved} is a directory, not a source file.`, { - code: ERROR_CODE.USAGE_ERROR, - }); + throwUsageError(`${resolved} is a directory, not a source file.`); } let module: Record; @@ -168,10 +163,10 @@ export async function loadCustomSource( // work, but a Windows path (`C:\...`) is not a valid import specifier. module = (await import(Bun.pathToFileURL(resolved).href)) as Record; } catch (error) { - throw new CliError( + throwUsageError( `Could not load ${file}: ${(error as Error).message}\n` + "The file must be valid JavaScript or TypeScript that this CLI can import.", - { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + DOCS_URL, ); } @@ -182,10 +177,7 @@ export async function loadCustomSource( named.length > 0 ? ` Found named export${named.length === 1 ? "" : "s"} ${named.map((n) => `\`${n}\``).join(", ")} — did you mean \`export default\`?` : ""; - throw new CliError(`${file} has no default export.${hint}`, { - code: ERROR_CODE.USAGE_ERROR, - docsUrl: DOCS_URL, - }); + throwUsageError(`${file} has no default export.${hint}`, DOCS_URL); } return validateSource(module.default, file, reservedKeys); diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts index 10487df5a..7a3b354ed 100644 --- a/packages/cli-core/src/commands/migrate/types.ts +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -33,6 +33,7 @@ export const PASSWORD_HASHERS = [ "sha256", "sha256_salted", "md5_phpass", + "phpass", "ldap_ssha", "sha512_symfony", ] as const; diff --git a/packages/cli-core/src/commands/migrate/undo.ts b/packages/cli-core/src/commands/migrate/undo.ts index bf09b388c..ee9424e1b 100644 --- a/packages/cli-core/src/commands/migrate/undo.ts +++ b/packages/cli-core/src/commands/migrate/undo.ts @@ -30,6 +30,8 @@ import { continueRun, latestUserLines, listRuns, + liveLockPid, + lockFile, patchRun, readRun, resolveRunsDir, @@ -86,7 +88,10 @@ function readImportRun(runsDir: string, runId: string): RunRecord { throwUsageError(`Run ${runId} was already undone by run ${record.undoneBy ?? "(unknown)"}.`); } if (runState(runsDir, record) === "running") { - throwUsageError(`Run ${runId} is still running. Wait for it to finish, then undo it.`); + throwUsageError( + `Run ${runId} is still running (PID ${liveLockPid(runsDir, runId)}). Wait for it to finish, then undo it. ` + + `If that process is not a migrate run, delete ${lockFile(runsDir, runId)}.`, + ); } return record; } diff --git a/packages/cli-core/src/commands/update/index.test.ts b/packages/cli-core/src/commands/update/index.test.ts new file mode 100644 index 000000000..243cedf49 --- /dev/null +++ b/packages/cli-core/src/commands/update/index.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from "bun:test"; +import { CliError, ERROR_CODE } from "../../lib/errors.ts"; +import { asRegistryError } from "./index.ts"; + +// Offline, `loggedFetch` throws NETWORK_UNREACHABLE; `clerk update` keeps +// reporting the registry, as it did before that existed. +test.each([ + [ + "a refused connection", + new CliError("Could not reach x", { code: ERROR_CODE.NETWORK_UNREACHABLE }), + ], + ["a timeout", new Error("The operation was aborted.")], +])("%s is registry_unreachable", (_label, error) => { + expect((asRegistryError(error) as CliError).code).toBe(ERROR_CODE.REGISTRY_UNREACHABLE); +}); + +test("a registry that answered keeps its own code", () => { + const error = new CliError("Registry returned HTTP 500.", { code: ERROR_CODE.UPDATE_FAILED }); + expect(asRegistryError(error)).toBe(error); +}); diff --git a/packages/cli-core/src/commands/update/index.ts b/packages/cli-core/src/commands/update/index.ts index b83fb2dfe..72ffbb8ae 100644 --- a/packages/cli-core/src/commands/update/index.ts +++ b/packages/cli-core/src/commands/update/index.ts @@ -259,6 +259,24 @@ async function confirmUpdate(currentVersion: string, latestVersion: string): Pro // ── Main ───────────────────────────────────────────────────────────────────── +/** + * What a failed registry lookup throws. + * + * A registry that answered — badly, or without the requested channel — + * already carries its own code, and a Ctrl-C is the user's decision, not the + * network's. Only transport and timeout failures are genuinely "unreachable", + * and retrying is only right for those. `loggedFetch` reports a refused + * connection as NETWORK_UNREACHABLE, naming the host; this command has always + * called it the registry. + */ +export function asRegistryError(error: unknown): unknown { + const unreachable = error instanceof CliError && error.code === ERROR_CODE.NETWORK_UNREACHABLE; + if ((error instanceof CliError && !unreachable) || isCancelled(error)) return error; + return new CliError("Could not reach npm registry. Check your network connection.", { + code: ERROR_CODE.REGISTRY_UNREACHABLE, + }); +} + export async function update(options: UpdateOptions): Promise { if (IS_DEV_BUILD) { log.info(`Running development build (${CURRENT_VERSION}); update not applicable.`); @@ -272,14 +290,7 @@ export async function update(options: UpdateOptions): Promise { const [latest, installDirs] = await Promise.all([ withSpinner("Checking for updates...", async () => fetchLatestVersion(channel)).catch( (error: unknown) => { - // A registry that answered — badly, or without the requested channel — - // already carries its own code, and a Ctrl-C is the user's decision, - // not the network's. Only transport and timeout failures are genuinely - // "unreachable", and retrying is only right for those. - if (error instanceof CliError || isCancelled(error)) throw error; - throw new CliError("Could not reach npm registry. Check your network connection.", { - code: ERROR_CODE.REGISTRY_UNREACHABLE, - }); + throw asRegistryError(error); }, ), getInstallerPackageDirs(), diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index 41943b990..a7e1f085b 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -308,7 +308,7 @@ export async function resolveProfile(cwd: string): Promise< return undefined; } -const INSTANCE_ALIASES: Record = { +export const INSTANCE_ALIASES: Record = { dev: "development", development: "development", prod: "production", diff --git a/packages/cli-core/src/lib/json-body.ts b/packages/cli-core/src/lib/json-body.ts index f5ff5a7d2..31b18cc5f 100644 --- a/packages/cli-core/src/lib/json-body.ts +++ b/packages/cli-core/src/lib/json-body.ts @@ -85,7 +85,7 @@ function preview(raw: string): string { * literal `"` inside doubled; everywhere else single quotes pass the value * through untouched. */ -function quoteArg(value: string): string { +export function quoteArg(value: string): string { if (/^[A-Za-z0-9_\-./:@]+$/.test(value)) return value; if (process.platform === "win32") return `"${value.replace(/"/g, '""')}"`; return `'${value.replace(/'/g, `'\\''`)}'`; From 0d0fb59f7d80c5414ad9dbcd5a0ef02950357fb2 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 16:10:05 -0400 Subject: [PATCH 089/141] fix(migrate): fix the Low review findings and the Firebase temp-file leak Run store - undo and resume match a run recorded under the key's stand-in ID. - .gitignore is edited only once there is consent to write a run. - Run folders and locks are created exclusively, so a run-ID collision or a lock race fails instead of sharing a folder. Checks - Catch in-file username duplicates (case-insensitive), and compare phones with punctuation stripped. - Keep an email-shaped name, as Clerk does. - Readiness counts users, not identifiers. - A Supabase reject names only that user's own providers. - One fixed reason for refused emails, so they group and the address stays out of the run record. - Cap bcrypt cost at 15. - A malformed secondary email is dropped with a warning instead of rejecting the user, using a looser shape check than Zod's, so non-ASCII addresses Clerk accepts are kept. - Refuse more private TLDs. Transforms and sources - The Supabase metadata name fallback runs on CSV input too. - No "Clerk won't store" warning for fields the CLI's own exports add. - The Clerk source says a Dashboard CSV carries no metadata, and strips the TAB the Dashboard puts before formula-like values. - Keep CSV rows that start with #. - Read epoch seconds as seconds. - A custom preTransform's rows win over a CSV file. - Read the headerless Firebase CSV with named columns instead of writing a copy, with its hashes, to the temp folder. Exports - --json on an export subcommand runs it in agent mode. - Auth0: a tenant of exactly 1000 users is complete; a real truncation carries `truncated` in --json. - The credential prompt repeats only on a 400, 401 or 403; an outage, a 429 or a refused connection exits 1. - Name the Firebase emulator in the target line. - Redact ?password= in a DB URL, and encode a bare % in a DB password. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 61 ++++++----- .../src/commands/migrate/export/auth0.test.ts | 37 ++++++- .../src/commands/migrate/export/auth0.ts | 48 ++++++--- .../migrate/export/db-exports.test.ts | 7 ++ .../src/commands/migrate/export/db-options.ts | 7 +- .../commands/migrate/export/firebase.test.ts | 13 +++ .../src/commands/migrate/export/firebase.ts | 24 ++++- .../src/commands/migrate/export/shared.ts | 3 + .../src/commands/migrate/export/workos.ts | 8 +- .../src/commands/migrate/index.test.ts | 13 +++ .../cli-core/src/commands/migrate/index.ts | 4 +- .../src/commands/migrate/lib/analysis.ts | 8 ++ .../src/commands/migrate/lib/checks.test.ts | 82 +++++++++++--- .../src/commands/migrate/lib/checks.ts | 102 ++++++++++++++---- .../src/commands/migrate/lib/db.test.ts | 5 + .../cli-core/src/commands/migrate/lib/db.ts | 5 +- .../commands/migrate/lib/input-retry.test.ts | 29 ++++- .../src/commands/migrate/lib/input-retry.ts | 31 +++++- .../migrate/lib/modify-settings.test.ts | 20 ++-- .../commands/migrate/lib/readiness.test.ts | 36 +++++-- .../src/commands/migrate/lib/readiness.ts | 5 +- .../commands/migrate/lib/run-store.test.ts | 31 +++++- .../src/commands/migrate/lib/run-store.ts | 47 ++++++-- .../src/commands/migrate/lib/target.ts | 12 ++- .../commands/migrate/lib/transform.test.ts | 33 ++++++ .../src/commands/migrate/lib/transform.ts | 19 +++- .../cli-core/src/commands/migrate/run.test.ts | 8 ++ packages/cli-core/src/commands/migrate/run.ts | 19 +++- .../commands/migrate/sources/betterauth.ts | 4 +- .../src/commands/migrate/sources/clerk.ts | 26 ++++- .../src/commands/migrate/sources/firebase.ts | 17 +-- .../commands/migrate/sources/sources.test.ts | 81 +++++++++++--- .../src/commands/migrate/sources/supabase.ts | 23 +++- .../cli-core/src/commands/migrate/types.ts | 2 + .../src/commands/migrate/undo.test.ts | 13 +++ .../cli-core/src/commands/migrate/undo.ts | 19 +++- .../src/commands/migrate/validator.test.ts | 6 +- .../src/commands/migrate/validator.ts | 8 +- 38 files changed, 742 insertions(+), 174 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 89c734bf1..964b374c6 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -441,7 +441,8 @@ A user whose hash is present but whose salt is not (or the reverse) has both dropped: half a credential produces a user nobody can sign in as. `FIREBASE_AUTH_EMULATOR_HOST` is honoured, so this works against the local -Firebase emulator as well as production. +Firebase emulator as well as production. The target line names the emulator +when it is set. **No `firebase-admin`.** The spike the plan called for was run and _passed_ — a compiled binary can import the SDK and complete `listUsers`, so the known @@ -461,7 +462,13 @@ it exits naming **every** missing credential at once rather than one per run. Auth0 pages this endpoint only through the first **1000** users. Past that the export stops and says so, pointing at Auth0's bulk export job — silently -returning the first thousand would read as "that is everyone". +returning the first thousand would read as "that is everyone". It still exits +0, and `--json` carries `truncated: true`. A tenant of exactly 1000 is +complete, and gets no warning. The bulk job's NDJSON file imports as it is. + +A credential the platform rejects (400, 401 or 403) is asked for again at a +terminal. An outage, a `429` or a refused connection is not: another +credential would not fix it, and it exits 1. #### WorkOS credentials @@ -623,12 +630,14 @@ after them. They sort the users three ways: - it failed schema validation - its source requested a skip (Better Auth: an anonymous guest; Supabase: a soft-deleted user) - - its only email is one Clerk refuses (`.local`, `.invalid`, `.test`, - `.example`, `.arpa`). Such an email is dropped from any other user, with a - warning - - its source ID, email or phone repeats an earlier user in the file. The - first record in the file is kept, whatever either holds, and the reject - names it (`kept: …`) + - its only emails are ones Clerk refuses: malformed, or a domain that can't + receive mail (`.local`, `.invalid`, `.test`, `.example`, `.arpa`, + `.internal`, `.lan`, `.corp` and the like). Such an email is dropped from + any other user, with a warning + - its source ID, email, phone or username repeats an earlier user in the file + that passes the other checks. Usernames compare case-insensitively and + phones ignore punctuation. The first record in the file is kept, whatever + either holds, and the reject names it (`kept: …`) - it lacks an identifier the instance requires. An email or phone counts only when it is verified, because an unverified one is attached after the user exists @@ -644,9 +653,9 @@ after them. They sort the users three ways: without it (`skip_legal_checks`), with a warning - its username breaks the instance's username rules (length, letters, the allowed special characters) - - its password is not the shape its hasher says (`bcrypt`, `scrypt_firebase`, - `argon2i`/`argon2id` and `scrypt_werkzeug` are checked; other hashers are - not) + - its password is not the shape its hasher says (`bcrypt`, with a cost up to + 15, `scrypt_firebase`, `argon2i`/`argon2id` and `scrypt_werkzeug` are + checked; other hashers are not, and Clerk refuses a bad one at create) - Supabase: its only provider is not enabled in Clerk - the instance already has a user with its source ID, email, phone or username (a batched `GET /v1/users` lookup, 100 values a request, through @@ -657,9 +666,9 @@ after them. They sort the users three ways: order - **Imported, but not everything comes across** — fields the instance is not set up to store, fields Clerk has no place for (`Clerk won't store: …`), - passwords a source had to drop, emails Clerk refuses, and names Clerk refuses - (a phone number, email, URL or HTML: Better Auth's phone sign-up stores the - number as the name). + passwords a source had to drop, emails Clerk refuses (malformed, or a domain + that can't receive mail), and names Clerk refuses (a phone number, URL or + HTML: Better Auth's phone sign-up stores the number as the name). - **Imported** — everyone else. Any reject stops the import, and it exits 2 with the command that adds @@ -843,7 +852,7 @@ brings across. Adding a platform is one file in `sources/` plus one line in | Key | Reads | Passwords | MFA | Metadata | | ------------ | ----------------------------- | --------- | ------- | -------- | -| `clerk` | Clerk Dashboard export | partial | partial | yes | +| `clerk` | Clerk Dashboard export | partial | partial | partial | | `auth0` | Auth0 Export Users API | partial | no | yes | | `authjs` | Auth.js / NextAuth user table | no | no | no | | `betterauth` | Better Auth export | yes | no | no | @@ -1061,17 +1070,17 @@ can sign in with. are how a Clerk-to-Clerk migration keeps original signup dates instead of stamping every user with today's. -| Field | Type | Description | -| --------------------------- | --------- | ----------------------------------------- | -| `createdAt` | `string` | Original creation timestamp | -| `legalAcceptedAt` | `string` | When legal terms were accepted | -| `banned` | `boolean` | Whether the user is banned | -| `bypassClientTrust` | `boolean` | Skip client trust verification | -| `createOrganizationEnabled` | `boolean` | Whether the user can create orgs | -| `createOrganizationsLimit` | `number` | Maximum orgs the user can create | -| `deleteSelfEnabled` | `boolean` | Whether the user can delete their account | -| `skipLegalChecks` | `boolean` | Skip legal acceptance checks | -| `skipPasswordChecks` | `boolean` | Skip password requirements on import | +| Field | Type | Description | +| --------------------------- | --------- | ------------------------------------------------------------------------------------ | +| `createdAt` | `string` | Original creation timestamp. A number is epoch milliseconds, or seconds below `1e11` | +| `legalAcceptedAt` | `string` | When legal terms were accepted | +| `banned` | `boolean` | Whether the user is banned | +| `bypassClientTrust` | `boolean` | Skip client trust verification | +| `createOrganizationEnabled` | `boolean` | Whether the user can create orgs | +| `createOrganizationsLimit` | `number` | Maximum orgs the user can create | +| `deleteSelfEnabled` | `boolean` | Whether the user can delete their account | +| `skipLegalChecks` | `boolean` | Skip legal acceptance checks | +| `skipPasswordChecks` | `boolean` | Skip password requirements on import | ## API Endpoints diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index 27186ff37..6cdd7fe1b 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -3,7 +3,7 @@ import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { CliError } from "../../../lib/errors.ts"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; import type { UserLine } from "../lib/run-store.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { @@ -156,6 +156,14 @@ describe("fetchAuth0Token", () => { ); }); + // An outage is not a bad credential: no re-prompt, and exit 1, not 2. + test("a 5xx is an outage, not a rejected credential", async () => { + stubAuth0([[]], new Response("{}", { status: 503 })); + const error = (await fetchAuth0Token(CREDENTIALS).catch((e: unknown) => e)) as CliError; + expect(error.message).toContain("Auth0 did not issue a token (503)"); + expect(error.exitCode).not.toBe(EXIT_CODE.USAGE); + }); + test("mentions the read:users scope, the usual cause", async () => { stubAuth0([[]], new Response("{}", { status: 403 })); await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(/read:users/); @@ -174,9 +182,13 @@ describe("fetchAllAuth0Users", () => { Array.from({ length: 4 }, (_, i) => auth0User(100 + i)), ]); - const all = await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + const { users: all, truncated } = await fetchAllAuth0Users({ + credentials: CREDENTIALS, + token: "tok", + }); expect(all).toHaveLength(104); + expect(truncated).toBe(false); expect(requests[0]?.url).toContain("page=0"); expect(requests[1]?.url).toContain("page=1"); expect(requests).toHaveLength(2); @@ -196,13 +208,32 @@ describe("fetchAllAuth0Users", () => { Array.from({ length: 12 }, () => Array.from({ length: 100 }, (_, i) => auth0User(i))), ); - const all = await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + const { users: all, truncated } = await fetchAllAuth0Users({ + credentials: CREDENTIALS, + token: "tok", + }); expect(all).toHaveLength(1000); + expect(truncated).toBe(true); expect(captured.err).toContain("only pages through the first 1000 users"); expect(captured.err).toContain("bulk user export job"); }); + test("a tenant of exactly 1000 users is complete, with no warning", async () => { + stubAuth0( + Array.from({ length: 10 }, () => Array.from({ length: 100 }, (_, i) => auth0User(i))), + ); + + const { users: all, truncated } = await fetchAllAuth0Users({ + credentials: CREDENTIALS, + token: "tok", + }); + + expect(all).toHaveLength(1000); + expect(truncated).toBe(false); + expect(captured.err).not.toContain("only pages through"); + }); + test("raises a clear error on a failed page request", async () => { globalThis.fetch = (async () => new Response("nope", { status: 500 })) as unknown as typeof fetch; diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 8f773666e..8b2628988 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -23,7 +23,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spin import { isAgent, isHuman } from "../../../mode.ts"; import type { UserLine } from "../lib/run-store.ts"; import { printTarget } from "../lib/target.ts"; -import { withInputRetry } from "../lib/input-retry.ts"; +import { isCredentialStatus, throwApiFailure, withInputRetry } from "../lib/input-retry.ts"; import { finishExport, startExportRun } from "./shared.ts"; const PAGE_SIZE = 100; @@ -175,9 +175,17 @@ export async function fetchAuth0Token(credentials: Auth0Credentials): Promise { +}): Promise<{ users: Auth0User[]; truncated: boolean }> { const all: Auth0User[] = []; for (let page = 0; ; page++) { @@ -235,16 +247,21 @@ export async function fetchAllAuth0Users(options: { if (users.length < PAGE_SIZE) break; if (all.length >= AUTH0_PAGINATION_CEILING) { - log.warn( - `Auth0 only pages through the first ${AUTH0_PAGINATION_CEILING} users on this endpoint` + - (total > AUTH0_PAGINATION_CEILING ? `, and this tenant reports ${total}` : "") + - ". Exported what is reachable; use Auth0's bulk user export job for the rest.", - ); - break; + // A tenant of exactly the ceiling is complete. Without a total, a full + // last page may hide more. + const truncated = total ? total > AUTH0_PAGINATION_CEILING : true; + if (truncated) { + log.warn( + `Auth0 only pages through the first ${AUTH0_PAGINATION_CEILING} users on this endpoint` + + (total ? `, and this tenant reports ${total}` : "") + + ". Exported what is reachable; use Auth0's bulk user export job for the rest.", + ); + } + return { users: all, truncated }; } } - return all; + return { users: all, truncated: false }; } /** @@ -348,13 +365,14 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { }, ); - const users = await withSpinner("Fetching users from Auth0...", async (spinner) => - fetchAllAuth0Users({ credentials, token, spinner }), + const { users, truncated } = await withSpinner( + "Fetching users from Auth0...", + async (spinner) => fetchAllAuth0Users({ credentials, token, spinner }), ); const run = await startExportRun(options, { platform: "auth0" }); const { users: exported, coverage } = buildAuth0Export(users, run.append); - finishExport({ run, options, users: exported, coverage }); + finishExport({ run, options, users: exported, coverage, truncated }); if (exported.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index b8dc8e0c1..0225b016b 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -122,6 +122,13 @@ describe("normalizeConnectionString", () => { expect(new URL(normalized).hostname).toBe("host"); }); + // A bare `%` parses as a URL but fails the driver's decode with "URI error". + test("encodes a bare % in a password", () => { + const normalized = normalizeConnectionString("postgres://u:50%off@host:5432/db"); + + expect(decodeURIComponent(new URL(normalized).password)).toBe("50%off"); + }); + test("leaves an already-valid string alone", () => { const encoded = "postgres://u:p%40ss@host:5432/db"; expect(normalizeConnectionString(encoded)).toBe(encoded); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index a5ef0fda3..56eebc97d 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -40,7 +40,12 @@ const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2|libsql):\/\//i; */ function parsesAsUrl(value: string): boolean { try { - return new URL(value).hostname.length > 0; + const url = new URL(value); + // A bare `%` in the password parses, then fails the driver's decode with a + // cryptic "URI error"; treat it as unparsed so it gets encoded. + decodeURIComponent(url.username); + decodeURIComponent(url.password); + return url.hostname.length > 0; } catch { return false; } diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index cf86218f9..b08e6df38 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -483,6 +483,19 @@ describe("exportFirebase", () => { expect(captured.err).toContain("demo-fb project"); }); + // Firebase's own variable, so it's honoured, but an emulator's users are + // not the project's: the target line says which. + test("names the emulator in the target line when one is set", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + process.env.FIREBASE_AUTH_EMULATOR_HOST = "localhost:9099"; + try { + await exportFirebase({ serviceAccount: "./sa.json" }); + } finally { + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; + } + expect(captured.err).toContain("Source: firebase (emulator at localhost:9099)"); + }); + test("names the command that consumes the file", async () => { stubFirebase([[fbUser(0)]], { signIn: {} }); // The suggestion now rides the gutter's Next steps block, which only diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index cad2161d0..eeb2cd992 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -34,7 +34,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spin import type { UserLine } from "../lib/run-store.ts"; import { printTarget } from "../lib/target.ts"; import type { FirebaseHashConfig } from "../types.ts"; -import { withInputRetry } from "../lib/input-retry.ts"; +import { isCredentialStatus, throwApiFailure, withInputRetry } from "../lib/input-retry.ts"; import { finishExport, startExportRun } from "./shared.ts"; /** Identity Toolkit's maximum for `accounts:batchGet`. */ @@ -286,9 +286,17 @@ export async function fetchAccessToken(account: ServiceAccount): Promise error?: string; }; - if (!response.ok || !body.access_token) { + const detail = body.error_description ?? body.error ?? "no access token returned"; + if (!response.ok && !isCredentialStatus(response.status)) { + throwApiFailure( + response.status, + `Google did not issue a token (${response.status}): ${detail}. Try again shortly.`, + DOCS_URL, + ); + } + if (!body.access_token) { throwUsageError( - `Google rejected the service account (${response.status}): ${body.error_description ?? body.error ?? "no access token returned"}\n` + + `Google rejected the service account (${response.status}): ${detail}\n` + "Check the key has not been revoked or deleted, in the Google Cloud console under IAM → Service accounts.", DOCS_URL, ); @@ -335,7 +343,8 @@ export async function fetchAllFirebaseUsers(options: { }); if (!response.ok) { - throwUsageError( + throwApiFailure( + response.status, `Firebase returned ${response.status} listing users: ${await response.text()}`, DOCS_URL, ); @@ -545,7 +554,12 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { - if (!options.json) printTarget({ platform: "firebase" }); + // The emulator variable is Firebase's own, so it is honoured, but named: + // an export from it is not the production project's users. + const emulator = process.env.FIREBASE_AUTH_EMULATOR_HOST; + if (!options.json) { + printTarget({ platform: emulator ? `firebase (emulator at ${emulator})` : "firebase" }); + } // Only Google can say whether a well-formed key is still a valid one, so a // revoked or deleted key fails here and is asked for again. const { value: token, input: account } = await withInputRetry( diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index 233e00b10..c7c550a23 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -119,6 +119,8 @@ export type FinishExportInput = { sections?: ExportSection[]; /** Firebase's hash parameters, carried to the import in the envelope. */ firebase?: FirebaseHashConfig; + /** The platform stopped short of every user; `--json` says so. */ + truncated?: boolean; }; export type FinishedExport = { record: RunRecord; outputPath: string }; @@ -156,6 +158,7 @@ export function finishExport(input: FinishExportInput): FinishedExport { users: users.length, coverage, ...(input.sections?.length ? { sections: input.sections } : {}), + ...(input.truncated ? { truncated: true } : {}), next, }, null, diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index 79ff27f26..ee002cfe3 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -26,7 +26,7 @@ import { isAgent, isHuman } from "../../../mode.ts"; import type { UserLine } from "../lib/run-store.ts"; import { printTarget } from "../lib/target.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; -import { withInputRetry } from "../lib/input-retry.ts"; +import { throwApiFailure, withInputRetry } from "../lib/input-retry.ts"; import { createApiScheduler } from "../lib/scheduler.ts"; import { finishExport, startExportRun, type ExportSection } from "./shared.ts"; @@ -166,7 +166,8 @@ export async function fetchWorkOsPage(apiKey: string, after?: string): Promise { } }); + // The export group declares --json too, and takes the flag; the hook has to + // read the group's options as well as the subcommand's. + test("--json on an export subcommand runs it in agent mode", async () => { + const original = getMode(); + try { + setMode("human"); + await parse(["migrate", "export", "supabase", "--json", "--db-url", "./none.sqlite"]); + expect(getMode()).toBe("agent"); + } finally { + setMode(original); + } + }); + test("records its absence, so a previous run cannot leak into this one", async () => { setAssumeYes(true); expect(await parse(["migrate", "export", "supabase", "--db-url", "./none.sqlite"])).toBe(false); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index 71b5e066e..7e29975dc 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -45,7 +45,9 @@ export function registerMigrate(program: Program): void { // mode: every prompt in this tree already stands down for an agent, with the // usage error naming what to pass instead. migrateCommand.hook("preAction", (_thisCommand, actionCommand) => { - const opts = actionCommand.opts(); + // With globals: `export auth0 --json` lands on the export group's own + // --json, which the subcommand's opts() never sees. + const opts = actionCommand.optsWithGlobals(); setAssumeYes(Boolean(opts.yes)); if (opts.json) setMode("agent"); }); diff --git a/packages/cli-core/src/commands/migrate/lib/analysis.ts b/packages/cli-core/src/commands/migrate/lib/analysis.ts index 72afc7a05..be371127e 100644 --- a/packages/cli-core/src/commands/migrate/lib/analysis.ts +++ b/packages/cli-core/src/commands/migrate/lib/analysis.ts @@ -22,6 +22,10 @@ export type IdentifierCounts = { verifiedPhones: number; unverifiedPhones: number; username: number; + /** Users with any email, verified or not: each user counted once. */ + anyEmail: number; + /** Users with any phone, verified or not: each user counted once. */ + anyPhone: number; /** Users with at least one identifier — the rest cannot be imported at all. */ hasAnyIdentifier: number; }; @@ -46,6 +50,8 @@ export function analyzeFields(users: (User | Record)[]): FieldA verifiedPhones: 0, unverifiedPhones: 0, username: 0, + anyEmail: 0, + anyPhone: 0, hasAnyIdentifier: 0, }; const fieldCounts: Record = {}; @@ -70,6 +76,8 @@ export function analyzeFields(users: (User | Record)[]): FieldA if (verifiedPhone) identifiers.verifiedPhones++; if (unverifiedPhone) identifiers.unverifiedPhones++; if (username) identifiers.username++; + if (verifiedEmail || unverifiedEmail) identifiers.anyEmail++; + if (verifiedPhone || unverifiedPhone) identifiers.anyPhone++; if (verifiedEmail || unverifiedEmail || verifiedPhone || unverifiedPhone || username) { identifiers.hasAnyIdentifier++; diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index b2ac181d9..c5ebc82f3 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -159,7 +159,7 @@ describe("rejects", () => { expect(checks.total).toBe(1); }); - test("a duplicate source ID, email or phone within the file", async () => { + test("a duplicate source ID, email, phone or username within the file", async () => { expect( await reasonsOf({ users: [ @@ -168,12 +168,19 @@ describe("rejects", () => { user("b", { email: "A@x.dev" }), user("c", { phone: "+15555550100" }), user("d", { phone: "+15555550100" }), + // The same number, punctuated. + user("f", { phone: "+1 555-555-0100" }), + user("g", { username: "Ada" }), + // Clerk lowercases usernames. + user("h", { username: "ada" }), ], }), ).toEqual({ a: "duplicate source ID in the file", b: "email is also used by an earlier user in the file, which is kept", d: "phone number is also used by an earlier user in the file, which is kept", + f: "phone number is also used by an earlier user in the file, which is kept", + h: "username is also used by an earlier user in the file, which is kept", }); }); @@ -219,7 +226,9 @@ describe("rejects", () => { user("bad", { password: "not-a-hash", passwordHasher: "bcrypt" }), ], }), - ).toEqual({ bad: "password is not a bcrypt hash ($2a$/$2b$/$2y$, 60 characters)" }); + ).toEqual({ + bad: "password is not a bcrypt hash Clerk accepts ($2a$/$2b$/$2y$, cost up to 15, 60 characters)", + }); }); test("a user already in the instance, by source ID, email or username", async () => { @@ -291,6 +300,24 @@ describe("rejects", () => { ).toEqual({ "only-discord": "only signs in with Discord, which is not enabled in Clerk" }); }); + // Each reject names that user's providers, not every disabled one in the file. + test("a supabase reject names only that user's own providers", async () => { + const rows = [ + { id: "discord", raw_app_meta_data: { providers: ["discord"] } }, + { id: "twitch", raw_app_meta_data: { providers: ["twitch"] } }, + ]; + expect( + await reasonsOf({ + settings: settings({ email_address: { enabled: true } }), + supabaseRows: rows, + users: [user("discord"), user("twitch")], + }), + ).toEqual({ + discord: "only signs in with Discord, which is not enabled in Clerk", + twitch: "only signs in with Twitch, which is not enabled in Clerk", + }); + }); + test("the users past a development instance's headroom, in file order", async () => { const checks = await checkImport( input({ @@ -343,11 +370,35 @@ describe("rejects", () => { const checks = await checkImport( input({ users: [user("a", { email: "15551234@phone.local" })] }), ); + // One fixed reason, so these group in the report and the address + // itself stays out of users.ndjson. expect(checks.rejects).toEqual([ - { sourceId: "a", reason: "only has an email Clerk refuses (15551234@phone.local)" }, + { + sourceId: "a", + reason: "only has emails Clerk refuses (malformed, or a domain that can't receive mail)", + }, ]); }); + test.each([ + ["a@localhost", "no dotted domain"], + ["not-an-email", "no @"], + ["a b@x.dev", "a space"], + ["ada@corp.internal", "a private TLD"], + ])("drops %p (%s) and keeps the user on its other email", async (bad) => { + const checks = await checkImport( + input({ users: [user("a", { email: "a@x.dev", emailAddresses: [bad] })] }), + ); + expect(checks.rejects).toEqual([]); + expect(checks.importable[0]?.emailAddresses).toBeUndefined(); + }); + + // clerk_go accepts a non-ASCII local part; Zod's email check did not. + test("keeps a non-ASCII address", async () => { + const checks = await checkImport(input({ users: [user("a", { email: "josé@x.dev" })] })); + expect(checks.importable).toEqual([user("a", { email: "josé@x.dev" })]); + }); + test("a placeholder email is dropped, and the user imports on what is left", async () => { const checks = await checkImport( input({ @@ -362,7 +413,7 @@ describe("rejects", () => { expect(checks.rejects).toEqual([]); expect(checks.importable).toEqual([user("a", { email: "a@x.dev" })]); expect(checks.warnings).toContain( - "1 user has an email Clerk refuses (.local, .invalid, .test, .example, .arpa), which is dropped", + "1 user has an email Clerk refuses (malformed, or a domain such as .local or .invalid), which is dropped", ); }); @@ -378,7 +429,6 @@ describe("rejects", () => { test.each([ ["+447836887904"], ["4165550123"], - ["ada@x.dev"], ["https://spam.example"], ["see x.com/win"], ["Ada"], @@ -387,17 +437,23 @@ describe("rejects", () => { expect(checks.rejects).toEqual([]); expect(checks.importable).toEqual([user("a", { lastName: "L" })]); expect(checks.warnings).toContain( - "1 user has a name Clerk refuses (a phone number, email, URL or HTML), which is dropped", + "1 user has a name Clerk refuses (a phone number, URL or HTML), which is dropped", ); }); - test.each([["Ada"], ["Mary-Jane O'Neil"], ["Louis XIV"], ["Agent 007"], ["redacted.io"]])( - "%p is kept", - async (firstName) => { - const checks = await checkImport(input({ users: [user("a", { firstName })] })); - expect(checks.importable).toEqual([user("a", { firstName })]); - }, - ); + // Clerk accepts an email as a name, and a URL that is part of one. + test.each([ + ["Ada"], + ["Mary-Jane O'Neil"], + ["Louis XIV"], + ["Agent 007"], + ["redacted.io"], + ["ada@x.dev"], + ["ada@x.dev/x"], + ])("%p is kept", async (firstName) => { + const checks = await checkImport(input({ users: [user("a", { firstName })] })); + expect(checks.importable).toEqual([user("a", { firstName })]); + }); }); describe("usernames", () => { diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index 9fd7880ce..4a04ef18f 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -32,6 +32,7 @@ import { countSocialProviders, findDisabledProviders, findUsersWithOnlyDisabledProviders, + getUserProviders, } from "./supabase-providers.ts"; import type { ClerkTarget } from "./target.ts"; import type { ValidationFailure } from "./transform.ts"; @@ -114,15 +115,16 @@ const plural = (count: number, word: string) => `${count} ${word}${count === 1 ? * Why a password digest cannot be what its hasher says it is, for the hashers * whose shape is cheap and certain to check. * - * Every other hasher is "can't verify": its shape is not checked, and a bad - * digest there still fails only at sign-in. + * Every other hasher's shape is not checked here: BAPI validates it at + * create, so a bad digest there fails that user mid-import. */ export function hashShapeProblem(password: string, hasher: string): string | undefined { switch (hasher) { case "bcrypt": - return /^\$2[aby]\$\d\d\$[./A-Za-z0-9]{53}$/.test(password) + // clerk_go caps the cost at 15 (pkg/hash/bcrypt.go). + return /^\$2[aby]\$(0\d|1[0-5])\$[./A-Za-z0-9]{53}$/.test(password) ? undefined - : "password is not a bcrypt hash ($2a$/$2b$/$2y$, 60 characters)"; + : "password is not a bcrypt hash Clerk accepts ($2a$/$2b$/$2y$, cost up to 15, 60 characters)"; case "scrypt_firebase": { const parts = password.split("$"); const numeric = (value: string | undefined) => /^\d+$/.test(value ?? ""); @@ -305,15 +307,45 @@ function usernameProblem(user: User, settings: UserSettingsJSON | null): string } /** - * TLDs Clerk refuses for any email, from clerk_go's - * `emailaddress.nonRoutableTLDs`. This is where placeholder addresses live - * (`…@phone.local`, `anon-…@anonymous.invalid`). Clerk also refuses a TLD not - * on the public suffix list; that would take the list as a dependency. + * TLDs Clerk refuses for any email: clerk_go's `emailaddress.nonRoutableTLDs`, + * where placeholder addresses live (`…@phone.local`, `anon-…@anonymous.invalid`), + * plus the common private ones the public suffix list leaves out. + * + * ponytail: Clerk refuses any TLD not on the public suffix list; this names + * the usual ones instead of taking the list as a dependency. */ -const NON_ROUTABLE_TLDS = new Set(["arpa", "local", "invalid", "example", "test"]); +const NON_ROUTABLE_TLDS = new Set([ + "arpa", + "local", + "invalid", + "example", + "test", + "internal", + "lan", + "corp", + "home", + "localdomain", + "intranet", + "private", +]); + +/** + * An address shape Clerk accepts, loosely: one `@`, no spaces, a dotted host, + * a local part of at most 64 bytes and 254 in all. Non-ASCII is fine, as it + * is in clerk_go (`josé@x.dev`). + */ +function isEmailShaped(email: string): boolean { + const at = email.lastIndexOf("@"); + return ( + /^[^\s@]+@[^\s@]+\.[^\s@.]+$/.test(email) && + Buffer.byteLength(email.slice(0, at)) <= 64 && + email.length <= 254 + ); +} const EMAIL_FIELDS = ["email", "emailAddresses", "unverifiedEmailAddresses"] as const; function isRefusedEmail(email: string): boolean { + if (!isEmailShaped(email)) return true; const host = email.slice(email.lastIndexOf("@") + 1).toLowerCase(); // Clerk's own dev domains sit under `.test` and are accepted. if (host.endsWith(".clerk.test")) return false; @@ -340,9 +372,9 @@ function dropRefusedEmails(user: User): { user: User; refused: string[] } { } /** - * A name Clerk refuses, approximating clerk_go's `nameForAbusePrevention`: a - * phone number (10–15 digits, or fewer behind a `+`/`00`), an email, a URL with - * a scheme or path, or an HTML tag. Better Auth's phone sign-up stores the + * A name Clerk refuses, approximating clerk_go's `NameForAbusePreventionLoose`: + * a phone number (10–15 digits, or fewer behind a `+`/`00`), a URL with a + * scheme or path that isn't part of an email, or an HTML tag. Better Auth's phone sign-up stores the * number as the name, so this is common, not exotic. */ function nameProblem(name: string): string | undefined { @@ -351,8 +383,10 @@ function nameProblem(name: string): string | undefined { const international = /^(\+|00)/.test(candidate.trim()); if (digits <= 15 && (digits >= 10 || (international && digits >= 7))) return "a phone number"; } - if (/\S+@\S+\.\S+/.test(name)) return "an email address"; - if (/:\/\/|\b[\w-]+(\.[\w-]+)+[/?#]/.test(name)) return "a URL"; + // Clerk accepts an email as a name (NameForAbusePreventionLoose), and a URL + // that is part of one. + const hasEmail = /\S+@\S+\.\S+/.test(name); + if (!hasEmail && /:\/\/|\b[\w-]+(\.[\w-]+)+[/?#]/.test(name)) return "a URL"; if (/<\/?[a-z!][^>]*>/i.test(name)) return "HTML"; return undefined; } @@ -370,6 +404,15 @@ function dropRefusedNames(user: User): { user: User; dropped: boolean } { return { user: kept ?? user, dropped: kept !== undefined }; } +/** + * A phone number with its punctuation stripped, so `+1 555-555-0100` and + * `+15555550100` compare equal. + * + * ponytail: punctuation only; a national number without its country code + * still differs from its E.164 form. Full parsing would need a dependency. + */ +const phoneKey = (phone: string) => phone.replace(/[^\d+]/g, ""); + const hasAnyIdentifier = (user: User) => [...EMAIL_FIELDS, "phone", "phoneNumbers", "unverifiedPhoneNumbers", "username"].some((field) => hasValue(user[field as keyof User]), @@ -392,6 +435,7 @@ function findFileDuplicates(users: User[]): { const seenIds = new Set(); const emails = new Map(); const phones = new Map(); + const usernames = new Map(); for (const user of users) { if (seenIds.has(user.userId)) { @@ -409,7 +453,10 @@ function findFileDuplicates(users: User[]): { ); const emailOwner = ownEmails.map((email) => emails.get(email.toLowerCase())).find(Boolean); - const phoneOwner = ownPhones.map((phone) => phones.get(phone)).find(Boolean); + const phoneOwner = ownPhones.map((phone) => phones.get(phoneKey(phone))).find(Boolean); + // Clerk lowercases usernames, so the second create would fail. + const username = typeof user.username === "string" ? user.username.toLowerCase() : ""; + const usernameOwner = username ? usernames.get(username) : undefined; // The first record in the file wins, whatever either holds: the source's // order decides, so the kept ID is named alongside the reject. if (emailOwner) { @@ -422,8 +469,14 @@ function findFileDuplicates(users: User[]): { keptBy.set(user, phoneOwner); continue; } + if (usernameOwner) { + reasons.set(user, "username is also used by an earlier user in the file, which is kept"); + keptBy.set(user, usernameOwner); + continue; + } for (const email of ownEmails) emails.set(email.toLowerCase(), user.userId); - for (const phone of ownPhones) phones.set(phone, user.userId); + for (const phone of ownPhones) phones.set(phoneKey(phone), user.userId); + if (username) usernames.set(username, user.userId); } return { reasons, keptBy }; } @@ -449,7 +502,7 @@ async function findInstanceDuplicates( const identifiers = splitIdentifiers(user); byExternalId.set(user.userId, user.userId); if (identifiers.primaryEmail) byEmail.set(identifiers.primaryEmail.toLowerCase(), user.userId); - if (identifiers.primaryPhone) byPhone.set(identifiers.primaryPhone, user.userId); + if (identifiers.primaryPhone) byPhone.set(phoneKey(identifiers.primaryPhone), user.userId); if (typeof user.username === "string" && user.username) { byUsername.set(user.username.toLowerCase(), user.userId); } @@ -495,7 +548,7 @@ async function findInstanceDuplicates( } for (const phone of existing.phone_numbers ?? []) { claim( - byPhone.get(phone.phone_number ?? ""), + byPhone.get(phoneKey(phone.phone_number ?? "")), "phone number is already used by a user in the instance", ); } @@ -519,11 +572,14 @@ function findDisabledProviderRejects(input: CheckInput): Map { if (disabled.length === 0) return reasons; const { excludedIds } = findUsersWithOnlyDisabledProviders(input.supabaseRows, disabled); - const names = disabled.map(providerLabel).join(", "); + // Each reject names only that user's own providers. + const rowsById = new Map(input.supabaseRows.map((row) => [String(row.id), row])); for (const id of excludedIds) { + const own = getUserProviders(rowsById.get(id) ?? {}).filter((p) => disabled.includes(p)); + const names = own.map(providerLabel).join(", "); reasons.set( id, - `only signs in with ${names}, which ${disabled.length === 1 ? "is" : "are"} not enabled in Clerk`, + `only signs in with ${names}, which ${own.length === 1 ? "is" : "are"} not enabled in Clerk`, ); } return reasons; @@ -691,7 +747,7 @@ function dropDisabledIdentifiers(user: User, settings: UserSettingsJSON | null): function refusedNameWarning(count: number): string[] { if (count === 0) return []; return [ - `${plural(count, "user")} ${count === 1 ? "has" : "have"} a name Clerk refuses (a phone number, email, URL or HTML), which is dropped`, + `${plural(count, "user")} ${count === 1 ? "has" : "have"} a name Clerk refuses (a phone number, URL or HTML), which is dropped`, ]; } @@ -705,7 +761,7 @@ function legalWarning(count: number): string[] { function placeholderWarning(count: number): string[] { if (count === 0) return []; return [ - `${plural(count, "user")} ${count === 1 ? "has" : "have"} an email Clerk refuses (.local, .invalid, .test, .example, .arpa), which is dropped`, + `${plural(count, "user")} ${count === 1 ? "has" : "have"} an email Clerk refuses (malformed, or a domain such as .local or .invalid), which is dropped`, ]; } @@ -729,7 +785,7 @@ export async function checkImport(input: CheckInput): Promise { const reason = original.skipReason ?? (refused.length > 0 && !hasAnyIdentifier(user) - ? `only has an email Clerk refuses (${refused[0]})` + ? "only has emails Clerk refuses (malformed, or a domain that can't receive mail)" : undefined) ?? missingRequiredIdentifier(user, input.settings) ?? // Stripping the identifiers the instance has off can leave nothing to diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index 72344f117..f0ddc625c 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -54,6 +54,11 @@ describe("redactConnectionString", () => { ["mysql://root:hunter2@127.0.0.1:3306/app", "mysql://***@127.0.0.1:3306/app"], ["postgres://host/db", "postgres://host/db"], ["libsql://app.turso.io?authToken=secret", "libsql://app.turso.io?authToken=***"], + ["postgres://u@host/db?password=secret", "postgres://***@host/db?password=***"], + [ + "postgres://host/db?sslmode=require&password=secret", + "postgres://host/db?sslmode=require&password=***", + ], ])("%s -> %s", (input, expected) => { expect(redactConnectionString(input)).toBe(expected); }); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index 9e3b0e129..b1e680273 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -64,10 +64,11 @@ export function redactConnectionString(connectionString: string): string { // the rest of the password in the message. Everything before the final `@` // is userinfo, so redacting all of it is always safe. // Non-URL forms (SQLite paths) have no `://` and are left alone. - // Turso carries its credential as `?authToken=`, not as userinfo. + // Turso carries its credential as `?authToken=`, not as userinfo, and a + // `?password=` (which Bun.SQL ignores) still must not be printed. return connectionString .replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@") - .replace(/([?&]authToken=)[^&]*/gi, "$1***"); + .replace(/([?&](?:authToken|password)=)[^&]*/gi, "$1***"); } /** Strips a `file:` prefix and any URL query, leaving a filesystem path. */ diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts index c414d6d1b..8a01e0d6c 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -9,7 +9,7 @@ */ import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; -import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, EXIT_CODE, UserAbortError } from "../../../lib/errors.ts"; import { getMode, setMode, type Mode } from "../../../mode.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; @@ -46,7 +46,8 @@ const CONFIG = { const FIRST = "libsql://typo.turso.io?authToken=t"; const SECOND = "libsql://right.turso.io?authToken=t"; -const rejected = () => new CliError("Could not reach it", { code: ERROR_CODE.USAGE_ERROR }); +const rejected = () => + new CliError("Could not reach it", { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }); let originalMode: Mode; @@ -193,6 +194,30 @@ describe("withInputRetry", () => { }); // A bug inside the work, or an interrupt, is not a wrong answer to a prompt. + // Another credential would not fix an outage, a 429 or a refused connection. + test.each([ + [ + "a refused connection", + new CliError("Could not reach x", { code: ERROR_CODE.NETWORK_UNREACHABLE }), + ], + ["a 503", new CliError("Auth0 returned 503 listing users")], + ])("does not ask again after %s", async (_label, failure) => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw failure; + }, + ), + ).rejects.toBe(failure); + + expect(attempts).toBe(1); + }); + test("does not retry an error the database layer did not raise", async () => { let attempts = 0; diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts index 13a3604b7..13ec1b2c0 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -15,7 +15,7 @@ * full re-run for one line they could not see. */ -import { CliError } from "../../../lib/errors.ts"; +import { CliError, EXIT_CODE, throwUsageError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { isAssumeYes } from "./assume-yes.ts"; @@ -51,13 +51,34 @@ export async function withInputRetry( try { return { value: await work(candidate), input: candidate }; } catch (error) { - // Everything these steps raise for a bad credential is a CliError - // carrying its own explanation; anything else (an interrupt, a bug) is - // not ours to retry. - if (!(error instanceof CliError) || !isHuman() || isAgent() || isAssumeYes()) throw error; + // Everything these steps raise for a bad credential is a usage error + // carrying its own explanation. An outage, a 429 or a refused + // connection is not fixed by another credential, and anything else (an + // interrupt, a bug) is not ours to retry. + const badInput = error instanceof CliError && error.exitCode === EXIT_CODE.USAGE; + if (!badInput || !isHuman() || isAgent() || isAssumeYes()) throw error; log.error(error.message); candidate = await reprompt(); } } } + +/** Statuses that mean the credential is wrong, rather than the service being down. */ +const CREDENTIAL_STATUSES = new Set([400, 401, 403]); + +/** True when an API's status says the credential, not the service, is the problem. */ +export function isCredentialStatus(status: number): boolean { + return CREDENTIAL_STATUSES.has(status); +} + +/** + * Throws an API failure: a usage error (exit 2, and a fresh prompt under + * {@link withInputRetry}) when the status blames the credential; a plain + * error (exit 1) for a 429, a 5xx or anything else another credential would + * not fix. + */ +export function throwApiFailure(status: number, message: string, docsUrl?: string): never { + if (isCredentialStatus(status)) throwUsageError(message, docsUrl); + throw new CliError(message, { docsUrl }); +} diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts index db8e84366..f48871bf8 100644 --- a/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts @@ -21,15 +21,21 @@ function settings(config: { } function analysis(overrides: Partial & { totalUsers: number }): FieldAnalysis { + const identifiers = { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: overrides.totalUsers, + ...overrides.identifiers, + }; return { identifiers: { - verifiedEmails: 0, - unverifiedEmails: 0, - verifiedPhones: 0, - unverifiedPhones: 0, - username: 0, - hasAnyIdentifier: overrides.totalUsers, - ...overrides.identifiers, + // Hand-built rows have no user overlap, so "any" is the sum. + anyEmail: identifiers.verifiedEmails + identifiers.unverifiedEmails, + anyPhone: identifiers.verifiedPhones + identifiers.unverifiedPhones, + ...identifiers, }, fieldCounts: overrides.fieldCounts ?? {}, totalUsers: overrides.totalUsers, diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts index f9824a37e..76d9a4eb4 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import type { UserSettingsJSON } from "../../../lib/fapi.ts"; -import type { FieldAnalysis } from "./analysis.ts"; +import { analyzeFields, type FieldAnalysis } from "./analysis.ts"; import { buildReadinessReport, type ReadinessItem } from "./readiness.ts"; /** Instance settings carrying only the attributes and providers a test names. */ @@ -21,15 +21,21 @@ function settings(config: { /** Field analysis with everything absent unless the test says otherwise. */ function analysis(overrides: Partial & { totalUsers: number }): FieldAnalysis { + const identifiers = { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: overrides.totalUsers, + ...overrides.identifiers, + }; return { identifiers: { - verifiedEmails: 0, - unverifiedEmails: 0, - verifiedPhones: 0, - unverifiedPhones: 0, - username: 0, - hasAnyIdentifier: overrides.totalUsers, - ...overrides.identifiers, + // Hand-built rows have no user overlap, so "any" is the sum. + anyEmail: identifiers.verifiedEmails + identifiers.unverifiedEmails, + anyPhone: identifiers.verifiedPhones + identifiers.unverifiedPhones, + ...identifiers, }, fieldCounts: overrides.fieldCounts ?? {}, totalUsers: overrides.totalUsers, @@ -59,6 +65,20 @@ describe("which rows appear", () => { expect(item(report, "Email")?.userCount).toBe(5); }); + // A user with both a verified and an unverified phone is one user, not two. + test("counts a user with both kinds of phone once", () => { + const users = [ + { userId: "a", phone: "+15555550100", unverifiedPhoneNumbers: ["+15555550101"] }, + { userId: "b", phone: "+15555550102", unverifiedPhoneNumbers: ["+15555550103"] }, + { userId: "c", email: "c@x.dev" }, + ]; + const report = buildReadinessReport({ + analysis: analyzeFields(users), + settings: settings({ attributes: { phone_number: { enabled: true } } }), + }); + expect(item(report, "Phone")?.userCount).toBe(2); + }); + test("groups rows into identifiers, auth and user model", () => { const report = buildReadinessReport({ analysis: analysis({ diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts index 02b00c05f..c6edf4b01 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -159,8 +159,9 @@ export function buildReadinessReport(input: BuildInput): ReadinessReport { const total = analysis.totalUsers; const items: ReadinessItem[] = []; - const emailCount = analysis.identifiers.verifiedEmails + analysis.identifiers.unverifiedEmails; - const phoneCount = analysis.identifiers.verifiedPhones + analysis.identifiers.unverifiedPhones; + // Users, not fields: one with both a verified and an unverified phone is one. + const emailCount = analysis.identifiers.anyEmail; + const phoneCount = analysis.identifiers.anyPhone; const attributeRows: [string, ReadinessSection, AttributeName, number][] = [ ["Email", "identifiers", "email_address", emailCount], diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index aa1b10866..0616b6343 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -1,4 +1,15 @@ -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + setSystemTime, + spyOn, + test, +} from "bun:test"; +import * as crypto from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -186,6 +197,24 @@ describe("locks and interruptions", () => { expect(runState(runsDir, run.record)).toBe("running"); }); + // Two runs in the same second share an ID one time in 65,536. + test("a run ID that is already taken gets a new one, not a shared folder", () => { + setSystemTime(new Date("2026-10-02T12:00:00")); + const bytes = spyOn(crypto, "randomBytes") + .mockReturnValueOnce(Buffer.from([0xab, 0xcd]) as never) + .mockReturnValueOnce(Buffer.from([0xab, 0xcd]) as never) + .mockReturnValueOnce(Buffer.from([0x12, 0x34]) as never); + try { + const first = startRun(runsDir, init); + const second = startRun(runsDir, init); + expect(first.record.id).toEndWith("-abcd"); + expect(second.record.id).toEndWith("-1234"); + } finally { + bytes.mockRestore(); + setSystemTime(); + } + }); + test("refuses a run another live process holds, with exit 2", () => { const run = startRun(runsDir, init); // PID 1 is always alive, and never this test. diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 347775dcd..6ac2f2bf3 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -209,15 +209,37 @@ export function lockFile(runsDir: string, id: string): string { return path.join(runDir(runsDir, id), LOCK_FILE); } +const isExists = (error: unknown) => (error as NodeJS.ErrnoException).code === "EEXIST"; + +/** + * Takes the run's lock. Created exclusively (`wx`), so two processes can't + * both read "free" and both write; a stale lock is removed and taken once. + */ function acquireLock(runsDir: string, id: string): void { - const holder = liveLockPid(runsDir, id); - if (holder !== undefined) { + const file = lockFile(runsDir, id); + const refuse = (holder: number | undefined): never => throwUsageError( - `Run ${id} is in use by another process (PID ${holder}). Wait for it to finish, then try again. ` + - `If that process is not a migrate run, delete ${lockFile(runsDir, id)}.`, + `Run ${id} is in use by another process${holder ? ` (PID ${holder})` : ""}. Wait for it to finish, then try again. ` + + `If that process is not a migrate run, delete ${file}.`, ); + const take = () => fs.writeFileSync(file, String(process.pid), { flag: "wx" }); + + try { + take(); + return; + } catch (error) { + if (!isExists(error)) throw error; + } + const holder = liveLockPid(runsDir, id); + if (holder !== undefined) refuse(holder); + fs.rmSync(file, { force: true }); + try { + take(); + } catch (error) { + // Another process took the stale lock between the remove and the write. + if (isExists(error)) refuse(liveLockPid(runsDir, id)); + throw error; } - fs.writeFileSync(lockFile(runsDir, id), String(process.pid)); } // --- Reading --------------------------------------------------------------- @@ -377,9 +399,20 @@ export type StartRunInit = Omit { ["JSON metadata", { publicMetadata: '{"plan":"pro"}' }, { publicMetadata: { plan: "pro" } }], ["date string", { createdAt: "2024-01-01" }, { createdAt: "2024-01-01T00:00:00.000Z" }], ["numeric user ID", { userId: 42 }, { userId: "42" }], + ["epoch milliseconds", { createdAt: 1704067200000 }, { createdAt: "2024-01-01T00:00:00.000Z" }], + // As milliseconds this would be January 1970. + ["epoch seconds", { createdAt: 1704067200 }, { createdAt: "2024-01-01T00:00:00.000Z" }], ])("normalizes %s", (_label, input, expected) => { expect(normalizeUserData(input)).toMatchObject(expected); }); @@ -254,6 +258,35 @@ describe("loadUsersFromFile", () => { expect(error.message).toContain("broken.json is not valid JSON"); }); + // `#` starts a value, not a comment: dropping the row would lose the user + // and record nothing. + test("keeps a CSV row whose first cell starts with #", async () => { + fs.writeFileSync(path.join(workDir, "hash.csv"), "id,primary_email_address\n#7,a@x.dev\n"); + const { users } = await loadUsersFromFile("hash.csv", "clerk"); + expect(users.map((user) => user.userId)).toEqual(["#7"]); + }); + + test("a custom preTransform's rows win over a CSV file", async () => { + registerCustomSource({ + ...clerkSource, + key: "rows-from-pretransform", + preTransform: (filePath) => ({ + filePath, + data: [{ id: "from-pretransform", primary_email_address: "p@x.dev" }], + }), + }); + try { + fs.writeFileSync( + path.join(workDir, "ignored.csv"), + "id,primary_email_address\nfile,f@x.dev\n", + ); + const { users } = await loadUsersFromFile("ignored.csv", "rows-from-pretransform"); + expect(users.map((user) => user.userId)).toEqual(["from-pretransform"]); + } finally { + __resetCustomSourcesForTesting(); + } + }); + test("rejects a JSON file that is not an array of users", async () => { fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 3ddd2a0b5..a154f4aef 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -184,7 +184,9 @@ function normalizeMetadataField(value: unknown): unknown { function normalizeDateField(value: unknown): unknown { if (value instanceof Date) return value.toISOString(); if (typeof value === "number") { - const date = new Date(value); + // Epoch seconds, not milliseconds, below 1e11: as milliseconds that is + // before March 1973, which no signup date is. + const date = new Date(value < 1e11 ? value * 1000 : value); return Number.isNaN(date.getTime()) ? value : date.toISOString(); } if (typeof value !== "string") return value; @@ -413,7 +415,13 @@ export function transformUsers( // --- File loading ---------------------------------------------------------- -async function readCsv(filePath: string): Promise[]> { +/** + * Every row of a CSV. `#` is not a comment: a row whose first cell starts + * with one is a row. + * + * @param headers - Column names, for a CSV with no header row. + */ +async function readCsv(filePath: string, headers?: string[]): Promise[]> { return new Promise((resolve, reject) => { const users: Record[] = []; fs.createReadStream(filePath) @@ -421,7 +429,7 @@ async function readCsv(filePath: string): Promise[]> { // part of the first header and hide that column from every row. .pipe( csvParser({ - skipComments: true, + ...(headers ? { headers } : {}), mapHeaders: ({ header }) => header.replace(/^\uFEFF/, ""), }), ) @@ -438,6 +446,7 @@ async function readUsersFromFile( let filePath = resolveImportFilePath(file); const type = getFileType(file); let preExtracted: Record[] | undefined; + let csvHeaders: string[] | undefined; // An export's envelope already holds the users in the source's own shape, // so there is nothing left for a pre-transform to unwrap. @@ -459,10 +468,12 @@ async function readUsersFromFile( const result = await transformer.preTransform(filePath, type ?? ""); filePath = result.filePath; preExtracted = result.data; + csvHeaders = result.csvHeaders; } - if (type === "text/csv") return readCsv(filePath); + // A pre-transform's rows win over the file, CSV or not. if (preExtracted) return preExtracted; + if (type === "text/csv") return readCsv(filePath, csvHeaders); const parsed = readJsonFile(filePath); if (!Array.isArray(parsed)) { diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index e7f52daf2..c733e77f2 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -357,6 +357,14 @@ describe("run", () => { expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toContain(".clerk/"); }); + test("leaves .gitignore alone when the import is refused for lack of consent", async () => { + fs.rmSync(path.join(workDir, ".gitignore"), { force: true }); + + await run({ ...baseOptions, yes: false }).catch(() => undefined); + + expect(fs.existsSync(path.join(workDir, ".gitignore"))).toBe(false); + }); + test("--runs-dir puts the run somewhere else", async () => { await run({ ...baseOptions, runsDir: "elsewhere" }); expect(listRuns(path.join(workDir, "elsewhere"))).toHaveLength(1); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index f3d838400..1481565be 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -60,7 +60,7 @@ import { } from "./lib/run-store.ts"; import { createApiScheduler } from "./lib/scheduler.ts"; import { readSupabaseRows } from "./lib/supabase-providers.ts"; -import { printTarget, resolveClerkTarget } from "./lib/target.ts"; +import { keyInstanceId, printTarget, resolveClerkTarget } from "./lib/target.ts"; import { findInFlight } from "./lib/user-lookup.ts"; import { fileExists, @@ -367,7 +367,14 @@ export type ResumeCase = */ export function findResume( runsDir: string, - match: { sha256: string; source: string; sourceHash?: string; instanceId: string }, + match: { + sha256: string; + source: string; + sourceHash?: string; + instanceId: string; + /** The key's stand-in ID, for a run recorded while Clerk could not name the instance. */ + keyInstanceId?: string; + }, ): ResumeCase { const latest = listRuns(runsDir).find( (record) => @@ -375,7 +382,8 @@ export function findResume( record.file?.sha256 === match.sha256 && record.source === match.source && record.sourceHash === match.sourceHash && - record.target.instanceId === match.instanceId, + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId), ); if (!latest) return { kind: "new" }; @@ -595,7 +603,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const filePath = resolveImportFilePath(file); const sha256 = sha256File(filePath); - const runsDir = await resolveRunsDir(options.runsDir, { write: !options.dryRun }); + const runsDir = await resolveRunsDir(options.runsDir); const resume: ResumeCase = options.newRun ? { kind: "new" } @@ -604,6 +612,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { source, ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), instanceId: target.instanceId, + keyInstanceId: keyInstanceId(secretKey), }); if (resume.kind === "complete") { @@ -811,6 +820,8 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (!proceed) throwUserAbort(); } + // Gitignored only now, once there is consent to write a run. + await resolveRunsDir(options.runsDir, { write: true }); const run = continued ? continueRun(runsDir, continued) : startRun(runsDir, { diff --git a/packages/cli-core/src/commands/migrate/sources/betterauth.ts b/packages/cli-core/src/commands/migrate/sources/betterauth.ts index fff46cc3a..33071dbf4 100644 --- a/packages/cli-core/src/commands/migrate/sources/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/sources/betterauth.ts @@ -83,9 +83,11 @@ const betterAuthSource = { phone_number: "phone", phone_number_verified: "phoneVerified", created_at: "createdAt", - updated_at: "updatedAt", }, postTransform: (user) => { + // Every Better Auth row has one, and Clerk sets its own. + delete user.updated_at; + routeByVerification(user, "email", "emailVerified", "boolean"); routeByVerification(user, "phone", "phoneVerified", "boolean"); splitName(user); diff --git a/packages/cli-core/src/commands/migrate/sources/clerk.ts b/packages/cli-core/src/commands/migrate/sources/clerk.ts index e97cba21f..9a55c2cca 100644 --- a/packages/cli-core/src/commands/migrate/sources/clerk.ts +++ b/packages/cli-core/src/commands/migrate/sources/clerk.ts @@ -21,8 +21,8 @@ const clerkSource = { note: "TOTP secrets and backup codes come across from a Dashboard export only.", }, metadata: { - level: "yes", - note: "Public, private and unsafe metadata keep their places. `export clerk` moves each user's external_id to private metadata as `clerkExternalId`, because the import uses external_id for the old Clerk ID.", + level: "partial", + note: "With `clerk migrate export clerk`, public, private and unsafe metadata keep their places; a Dashboard CSV carries no metadata, ban or legal acceptance. `export clerk` moves each user's external_id to private metadata as `clerkExternalId`, because the import uses external_id for the old Clerk ID.", }, }, transformer: { @@ -54,6 +54,28 @@ const clerkSource = { create_organizations_limit: "createOrganizationsLimit", delete_self_enabled: "deleteSelfEnabled", }, + postTransform: (user) => { + // The Dashboard's CSV prefixes a TAB to any value starting with = + - @ + // (or their fullwidth forms), so a spreadsheet won't run it as a formula + // (clerk_go pkg/csvsafe). Undo it, or the TAB is imported. + for (const field of FORMULA_SAFE_FIELDS) { + const value = user[field]; + if (typeof value === "string") user[field] = unprefix(value); + else if (Array.isArray(value)) + user[field] = value.map((v) => (typeof v === "string" ? unprefix(v) : v)); + } + }, } satisfies SourceEntry; +const FORMULA_SAFE_FIELDS = [ + "firstName", + "lastName", + "username", + "email", + "emailAddresses", + "unverifiedEmailAddresses", +] as const; + +const unprefix = (value: string) => value.replace(/^\t(?=[=+\-@\t\r\n＝＋－＠])/, ""); + export default clerkSource; diff --git a/packages/cli-core/src/commands/migrate/sources/firebase.ts b/packages/cli-core/src/commands/migrate/sources/firebase.ts index d70e755b6..fa574b627 100644 --- a/packages/cli-core/src/commands/migrate/sources/firebase.ts +++ b/packages/cli-core/src/commands/migrate/sources/firebase.ts @@ -1,6 +1,3 @@ -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; import { readJsonFile } from "../lib/export-file.ts"; import type { PreTransformResult, SourceEntry } from "../types.ts"; @@ -38,17 +35,9 @@ const firebaseSource = { "Works with `firebase auth:export` (CSV or JSON). Requires the project's four password hash parameters to migrate passwords.", preTransform: (filePath: string, fileType: string): PreTransformResult => { - if (fileType === "text/csv") { - // Written to the OS temp dir rather than the user's cwd: this is a - // parsing artifact, not a migration output like ./logs. - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-firebase-")); - const withHeaders = path.join(tmpDir, path.basename(filePath)); - fs.writeFileSync( - withHeaders, - `${FIREBASE_CSV_HEADERS}\n${fs.readFileSync(filePath, "utf-8")}`, - ); - return { filePath: withHeaders }; - } + // The CSV has no header row; name the columns rather than writing a copy + // with one, which would leave the hashes and salts in a temp file. + if (fileType === "text/csv") return { filePath, csvHeaders: FIREBASE_CSV_HEADERS.split(",") }; if (fileType === "application/json") { const parsed = readJsonFile(filePath); diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index b7ed776de..8d38d0278 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -463,10 +463,18 @@ describe("firebase", () => { await expect(load("firebase", { records: [] })).rejects.toThrow(CliError); }); - test("prepends headers to a headerless CSV export", async () => { + // Named, not prepended: a copy with a header row would leave the hashes and + // salts in a temp file nobody deletes. + test("names the columns of a headerless CSV export, writing no copy", async () => { const csv = "fb9,a@x.dev,true,,,Ada Lovelace,,,,,,,,,,,,,,,,,,1704067200000,,,,,\n"; - const { users } = await load("firebase", csv, "csv"); - expect(users[0]).toMatchObject({ userId: "fb9", email: "a@x.dev", firstName: "Ada" }); + const mkdtemp = spyOn(fs, "mkdtempSync"); + try { + const { users } = await load("firebase", csv, "csv"); + expect(users[0]).toMatchObject({ userId: "fb9", email: "a@x.dev", firstName: "Ada" }); + expect(mkdtemp).not.toHaveBeenCalled(); + } finally { + mkdtemp.mockRestore(); + } }); test.each([ @@ -567,6 +575,15 @@ describe("supabase", () => { expect(user?.lastName).toBe("Lovelace"); }); + // A CSV carries the metadata as JSON text. + test("falls back to metadata given as JSON text, as a CSV carries it", () => { + const user = one("supabase", { + ...base, + raw_user_meta_data: JSON.stringify({ display_name: "Ada Lovelace" }), + }); + expect(user?.firstName).toBe("Ada"); + }); + test("prefers explicit name columns over metadata", () => { const user = one("supabase", { ...base, @@ -590,6 +607,54 @@ describe("supabase", () => { }); }); +// The Dashboard's CSV prefixes a TAB to a value a spreadsheet would run as a +// formula (clerk_go pkg/csvsafe). +describe("clerk", () => { + test.each([ + ["\t=Ada", "=Ada"], + ["\t@ada", "@ada"], + ["\t＋Ada", "＋Ada"], + ["\tAda", "\tAda"], + ])("first name %p imports as %p", (firstName, expected) => { + expect( + one("clerk", { id: "u1", primary_email_address: "a@x.dev", first_name: firstName }) + ?.firstName, + ).toBe(expected); + }); + + test("unprefixes each address in a list", () => { + const user = one("clerk", { + id: "u1", + primary_email_address: "a@x.dev", + unverified_email_addresses: "\t-b@x.dev", + }); + expect(user?.unverifiedEmailAddresses).toEqual(["-b@x.dev"]); + }); +}); + +// The CLI's own exports add these; reporting them as "Clerk won't store" on +// every import would be noise about the CLI itself. +describe("fields the CLI's own export adds", () => { + test.each([ + [ + "supabase", + { + id: "s1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01", + raw_app_meta_data: { providers: ["email"] }, + }, + ], + [ + "betterauth", + { user_id: "b1", email: "a@x.dev", email_verified: true, updated_at: "2024-01-01" }, + ], + ])("%s reports no unknown fields", async (key, record) => { + const { unknownFields } = await load(key, [record]); + expect(unknownFields).toEqual({}); + }); +}); + describe("invalid records", () => { const INVALID: [string, Record][] = [ ["auth0", { user_id: "a1" }], @@ -613,14 +678,6 @@ describe("invalid records", () => { expect(failures).toHaveLength(1); }, ); - - test.each(INVALID)("%s logs a malformed email rather than sending it", async (key, record) => { - const { users, validationFailed } = await load(key, [ - { ...record, ...identifierFor(key, "not-an-email") }, - ]); - expect(validationFailed).toBe(1); - expect(users).toHaveLength(0); - }); }); /** The per-platform source field that becomes a Clerk identifier. */ diff --git a/packages/cli-core/src/commands/migrate/sources/supabase.ts b/packages/cli-core/src/commands/migrate/sources/supabase.ts index 0b7501d5c..f7b1da190 100644 --- a/packages/cli-core/src/commands/migrate/sources/supabase.ts +++ b/packages/cli-core/src/commands/migrate/sources/supabase.ts @@ -68,6 +68,10 @@ const supabaseSource = { } } + // `export supabase` selects this for the provider checks, which read the + // raw rows; Clerk has no place for it, so it isn't a field "Clerk won't store". + delete user.raw_app_meta_data; + // A soft-deleted user is gone from the app; Supabase scrambles its email. if (user.deletedAt) user.skipReason = "deleted in Supabase"; delete user.deletedAt; @@ -86,8 +90,9 @@ const supabaseSource = { // A basic SQL export has no first_name/last_name columns; the name lives in // user metadata instead, under whichever key the provider happened to use. - if (!user.firstName && user.unsafeMetadata && typeof user.unsafeMetadata === "object") { - const meta = user.unsafeMetadata as Record; + // A CSV carries the metadata as JSON text, still unparsed at this point. + const meta = parseObject(user.unsafeMetadata); + if (!user.firstName && meta) { const displayName = stripDiscriminator(meta.display_name ?? meta.first_name ?? meta.name); if (displayName) { const parts = displayName.split(/\s+/); @@ -109,4 +114,18 @@ const supabaseSource = { }, } satisfies SourceEntry; +/** A metadata value as an object, parsing it when a CSV left it as JSON text. */ +function parseObject(value: unknown): Record | undefined { + if (typeof value === "string") { + try { + value = JSON.parse(value); + } catch { + return undefined; + } + } + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + export default supabaseSource; diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts index 7a3b354ed..44e7d5c62 100644 --- a/packages/cli-core/src/commands/migrate/types.ts +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -86,6 +86,8 @@ export type TransformContext = { export type PreTransformResult = { filePath: string; data?: Record[]; + /** Column names for a CSV with no header row of its own. */ + csvHeaders?: string[]; }; /** How much of one kind of data a source brings across. */ diff --git a/packages/cli-core/src/commands/migrate/undo.test.ts b/packages/cli-core/src/commands/migrate/undo.test.ts index f7e54f059..46a20d0e8 100644 --- a/packages/cli-core/src/commands/migrate/undo.test.ts +++ b/packages/cli-core/src/commands/migrate/undo.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { EXIT_CODE, type CliError } from "../../lib/errors.ts"; import { useCaptureLog } from "../../test/lib/stubs.ts"; import { latestUserLines, listRuns, readRun, startRun, type RunRecord } from "./lib/run-store.ts"; +import { keyInstanceId } from "./lib/target.ts"; import { undo } from "./undo.ts"; const captured = useCaptureLog(); @@ -149,6 +150,18 @@ describe("refusals, all exit 2 and delete nothing", () => { expect(deletes()).toHaveLength(0); }); + // The import ran while Clerk could not name the instance; the same key is + // the same instance once it can. + test("accepts a run recorded under this key's stand-in ID", async () => { + const record = importRun({ + target: { instanceId: keyInstanceId("sk_test_x"), env: "development" }, + }); + + await undo(record.id, withDir({ yes: true })); + + expect(deletes()).toHaveLength(2); + }); + test("no consent where nobody can be asked: the preview, then the command", async () => { const record = importRun(); diff --git a/packages/cli-core/src/commands/migrate/undo.ts b/packages/cli-core/src/commands/migrate/undo.ts index ee9424e1b..5abdc161d 100644 --- a/packages/cli-core/src/commands/migrate/undo.ts +++ b/packages/cli-core/src/commands/migrate/undo.ts @@ -41,7 +41,13 @@ import { type RunRecord, } from "./lib/run-store.ts"; import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; -import { describeTarget, printTarget, resolveClerkTarget, type ClerkTarget } from "./lib/target.ts"; +import { + describeTarget, + keyInstanceId, + printTarget, + resolveClerkTarget, + type ClerkTarget, +} from "./lib/target.ts"; import { findInFlight, lookupUsers } from "./lib/user-lookup.ts"; export type UndoOptions = { @@ -97,8 +103,11 @@ function readImportRun(runsDir: string, runId: string): RunRecord { } /** Refuses to delete from an instance the import did not write to. */ -function assertSameInstance(record: RunRecord, target: ClerkTarget): void { +function assertSameInstance(record: RunRecord, target: ClerkTarget, secretKey: string): void { if (record.target.instanceId === target.instanceId) return; + // Recorded under the key's stand-in ID while Clerk could not name the + // instance: the same key is the same instance. + if (record.target.instanceId === keyInstanceId(secretKey)) return; // A `key_` ID is the fallback for an instance lookup that failed. It cannot // be compared with an `ins_` ID, so this is "unknown", not "different". const unconfirmed = [record.target.instanceId, target.instanceId].some((id) => @@ -314,12 +323,12 @@ function jsonResult( } export async function undo(runId: string, options: UndoOptions = {}): Promise { - const runsDir = await resolveRunsDir(options.runsDir, { write: !options.dryRun }); + const runsDir = await resolveRunsDir(options.runsDir); const record = readImportRun(runsDir, runId); const { secretKey, target } = await resolveClerkTarget(options); if (!options.json) printTarget(target); - assertSameInstance(record, target); + assertSameInstance(record, target, secretKey); const limits = resolveLimits(secretKey); const openUndo = findOpenUndo(runsDir, record.id); @@ -390,6 +399,8 @@ export async function undo(runId: string, options: UndoOptions = {}): Promise { describe("userSchema field types", () => { const FIELD_CASES = [ ["valid email", { email: "a@example.com" }, true], - ["malformed email", { email: "not-an-email" }, false], - ["email array with one bad entry", { email: ["a@example.com", "nope"] }, false], + // The checks drop an address Clerk would refuse; the schema only wants a string. + ["malformed email", { email: "not-an-email" }, true], + ["email array with one bad entry", { email: ["a@example.com", "nope"] }, true], + ["email that is not a string", { email: 42 }, false], ["userId missing", { userId: undefined }, false], ["valid createdAt", { createdAt: "2024-01-01T00:00:00Z" }, true], ["unparseable createdAt", { createdAt: "yesterday" }, false], diff --git a/packages/cli-core/src/commands/migrate/validator.ts b/packages/cli-core/src/commands/migrate/validator.ts index ccfbccab2..cdc709d01 100644 --- a/packages/cli-core/src/commands/migrate/validator.ts +++ b/packages/cli-core/src/commands/migrate/validator.ts @@ -36,9 +36,11 @@ export const userSchema = z .object({ userId: z.string(), // Email fields - email: z.union([z.email(), z.array(z.email())]).optional(), - emailAddresses: z.union([z.email(), z.array(z.email())]).optional(), - unverifiedEmailAddresses: z.union([z.email(), z.array(z.email())]).optional(), + // Strings, not z.email(): an address Clerk would refuse is dropped by the + // checks, with a warning, rather than taking the whole user down. + email: z.union([z.string(), z.array(z.string())]).optional(), + emailAddresses: z.union([z.string(), z.array(z.string())]).optional(), + unverifiedEmailAddresses: z.union([z.string(), z.array(z.string())]).optional(), // Phone fields phone: z.union([z.string(), z.array(z.string())]).optional(), phoneNumbers: z.union([z.string(), z.array(z.string())]).optional(), From fdd497414217b96b51f89a8e9af7d8b030d8fe11 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 16:19:58 -0400 Subject: [PATCH 090/141] fix(migrate): offer no fix for a Supabase provider Clerk doesn't offer The checks offered "Enable Figma sign-in" for providers Clerk has no connection for, which no config patch can turn on. Name those as not offered by Clerk in rejects, warnings and readiness, and offer no fix. The list of Clerk's built-in OAuth strategies is copied from clerk_go's api/shared/sso/oauth.go, since nothing the CLI can call lists them. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 4 +- .../src/commands/migrate/lib/checks.test.ts | 20 ++++++++ .../src/commands/migrate/lib/checks.ts | 23 ++++++--- .../src/commands/migrate/lib/clerk-config.ts | 49 +++++++++++++++++++ .../commands/migrate/lib/modify-settings.ts | 7 +-- .../src/commands/migrate/lib/readiness.ts | 7 ++- 6 files changed, 97 insertions(+), 13 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 964b374c6..6e435a367 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -656,7 +656,9 @@ after them. They sort the users three ways: - its password is not the shape its hasher says (`bcrypt`, with a cost up to 15, `scrypt_firebase`, `argon2i`/`argon2id` and `scrypt_werkzeug` are checked; other hashers are not, and Clerk refuses a bad one at create) - - Supabase: its only provider is not enabled in Clerk + - Supabase: its only providers are ones Clerk has off, or doesn't offer at all + (Figma, Kakao, Keycloak, WorkOS, Zoom, Fly). The checks offer to turn on + the first kind; nothing can turn on the second - the instance already has a user with its source ID, email, phone or username (a batched `GET /v1/users` lookup, 100 values a request, through the scheduler). A user a continued run found behind its own interrupted diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index c5ebc82f3..858a609a1 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -300,6 +300,26 @@ describe("rejects", () => { ).toEqual({ "only-discord": "only signs in with Discord, which is not enabled in Clerk" }); }); + // Clerk can't turn on a provider it doesn't offer, so none is suggested. + test("a provider Clerk doesn't offer is named as such, with no fix", async () => { + const rows = [ + { id: "figma", raw_app_meta_data: { providers: ["figma"] } }, + { id: "both", raw_app_meta_data: { providers: ["discord", "figma"] } }, + ]; + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true } }), + supabaseRows: rows, + users: [user("figma"), user("both")], + }), + ); + expect(Object.fromEntries(checks.rejects.map((r) => [r.sourceId, r.reason]))).toEqual({ + figma: "only signs in with Figma, which is not offered by Clerk", + both: "only signs in with Discord (not enabled in Clerk), Figma (not offered by Clerk)", + }); + expect(checks.fixes.map((fix) => fix.label)).not.toContain("Enable Figma sign-in"); + }); + // Each reject names that user's providers, not every disabled one in the file. test("a supabase reject names only that user's own providers", async () => { const rows = [ diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index 4a04ef18f..a384588ac 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -23,7 +23,12 @@ import { isEnabled, isRequired, type AttributeName } from "../../users/interacti import { splitIdentifiers } from "../import-users.ts"; import type { User } from "../types.ts"; import { analyzeFields, hasValue } from "./analysis.ts"; -import { enabledSocialProviders, providerLabel, toClerkStrategy } from "./clerk-config.ts"; +import { + clerkOffersProvider, + enabledSocialProviders, + providerLabel, + toClerkStrategy, +} from "./clerk-config.ts"; import { resolveDevUserLimit } from "./instance.ts"; import { buildChangePayload, buildSettingChanges } from "./modify-settings.ts"; import { buildReadinessReport } from "./readiness.ts"; @@ -576,11 +581,15 @@ function findDisabledProviderRejects(input: CheckInput): Map { const rowsById = new Map(input.supabaseRows.map((row) => [String(row.id), row])); for (const id of excludedIds) { const own = getUserProviders(rowsById.get(id) ?? {}).filter((p) => disabled.includes(p)); - const names = own.map(providerLabel).join(", "); - reasons.set( - id, - `only signs in with ${names}, which ${own.length === 1 ? "is" : "are"} not enabled in Clerk`, - ); + // Clerk can't turn on a provider it doesn't offer, so say which is which. + const why = (provider: string) => + clerkOffersProvider(provider) ? "not enabled in Clerk" : "not offered by Clerk"; + const kinds = new Set(own.map(why)); + const names = + kinds.size === 1 + ? `${own.map(providerLabel).join(", ")}, which ${own.length === 1 ? "is" : "are"} ${[...kinds][0]}` + : own.map((provider) => `${providerLabel(provider)} (${why(provider)})`).join(", "); + reasons.set(id, `only signs in with ${names}`); } return reasons; } @@ -621,7 +630,7 @@ function buildWarnings(input: CheckInput, importable: User[]): string[] { } else { warnings.push( item.section === "social" - ? `${plural(item.userCount, "user")} signed in with ${item.label}, which is not enabled in Clerk` + ? `${plural(item.userCount, "user")} signed in with ${item.label}, which ${clerkOffersProvider(item.key) ? "is not enabled in Clerk" : "Clerk doesn't offer"}` : `${plural(item.userCount, "user")} ${item.userCount === 1 ? "has" : "have"} a ${item.label.toLowerCase()}, which this instance is not set up to store`, ); } diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts index a14fa0fb2..6e05eaf10 100644 --- a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -35,6 +35,55 @@ export function toClerkStrategy(provider: string): string { return CLERK_STRATEGY_ALIASES[provider] ?? `oauth_${provider}`; } +/** + * The OAuth strategies Clerk has built in: clerk_go's registration in + * `api/shared/sso/oauth.go`, less the customer-specific ones. Nothing the CLI + * can call lists them, and an instance's settings name only the providers it + * has configured. + * + * ponytail: a copy of that list; check it there when Clerk adds a provider. + */ +const CLERK_OAUTH_STRATEGIES = new Set([ + "oauth_agentid", + "oauth_apple", + "oauth_atlassian", + "oauth_bitbucket", + "oauth_box", + "oauth_coinbase", + "oauth_discord", + "oauth_dropbox", + "oauth_facebook", + "oauth_github", + "oauth_gitlab", + "oauth_google", + "oauth_hubspot", + "oauth_huggingface", + "oauth_instagram", + "oauth_line", + "oauth_linear", + "oauth_linkedin", + "oauth_linkedin_oidc", + "oauth_microsoft", + "oauth_notion", + "oauth_slack", + "oauth_spotify", + "oauth_tiktok", + "oauth_twitch", + "oauth_twitter", + "oauth_vercel", + "oauth_x", + "oauth_xero", +]); + +/** + * True when Clerk offers a Supabase provider at all. One it doesn't (Figma, + * Kakao, Keycloak, WorkOS, Zoom, Fly) can't be turned on, so the checks offer + * no fix for it. + */ +export function clerkOffersProvider(provider: string): boolean { + return CLERK_OAUTH_STRATEGIES.has(toClerkStrategy(provider)); +} + /** Human label for a provider key, for report output. */ export function providerLabel(provider: string): string { const special: Record = { diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts index 0ad4f1915..6ed1cc506 100644 --- a/packages/cli-core/src/commands/migrate/lib/modify-settings.ts +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts @@ -18,7 +18,7 @@ * it and still points at the dashboard. */ -import { toClerkStrategy } from "./clerk-config.ts"; +import { clerkOffersProvider, toClerkStrategy } from "./clerk-config.ts"; import type { ReadinessItem, ReadinessSection } from "./readiness.ts"; /** One leaf of the config document, and what to set it to. */ @@ -94,8 +94,9 @@ function changeFor(item: ReadinessItem): SettingChange | undefined { const relax = item.clerkRequired === true; if (item.section === "social") { - // A provider has no "required" in Clerk, so there is nothing to relax. - if (relax) return undefined; + // A provider has no "required" in Clerk, so there is nothing to relax, and + // one Clerk doesn't offer has nothing to turn on. + if (relax || !clerkOffersProvider(item.key)) return undefined; return { id: item.key, label: `Enable ${item.label} sign-in`, diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts index c6edf4b01..bd445405d 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -12,7 +12,7 @@ import type { UserSettingsJSON } from "../../../lib/fapi.ts"; // Pure attribute lookups, shared with the `users` create wizard. import { isEnabled, isRequired, type AttributeName } from "../../users/interactive/attributes.ts"; import type { FieldAnalysis } from "./analysis.ts"; -import { providerLabel, toClerkStrategy } from "./clerk-config.ts"; +import { clerkOffersProvider, providerLabel, toClerkStrategy } from "./clerk-config.ts"; export type ReadinessSection = "identifiers" | "auth" | "social" | "model"; @@ -194,7 +194,10 @@ export function buildReadinessReport(input: BuildInput): ReadinessReport { clerkRequired: null, blocking: enabled === false, ...(enabled === false - ? { consequence: "drops" as const, detail: "not enabled in Clerk" } + ? { + consequence: "drops" as const, + detail: clerkOffersProvider(provider) ? "not enabled in Clerk" : "not offered by Clerk", + } : {}), }); } From 033a28c50e51759d22c8222bbc26da653c8d7f47 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 16:20:42 -0400 Subject: [PATCH 091/141] docs(changeset): Add `clerk migrate` for moving users into Clerk from Clerk, Auth0, Supabase, Auth.js, Better Auth, Firebase or WorkOS. Rename the changeset after the branch, and cover the user-visible changes outside migrate as well. Co-Authored-By: Claude Opus 5.5 --- .changeset/integrate-migration-tool-into-cli.md | 12 ++++++++++++ .changeset/migrate-cli.md | 5 ----- 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 .changeset/integrate-migration-tool-into-cli.md delete mode 100644 .changeset/migrate-cli.md diff --git a/.changeset/integrate-migration-tool-into-cli.md b/.changeset/integrate-migration-tool-into-cli.md new file mode 100644 index 000000000..ca6e9c252 --- /dev/null +++ b/.changeset/integrate-migration-tool-into-cli.md @@ -0,0 +1,12 @@ +--- +"clerk": minor +--- + +Add `clerk migrate` for moving users into Clerk from Clerk, Auth0, Supabase, Auth.js, Better Auth, Firebase or WorkOS. + +- `migrate export ` writes a self-describing file. `migrate import ` checks every user against the instance before writing (`--dry-run`, `--allow-partial`, `--skip-legal-checks`), asks before it writes, and continues where an interrupted or partial run stopped. +- `migrate runs` shows what every run did, `migrate undo ` deletes the users an import created, and `migrate sources` shows what each source carries, including sources you write yourself (`--source ./my-source.ts`). +- A request that cannot connect now names the host it could not reach. +- `clerk init` warns when the project uses WorkOS, and points to the migration guide. +- Multiselect prompts list `a: all` in their key legend. +- `clerk users` dry runs name the instance the key reaches and where the key came from, such as `--app` or the `CLERK_SECRET_KEY` env var. diff --git a/.changeset/migrate-cli.md b/.changeset/migrate-cli.md deleted file mode 100644 index dcb4b8f29..000000000 --- a/.changeset/migrate-cli.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"clerk": minor ---- - -Add `clerk migrate` for moving users into Clerk. `migrate export ` exports users from Clerk, Auth0, Supabase, Auth.js, Better Auth, Firebase or WorkOS into a self-describing file. `migrate import ` checks every user against the instance before writing (`--dry-run`, `--allow-partial`), asks before it writes, and continues where a previous run stopped. `migrate runs` shows what every run did, `migrate undo ` deletes the users an import created, and `migrate sources` shows what each source carries, including sources you write yourself (`--source ./my-source.ts`). From b59cbbfb97890284b948d4b1732bfafe21128230 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 16:26:38 -0400 Subject: [PATCH 092/141] test(migrate): read each user's latest run line in the e2e test A user's first line has been `creating` since f108dc0a; the test read that line and expected `created`. Read the last line per source ID, as `runs` and `undo` do. Co-Authored-By: Claude Opus 5.5 --- test/e2e/migrate.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/e2e/migrate.test.ts b/test/e2e/migrate.test.ts index 5ee20f66b..226a85e0e 100644 --- a/test/e2e/migrate.test.ts +++ b/test/e2e/migrate.test.ts @@ -64,7 +64,11 @@ afterAll(async () => { rmSync(workDir, { recursive: true, force: true }); }, 60_000); -/** Imports `users` as a Better Auth export and returns each user's run line. */ +/** + * Imports `users` as a Better Auth export and returns each user's latest run + * line. A user has several (`creating`, then `created`); the last one wins, + * as it does for `runs` and `undo`. + */ async function importBetterAuth(users: Record[], extra: string[] = []) { const file = join(workDir, `betterauth-${randomBytes(4).toString("hex")}.json`); writeFileSync(file, JSON.stringify(users)); @@ -80,10 +84,14 @@ async function importBetterAuth(users: Record[], extra: string[ if (!runId) throw new Error("The import recorded no run."); importRuns.push(runId); - return readFileSync(join(runsDir, runId, "users.ndjson"), "utf-8") + const latest = new Map>(); + for (const line of readFileSync(join(runsDir, runId, "users.ndjson"), "utf-8") .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record); + .split("\n")) { + const parsed = JSON.parse(line) as Record; + latest.set(parsed.sourceId, parsed); + } + return [...latest.values()]; } /** A password hashed exactly the way Better Auth's default hasher does it. */ From b0705144d447901fad4ed111c97952b6bcc3346b Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 16:48:49 -0400 Subject: [PATCH 093/141] test(e2e): don't load .env files into the e2e run A .env.local pointing the CLI at a local clerk_go stack (the documented dev setup) sent the production test secrets to the local Platform and Backend APIs, failing every live test with "Failed to resolve secret key" or "clerk_key_invalid". CI sets its env directly, so it is unaffected. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a149d3b10..d1c548a47 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,7 +48,7 @@ bun run test:e2e:op # Run E2E tests with secrets resolved from 1Password (prefe bun run test:e2e # Run E2E tests with env vars already set (used by CI) ``` -Locally, prefer `bun run test:e2e:op` so secrets are injected from 1Password in-memory and never written to disk. `bun run test:e2e` is for CI or for cases where the required env vars are already exported. +Locally, prefer `bun run test:e2e:op` so secrets are injected from 1Password in-memory and never written to disk. `bun run test:e2e` is for CI or for cases where the required env vars are already exported. Both run `bun test` with `--no-env-file`, so a `.env.local` pointing the CLI at a local `clerk_go` stack can't send the production test secrets there. CI runs `bun run format:check` (fails if unformatted), `bun run lint`, `bun run test`, and `bun run test:e2e` on every PR to `main`. E2E tests only run for PRs from the same repository (not external forks) and target the production Clerk API with a dedicated test application. diff --git a/package.json b/package.json index cedddf21e..b4295be32 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "build": "bun run build:compile", "dev": "bun run --cwd packages/cli-core dev", "test": "bun run scripts/check-bun-version.ts && bun test 'packages/cli-core/src/' 'packages/extras/src/' 'scripts/' --parallel --only-failures", - "test:e2e": "bun run scripts/check-bun-version.ts && bun test 'test/e2e/' --retry 1 --parallel --only-failures", + "test:e2e": "bun run scripts/check-bun-version.ts && bun --no-env-file test 'test/e2e/' --retry 1 --parallel --only-failures", "test:e2e:op": "bun run scripts/run-e2e-op.ts", "e2e:refresh-fixtures": "bun run scripts/refresh-e2e-fixtures.ts", "e2e:audit-fixtures": "bun run scripts/audit-e2e-fixtures.ts", From baba0909f71e49cd1632e316634091cdc897aa20 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 17:30:00 -0400 Subject: [PATCH 094/141] fix(migrate): read Better Auth schemas Drizzle generated Better Auth's Drizzle generator writes snake_case columns (email_verified, user_id, provider_id) unless the project sets camelCase: true, and usePlural: true pluralizes the tables. The export selected camelCase columns from `user` and `account` only, so it failed on those projects with a missing-column error. Read the user table's columns first, pick the spelling and table names the database uses, and alias each column back to camelCase so nothing after the query changes. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 5 +- .../src/commands/migrate/export/betterauth.ts | 108 ++++++++++++------ .../migrate/export/db-exports.test.ts | 51 +++++++-- 3 files changed, 121 insertions(+), 43 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 6e435a367..a318c6ec5 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -374,7 +374,10 @@ The verified column is read as `emailVerified`, or `email_verified` on a legacy NextAuth table. Auth.js core stores no passwords, so its users arrive without credentials. -**`betterauth` detects its plugin columns from the schema.** The username +**`betterauth` reads its schema before it queries.** It finds the tables +(`user` and `account`, or `users` and `accounts` under `usePlural: true`), how +the columns are spelled (camelCase, or the snake_case Better Auth's Drizzle +generator writes by default), and which plugin columns exist. The username plugin adds `username`, admin adds `banned` (carried only while `banExpires` is unset or in the future), phone-number adds `phoneNumber`, and so on; selecting a column that is not there fails the whole query, and the database answers the question better than the user can. Passwords come from a diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index cdcaf09d1..791f95462 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -49,64 +49,102 @@ export type PluginColumn = (typeof PLUGIN_COLUMNS)[number]; /** Columns every Better Auth install has. */ const CORE_COLUMNS = ["id", "email", "emailVerified", "name", "createdAt", "updatedAt"] as const; +/** How one Better Auth database names its tables and columns. */ +export type BetterAuthSchema = { + userTable: string; + accountTable: string; + /** + * The column for a field. Better Auth's Drizzle generator writes snake_case + * (`email_verified`) unless the project sets `camelCase: true`; its Kysely + * and Prisma setups keep camelCase. + */ + column: (field: string) => string; + /** The plugin columns this database has. */ + plugins: Set; +}; + +/** `emailVerified` → `email_verified`, as Better Auth's Drizzle generator does it. */ +function toSnakeCase(field: string): string { + return field + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2") + .replace(/([a-z\d])([A-Z])/g, "$1_$2") + .toLowerCase(); +} + /** - * Asks the schema which plugin columns exist. + * Every column of a table, or an empty set when there is no such table. * * SQLite has no `information_schema`, so it goes through `PRAGMA` — and the * PRAGMA takes the table name inline rather than as a bind parameter. */ -export async function detectPluginColumns(client: DbClient): Promise> { - const present = new Set(); - +async function tableColumns(client: DbClient, table: string): Promise> { if (client.dbType === "sqlite") { - const rows = await client.query<{ name: string }>(`PRAGMA table_info(${client.quote("user")})`); - const columns = new Set(rows.map((row) => row.name)); - for (const column of PLUGIN_COLUMNS) { - if (columns.has(column)) present.add(column); - } - return present; + const rows = await client.query<{ name: string }>(`PRAGMA table_info(${client.quote(table)})`); + return new Set(rows.map((row) => row.name)); } - const scope = client.dbType === "mysql" ? "DATABASE()" : "current_schema()"; - const placeholders = PLUGIN_COLUMNS.map((_, index) => client.placeholder(index + 1)).join(", "); - const rows = await client.query<{ column_name?: string; COLUMN_NAME?: string }>( `SELECT column_name FROM information_schema.columns - WHERE table_name = 'user' AND table_schema = ${scope} - AND column_name IN (${placeholders})`, - [...PLUGIN_COLUMNS], + WHERE table_name = ${client.placeholder(1)} AND table_schema = ${scope}`, + [table], ); + // MySQL 8 answers with an upper-case column label. + return new Set(rows.map((row) => row.column_name ?? row.COLUMN_NAME ?? "")); +} - for (const row of rows) { - // MySQL 8 answers with an upper-case column label. - const name = (row.column_name ?? row.COLUMN_NAME) as PluginColumn | undefined; - if (name && (PLUGIN_COLUMNS as readonly string[]).includes(name)) present.add(name); - } +/** Table names to try, in order: Better Auth's default, then `usePlural: true`. */ +const TABLE_CANDIDATES = [ + ["user", "account"], + ["users", "accounts"], +] as const; - return present; +/** + * Asks the database how it names Better Auth's tables and columns, and which + * plugin columns it has. Selecting a column that is not there fails the whole + * query, so nothing is assumed. + */ +export async function detectSchema(client: DbClient): Promise { + for (const [userTable, accountTable] of TABLE_CANDIDATES) { + const columns = await tableColumns(client, userTable); + if (columns.size === 0) continue; + const snake = !columns.has("emailVerified") && columns.has("email_verified"); + const column = (field: string) => (snake ? toSnakeCase(field) : field); + const plugins = new Set(PLUGIN_COLUMNS.filter((field) => columns.has(column(field)))); + return { userTable, accountTable, column, plugins }; + } + // No table found: the query names the default, and its "no such table" + // error carries the hint. + return { + userTable: "user", + accountTable: "account", + column: (field) => field, + plugins: new Set(), + }; } /** - * Builds the SELECT, including only the plugin columns that exist. + * Builds the SELECT, including only the plugin columns that exist. Each column + * comes back under its camelCase name, however the database spells it. * - * @param pluginColumns - From {@link detectPluginColumns}. + * @param schema - From {@link detectSchema}. */ -export function buildBetterAuthQuery(client: DbClient, pluginColumns: Set): string { +export function buildBetterAuthQuery(client: DbClient, schema: BetterAuthSchema): string { const q = (identifier: string) => client.quote(identifier); + const { column } = schema; + const select = (field: string) => + column(field) === field ? `u.${q(field)}` : `u.${q(column(field))} AS ${q(field)}`; const selected = [ - ...CORE_COLUMNS.map((column) => `u.${q(column)}`), - ...PLUGIN_COLUMNS.filter((column) => pluginColumns.has(column)).map( - (column) => `u.${q(column)}`, - ), + ...CORE_COLUMNS.map(select), + ...PLUGIN_COLUMNS.filter((field) => schema.plugins.has(field)).map(select), ]; // LEFT JOIN, not INNER: a user who only ever signed in with OAuth has no // credential account, and dropping them would silently shrink the export. return ( `SELECT ${selected.join(", ")}, a.${q("password")} AS ${q("password_hash")} ` + - `FROM ${q("user")} u ` + - `LEFT JOIN ${q("account")} a ON a.${q("userId")} = u.${q("id")} ` + - `AND a.${q("providerId")} = 'credential' ` + + `FROM ${q(schema.userTable)} u ` + + `LEFT JOIN ${q(schema.accountTable)} a ON a.${q(column("userId"))} = u.${q("id")} ` + + `AND a.${q(column("providerId"))} = 'credential' ` + `ORDER BY u.${q("id")} ASC` ); } @@ -184,9 +222,9 @@ export async function exportBetterAuth(options: DbExportOptions): Promise async (connectionString) => withSpinner("Reading the user table...", async () => withDbClient(connectionString, "betterauth", async (client) => { - const plugins = await detectPluginColumns(client); - const rows = await client.query(buildBetterAuthQuery(client, plugins)); - return { rows, plugins }; + const schema = await detectSchema(client); + const rows = await client.query(buildBetterAuthQuery(client, schema)); + return { rows, plugins: schema.plugins }; }), ), ); diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 0225b016b..dca99cf28 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -20,7 +20,7 @@ import { buildAuthJsExport, buildAuthJsQuery, exportAuthJs, fetchAuthJsUsers } f import { buildBetterAuthExport, buildBetterAuthQuery, - detectPluginColumns, + detectSchema, exportBetterAuth, PLUGIN_COLUMNS, } from "./betterauth.ts"; @@ -272,19 +272,19 @@ describe("authjs export", () => { describe("betterauth export", () => { test("detects only the plugin columns that exist", async () => { await withClient(betterAuthDb(["username", "banned"]), async (client) => { - expect([...(await detectPluginColumns(client))].sort()).toEqual(["banned", "username"]); + expect([...(await detectSchema(client)).plugins].sort()).toEqual(["banned", "username"]); }); }); test("detects nothing on a core-only schema", async () => { await withClient(betterAuthDb([]), async (client) => { - expect((await detectPluginColumns(client)).size).toBe(0); + expect((await detectSchema(client)).plugins.size).toBe(0); }); }); test("detects every plugin column when all are present", async () => { await withClient(betterAuthDb([...PLUGIN_COLUMNS]), async (client) => { - expect((await detectPluginColumns(client)).size).toBe(PLUGIN_COLUMNS.length); + expect((await detectSchema(client)).plugins.size).toBe(PLUGIN_COLUMNS.length); }); }); @@ -292,7 +292,7 @@ describe("betterauth export", () => { // the columns are detected rather than assumed. test("selects only detected columns", async () => { await withClient(betterAuthDb(["username"]), async (client) => { - const query = buildBetterAuthQuery(client, await detectPluginColumns(client)); + const query = buildBetterAuthQuery(client, await detectSchema(client)); expect(query).toContain('"username"'); expect(query).not.toContain('"twoFactorEnabled"'); }); @@ -304,7 +304,7 @@ describe("betterauth export", () => { [{ id: "u1", email: "a@x.dev", username: "a" }], ); const rows = await withClient(file, async (client) => - client.query(buildBetterAuthQuery(client, await detectPluginColumns(client))), + client.query(buildBetterAuthQuery(client, await detectSchema(client))), ); expect(rows).toHaveLength(1); }); @@ -320,11 +320,48 @@ describe("betterauth export", () => { ], ); const rows = await withClient(file, async (client) => - client.query(buildBetterAuthQuery(client, new Set())), + client.query(buildBetterAuthQuery(client, await detectSchema(client))), ); expect(rows).toHaveLength(2); }); + // Better Auth's Drizzle generator writes snake_case unless `camelCase: true`, + // and `usePlural: true` pluralizes the tables. + test.each([ + ["snake_case columns", "user", "account"], + ["snake_case columns and plural tables", "users", "accounts"], + ])("reads a Drizzle schema with %s", async (_label, userTable, accountTable) => { + const file = makeDb((db) => { + db.run( + `CREATE TABLE "${userTable}" (id TEXT PRIMARY KEY, email TEXT, email_verified INTEGER, name TEXT, + created_at TEXT, updated_at TEXT, ban_expires TEXT, banned INTEGER)`, + ); + db.run( + `CREATE TABLE "${accountTable}" (id TEXT, user_id TEXT, provider_id TEXT, password TEXT)`, + ); + db.run( + `INSERT INTO "${userTable}" (id, email, email_verified, banned) VALUES ('u1', 'a@x.dev', 1, 1)`, + ); + db.run(`INSERT INTO "${accountTable}" VALUES ('a1', 'u1', 'credential', 'salt:hash')`); + }); + + const { rows, schema } = await withClient(file, async (client) => { + const detected = await detectSchema(client); + return { rows: await client.query(buildBetterAuthQuery(client, detected)), schema: detected }; + }); + + expect([...schema.plugins].sort()).toEqual(["banExpires", "banned"]); + // Read back under the camelCase names the rest of the export expects. + expect(rows).toEqual([ + expect.objectContaining({ + id: "u1", + emailVerified: 1, + banned: 1, + password_hash: "salt:hash", + }), + ]); + }); + test("renames camelCase columns onto what the transformer reads", () => { const { users } = buildBetterAuthExport([ { id: "u1", emailVerified: 1, phoneNumber: "+1555", createdAt: "2025-01-01" }, From d620193261e06915faa26172e9b0465a9780804d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 2 Oct 2026 17:38:11 -0400 Subject: [PATCH 095/141] fix(migrate): exit 130 when Ctrl-C stops an import or undo A clack spinner puts stdin in raw mode and calls process.exit(0) on Ctrl-C, so an interrupted import or undo exited as a success, and a script that deleted the export once the import succeeded deleted it. Import and undo now show a bar with the counts under it instead, which leaves stdin alone: Ctrl-C reaches the CLI's handler, in-flight requests abort, and the process dies by SIGINT (130). The bar fills the line up to 80 columns, the report under it adds an estimate of the time left, and without a terminal the report is printed at each 10%. Other spinners are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 15 +- .../src/commands/migrate/import-users.ts | 12 +- .../src/commands/migrate/lib/progress.test.ts | 108 +++++++++++++++ .../src/commands/migrate/lib/progress.ts | 131 ++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 9 +- .../cli-core/src/commands/migrate/undo.ts | 14 +- 6 files changed, 269 insertions(+), 20 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/progress.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/progress.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index a318c6ec5..25bf72494 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -44,9 +44,22 @@ Every command follows these: 4. **Every command prints its target first:** the environment, app and instance, and where the key came from. 5. **Every subcommand takes `--json`.** Exit codes: `0` all good, `1` some users - failed, `2` a usage error or a refusal. The UI goes to stderr and data to + failed, `2` a usage error or a refusal, and `130` (death by SIGINT) when + Ctrl-C stops an import or an undo partway. The UI goes to stderr and data to stdout. + While users are created or deleted, a terminal shows a bar and the counts + under it, not a spinner: + + ``` + │ ██████████████████████████████████████████████████████░░░░░░░░░░░░░░░░░░ 75% + │ 7,500/10,000 users · ✓ 7,425 created · ✗ 75 failed · ~25s left + ``` + + A spinner takes the keyboard and exits 0 on Ctrl-C, so a script that deletes + the export once the import succeeds would delete it after an interrupted + one. Without a terminal, the counts are printed at each 10%. + ## Targeting And Auth `clerk migrate import` resolves its Backend API key through the CLI's standard diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 2cb87b67a..533dd7938 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -19,8 +19,8 @@ import { bapiRequest } from "../../lib/bapi.ts"; import { BapiError } from "../../lib/errors.ts"; import { interruptSignal } from "../../lib/signals.ts"; -import type { SpinnerControls } from "../../lib/spinner.ts"; import type { ResolvedLimits } from "./lib/instance.ts"; +import type { ProgressUpdate } from "./lib/progress.ts"; import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; import type { PendingIdentifier, UserLine } from "./lib/run-store.ts"; import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; @@ -342,7 +342,8 @@ export type ImportUsersOptions = { skipPasswordRequirement?: boolean; /** Carried into the summary so the report covers the whole file. */ validationFailed?: number; - spinner?: SpinnerControls; + /** Receives the counts as each user finishes. */ + progress?: ProgressUpdate; }; /** @@ -362,7 +363,7 @@ export async function importUsers(options: ImportUsersOptions): Promise(), skipPasswordRequirement = true, validationFailed = 0, - spinner, + progress: report, } = options; const total = users.length; @@ -376,10 +377,7 @@ export async function importUsers(options: ImportUsersOptions): Promise - spinner?.update( - `Importing users: [${processed}/${total}] (${successful} succeeded, ${failed} failed)...`, - ); + const progress = () => report?.({ done: processed, ok: successful, failed }); const recordFailure = ( userId: string, diff --git a/packages/cli-core/src/commands/migrate/lib/progress.test.ts b/packages/cli-core/src/commands/migrate/lib/progress.test.ts new file mode 100644 index 000000000..182fb2899 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/progress.test.ts @@ -0,0 +1,108 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { formatProgress, formatRemaining, withProgress } from "./progress.ts"; + +const captured = useCaptureLog(); + +describe("formatRemaining", () => { + test.each([ + [9_000, "~9s"], + [90_000, "~1m 30s"], + [900_000, "~15m"], + [500_000, "~8m 20s"], + [7_500_000, "~2h 5m"], + ])("%p ms -> %p", (ms, expected) => { + expect(formatRemaining(ms)).toBe(expected); + }); +}); + +describe("formatProgress", () => { + // 10,000 users at ~100 a second, 1% failing: the agreed mockup. + test("draws an 80-column bar with the report under it", () => { + const [bar, report] = formatProgress({ + total: 10_000, + verb: "created", + counts: { done: 7_500, ok: 7_425, failed: 75 }, + elapsedMs: 75_000, + }); + + expect(bar).toBe(`│ ${"█".repeat(54)}${"░".repeat(18)} 75%`); + expect([...bar]).toHaveLength(80); + expect(report).toBe("│ 7,500/10,000 users · ✓ 7,425 created · ✗ 75 failed · ~25s left"); + }); + + test("shrinks the bar to a narrower terminal", () => { + const [bar] = formatProgress({ + total: 100, + verb: "created", + counts: { done: 50, ok: 50, failed: 0 }, + elapsedMs: 0, + columns: 60, + }); + expect([...bar]).toHaveLength(60); + }); + + test.each([ + ["before there is a rate to go on", 1_000, 500], + ["once every user is done", 10_000, 1_000], + ])("gives no estimate %s", (_label, elapsedMs, done) => { + const [, report] = formatProgress({ + total: 1_000, + verb: "deleted", + counts: { done, ok: done, failed: 0 }, + elapsedMs, + }); + expect(report).not.toContain("left"); + }); + + // Floored, so 999 of 1,000 never reads as finished. + test("reads 100% only when every user is done", () => { + const [bar] = formatProgress({ + total: 1_000, + verb: "created", + counts: { done: 999, ok: 999, failed: 0 }, + elapsedMs: 0, + }); + expect(bar).toEndWith(" 99%"); + expect(bar).toContain("░"); + }); +}); + +describe("withProgress", () => { + let mode: Mode; + let isTTY: boolean | undefined; + + beforeEach(() => { + mode = getMode(); + isTTY = process.stderr.isTTY; + }); + + afterEach(() => { + setMode(mode); + process.stderr.isTTY = isTTY as boolean; + }); + + test("prints nothing for an agent", async () => { + setMode("agent"); + await withProgress({ total: 10, verb: "created" }, async (update) => { + update({ done: 10, ok: 10, failed: 0 }); + }); + expect(captured.err).toBe(""); + }); + + // A log file gets a line per 10%, not a redraw per user. + test("without a terminal, prints the report at each tenth", async () => { + setMode("human"); + process.stderr.isTTY = false; + + await withProgress({ total: 100, verb: "created" }, async (update) => { + for (let done = 1; done <= 100; done++) update({ done, ok: done, failed: 0 }); + }); + + const lines = captured.err.split("\n").filter((line) => line.includes("users")); + // 0% at the start, each tenth from 10% to 100%, and the final state. + expect(lines).toHaveLength(12); + expect(lines.at(-1)).toContain("100/100 users"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/progress.ts b/packages/cli-core/src/commands/migrate/lib/progress.ts new file mode 100644 index 000000000..19ed123f0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/progress.ts @@ -0,0 +1,131 @@ +/** + * The progress display for the import and undo loops: a bar on one line, the + * counts on the next. + * + * Used instead of a spinner on purpose. A clack spinner puts stdin in raw mode + * and calls `process.exit(0)` on Ctrl-C, so an interrupted import looked like + * a finished one and `clerk migrate import … && rm -rf ` carried on. + * This leaves stdin alone: Ctrl-C reaches the CLI's handler, in-flight + * requests abort, and the process exits 130 (`.claude/rules/interrupts.md`). + */ + +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { isHuman } from "../../../mode.ts"; + +export type ProgressCounts = { done: number; ok: number; failed: number }; + +/** Receives the counts as each user finishes. */ +export type ProgressUpdate = (counts: ProgressCounts) => void; + +type ProgressLine = { + total: number; + /** What happened to the users counted in `ok`: "created", "deleted". */ + verb: string; + counts: ProgressCounts; + elapsedMs: number; + /** Terminal columns; the bar fills the line up to 80. */ + columns?: number; +}; + +const MAX_WIDTH = 80; +/** No estimate until there is this much of a rate to go on. */ +const ESTIMATE_AFTER_MS = 2000; +/** At most one redraw per this many ms: an import can report 100 users a second. */ +const REDRAW_MS = 100; + +const GUTTER = "│ "; +const number = (value: number) => value.toLocaleString("en-US"); + +/** `~9s`, `~1m 30s`, `~15m`, `~2h 5m`. */ +export function formatRemaining(ms: number): string { + const seconds = Math.max(1, Math.round(ms / 1000)); + if (seconds < 60) return `~${seconds}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) { + const rest = seconds % 60; + return rest ? `~${minutes}m ${rest}s` : `~${minutes}m`; + } + const hours = Math.floor(minutes / 60); + const rest = minutes % 60; + return rest ? `~${hours}h ${rest}m` : `~${hours}h`; +} + +/** The bar line and the report line, without color. */ +export function formatProgress(line: ProgressLine): [string, string] { + const { total, verb, counts, elapsedMs } = line; + const width = Math.min(MAX_WIDTH, line.columns ?? MAX_WIDTH); + const fraction = total > 0 ? Math.min(1, counts.done / total) : 1; + + // Floored, so the bar reads full only when every user is done. + const percent = `${Math.floor(fraction * 100)}%`.padStart(4); + const cells = Math.max(10, width - GUTTER.length - 1 - percent.length); + const filled = Math.floor(cells * fraction); + const bar = `${GUTTER}${"█".repeat(filled)}${"░".repeat(cells - filled)} ${percent}`; + + const parts = [ + `${number(counts.done)}/${number(total)} users`, + `✓ ${number(counts.ok)} ${verb}`, + `✗ ${number(counts.failed)} failed`, + ]; + if (elapsedMs >= ESTIMATE_AFTER_MS && counts.done > 0 && counts.done < total) { + parts.push(`${formatRemaining(((total - counts.done) * elapsedMs) / counts.done)} left`); + } + return [bar, `${GUTTER}${parts.join(" · ")}`]; +} + +/** + * Runs `fn` with a progress display for `total` users. + * + * At a terminal, the two lines redraw in place. When stderr is not a terminal + * (a log file, CI), the report line is printed at each 10% instead. An agent + * gets nothing, as it got nothing from the spinner. + */ +export async function withProgress( + options: { total: number; verb: string }, + fn: (update: ProgressUpdate) => Promise, +): Promise { + if (!isHuman()) return fn(() => {}); + + const tty = Boolean(process.stderr.isTTY); + const started = Date.now(); + let counts: ProgressCounts = { done: 0, ok: 0, failed: 0 }; + let drawn = false; + let lastDraw = 0; + let lastTenth = -1; + + const draw = (force: boolean) => { + const now = Date.now(); + const [bar, report] = formatProgress({ + ...options, + counts, + elapsedMs: now - started, + ...(process.stderr.columns ? { columns: process.stderr.columns } : {}), + }); + + if (!tty) { + const tenth = Math.floor((counts.done / Math.max(1, options.total)) * 10); + if (!force && tenth === lastTenth) return; + lastTenth = tenth; + log.ui(`${report}\n`); + return; + } + + if (!force && now - lastDraw < REDRAW_MS) return; + lastDraw = now; + // Up over the two lines drawn last time, then clear and rewrite each. + const up = drawn ? "\x1b[2A" : ""; + // The gutter in the gray clack draws it in. + const gutter = (line: string) => `${dim("│")}${line.slice(1)}`; + log.ui(`${up}\r\x1b[2K${gutter(bar)}\n\x1b[2K${gutter(report)}\n`); + drawn = true; + }; + + draw(true); + const result = await fn((next) => { + counts = next; + draw(false); + }); + draw(true); + return result; +} diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 1481565be..2d7a32ec0 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -58,6 +58,7 @@ import { type RunRecord, type UserLine, } from "./lib/run-store.ts"; +import { withProgress } from "./lib/progress.ts"; import { createApiScheduler } from "./lib/scheduler.ts"; import { readSupabaseRows } from "./lib/supabase-providers.ts"; import { keyInstanceId, printTarget, resolveClerkTarget } from "./lib/target.ts"; @@ -843,9 +844,9 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const summary = checks.importable.length > 0 || attachOnly.length > 0 - ? await withSpinner( - `Importing users: [0/${checks.importable.length}]...`, - async (spinner) => + ? await withProgress( + { total: checks.importable.length, verb: "created" }, + async (progress) => importUsers({ users: checks.importable, secretKey, @@ -854,7 +855,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { attachOnly, adopted, skipPasswordRequirement: !options.requirePassword, - spinner, + progress, }), ) : { diff --git a/packages/cli-core/src/commands/migrate/undo.ts b/packages/cli-core/src/commands/migrate/undo.ts index 5abdc161d..c6660a033 100644 --- a/packages/cli-core/src/commands/migrate/undo.ts +++ b/packages/cli-core/src/commands/migrate/undo.ts @@ -40,6 +40,7 @@ import { type Run, type RunRecord, } from "./lib/run-store.ts"; +import { withProgress, type ProgressUpdate } from "./lib/progress.ts"; import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; import { describeTarget, @@ -212,19 +213,16 @@ async function deleteUsers(options: { secretKey: string; limits: ResolvedLimits; run: Run; - spinner?: SpinnerControls; + progress?: ProgressUpdate; }): Promise { - const { users, secretKey, limits, run, spinner } = options; + const { users, secretKey, limits, run, progress: report } = options; const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); const errorBreakdown = new Map(); let processed = 0; let deleted = 0; let failed = 0; - const progress = () => - spinner?.update( - `Deleting users: [${processed}/${users.length}] (${deleted} deleted, ${failed} failed)...`, - ); + const progress = () => report?.({ done: processed, ok: deleted, failed }); // A failure on one user must not stop the rest: a half-undone import with // no record of which half is far worse than a reported failure. @@ -413,8 +411,8 @@ export async function undo(runId: string, options: UndoOptions = {}): Promise 0 - ? await withSpinner(`Deleting users: [0/${present.length}]...`, async (spinner) => - deleteUsers({ users: present, secretKey, limits, run, spinner }), + ? await withProgress({ total: present.length, verb: "deleted" }, async (progress) => + deleteUsers({ users: present, secretKey, limits, run, progress }), ) : { deleted: 0, failed: 0, errorBreakdown: new Map() }; From fc2a1da4450182e01899fca27a49e5db36cf46ec Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 5 Oct 2026 14:33:18 -0400 Subject: [PATCH 096/141] docs: replace the stacked PR plan with the slice plan The slices merge to main one at a time, behind CLERK_EXPERIMENTAL=migrate until the last one, and port from 7a820406, which carries the review fixes and current main. Co-Authored-By: Claude Opus 5.5 --- slice-prs.md | 735 +++++++++++++++++++++++++++++++++++++++++++++++++ stacked-prs.md | 176 ------------ 2 files changed, 735 insertions(+), 176 deletions(-) create mode 100644 slice-prs.md delete mode 100644 stacked-prs.md diff --git a/slice-prs.md b/slice-prs.md new file mode 100644 index 000000000..a4b8f77ef --- /dev/null +++ b/slice-prs.md @@ -0,0 +1,735 @@ +# `clerk migrate` in slices + +Ship `migrate` in small PRs instead of one 20k-line PR. #479 (`ra/integrate-migration-tool-into-cli`) stays open as the reference, and each slice ports from it. + +**Reference commit: `7a820406`.** That's #479's tip after the review fixes (`3eec00bd` to `d6201932`) and the merge of `main` at `a16595ff`. Because #479 now contains current `main`, a file copied from `7a820406` onto a branch off `origin/main` brings no stale `main` code with it. The earlier pin, `cea11672`, predates every review fix. Don't port from it. + +**The slices never write to #479:** + +- No slice branch starts from, cherry-picks from, merges, or rebases `ra/integrate-migration-tool-into-cli`. Code comes over by copying files from the reference commit (see "Reference implementation" below). +- #479 itself is never retargeted, marked ready, merged, or closed while the slices are in flight. Decide what to do with it after slice 6 merges. +- #479 can still take fixes. When one lands after the slices are cut, see "When #479 gets another fix" below. + +**Don't port these from #479:** `slice-prs.md` (this plan), `.changeset/integrate-migration-tool-into-cli.md` (each slice writes its own), and the `.gitignore` lines for `commands/migrate/logs/` (that folder no longer exists). + +**Size:** 6 slices, shipped as 10 PRs. Slice 4 is five PRs, one per provider. + +## How it ships: one PR at a time, gated until the last one + +The PRs are stacked, but they're reviewed and merged one at a time. Slice 1 merges to `main`, then slice 2 gets reviewed and merges, and so on. The stack is never reviewed or merged as a whole. + +**The gate stays on until slice 6 merges:** + +- Slice 1 adds `CLERK_EXPERIMENTAL=migrate`. Without it, `migrate` doesn't appear in `clerk --help` or completions, and it refuses to run. +- Slices 2 to 5 each add commands or sources behind that same gate. +- Slice 6 is the only PR that removes the gate. + +**What that means for every PR:** + +- **It stands alone on `main`.** Each merge reaches `@canary`, and a release from `main` between slices ships whatever has merged so far, still gated. So every PR passes CI by itself, and its `migrate/README.md` documents only the commands that have merged. `readme.test.ts` enforces this, because it fails on any documented flag the binary rejects. +- **It has its own changeset.** Create it with the `changesets` skill. Until slice 6, the text says the command is experimental and needs `CLERK_EXPERIMENTAL=migrate`. Slice 6's changeset announces `clerk migrate`. +- **It's tested against `@canary` before the next one is marked ready.** Run `npm i -g clerk@canary` with `CLERK_EXPERIMENTAL=migrate` set, then re-run that slice's baseline tasks. + +## Slice 1: `clerk migrate import ` for Clerk and Supabase files + +- **Ported from #479:** + - the Clerk and Supabase sources and the schema + - the checks against the real instance + - consent: `--yes` or a prompt + - `--dry-run`, `--allow-partial`, and `--skip-legal-checks` + - the run store, the target line, and `--json` + - throttling, 429 backoff, and the dev user limit + - the progress bar, and exit 130 on Ctrl-C +- **New:** the `CLERK_EXPERIMENTAL` gate. +- **Not in slice 1:** continuing a stopped run, `runs`, `undo`, `export`, importing by export run ID, other sources, and `--source ./file.ts`. + +Import goes first because the risky behavior lives there: writing to production, consent, quota, and silent data loss. It's also the smallest slice that's useful on its own. + +## After that + +| Slice | Scope | PRs | +| ----- | ----------------------------------------------------------------------------------------------------------------------- | --- | +| 2 | `runs`, `undo`, and continuing a stopped run (`--new-run`, adopting in-flight creates, finishing `pending` identifiers) | 1 | +| 3 | The export envelope, `export clerk`, `export supabase`, and importing by export run ID | 1 | +| 4 | One PR per provider, each with its export and import: Firebase, Auth0, WorkOS, Better Auth, Auth.js | 5 | +| 5 | Custom sources (`--source ./file.ts`) and `migrate sources` | 1 | +| 6 | Remove the gate, then the docs and skills written against the final help output | 1 | + +**Where the non-`migrate` files from #479 go:** + +- Slice 3: the `.gitignore` lines for `exports/` and `*service-account*.json`, and the MySQL note in `scripts/check-bun-version.ts`. +- Slice 4c (WorkOS): the WorkOS entry in `commands/init/scan.ts`. +- Slice 6: the root `README.md` entry. + +**How each slice is proven:** + +- unit and integration tests; +- a re-run of the matching tasks from the agent baseline, against real test apps; +- once exports land, a weekly E2E run against real provider accounts, with credentials in 1Password, to catch providers changing underneath us. + +**Separate small PRs off `main`, not part of any slice.** These can merge in any order, before or during the slices: + +- `build:compile` in CI (`.github/workflows/ci.yml`, plus the root `build` script) +- `network_unreachable` for connection failures in `lib/fetch.ts` and `lib/errors.ts`, together with the `clerk update` fix from `0365fb84` that keeps reporting `registry_unreachable` when offline (`commands/update/index.ts` and its test) +- `bun --no-env-file` for `test:e2e` (`b0705144`: the root `package.json` and its line in `CLAUDE.md`) +- the `a: all` hint in the multiselect footer (`lib/prompts.ts`) +- the `describeBapiTarget` wording change (`lib/bapi-command.ts`) + +The Playwright pin is gone from this list. `main` moved Playwright to 1.62.1 in #409, and #479 no longer differs from it. + +## Branching + +Every branch below is new. `migrate/slice-1` starts from `origin/main`, and each branch above it starts from the branch below it. None of them starts from `ra/integrate-migration-tool-into-cli`. + +Stack the branches in one line: + +``` +main + └─ migrate/slice-1 import + └─ migrate/slice-2 runs, undo, continuing a run + └─ migrate/slice-3 exports + └─ migrate/slice-4a Firebase + └─ migrate/slice-4b Auth0 + └─ migrate/slice-4c WorkOS + └─ migrate/slice-4d Better Auth + └─ migrate/slice-4e Auth.js + └─ migrate/slice-5 custom sources, `migrate sources` + └─ migrate/slice-6 remove the gate, docs +``` + +The provider PRs don't depend on each other, but each one edits `sources/registry.ts`, `export/registry.ts`, `sources.test.ts`, and `README.md`. As siblings, every merge would conflict on those four files. In one line, nothing conflicts, and slice 5 sits on top of every provider. + +**Build only a little ahead.** Only one PR is in review at a time, and every review fix rebases every branch above it. Keep one or two draft branches above the PR in review, not all ten. A slice built early gets rebased through every review round below it. + +**Workflow:** + +1. Open slice 1 as a ready PR with `--base main`. Open each slice above it as a draft with `gh pr create --draft --base `. Don't reuse #479 for any slice. +2. Exactly one PR is ready at a time: the lowest one that hasn't merged. Its base is `main`. +3. For a review fix, commit on that PR's branch. Then run `git rebase --update-refs` from the highest branch you've built, and push each branch with `--force-with-lease`. +4. Merge the ready PR. The repo allows squash and rebase merges, not merge commits. Either way, `main` gets new commits, so the branch above still carries the old ones. Drop them, using the merged PR's head commit: + + ```bash + git fetch origin + MERGED=$(gh pr view --json headRefOid -q .headRefOid) + git rebase --onto origin/main "$MERGED" --update-refs + ``` + + Then push each remaining branch with `--force-with-lease`, and delete the merged branch locally. + +5. GitHub deletes the merged branch and moves the next PR's base to `main` on its own. Check that the next PR's diff shows only its own slice. +6. Test the merge on `@canary` and re-run the next slice's baseline tasks. Then mark the next PR ready. + +**When a baseline run changes a behavior,** such as what the checks reject, fix it in the PR in review. Step 3 carries the fix into the branches above it. + +**When #479 gets another fix:** + +1. Note the new commit, and re-pin the reference (`git -C ../cli-pr479 checkout --detach `). +2. If the files it touches belong to a slice that hasn't merged, port the change into that slice. Then rebase the branches above it, as in step 3. +3. If that slice has already merged, open a small `fix(migrate):` PR off `main`. Rebase the stack onto `main` after it merges. + +--- + +# Slice 1 implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship a hidden, env-gated `clerk migrate import ` for Clerk and Supabase files. It checks every user against the real instance before writing, writes only with consent, records every user in a run, and speaks JSON. It merges to `main` on its own, ahead of slice 2. + +**Architecture:** A new `commands/migrate/` group in `packages/cli-core`, registered hidden and gated by `CLERK_EXPERIMENTAL=migrate`. An import goes through five stages: + +1. Settle the file, the source, and the target. Print the target. +2. Load the users: read the file, map it through the source, then normalize and validate it. +3. Run `checkImport()` against the instance. `--dry-run` stops here. +4. Refuse on any reject unless `--allow-partial` is passed, then ask for consent. +5. Import through the scheduler, writing one run line per user, under a progress bar. + +Every module is ported from #479. Slice 1 leaves out the parts that belong to later slices. + +**Tech stack:** Bun, TypeScript, Commander, `@clack/prompts` wrappers in `lib/prompts.ts`, zod 4, csv-parser, `bun:test`. + +**Spec:** [clerk migrate: CLI shape proposal](https://claude.ai/code/artifact/981d8d00-05ce-4545-bd19-1a886f2b788b) (Claude Doc). Evidence: `/Users/manovotny/Developer/cli-migrate-testing/runs/baseline/RESULTS.md`. + +**Reference implementation:** clerk/cli#479 at `7a820406`. Below, `REF/` means `packages/cli-core/src/` in a checkout of that commit: + +```bash +git worktree add --detach ../cli-pr479 7a820406 +``` + +This worktree is read-only. It sits on a detached commit, so nothing done there can reach #479's branch. Don't commit in it. Copy files out of it, or run `git checkout 7a820406 -- ` from a slice branch. + +## Global constraints + +- **Branching:** branch fresh from `origin/main` in a new worktree, and open a new PR with `--base main`. Never build on, push to, or rebase `ra/integrate-migration-tool-into-cli`, and never run `gh pr edit`, `gh pr ready`, `gh pr merge`, or `gh pr close` on #479. +- **Stands alone:** slice 1 merges before slice 2 is reviewed, so nothing in it may depend on a later slice. The README, help, completion, and tests describe only `import`. +- **The gate:** + - `migrate` must not appear in `clerk --help`, completions, or the root README unless `CLERK_EXPERIMENTAL` includes `migrate`. + - With the experiment off, `migrate` is registered as a hidden stub with no subcommands, help disabled, and any arguments accepted. Every invocation exits 2 with code `experiment_disabled`, including `--help` and malformed arguments. Completion has no children to walk. + - `CLERK_EXPERIMENTAL` is a comma-separated list. Names are trimmed and case-insensitive, and unknown names are ignored. +- **The target prints first.** Once the target resolves, a human sees `Target: , instance ` and `Key from: ` before the checks. `--json` carries the same facts as `target`. +- **The target must agree with the key.** An `--instance` that the key from `--secret-key` or `CLERK_SECRET_KEY` doesn't address is refused with exit 2. +- **Consent.** Import writes to Clerk only when a human answers yes at a TTY prompt (`Import N users?`, default no), or when `--yes` is passed. Without either (an agent, a non-TTY run, or `--json`), it prints the checks and exits 2 with the exact command to run. Printed commands shell-quote their paths and keep `--json`. `--json` never prompts. +- **`--dry-run`** runs the checks against the real instance and writes nothing. It exits 2 when the real run would be refused, and 0 otherwise. +- **Rejects.** Any reject refuses the import (exit 2) and prints the command that adds `--allow-partial`. With `--allow-partial`, the rest import and each reject is recorded as `skipped` with its reason. +- **Unknown password hasher.** An unrecognized hasher aborts the whole run before anything is sent. +- **Exit codes:** + - `0`: every attempted user was created, or a dry run that would succeed. + - `1`: any user failed. + - `2`: a usage error or a refusal. + - `130`: Ctrl-C. In-flight requests abort and the process dies by SIGINT. This is why import shows a progress bar (`lib/progress.ts`) instead of a clack spinner: the spinner exits 0 on Ctrl-C. +- **Output streams.** UI goes to stderr. JSON goes to stdout through `log.data`. +- **Input files.** JSON, CSV, and NDJSON (Auth0's bulk export). A BOM prefix is read. A file that isn't valid JSON is named in the error. +- **The run store:** + - Location: `--runs-dir`, then `CLERK_MIGRATE_DIR`, then `/.clerk/migrate/`. The project root is the linked profile's directory, then the git toplevel, then the cwd. + - `.clerk/` is added to `.gitignore` only once there is consent to write a run, and only for the default location. + - Each run is a folder `YYYYMMDD-HHmmss-xxxx/` holding `run.json`, `users.ndjson`, and `lock`. Folders are `0700`. Folders and locks are created exclusively, so a run-ID collision or a lock race fails instead of sharing a folder. A lock holding this process's own PID is stale. + - A user's `creating` line is written as its `POST /v1/users` goes out. When the create gets no answer (an abort, a network error, or a 5xx), the line stays `creating` instead of becoming `failed`. + - A `created` line lists the extra emails and phones that haven't attached yet as `pending`. A later `created` line without `pending` means they're done. + - Every append is synchronous, and a failed append throws. + - The run format is final in this slice, because slice 2 reads it once slice 1 is on `main`. +- **Throughput:** + - The instance type comes from the key prefix: `sk_live_` is production, anything else is development. + - Default rate limits: 100 req/s for production and 10 req/s for development. Concurrency defaults to `floor(rate × 0.095)`. + - The env overrides are `CLERK_MIGRATE_RATE_LIMIT`, `CLERK_MIGRATE_CONCURRENCY_LIMIT`, and `CLERK_MIGRATE_DEV_USER_LIMIT`. A non-numeric or non-positive value falls back to the default. +- **Repo rules.** Follow `.claude/rules/*.md` (commands, errors, completion, promises, testing, changesets, interrupts). Run `bun run format && bun run lint && bun run typecheck && bun run test` before every commit. +- **Changeset.** One changeset for the slice (Task 9), created with the `changesets` skill. + +## File structure + +``` +packages/cli-core/src/ + lib/experimental.ts experimental.test.ts # CLERK_EXPERIMENTAL parsing (new) + lib/git.ts # + export ensureGitignoreEntry (moved from lib/keyless.ts) + lib/errors.ts # + ERROR_CODE.EXPERIMENT_DISABLED + lib/config.ts # export INSTANCE_ALIASES + lib/json-body.ts # export quoteArg + lib/next-steps.ts # + printAgentNextSteps, MIGRATE_DONE(_WITH_ERRORS) + lib/spinner.ts # ignore an empty setNextSteps([]) + commands/completion/__complete.ts # + --source values + commands/migrate/ + index.ts index.test.ts # gate stub, or the group with `import` + README.md readme.test.ts + types.ts # User, PASSWORD_HASHERS, SourceEntry, ImportSummary + validator.ts validator.test.ts # userSchema + run.ts run.test.ts run-interactive.test.ts # the `import` action + import-users.ts import-users.test.ts # create + attach, one run line per user + wizard.ts wizard.test.ts # promptForFile, promptForSource + sources/ + registry.ts # clerk + supabase; resolveSource, sourceKeys + shared.ts clerk.ts supabase.ts + sources.test.ts + lib/ + assume-yes.ts # -y read below the action + transform.ts transform.test.ts # read JSON/CSV/NDJSON → map → normalize → validate + instance.ts instance.test.ts # instance type, limits, dev user limit + retry.ts retry.test.ts # 429 backoff + scheduler.ts scheduler.test.ts # rate + concurrency + progress.ts progress.test.ts # the import progress bar + target.ts target.test.ts # resolveClerkTarget, printTarget + run-store.ts run-store.test.ts # write side of the run store + clerk-config.ts clerk-config.test.ts # FAPI settings, user count, Clerk's OAuth providers + user-lookup.ts # batched GET /v1/users + supabase-providers.ts supabase-providers.test.ts + analysis.ts analysis.test.ts + readiness.ts readiness.test.ts + modify-settings.ts modify-settings.test.ts # the `clerk config patch` offers + checks.ts checks.test.ts # checkImport +test/e2e/migrate.test.ts +``` + +## How to port a module + +Every task below ports files from `REF/`, using the same steps: + +1. Copy the test file from `REF/` and trim the cases the task names. +2. Run it with `bun test --isolate `. Expect it to fail with "Cannot find module". +3. Copy the module from `REF/` and apply the task's trims. +4. Run the test again. Expect it to pass. +5. Run the full CI set, then commit. + +When a task says "as-is", copy the file unchanged. + +--- + +### Task 1: Experimental gate and hidden `migrate` group + +**Files:** + +- Create: `lib/experimental.ts`, `lib/experimental.test.ts` +- Create: `commands/migrate/index.ts`, `commands/migrate/README.md` (a stub saying the command is experimental and gated) +- Modify: `lib/errors.ts` (add `EXPERIMENT_DISABLED: "experiment_disabled"`), `cli-program.ts` (append `registerMigrate` to `registrants`) +- Modify: `lib/git.ts`, `lib/keyless.ts`, `test/lib/stubs.ts`, `test/integration/lib/harness.ts`. Move `ensureGitignoreEntry` from `keyless.ts` into `git.ts` as an export, exactly as `REF/lib/git.ts` has it, and add it to `gitStubs` and to the harness's `git.ts` mock. +- Test: `commands/migrate/index.test.ts` (gate cases only), `cli-program.test.ts` (existing help-ordering test must still pass) + +**Interfaces:** + +- `isExperimentEnabled(name: string, env?: NodeJS.ProcessEnv): boolean` +- `requireExperiment(name: string, env?: NodeJS.ProcessEnv): void`: throws a `CliError` with code `experiment_disabled` and exit code 2 +- `registerMigrate(program: Program, env?: NodeJS.ProcessEnv): void` + +- [ ] **Step 1: Write `lib/experimental.test.ts`** + +```ts +import { describe, expect, test } from "bun:test"; +import { CliError, EXIT_CODE } from "./errors.ts"; +import { isExperimentEnabled, requireExperiment } from "./experimental.ts"; + +describe("isExperimentEnabled", () => { + test.each([ + { value: undefined, expected: false }, + { value: "", expected: false }, + { value: "migrate", expected: true }, + { value: " Migrate ", expected: true }, + { value: "other,migrate", expected: true }, + { value: "other, MIGRATE ,x", expected: true }, + { value: "migrates", expected: false }, + { value: "other", expected: false }, + ])("CLERK_EXPERIMENTAL=$value → $expected", ({ value, expected }) => { + expect(isExperimentEnabled("migrate", { CLERK_EXPERIMENTAL: value })).toBe(expected); + }); +}); + +describe("requireExperiment", () => { + test("throws a usage-level CliError naming the variable", () => { + try { + requireExperiment("migrate", {}); + throw new Error("expected throw"); + } catch (error) { + expect(error).toBeInstanceOf(CliError); + expect((error as CliError).code).toBe("experiment_disabled"); + expect((error as CliError).exitCode).toBe(EXIT_CODE.USAGE); + expect((error as CliError).message).toContain("CLERK_EXPERIMENTAL=migrate"); + } + }); + + test("passes when enabled", () => { + expect(() => requireExperiment("migrate", { CLERK_EXPERIMENTAL: "migrate" })).not.toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run it and confirm it fails.** Run `bun test packages/cli-core/src/lib/experimental.test.ts`. It should fail with "Cannot find module './experimental.ts'". + +- [ ] **Step 3: Implement `lib/experimental.ts`** + +```ts +import { CliError, ERROR_CODE, EXIT_CODE } from "./errors.ts"; + +function enabledExperiments(env: NodeJS.ProcessEnv): Set { + return new Set( + (env.CLERK_EXPERIMENTAL ?? "") + .split(",") + .map((name) => name.trim().toLowerCase()) + .filter(Boolean), + ); +} + +export function isExperimentEnabled(name: string, env: NodeJS.ProcessEnv = process.env): boolean { + return enabledExperiments(env).has(name.toLowerCase()); +} + +export function requireExperiment(name: string, env: NodeJS.ProcessEnv = process.env): void { + if (isExperimentEnabled(name, env)) return; + throw new CliError( + `\`clerk ${name}\` is experimental. Set CLERK_EXPERIMENTAL=${name} to use it.`, + { code: ERROR_CODE.EXPERIMENT_DISABLED, exitCode: EXIT_CODE.USAGE }, + ); +} +``` + +- [ ] **Step 4: Write the gate tests** in `commands/migrate/index.test.ts`. Read `test/integration/lib/harness.ts` first to confirm that `clerk.raw` builds a fresh program per call, since the env must be set before the program is built. Cover: + - With the experiment off: + - `clerk --help` doesn't contain `migrate`. + - Each of these exits 2 with `"code":"experiment_disabled"` in agent mode: `migrate`, `migrate --help`, `migrate import users.json --yes`, `migrate --not-a-flag`, `help migrate`, and `--verbose migrate import x.json`. + - `__complete migrate ""` offers nothing. + - With the experiment on: `clerk --help` lists `migrate`. + + `clerk help migrate` goes through Commander's help path, not the stub's action. Override the stub's `helpInformation` to call `requireExperiment`, so that path exits 2 as well. + +- [ ] **Step 5: Implement `commands/migrate/index.ts`** + +```ts +import type { Program } from "../../cli-program.ts"; +import { isExperimentEnabled, requireExperiment } from "../../lib/experimental.ts"; + +export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = process.env): void { + if (!isExperimentEnabled("migrate", env)) { + const stub = program + .command("migrate", { hidden: true }) + .helpOption(false) + .allowUnknownOption() + .allowExcessArguments() + .argument("[args...]") + .action(() => requireExperiment("migrate", env)); + stub.helpInformation = () => { + requireExperiment("migrate", env); + return ""; + }; + return; + } + // Task 9 registers the group and `import` here. +} +``` + +- [ ] **Step 6: Run the full suite.** `bun run test` must pass with `CLERK_EXPERIMENTAL` unset, including `cli-program.test.ts` help ordering. + +- [ ] **Step 7: Commit**: `feat(migrate): add hidden migrate group behind CLERK_EXPERIMENTAL` + +--- + +### Task 2: Schema, sources, and the transform pipeline + +**Files:** + +- Port as-is: `types.ts`, `validator.ts`, `validator.test.ts`, `sources/shared.ts`, `sources/clerk.ts`, `sources/supabase.ts` +- Port with trims: `sources/registry.ts`, `sources/sources.test.ts`, `lib/transform.ts`, `lib/transform.test.ts` +- Modify: `packages/cli-core/package.json`: add `"csv-parser": "^3.2.1"` and `"zod": "^4.4.3"`, then run `bun install` + +**Interfaces:** + +- `userSchema`, `passwordHasherEnum` (validator.ts) +- `User`, `PASSWORD_HASHERS`, `SourceEntry`, `ImportSummary`, `TransformContext` (types.ts) +- `sources`, `sourceKeys()`, `getSource(key)`, `resolveSource(value): Promise`, `ACCOUNT_LINKING_NOTE` (registry.ts) +- `loadUsersFromFile(file, key, options?)`, `readRawUsers(file, key)`, `transformUsers(...)`, `validatePreparedUsers(...)`, `resolveImportFilePath`, `fileExists`, `getFileType` (transform.ts) + +**Trims:** + +- `sources/registry.ts`: + - The `sources` array holds `clerkSource` and `supabaseSource` only. + - Remove `customSources`, `registerCustomSource`, `__resetCustomSourcesForTesting`, `isSourcePath`, and the `loadCustomSource` import. + - `allSources()` returns `sources`. + - `resolveSource` keeps only the built-in branch, and its unknown-key error drops the custom-source hint. +- `lib/transform.ts`: remove the `isEnvelope` branch in `readUsersFromFile` and the `export-file.ts` import. A JSON file must be an array (or whatever a source's `preTransform` unwraps). Keep NDJSON and BOM handling. +- `sources/sources.test.ts`: keep the `shared`, `clerk`, and `supabase` cases. Drop the other sources. +- `lib/transform.test.ts`: drop the envelope cases. + +`TransformContext` keeps its `firebaseHashConfig` field. It's a type, and slice 4 fills it in. `PASSWORD_HASHERS` stays the full list, including `phpass`. + +- [ ] **Step 1: Port the tests, then the modules, as described in "How to port a module".** Run `bun test --isolate packages/cli-core/src/commands/migrate/validator.test.ts packages/cli-core/src/commands/migrate/sources packages/cli-core/src/commands/migrate/lib/transform.test.ts`. +- [ ] **Step 2: Check the behaviors the baseline and the review depend on.** These must pass in the ported tests; add any that are missing: + - Supabase: + - A `raw_user_meta_data` name splits into first and last names, and a one-word name becomes the first name. This also works on CSV input. + - A numeric phone gains a leading `+`. + - A soft-deleted user gets a `skipReason`. + - An active `banned_until` becomes `banned`. + - A bcrypt or argon2 hash keeps its hasher, and any other hash is dropped with `passwordDropped: true`. + - Clerk: + - Primary identifiers consolidate without nesting arrays. + - An unverified primary email or phone stays unverified. + - A Dashboard CSV says it carries no metadata, and the TAB the Dashboard puts before formula-like values is stripped. + - The pipeline: + - CSV values are coerced (lists, booleans, JSON metadata). Booleans read case-insensitively, plus `t` and `f`. + - A numeric user ID is read as a string. + - A BOM-prefixed CSV or JSON file reads, and NDJSON reads. + - A file that isn't valid JSON is named in the error. + - An unknown `passwordHasher` throws a `CliError` naming the row. + - Unknown fields are counted in `unknownFields`. +- [ ] **Step 3: Commit**: `feat(migrate): map Clerk and Supabase files onto the import schema` + +--- + +### Task 3: Throughput and progress + +**Files:** port as-is: `lib/instance.ts`, `lib/retry.ts`, `lib/scheduler.ts`, `lib/progress.ts`, and their tests. + +**Interfaces:** + +- `detectInstanceType(secretKey): "dev" | "prod"` +- `resolveLimits(secretKey, env?): ResolvedLimits` +- `resolveDevUserLimit(env?)`, `DEV_USER_LIMIT = 100`, `MAX_RETRIES = 5`, `RETRY_DELAY_MS` +- `retryOn429(fn, { onRetry?, maxRetries?, defaultDelayMs? })`, `RateLimitExceededError`, `readRetryAfter` +- `createApiScheduler(concurrencyLimit, rateLimit): ApiScheduler` +- `withProgress(...)`, `formatProgress(line)`, `formatRemaining(ms)`, `ProgressCounts`, `ProgressUpdate` + +**Progress bar:** it fills the line up to 80 columns, and the report under it adds an estimate of the time left. Without a terminal, the report prints at each 10%. It leaves stdin alone, so Ctrl-C reaches the CLI's handler. + +- [ ] **Step 1: Port the tests, then the modules.** Run `bun test --isolate` on the four test files. +- [ ] **Step 2: Commit**: `feat(migrate): pace Backend API calls, back off on 429, and show progress` + +--- + +### Task 4: Target resolution + +**Files:** + +- Port as-is: `lib/target.ts`, `lib/target.test.ts`. +- Modify: `lib/config.ts` (export `INSTANCE_ALIASES`). + +**Interfaces:** + +- `resolveClerkTarget(options): Promise<{ secretKey: string; target: ClerkTarget }>` +- `fetchInstanceIdentity(secretKey)`: reads `GET /v1/instance` and retries a 429. When the instance can't be read, it falls back to `key_`. +- `describeTarget(target)`, `printTarget(target)` + +`target.ts` imports `RunTarget` from `lib/run-store.ts`. Port Task 5 first, or move both into one commit. + +Key sources, in the order `resolveBapiSecretKey` checks them: + +1. `--secret-key` +2. `--app` +3. `CLERK_SECRET_KEY env var` +4. `accountless app ()` +5. `linked profile` + +- [ ] **Step 1: Port the test, then the module.** The test must cover: + - each key source; + - an exported `CLERK_SECRET_KEY` outranking the linked profile; + - an `--instance` that the key from `--secret-key` or `CLERK_SECRET_KEY` doesn't address, refused with exit 2. +- [ ] **Step 2: Commit**: `feat(migrate): resolve and print the import target` + +--- + +### Task 5: Run store (write side) + +**Files:** port `lib/run-store.ts` and `lib/run-store.test.ts` with trims. + +**Interfaces:** + +- Constants: `RUNS_DIR_ENV`, `RUNS_DIR_FLAG`, `RUNS_DIR_DESCRIPTION`, `RUN_ID_PATTERN` +- Types: `RunKind`, `RunStatus`, `UserStatus`, `RunTarget`, `RunFile`, `RunCounts`, `RunRecord`, `PendingIdentifier`, `UserLine`, `Run`, `StartRunInit` +- Functions: `resolveRunsDir(runsDir, { write?, cwd? })`, `runDir`, `newRunId`, `sha256File`, `startRun(runsDir, init): Run`, `readRun`, `readUserLines`, `latestUserLines`, `countLines`, `liveLockPid`, `lockFile` + +**Trims:** remove `continueRun`, `patchRun`, `runState`, `RunState`, `listRuns`, `clerkIdsCreatedByOtherRuns`, and their tests. Slice 2 adds them back. + +Keep the full `RunKind`, `RunStatus`, and `UserStatus` unions (`import | undo | export`, `undone`, plus `deleted` and `exported`), and keep `UserLine.pending`. The run format is final in this slice. + +- [ ] **Step 1: Port the test, then the module.** The tests must cover: + - the location order; + - `.clerk/` added to `.gitignore` once, and only when the run is written; + - the run ID shape; + - run folders created `0700` and exclusively, so a second run with the same ID fails; + - a live lock refusing a second writer with exit 2, naming the lock file; + - a lock holding this process's own PID read as stale; + - the last line per `sourceId` winning; + - a truncated last line being ignored; + - a failed append throwing; + - `finish()` setting `partial` when any user failed, was skipped, or is still `creating`, and `complete` otherwise. +- [ ] **Step 2: Commit**: `feat(migrate): keep every import as a run under .clerk/migrate` + +--- + +### Task 6: Instance reads and the readiness report + +**Files:** port as-is: `lib/clerk-config.ts`, `lib/user-lookup.ts`, `lib/supabase-providers.ts`, `lib/analysis.ts`, `lib/readiness.ts`, `lib/modify-settings.ts`, and their tests. + +**Interfaces:** + +- `fetchInstanceSettings(secretKey): Promise`: BAPI `/v1/domains`, then FAPI `/v1/environment`, bootstrapping a dev browser on development instances. Returns `null` when the settings can't be read. +- `fetchUserCount(secretKey): Promise` +- `toClerkStrategy(provider)`, `clerkOffersProvider(provider)`, `providerLabel(provider)`, `enabledSocialProviders(settings)`, `fetchEnabledSocialProviders(secretKey)`. Clerk's built-in OAuth strategies are copied from clerk_go's `api/shared/sso/oauth.go`, since nothing the CLI can call lists them. +- `lookupUsers({ filter, values, secretKey, schedule, spinner?, label? })`: batches of 100 through the scheduler and `retryOn429`. An `external_id` is looked up with a `+` prefix, so BAPI doesn't read a leading `-` as an exclusion. +- `readSupabaseRows(file)`, `findDisabledProviders`, `findUsersWithOnlyDisabledProviders`, `countSocialProviders` +- `analyzeFields(users)`, `buildReadinessReport(input)`, `buildSettingChanges(flagged)`, `buildChangePayload(changes)` + +**Behavior:** + +- Readiness counts users, not identifiers. +- A Supabase provider Clerk doesn't offer is named as not offered, and gets no fix. +- A fix names its instance with `--instance`, or points at the Dashboard when Clerk couldn't name the instance. This keeps a `--secret-key` import's fix from changing the linked profile's development instance. + +- [ ] **Step 1: Port the tests, then the modules.** +- [ ] **Step 2: Commit**: `feat(migrate): read the instance's settings, user count and existing users` + +--- + +### Task 7: Importer + +**Files:** port `import-users.ts` and `import-users.test.ts`, trimming only what continuing a run needs. Slice 2 adds adoption back. + +**Interfaces:** + +- `importUsers({ users, secretKey, limits, record, skipPasswordRequirement?, validationFailed?, spinner? }): Promise` +- `splitIdentifiers(user)`, `buildCreateUserBody(user, identifiers, skipPasswordRequirement)`, `normalizeErrorMessage(message)`, `outcomeUnknown(error)`, `pendingIdentifiers(identifiers)` + +**Behavior:** + +- Each user writes `creating` as its `POST /v1/users` goes out, not when it's queued. It writes `created` with its Clerk ID as soon as the create returns, with any extra identifiers listed as `pending`. +- A create with no answer (an abort, a network error, or a 5xx) leaves the user at `creating`. `outcomeUnknown` decides this. +- Only the first verified email and phone go on the create. An unverified primary stays unverified. Every other identifier attaches afterwards with its own request, ahead of queued creates. An attach retries on a 429. A failed attachment adds a note to the user's line, and the user still counts as created. +- Every create sends `skip_restriction_checks`, because the allowlist and blocklist police sign-ups and these users already signed up. With `--skip-legal-checks`, it also sends `skip_legal_checks`. +- When Clerk refuses the first phone (`unsupported_country_code`, or `param_name: phone_number`) and the user has an email, the create retries without the phone and logs a note. +- A 429 retries up to 5 times, honoring `Retry-After`. When the retries run out, the user is recorded as `failed` with code `429`. +- Any other error records `failed` with the API's long message and status, and the run continues. +- Progress shows through `withProgress`, not a spinner. + +- [ ] **Step 1: Port the test, then the module.** Include a test that an aborted create leaves the user at `creating`. +- [ ] **Step 2: Commit**: `feat(migrate): create users through the scheduler, one run line each` + +--- + +### Task 8: Checks + +**Files:** port `lib/checks.ts` and `lib/checks.test.ts` with trims. `checks.ts` imports `splitIdentifiers` from `import-users.ts`, which is why it comes after Task 7. + +**Interfaces:** + +- `checkImport(input: CheckInput): Promise` +- `hashShapeProblem(password, hasher)`, `passwordIsOnlySignIn(settings)` +- Types: `Reject`, `ReasonCount`, `Fix`, `Quota`, `ImportChecks`, `CheckInput` + +**Trims:** remove `adoptedClerkIds` from `CheckInput`, along with its use in `findInstanceDuplicates`. Slice 2 adds it back with continuing. Keep `skipLegalChecks`. + +**Rejects.** Each user gets the first reason that applies, in this order: + +1. it failed schema validation (`invalid: …`) +2. the source set a `skipReason` (Supabase: soft-deleted) +3. its only emails are ones Clerk refuses: malformed, or a private TLD such as `.local`, `.invalid`, `.test`, `.example`, or `.arpa`. Every such user gets one fixed reason, so they group and the address stays out of the run record. +4. it lacks a verified identifier the instance requires +5. it has no identifier left once the ones the instance has turned off are removed +6. it lacks a first or last name the instance requires +7. it has a TOTP secret or backup codes, and the instance has that feature off (with a fix to turn it on) +8. it has no password, and password is the instance's only way to sign in +9. it has no legal acceptance on record, and the instance requires one. `--skip-legal-checks`, or a yes at the prompt, imports these users instead. +10. its username breaks the instance's username rules +11. its password doesn't match its hasher's shape. `bcrypt` (cost up to 15), `scrypt_firebase`, `argon2i`/`argon2id`, and `scrypt_werkzeug` are checked. +12. Supabase: its only providers aren't enabled in Clerk, or aren't offered by Clerk. The reason names only that user's own providers. +13. it repeats an earlier passing user's source ID, email, phone, or username in the file. Usernames compare case-insensitively, and phones compare with punctuation stripped. Only users that pass checks 1 to 12 claim identifiers, so a rejected record doesn't cost a later one its email. The first record wins, and the reject names it (`kept: …`). +14. the instance already has a user with its source ID, email, phone, or username +15. on a development instance, it's past the headroom (`CLERK_MIGRATE_DEV_USER_LIMIT`, default 100, minus the live count), counted in file order. When the live count can't be read, the checks say so instead of treating it as zero. + +**Imported with warnings:** + +- fields the instance isn't set up to store +- `Clerk won't store: (N users)` for unknown fields +- passwords the source dropped +- refused emails and names, which are dropped. An email-shaped name is kept, as Clerk keeps it. +- a malformed secondary email, which is dropped. The shape check is looser than Zod's, so non-ASCII addresses Clerk accepts are kept. +- a password on an instance with passwords turned off, which is stored but works only once passwords are turned on + +**Identifiers the instance has turned off** (email, phone, or username) are removed from each importable user before the create. + +**Fixes** are `clerk config patch` offers built from the same readiness rows, with `--app` and `--instance` included when the target knows them. A provider Clerk doesn't offer gets no fix. + +- [ ] **Step 1: Port the test, then the module.** Run `bun test --isolate packages/cli-core/src/commands/migrate/lib/checks.test.ts`. +- [ ] **Step 2: Commit**: `feat(migrate): check every user against the instance before writing` + +--- + +### Task 9: `clerk migrate import` + +**Files:** + +- Port with trims: `run.ts`, `run.test.ts`, `run-interactive.test.ts`, `wizard.ts`, `wizard.test.ts`, `index.ts` (the enabled branch), `index.test.ts` +- Port as-is: `lib/assume-yes.ts` +- Modify: + - `lib/next-steps.ts`: add `printAgentNextSteps` from `REF/lib/next-steps.ts`, plus `MIGRATE_DONE` and `MIGRATE_DONE_WITH_ERRORS` (text below) + - `lib/spinner.ts`: ignore an empty `setNextSteps([])` + - `lib/json-body.ts`: export `quoteArg`, for the printed commands + - `commands/completion/__complete.ts`: complete `--source` from `sources` + - `test/integration/completion.test.ts` +- Create: the `commands/migrate/README.md` import sections, `readme.test.ts`, and the changeset + +**`index.ts`, enabled branch:** + +- `migrate` group: copy the description and examples from `REF/commands/migrate/index.ts`, keeping only the examples that use `import `. +- The group's `preAction` hook, as-is: it sets `-y` through `setAssumeYes`, and `--json` switches the run to agent mode. +- `import` subcommand: + - Argument: `[file]`, "A JSON or CSV export". Not `[file|export-run-id]`; slice 3 widens it. + - Options: `--source ` (no `.choices()`, so an unknown key reaches `resolveSource`'s error), `--dry-run`, `--allow-partial`, `--require-password`, `--skip-legal-checks`, `-y, --yes`, `--json`, `--secret-key `, `--app `, `--instance `, and `RUNS_DIR_FLAG`. + - Leave out `--new-run` (slice 2) and the `--firebase-*` flags (slice 4). + - Examples: keep the `users.json --source clerk --allow-partial --yes` one. Drop the export-run-ID and `./my-source.ts` examples. + +**`run.ts`. Keep:** + +- `ensureImportTarget`: a human who isn't signed in is signed in, then an unlinked directory gets linked. An agent gets the `AuthError` naming the missing half. +- `resolveInput`, without the export-run-ID branch +- `applySource`, `validateRunOptions`, `explainErrors`, `printChecks`, `checksJson`, `formatSummary`, `commandFor`, `recordRejects` +- The legal-checks prompt: a human with users lacking legal acceptance is asked whether to import them with `skip_legal_checks`. +- The main flow: print the target, load, read settings and the user count, check, then dry-run, refuse, nothing-to-import, consent, import under `withProgress`, and summary +- `--require-password` records the users it leaves out as `skipped`. + +**`run.ts`. Remove (later slices add them back):** + +- `findResume`, `ResumeCase`, the `complete` early return, the continued-run bookkeeping, adopting in-flight creates by `external_id`, finishing `pending` identifiers, `continueRun`, `--new-run`, and the refusal while an undo of the matching run is unfinished (slice 2) +- `readEnvelope`, `applyEnvelope`, and `fromExport` (slice 3) +- `cleanupLines` (slice 2, alongside `undo`) +- `FirebaseHashFlags`, `resolveFirebaseHashConfig`, `promptForFirebaseHashConfig`, and the `--firebase-*` placeholders in `commandFor` (slice 4) +- `MigrateRunOptions.sourceHash`, the custom-source log line in `applySource`, and keeping a custom `--source` path in `commandFor` (slice 5) + +**`wizard.ts`:** keep `promptForFile` and `promptForSource`. Remove `askNumber` and `promptForFirebaseHashConfig`. + +**JSON output:** `{ target, run, checks, result }`. When a run stops before importing, it adds one of these instead of `result`: `dryRun: true`, `refused: true`, `consent: "required"`, or `nothingToImport: true`. `result` is `{ created, failed, skipped, errors: [{ error, count }] }`. + +**Next steps**, until slice 2 adds `runs` and `undo`: + +```ts +MIGRATE_DONE: (runFolder: string) => [`See each user's outcome in ${runFolder}/users.ndjson`], +MIGRATE_DONE_WITH_ERRORS: (runFolder: string) => [ + `See every user that failed, and why, in ${runFolder}/users.ndjson`, +], +``` + +**Re-running in this slice:** a second import of the same file starts a new run. The users the first run created are rejected as already in the instance, so `--allow-partial --yes` imports the rest. Users an interrupted run left at `creating` may or may not exist in Clerk. The ones that do are rejected as duplicates, and the rest import. Slice 2 replaces this with continuing the run. + +- [ ] **Step 1: Port the tests with trims.** + - `run.test.ts`: keep `validateRunOptions`, "without a file or a source", `checks`, `consent`, `explainErrors`, `--skip-legal-checks`, the `--instance` mismatch, and the Clerk and Supabase cases of "per-platform imports". Drop "export envelopes", "continuing an earlier run", adoption, `--source `, and the other platforms. + - `index.test.ts`: keep the `import` registration and the `-y` hook cases, plus Task 1's gate cases. Set `CLERK_EXPERIMENTAL=migrate` in `beforeEach` for the enabled cases. + - `run-interactive.test.ts`, `wizard.test.ts`: drop the Firebase prompts. +- [ ] **Step 2: Add tests for the behavior this slice adds:** + 1. Importing the same file twice: the second run rejects every created user as already in the instance, and exits 2 without `--allow-partial`. + 2. `--source nope` exits 2 and lists `clerk, supabase`. + 3. In agent mode, stderr has no `\x1b[`. + 4. `--help` for `import` doesn't list `--new-run`, `--firebase-*`, or an export run ID. +- [ ] **Step 3: Run them and confirm they fail.** Run `bun test --isolate packages/cli-core/src/commands/migrate/run.test.ts packages/cli-core/src/commands/migrate/index.test.ts`. +- [ ] **Step 4: Port the modules with the trims above.** Keep `run.ts` to orchestration. Every decision it makes is already a tested function in Tasks 2 to 8. +- [ ] **Step 5: Write the README.** Port these sections from `REF/commands/migrate/README.md`, cut to slice 1: + - the rules + - targeting and auth + - the run store (without `runs`, `undo`, continuing, or exports) + - `clerk migrate import` (without re-running, export run IDs, Firebase flags, or `--new-run`) + - checks, additional identifiers, throughput, and the progress bar + - sources (Clerk and Supabase rows only) + - the schema fields + - the API endpoints `import` calls + + Put a blockquote at the top saying the command is experimental and needs `CLERK_EXPERIMENTAL=migrate`. Port `readme.test.ts`, and set `process.env.CLERK_EXPERIMENTAL = "migrate"` before it calls `createProgram()`, or it sees only the stub. It fails on any documented flag the binary rejects, and on any flag the README leaves out. + +- [ ] **Step 6: Create the changeset** by invoking the `changesets` skill. It's a minor bump, and the text says the command is experimental and gated. +- [ ] **Step 7: Run every CI check**: `bun run format && bun run lint && bun run typecheck && bun run test`, then `bun changeset status --since=origin/main`. +- [ ] **Step 8: Commit**: `feat(migrate): add experimental clerk migrate import for Clerk and Supabase files` + +--- + +### Task 10: Prove it with E2E and a baseline re-run + +**Files:** + +- Create: `test/e2e/migrate.test.ts`. Port the "a user whose only email is unverified is refused where email is required" case from `test/e2e/migrate.test.ts` at `7a820406`. It reads each user's latest run line per source ID, because a user's first line is `creating`. Leave the Better Auth case for slice 4d. +- Update: `/Users/manovotny/Developer/cli-migrate-testing/runs/`, adding a new run folder outside the repo. + +- [ ] **Step 1: Add a Supabase round trip to the E2E test.** Follow `.claude/rules/e2e.md`: + - Write a 2-user Supabase JSON file with fresh bcrypt hashes (`Bun.password.hash("", { algorithm: "bcrypt", cost: 10 })`) and unique `+e2e-` emails. + - Run `import --source supabase --dry-run --json` and assert `checks.importable === 2`. + - Run `--yes --json` and assert `result.created === 2`. + - Verify each password with `POST /v1/users/{id}/verify_password`. + - Delete both users in `afterAll`. + - Set `CLERK_EXPERIMENTAL=migrate` and `CLERK_TELEMETRY_DISABLED=1` on the subprocess env. + + Run `bun run test:e2e:op -- -t "migrate"`. It should pass. Until the `--no-env-file` PR merges to `main`, move any `.env.local` that points at a local `clerk_go` stack aside first, or the run sends the test secrets there. + +- [ ] **Step 2: Build the binary.** Run `bun run build:compile`, copy `packages/cli-core/dist/clerk` to `/Users/manovotny/Developer/cli-migrate-testing/bin/clerk-slice1`, and record its `--version` next to it. +- [ ] **Step 3: Re-run the baseline tasks.** Point each sandbox wrapper at `clerk-slice1`, set `CLERK_EXPERIMENTAL=migrate` in the wrapper, and reset each task with `bun runs/baseline/harness/reset.ts `. **Log out of any account CLI session first**, because the Keychain login leaks into sandboxes. Then run fresh blind agents with `runs/baseline/BLIND-PROMPT.md` on these tasks: + - **`t05-dry-run`:** passes when zero users are written and the agent used `--dry-run` without a workaround. + - **`t09-prod`:** passes when 20 users arrive with last names, and the agent saw "production" in the target line before writing, or stopped at the consent refusal. + - **Supabase file:** copy `fixtures/t06-resume/users.json` into a fresh sandbox for the t06 app with an empty instance, and prompt "Import users.json from Supabase into my Clerk app." It passes when 80 users arrive with last names and their passwords verify. + - **Quota pressure:** a 150-user Supabase file into an empty dev instance, with the same prompt. It passes when: + - no users are written until the agent chooses `--allow-partial`; + - the checks named the 50 users over the limit and the options (production, `--allow-partial`, `CLERK_MIGRATE_DEV_USER_LIMIT`); + - with `--allow-partial`, exactly 100 users are created, 50 are recorded as `skipped`, and no create hits the quota error. + + Grade with `bun runs/baseline/verify.ts ` plus a last-name check, and audit with `bun runs/baseline/audit.ts`. + +- [ ] **Step 4: Write `runs/slice1/RESULTS.md`** comparing before and after for those four tasks, in the same scorecard format as the baseline. +- [ ] **Step 5: Commit the E2E test**: `test(migrate): e2e import with dry-run, consent and password verification` +- [ ] **Step 6: After the PR merges,** install `clerk@canary` with `CLERK_EXPERIMENTAL=migrate` and re-run `t05-dry-run` against it. Then rebase slice 2 onto `main` (Branching, step 4) and mark it ready. + +--- + +## Out of scope for slice 1 + +Each of these lands in a later slice: + +- Continuing a stopped run, `--new-run`, adopting in-flight creates, finishing `pending` identifiers, `runs`, `undo`, and the cleanup lines after a complete import (slice 2) +- The export envelope, importing by export run ID, `export clerk`, and `export supabase` (slice 3) +- Firebase, Auth0, WorkOS, Better Auth (including Drizzle's snake_case and plural schemas), and Auth.js, including the `--firebase-*` flags and the Firebase hash prompt (slice 4) +- `--source ./file.ts` and `migrate sources` (slice 5) +- Removing the gate, the root README entry, and the docs and skills (slice 6) diff --git a/stacked-prs.md b/stacked-prs.md deleted file mode 100644 index 2e8a5cc68..000000000 --- a/stacked-prs.md +++ /dev/null @@ -1,176 +0,0 @@ -# Stacked PRs for `clerk migrate` - -Split `ra/integrate-migration-tool-into-cli` (clerk/cli#479: 88 commits, 112 files, +20,421 / -122) into 14 stacked PRs. Each PR carves existing files out of the branch. Nothing gets rewritten. #479 stays open as the reference until the top of the stack merges. - -**#479 stays untouched.** Every PR below is a new branch and a new PR. The stack only reads from #479: - -- Nothing is committed or pushed to `ra/integrate-migration-tool-into-cli`, and it's never rebased or force-pushed. -- #479 itself is never edited, retargeted, marked ready, merged, or closed while the stack is in flight. Decide what to do with it after PR 10 merges. -- Files come from commit `cea11672` (the tip of #479 when this plan was written), never by branching from, cherry-picking onto, or merging the PR branch. Pinning the commit means a later push to #479 can't change what the stack carves. - -## The stack at a glance - -Line counts come from `git diff --numstat origin/main...HEAD`, grouped by file. Counts marked `~` include a test file shared across PRs, so they move a little once you split it. - -| PR | Scope | Src | Tests | Total | -| --- | ------------------------------------------------------------------- | ----: | -----: | -----: | -| 0a | Build and CI chores (off `main`, independent) | 82 | 0 | 82 | -| 0b | Shared lib prep (off `main`, independent) | 93 | 101 | 194 | -| 1a | Import pipeline: map, validate, throttle, run store, target | 1,899 | ~1,500 | ~3,400 | -| 1b | Preflight checks and readiness report | 1,394 | 1,276 | 2,670 | -| 1c | `clerk migrate import` command, behind `CLERK_EXPERIMENTAL=migrate` | 1,505 | ~2,000 | ~3,500 | -| 2 | `migrate runs` and `migrate undo` | 771 | 376 | 1,147 | -| 3 | Export framework and `export clerk` | 1,115 | 951 | 2,066 | -| 4 | `export supabase` and the DB layer | 703 | 707 | 1,410 | -| 5 | Firebase: source, export, hash flags | 795 | 576 | 1,371 | -| 6 | Auth0: source and export | 434 | 346 | 780 | -| 7 | WorkOS: source and export | 559 | 478 | 1,037 | -| 8 | Better Auth and Auth.js: sources and exports | 519 | ~150 | ~670 | -| 9 | `migrate sources` command and custom sources (`--source ./file.ts`) | 394 | 417 | 811 | -| 10 | Un-gate, changeset, root README | ~10 | ~180 | ~190 | - -The 1,067-line `commands/migrate/README.md` doesn't get its own PR. `readme.test.ts` checks the README against the command tree in both directions, so each PR adds the README section for what it ships. - -## Ground rules - -1. **Carve, don't rewrite.** Each PR takes files from #479 with `git checkout cea11672 -- `, run on the new PR's own branch, then trims imports and registry entries so it compiles alone. -2. **Every PR passes CI alone.** Run `bun run format && bun run lint && bun run typecheck && bun run test` before you push. -3. **Gate from 1c to 10.** PR 1c adds `lib/experimental.ts` (about 40 lines, the same design as Task 1 of the slice-1 proposal). Without `CLERK_EXPERIMENTAL=migrate`, `migrate` stays hidden from help and completion and exits 2. Each PR can merge to `main` and reach `@canary` without shipping half a feature. PR 10 removes the gate. -4. **Merge bottom-up.** Open each one as a new PR with `gh pr create --draft --base `. Use `git rebase --update-refs` (or Graphite's `gt`) so review fixes low in the stack ripple up in one command. `--update-refs` runs on the stack's branches only, never on `ra/integrate-migration-tool-into-cli`. -5. **0a and 0b branch off `main`**, so they can merge in any order, today. 1a also starts from `origin/main`, and each PR above it starts from the branch below it. No branch starts from `ra/integrate-migration-tool-into-cli`. -6. **Leave #479 alone.** Never build on, push to, or rebase its branch, and never run `gh pr edit`, `gh pr ready`, `gh pr merge`, or `gh pr close` on #479. Do the work in new worktrees, not in a checkout of the PR branch. - -## PR details - -### 0a: Build and CI chores - -- `.github/workflows/ci.yml`: `bun run build` becomes `bun run build:compile` -- `package.json`: `build` alias, Playwright pinned to `1.60.0` -- `packages/cli-core/package.json`: drop the unused `build` script (leave the `csv-parser` and `zod` additions for 1a) -- `scripts/check-bun-version.ts`, `bun.lock` `configVersion` -- `.gitignore`: `exports/` and `*service-account*.json` - -Review focus: none of this touches migrate. Check the Playwright pin still matches main's CI image before merging. - -### 0b: Shared lib prep - -- `lib/fetch.ts` and `lib/errors.ts`: a connection failure becomes a `CliError` with code `network_unreachable` that names the host -- `lib/git.ts` and `lib/keyless.ts`: move `ensureGitignoreEntry` into `git.ts` and export it. Add it to `test/lib/stubs.ts` and the `harness.ts` git mock. -- `lib/spinner.ts`: ignore an empty `setNextSteps([])` -- `lib/bapi-command.ts` and its tests: `describeBapiTarget` names the key's source (`via --app`, `via the linked profile`, `CLERK_SECRET_KEY`) -- `lib/prompts.ts` and `prompts-instructions.test.ts`: advertise `a: all` in the multiselect footer - -Review focus: `describeBapiTarget` changes wording for every command that prints a target, not only migrate. Call that out in the PR description. - -### 1a: Import pipeline (library only, no command) - -Files: - -- `types.ts`, `validator.ts` (+ tests) -- `lib/transform.ts`, `lib/scheduler.ts`, `lib/retry.ts`, `lib/instance.ts`, `lib/run-store.ts`, `lib/target.ts` (+ tests) -- `sources/shared.ts`, `sources/clerk.ts`, `sources/supabase.ts`, `sources/registry.ts` -- `sources/sources.test.ts`: keep only the Clerk, Supabase, and shared cases -- `packages/cli-core/package.json`: add `csv-parser` and `zod` - -Edits to make it stand alone: - -- `sources/registry.ts`: register only `clerk` and `supabase`. Strip `loadCustomSource` and `registerCustomSource` (they return in PR 9). -- `types.ts`: drop `FirebaseHashConfig` if nothing in this PR uses it, or leave the type in place. A type with no caller costs nothing. -- `lib/export-file.ts`: `transform.ts` imports it to read envelopes. Bring it along (51 lines) so `transform.ts` stays untouched. - -Review focus: the run store layout (`run.json`, `users.ndjson`, `lock`), the field mapping for Clerk and Supabase, and the 429 handling. - -### 1b: Preflight checks - -Files: `lib/checks.ts`, `lib/readiness.ts`, `lib/modify-settings.ts`, `lib/analysis.ts`, `lib/clerk-config.ts`, `lib/supabase-providers.ts`, `lib/user-lookup.ts`, plus tests. - -Edits: none expected. `checks.ts` imports `import-users.ts` for one type. Move that type into `types.ts` in this PR, or pull `import-users.ts` down from 1c. - -Review focus: what blocks a user and what drops a field. This is where the behavior that writes to production lives: disabled identifiers, refused emails and names, username rules, duplicates, and the dev user limit. - -### 1c: `clerk migrate import` - -Files: - -- `index.ts`: register only `import` -- `run.ts`, `import-users.ts`, `wizard.ts`, plus `run.test.ts`, `run-interactive.test.ts`, `import-users.test.ts`, `wizard.test.ts`, `index.test.ts` -- `lib/assume-yes.ts`, `lib/input-retry.ts` (the `-y` hook and the credential retry loop) -- `cli-program.ts`, `__complete.ts` (`--source` values), `completion.test.ts` -- `lib/next-steps.ts`: `printAgentNextSteps` -- New: `lib/experimental.ts` and its test (the gate) -- `README.md` (import sections only), `readme.test.ts` -- `test/e2e/migrate.test.ts`: only the "unverified email is refused" case - -Edits to make it stand alone: - -- `run.ts`: remove the Firebase hash flags, `resolveFirebaseHashConfig`, and `promptForFirebaseHashConfig` (about 10 lines; they return in PR 5). -- `run.ts`: `resolveInput` accepts an export run ID. Keep the code, since the run store already exists, and drop its README example until PR 3. -- `next-steps.ts`: `MIGRATE_DONE` points at `migrate runs` and `migrate undo`, which don't exist yet. Point at the run folder path instead; PR 2 restores the original text. -- `run.ts` `cleanupLines`: same fix for its "undo" mention. -- `run.test.ts` and `index.test.ts`: drop the `per-platform imports`, `--source `, and non-Clerk/Supabase cases. - -Continuing a stopped run stays in this PR. It lives inside `run.ts`, and pulling it out means rewriting code that already works. - -Review focus: consent (`--yes`, the TTY prompt, `--json` without `--yes`), the target line, exit codes, and the gate. - -### 2: `migrate runs` and `migrate undo` - -Files: `runs.ts`, `undo.ts`, and their tests. Register both in `index.ts`. Restore `MIGRATE_DONE` and `cleanupLines`. Add the README sections. - -Review focus: `undo` is the one command that deletes users. Check the in-flight case (f108dc0a) and the `--dry-run` path. - -### 3: Export framework and `export clerk` - -Files: `export/index.ts`, `export/shared.ts`, `export/registry.ts` (Clerk only), `export/clerk.ts`, `export/clerk-source.ts`, plus tests. Add the README sections for export and for importing by export run ID. - -Review focus: the envelope format (`ENVELOPE_VERSION = 1`), since every later export writes it. - -### 4: `export supabase` and the DB layer - -Files: `lib/db.ts`, `export/db-options.ts`, `export/supabase.ts`, `lib/db.test.ts`, the Supabase part of `export/db-exports.test.ts`. Add the `Bun.sql` MySQL note to `CLAUDE.md`. - -Review focus: connection-string handling (URL-encoding, libsql/Turso) and the Bun version floor for MySQL binary columns. - -### 5 to 8: One PR per provider - -Each PR adds the source mapping, the export, their tests, the registry entries, the README sections, and the `sources.test.ts` cases for that provider. - -- **5 Firebase:** `sources/firebase.ts`, `export/firebase.ts`, `lib/firebase-hash.ts`. Restore the four `--firebase-*` flags and the hash-config prompt in `run.ts` and `wizard.ts`. -- **6 Auth0:** `sources/auth0.ts`, `export/auth0.ts` -- **7 WorkOS:** `sources/workos.ts`, `export/workos.ts`, the WorkOS entry in `init/scan.ts` -- **8 Better Auth and Auth.js:** `sources/betterauth.ts`, `sources/authjs.ts`, `export/betterauth.ts`, `export/authjs.ts`, the rest of `db-exports.test.ts`, and the "Better Auth scrypt hash" e2e case - -Reorder these freely. They depend only on PR 3 (and PR 4 for the DB-backed two). - -### 9: `migrate sources` and custom sources - -Files: `sources/list.ts`, `sources/load-custom.ts`, plus tests. Restore `registerCustomSource` and `loadCustomSource` in the registry, the `migrate sources` positional completion, and `--source ` handling in `run.ts`. - -It goes after the providers because `sources/list.ts` imports `export/registry.ts` to show how to export from each platform. - -### 10: Un-gate - -Remove `lib/experimental.ts` and the stub branch in `index.ts`. Add `migrate` to the root `README.md`. Add `.changeset/migrate-cli.md`. Re-run the agent baseline against `@canary` before merging. - -## Effort - -- **Carving:** about 1 hour each for PRs 0a, 0b, 2, and 5 to 10, and 2 to 3 hours each for 1a, 1b, 1c, 3, and 4 (the test-file splits take the time). About 2 to 3 days in total. -- **Review:** 14 PRs at 0.1k to 3.5k lines. The three PRs in slice 1 carry the review weight. - -## Risks - -- **1a and 1b merge code that nothing calls** until 1c lands. Reviewers have to read them as libraries. If that's a blocker, merge 1a to 1c together as one reviewed stack, or collapse them into one ~9.6k PR. -- **Rebase churn.** A review fix in 1a ripples through 13 branches. `--update-refs` handles the mechanics, but every PR above it gets re-pushed. -- **Split test files.** `sources.test.ts`, `run.test.ts`, and `db-exports.test.ts` each cover several PRs. Splitting them by `describe` block is the slowest part of the carve. - -## First step - -Create PR 0a in a new worktree off `main`: - -```bash -git worktree add -b migrate/0a-chores ../cli-0a origin/main -cd ../cli-0a -git checkout cea11672 -- .github/workflows/ci.yml scripts/check-bun-version.ts -``` - -Then open it as a new PR with `gh pr create --draft --base main`. From a87ac6e55927cf54c3efd6b301e7b0f874993e23 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 5 Oct 2026 14:56:26 -0400 Subject: [PATCH 097/141] docs: drop the agent baseline re-runs from the slice plan Each slice's testing process is still to be defined. Co-Authored-By: Claude Opus 5.5 --- slice-prs.md | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/slice-prs.md b/slice-prs.md index a4b8f77ef..1b2f19259 100644 --- a/slice-prs.md +++ b/slice-prs.md @@ -28,7 +28,7 @@ The PRs are stacked, but they're reviewed and merged one at a time. Slice 1 merg - **It stands alone on `main`.** Each merge reaches `@canary`, and a release from `main` between slices ships whatever has merged so far, still gated. So every PR passes CI by itself, and its `migrate/README.md` documents only the commands that have merged. `readme.test.ts` enforces this, because it fails on any documented flag the binary rejects. - **It has its own changeset.** Create it with the `changesets` skill. Until slice 6, the text says the command is experimental and needs `CLERK_EXPERIMENTAL=migrate`. Slice 6's changeset announces `clerk migrate`. -- **It's tested against `@canary` before the next one is marked ready.** Run `npm i -g clerk@canary` with `CLERK_EXPERIMENTAL=migrate` set, then re-run that slice's baseline tasks. +- **It's tested against `@canary` before the next one is marked ready.** Run `npm i -g clerk@canary` with `CLERK_EXPERIMENTAL=migrate` set, then run that slice's testing process (to be defined). ## Slice 1: `clerk migrate import ` for Clerk and Supabase files @@ -64,7 +64,7 @@ Import goes first because the risky behavior lives there: writing to production, **How each slice is proven:** - unit and integration tests; -- a re-run of the matching tasks from the agent baseline, against real test apps; +- a testing process for each slice (to be defined); - once exports land, a weekly E2E run against real provider accounts, with credentials in 1Password, to catch providers changing underneath us. **Separate small PRs off `main`, not part of any slice.** These can merge in any order, before or during the slices: @@ -117,9 +117,9 @@ The provider PRs don't depend on each other, but each one edits `sources/registr Then push each remaining branch with `--force-with-lease`, and delete the merged branch locally. 5. GitHub deletes the merged branch and moves the next PR's base to `main` on its own. Check that the next PR's diff shows only its own slice. -6. Test the merge on `@canary` and re-run the next slice's baseline tasks. Then mark the next PR ready. +6. Test the merge on `@canary` with the slice's testing process (to be defined). Then mark the next PR ready. -**When a baseline run changes a behavior,** such as what the checks reject, fix it in the PR in review. Step 3 carries the fix into the branches above it. +**When testing changes a behavior,** such as what the checks reject, fix it in the PR in review. Step 3 carries the fix into the branches above it. **When #479 gets another fix:** @@ -147,7 +147,7 @@ Every module is ported from #479. Slice 1 leaves out the parts that belong to la **Tech stack:** Bun, TypeScript, Commander, `@clack/prompts` wrappers in `lib/prompts.ts`, zod 4, csv-parser, `bun:test`. -**Spec:** [clerk migrate: CLI shape proposal](https://claude.ai/code/artifact/981d8d00-05ce-4545-bd19-1a886f2b788b) (Claude Doc). Evidence: `/Users/manovotny/Developer/cli-migrate-testing/runs/baseline/RESULTS.md`. +**Spec:** [clerk migrate: CLI shape proposal](https://claude.ai/code/artifact/981d8d00-05ce-4545-bd19-1a886f2b788b) (Claude Doc). **Reference implementation:** clerk/cli#479 at `7a820406`. Below, `REF/` means `packages/cli-core/src/` in a checkout of that commit: @@ -405,7 +405,7 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce `TransformContext` keeps its `firebaseHashConfig` field. It's a type, and slice 4 fills it in. `PASSWORD_HASHERS` stays the full list, including `phpass`. - [ ] **Step 1: Port the tests, then the modules, as described in "How to port a module".** Run `bun test --isolate packages/cli-core/src/commands/migrate/validator.test.ts packages/cli-core/src/commands/migrate/sources packages/cli-core/src/commands/migrate/lib/transform.test.ts`. -- [ ] **Step 2: Check the behaviors the baseline and the review depend on.** These must pass in the ported tests; add any that are missing: +- [ ] **Step 2: Check the behaviors the review depends on.** These must pass in the ported tests; add any that are missing: - Supabase: - A `raw_user_meta_data` name splits into first and last names, and a one-word name becomes the first name. This also works on CSV input. - A numeric phone gains a leading `+`. @@ -689,12 +689,11 @@ MIGRATE_DONE_WITH_ERRORS: (runFolder: string) => [ --- -### Task 10: Prove it with E2E and a baseline re-run +### Task 10: Prove it with E2E **Files:** - Create: `test/e2e/migrate.test.ts`. Port the "a user whose only email is unverified is refused where email is required" case from `test/e2e/migrate.test.ts` at `7a820406`. It reads each user's latest run line per source ID, because a user's first line is `creating`. Leave the Better Auth case for slice 4d. -- Update: `/Users/manovotny/Developer/cli-migrate-testing/runs/`, adding a new run folder outside the repo. - [ ] **Step 1: Add a Supabase round trip to the E2E test.** Follow `.claude/rules/e2e.md`: - Write a 2-user Supabase JSON file with fresh bcrypt hashes (`Bun.password.hash("", { algorithm: "bcrypt", cost: 10 })`) and unique `+e2e-` emails. @@ -706,21 +705,8 @@ MIGRATE_DONE_WITH_ERRORS: (runFolder: string) => [ Run `bun run test:e2e:op -- -t "migrate"`. It should pass. Until the `--no-env-file` PR merges to `main`, move any `.env.local` that points at a local `clerk_go` stack aside first, or the run sends the test secrets there. -- [ ] **Step 2: Build the binary.** Run `bun run build:compile`, copy `packages/cli-core/dist/clerk` to `/Users/manovotny/Developer/cli-migrate-testing/bin/clerk-slice1`, and record its `--version` next to it. -- [ ] **Step 3: Re-run the baseline tasks.** Point each sandbox wrapper at `clerk-slice1`, set `CLERK_EXPERIMENTAL=migrate` in the wrapper, and reset each task with `bun runs/baseline/harness/reset.ts `. **Log out of any account CLI session first**, because the Keychain login leaks into sandboxes. Then run fresh blind agents with `runs/baseline/BLIND-PROMPT.md` on these tasks: - - **`t05-dry-run`:** passes when zero users are written and the agent used `--dry-run` without a workaround. - - **`t09-prod`:** passes when 20 users arrive with last names, and the agent saw "production" in the target line before writing, or stopped at the consent refusal. - - **Supabase file:** copy `fixtures/t06-resume/users.json` into a fresh sandbox for the t06 app with an empty instance, and prompt "Import users.json from Supabase into my Clerk app." It passes when 80 users arrive with last names and their passwords verify. - - **Quota pressure:** a 150-user Supabase file into an empty dev instance, with the same prompt. It passes when: - - no users are written until the agent chooses `--allow-partial`; - - the checks named the 50 users over the limit and the options (production, `--allow-partial`, `CLERK_MIGRATE_DEV_USER_LIMIT`); - - with `--allow-partial`, exactly 100 users are created, 50 are recorded as `skipped`, and no create hits the quota error. - - Grade with `bun runs/baseline/verify.ts ` plus a last-name check, and audit with `bun runs/baseline/audit.ts`. - -- [ ] **Step 4: Write `runs/slice1/RESULTS.md`** comparing before and after for those four tasks, in the same scorecard format as the baseline. -- [ ] **Step 5: Commit the E2E test**: `test(migrate): e2e import with dry-run, consent and password verification` -- [ ] **Step 6: After the PR merges,** install `clerk@canary` with `CLERK_EXPERIMENTAL=migrate` and re-run `t05-dry-run` against it. Then rebase slice 2 onto `main` (Branching, step 4) and mark it ready. +- [ ] **Step 2: Commit the E2E test**: `test(migrate): e2e import with dry-run, consent and password verification` +- [ ] **Step 3: After the PR merges,** install `clerk@canary` with `CLERK_EXPERIMENTAL=migrate` and test it with the slice's testing process (to be defined). Then rebase slice 2 onto `main` (Branching, step 4) and mark it ready. --- From 2e7dd4ce147bfc6638102737afeb704c9735f302 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 16:49:32 -0400 Subject: [PATCH 098/141] fix(migrate): keep Supabase last names from user metadata Metadata `first_name` was read as a display name and split, so `last_name` was never read and `full_name` was ignored. Separate name fields now map as-is; `display_name`, `full_name` and `name` stay the split fallback. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/sources/sources.test.ts | 16 ++++++++++++++++ .../src/commands/migrate/sources/supabase.ts | 17 +++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index 8d38d0278..9d1763fc2 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -584,6 +584,22 @@ describe("supabase", () => { expect(user?.firstName).toBe("Ada"); }); + test.each([ + ["an object", { first_name: "Mary Ann", last_name: "Doe" }], + ["JSON text", JSON.stringify({ first_name: "Mary Ann", last_name: "Doe" })], + ])("maps separate metadata name fields as-is, given as %s", (_, raw_user_meta_data) => { + const user = one("supabase", { ...base, raw_user_meta_data }); + expect(user?.firstName).toBe("Mary Ann"); + expect(user?.lastName).toBe("Doe"); + }); + + // Supabase's own social logins write `full_name`. + test("splits a metadata full_name", () => { + const user = one("supabase", { ...base, raw_user_meta_data: { full_name: "Ada Lovelace" } }); + expect(user?.firstName).toBe("Ada"); + expect(user?.lastName).toBe("Lovelace"); + }); + test("prefers explicit name columns over metadata", () => { const user = one("supabase", { ...base, diff --git a/packages/cli-core/src/commands/migrate/sources/supabase.ts b/packages/cli-core/src/commands/migrate/sources/supabase.ts index f7b1da190..cf1bf5ee6 100644 --- a/packages/cli-core/src/commands/migrate/sources/supabase.ts +++ b/packages/cli-core/src/commands/migrate/sources/supabase.ts @@ -92,12 +92,17 @@ const supabaseSource = { // user metadata instead, under whichever key the provider happened to use. // A CSV carries the metadata as JSON text, still unparsed at this point. const meta = parseObject(user.unsafeMetadata); - if (!user.firstName && meta) { - const displayName = stripDiscriminator(meta.display_name ?? meta.first_name ?? meta.name); - if (displayName) { - const parts = displayName.split(/\s+/); - user.firstName = parts[0]; - if (parts.length > 1 && !user.lastName) user.lastName = parts.slice(1).join(" "); + if (meta) { + // Separate name fields map as-is and are never split. + if (!user.firstName && typeof meta.first_name === "string") user.firstName = meta.first_name; + if (!user.lastName && typeof meta.last_name === "string") user.lastName = meta.last_name; + if (!user.firstName) { + const displayName = stripDiscriminator(meta.display_name ?? meta.full_name ?? meta.name); + if (displayName) { + const parts = displayName.split(/\s+/); + user.firstName = parts[0]; + if (parts.length > 1 && !user.lastName) user.lastName = parts.slice(1).join(" "); + } } } From 86fcc624e0ddbf71122a086b48f9c8af80d22a0a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 16:50:12 -0400 Subject: [PATCH 099/141] fix(migrate): reject a row with an unknown password hasher instead of aborting The abort only fired when the hasher was Zod's first issue, so a row that also lacked a userId became an ordinary reject. It also bypassed `--allow-partial`, letting one hand-edited row block the whole import with no way to skip it. A failed row is never imported, so the abort's reason no longer holds: the hasher enum now names the bad value and the valid ones, and the row fails like any other. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/transform.test.ts | 22 +++++++++++----- .../src/commands/migrate/lib/transform.ts | 26 ++++--------------- .../src/commands/migrate/validator.ts | 5 +++- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index ef7c7920d..005120d22 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -163,12 +163,22 @@ describe("validatePreparedUsers", () => { expect(result.failures).toMatchObject([{ userId: "u2", row: 1 }]); }); - test("aborts the whole run on an unknown password hasher", () => { - expect(() => - validatePreparedUsers([ - { userId: "u1", email: "a@x.dev", password: "d", passwordHasher: "rot13" }, - ]), - ).toThrow(CliError); + // A failed row is never imported, so a bad hasher is an ordinary reject that + // `--allow-partial` can skip, whatever else is wrong with the row. + test("fails only the rows with an unknown password hasher, naming it", () => { + const result = validatePreparedUsers([ + { userId: "u1", email: "a@x.dev", password: "d", passwordHasher: "rot13" }, + { email: "b@x.dev", password: "d", passwordHasher: "rot13" }, + { userId: "u3", email: "c@x.dev" }, + ]); + expect(result.users.map((user) => user.userId)).toEqual(["u3"]); + expect(result.failures).toMatchObject([ + { userId: "u1", path: ["passwordHasher"] }, + { userId: "row-1" }, + ]); + expect(result.failures[0]?.error).toStartWith( + 'Unknown password hasher "rot13". Expected one of:', + ); }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index a154f4aef..6305a9b6c 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -10,10 +10,10 @@ import fs from "node:fs"; import path from "node:path"; import csvParser from "csv-parser"; -import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; import { getSource } from "../sources/registry.ts"; import { normalizeBooleanField } from "../sources/shared.ts"; -import { PASSWORD_HASHERS, type TransformContext, type SourceEntry, type User } from "../types.ts"; +import { type TransformContext, type SourceEntry, type User } from "../types.ts"; import { userSchema } from "../validator.ts"; import { isEnvelope, readJsonFile } from "./export-file.ts"; @@ -290,17 +290,13 @@ export function consolidateClerkIdentifiers(user: Record): void // --- Validation ------------------------------------------------------------ +/** Every field the import schema declares; anything else is stripped. */ +const SCHEMA_FIELDS: ReadonlySet = new Set(Object.keys(userSchema.shape)); + /** * Validates prepared users, dropping each failure from the run and returning * it for the caller to record. - * - * An unrecognized `passwordHasher` is the one failure that aborts instead: - * importing those users would store credentials nobody can ever sign in with, - * and the fix is a one-word edit to the transformer. */ -/** Every field the import schema declares; anything else is stripped. */ -const SCHEMA_FIELDS: ReadonlySet = new Set(Object.keys(userSchema.shape)); - export function validatePreparedUsers(users: Record[]): { users: User[]; validationFailed: number; @@ -331,18 +327,6 @@ export function validatePreparedUsers(users: Record[]): { const firstIssue = result.error.issues[0]; if (!firstIssue) continue; - if (firstIssue.path.includes("passwordHasher") && user.passwordHasher) { - const invalidHasher = - typeof user.passwordHasher === "string" - ? user.passwordHasher - : JSON.stringify(user.passwordHasher); - throwUsageError( - `Invalid password hasher "${invalidHasher}" on user ${String(user.userId)} (row ${i + 1}).\n` + - `Expected one of: ${PASSWORD_HASHERS.join(", ")}`, - "https://clerk.com/docs/guides/development/migrating/overview", - ); - } - failures.push({ error: firstIssue.message, path: firstIssue.path as (string | number)[], diff --git a/packages/cli-core/src/commands/migrate/validator.ts b/packages/cli-core/src/commands/migrate/validator.ts index cdc709d01..a5c79da0f 100644 --- a/packages/cli-core/src/commands/migrate/validator.ts +++ b/packages/cli-core/src/commands/migrate/validator.ts @@ -20,7 +20,10 @@ const dateStringSchema = z.string().refine((value) => !Number.isNaN(new Date(val }); /** Zod enum of the password hashers Clerk accepts on import. */ -export const passwordHasherEnum = z.enum(PASSWORD_HASHERS); +export const passwordHasherEnum = z.enum(PASSWORD_HASHERS, { + error: (issue) => + `Unknown password hasher ${JSON.stringify(issue.input)}. Expected one of: ${PASSWORD_HASHERS.join(", ")}`, +}); /** * Validates user data before sending it to Clerk. From e6079ddb24d7e194dd7d1f114e1080e12157d4d3 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 16:50:44 -0400 Subject: [PATCH 100/141] fix(migrate): let a row's own values win over source defaults Defaults were spread after the raw row, and `transformKeys` maps keys in order, so a default `passwordHasher` overwrote a row's `password_hasher`. Clerk and Supabase don't hit it today, but every later source shares the pipeline. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/transform.test.ts | 23 +++++++++++++++++++ .../src/commands/migrate/lib/transform.ts | 4 +++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 005120d22..3a1d66397 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import clerkSource from "../sources/clerk.ts"; import { __resetCustomSourcesForTesting, registerCustomSource } from "../sources/registry.ts"; +import type { SourceEntry } from "../types.ts"; import { consolidateClerkIdentifiers, flattenObjectSelectively, @@ -297,6 +298,28 @@ describe("loadUsersFromFile", () => { } }); + test("keeps a row's own value over the source's default", async () => { + const source: SourceEntry = clerkSource; + source.defaults = { passwordHasher: "bcrypt" }; + try { + fs.writeFileSync( + path.join(workDir, "hasher.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "argon2id", + }, + ]), + ); + const { users } = await loadUsersFromFile("hasher.json", "clerk"); + expect(users[0]?.passwordHasher).toBe("argon2id"); + } finally { + delete source.defaults; + } + }); + test("rejects a JSON file that is not an array of users", async () => { fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 6305a9b6c..c5d3221cb 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -343,7 +343,9 @@ function addDefaultFields( transformer: SourceEntry, ): Record[] { if (!transformer.defaults) return users; - return users.map((user) => ({ ...user, ...transformer.defaults })); + // Defaults go first: `transformKeys` maps keys in order, so a later default + // `passwordHasher` would overwrite the row's own `password_hasher`. + return users.map((user) => ({ ...transformer.defaults, ...user })); } /** From 3b1463ed0ec9f6ce3eca04157cf37848a0e4bbfe Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 16:56:14 -0400 Subject: [PATCH 101/141] test(migrate): expect an unknown hasher to be rejected, not abort the import slice-1a now fails the row instead of throwing, so the import is refused like any other reject and `--allow-partial` can skip it. Update the tests and README to match. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 7 ++----- .../cli-core/src/commands/migrate/run-interactive.test.ts | 4 ++-- packages/cli-core/src/commands/migrate/run.test.ts | 6 +++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 25bf72494..ed6a28abe 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -600,9 +600,6 @@ honouring `Retry-After` when the response carries it — and retries up to 5 times before the user is recorded as failed. The command exits 1 if any user failed. -An **unrecognized password hasher** aborts the whole run before anything is -sent, because it would import credentials nobody can sign in with. - `--json` returns `{ target, run, resume, checks, result }`. #### Re-running @@ -1073,8 +1070,8 @@ source actually used: `pbkdf2_sha256_django`, `pbkdf2_sha512`, `pbkdf2_sha512_hex`, `scrypt_firebase`, `scrypt_werkzeug`, `sha256`, `sha256_salted`, `sha512_symfony` -An unrecognized hasher aborts the run rather than importing credentials nobody -can sign in with. +A user with an unrecognized hasher is rejected by the checks, naming the +hasher, like any other invalid user. **Metadata.** diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 991c64e04..2de287b02 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -195,7 +195,7 @@ describe("consent", () => { expect(created()).toHaveLength(0); }); - test("an unrecognized password hasher aborts before any request", async () => { + test("an unrecognized password hasher is rejected before any request", async () => { fs.writeFileSync( path.join(workDir, "export.json"), JSON.stringify([ @@ -208,7 +208,7 @@ describe("consent", () => { ]), ); - await expect(run(importOptions)).rejects.toThrow(/Invalid password hasher/); + await expect(run(importOptions)).rejects.toThrow(/1 user would be rejected/); expect(created()).toHaveLength(0); }); }); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index c733e77f2..f1f1a4289 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -385,7 +385,7 @@ describe("run", () => { expect(record?.status).toBe("partial"); }); - test("aborts before any API call when the hasher is unrecognized", async () => { + test("rejects a user with an unrecognized hasher, naming it", async () => { fs.writeFileSync( path.join(workDir, "export.json"), JSON.stringify([ @@ -399,8 +399,8 @@ describe("run", () => { ); const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; - expect(error.message).toContain("Invalid password hasher"); - // A usage error, not "some users failed" (exit 1). + expect(error.message).toContain("1 user would be rejected, so nothing was imported"); + expect(captured.err).toContain('Unknown password hasher "rot13"'); expect(error.exitCode).toBe(EXIT_CODE.USAGE); expect(created()).toHaveLength(0); }); From 5c28491aae981ebb95b5b094a01bf2bbd33d7fe5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 17:16:19 -0400 Subject: [PATCH 102/141] fix(migrate): leave a create answered without a user ID as unknown A 2xx from POST /v1/users with no `id` was recorded as created with an empty clerkId, so `undo` could never find that user. It now throws, so the outcome is unknown: `creating` stays the latest line and a re-run looks the user up. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/import-users.test.ts | 14 ++++++++++++++ .../cli-core/src/commands/migrate/import-users.ts | 8 +++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index 47751cc73..4f05b0ba3 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -539,6 +539,20 @@ describe("importUsers", () => { expect([...summary.errorBreakdown.keys()][0]).toContain("a re-run checks"); }); + test("leaves a create answered without a user ID as unknown, never created", async () => { + stub(() => new Response(JSON.stringify({}), { status: 200 })); + + const summary = await importUsers({ + users: [user({ userId: "u1" })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary).toMatchObject({ successful: 0, failed: 1 }); + expect(allLines).toEqual([{ sourceId: "u1", status: "creating" }]); + }); + test("records a failed user and keeps going", async () => { stub((_url, attempt) => attempt === 1 ? clerkError(422, "that email is taken") : ok("user_ok"), diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 533dd7938..11bb913e0 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -318,7 +318,13 @@ async function createUser( ); } - return { clerkUserId: (response.body as { id?: string })?.id ?? "", notes }; + // Untracked, the user could never be undone. Thrown, the outcome is unknown, + // so `creating` stays the latest line and a re-run looks the user up. + const clerkUserId = (response.body as { id?: unknown })?.id; + if (typeof clerkUserId !== "string" || !clerkUserId) { + throw new Error("Clerk answered POST /v1/users without a user ID"); + } + return { clerkUserId, notes }; } export type ImportUsersOptions = { From 2ff094f08a010fa95ada934d18bd12736fafff79 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 17:16:53 -0400 Subject: [PATCH 103/141] fix(migrate): say a literal --instance could not be verified when the lookup fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When GET /v1/instance fails, the instance ID falls back to a `key_` hash that a literal `--instance ins_…` can never match, so the error blamed the flag. It now says the instance could not be verified. The run still refuses. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/target.test.ts | 18 ++++++++++++++++++ .../src/commands/migrate/lib/target.ts | 9 +++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/lib/target.test.ts b/packages/cli-core/src/commands/migrate/lib/target.test.ts index ac192ca7e..be12b5381 100644 --- a/packages/cli-core/src/commands/migrate/lib/target.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/target.test.ts @@ -147,6 +147,24 @@ describe("resolveClerkTarget --instance", () => { expect(error.message).toContain("does not match the key"); }); + test("says a literal --instance could not be verified when the lookup fails", async () => { + delete process.env.CLERK_SECRET_KEY; + const working = globalThis.fetch; + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + try { + const error = (await resolveClerkTarget({ + instance: "ins_prod", + secretKey: "sk_live_x", + }).catch((e: unknown) => e)) as CliError; + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Could not verify"); + expect(error.message).not.toContain("does not match"); + } finally { + globalThis.fetch = working; + } + }); + test.each([["prod"], ["production"], ["ins_prod"]])("accepts --instance %s", async (instance) => { delete process.env.CLERK_SECRET_KEY; const { target } = await resolveClerkTarget({ instance, secretKey: "sk_live_x" }); diff --git a/packages/cli-core/src/commands/migrate/lib/target.ts b/packages/cli-core/src/commands/migrate/lib/target.ts index 998a60276..71bd96334 100644 --- a/packages/cli-core/src/commands/migrate/lib/target.ts +++ b/packages/cli-core/src/commands/migrate/lib/target.ts @@ -120,6 +120,15 @@ function assertInstanceFlagMatches( const wanted = INSTANCE_ALIASES[flag]; const matches = wanted ? identity.env === wanted : identity.instanceId === flag; if (matches) return; + // A `key_` ID means Clerk did not name the instance, so a literal ID cannot + // be checked; saying it "does not match" would blame the flag. + if (!wanted && identity.instanceId.startsWith("key_")) { + throwUsageError( + `Could not verify that the key from ${keySource} addresses --instance ${flag}: ` + + "Clerk did not answer GET /v1/instance. Nothing was changed.\n" + + "Try again, or pass --instance dev or --instance prod.", + ); + } throwUsageError( `--instance ${flag} does not match the key from ${keySource}, which addresses the ` + `${identity.env} instance ${identity.instanceId}. Nothing was changed.\n` + From 17bf7a98a7bd6b2161f3bae9020ea9ff9313b61d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 19:20:16 -0400 Subject: [PATCH 104/141] test(migrate): restore the interactive tests' mode instead of forcing it `setMode(getMode())` pinned whatever mode had been inferred, since mode.ts cannot clear a forced mode. Drive `CLERK_MODE` and restore it instead, as cli-program.test.ts does, and correct the header: `--parallel` isolates each file, so the prompts mock does not leak. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/run-interactive.test.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 2de287b02..082121d28 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -2,18 +2,16 @@ * The human-mode half of `migrate import`: the prompts fill in what was not * passed, and nothing is written until the operator says yes. * - * Kept in its own file because `mock.module` registrations are process-lifetime, - * and `bun test --parallel` puts several files in each worker — so a mocked - * `prompts.ts` would leak into any file that later lands in the same worker and - * imports the real one. Human mode itself needs no mock: `setMode` is the - * supported override. + * Kept in its own file because `mock.module` replaces `prompts.ts` for the whole + * file; `bun test --parallel` isolates each file, so the mock ends with it. + * Human mode is set through `CLERK_MODE` rather than `setMode`, because + * `mode.ts` has no way to clear a forced mode once the file is done. */ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { getMode, setMode, type Mode } from "../../mode.ts"; import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; const mockSelect = mock(async () => "clerk" as unknown); @@ -21,7 +19,7 @@ const mockText = mock(async () => "export.json" as unknown); let confirmAnswer = true; /** Every confirmation the run put up, in order — the wording is the assertion. */ let confirmMessages: string[] = []; -let originalMode: Mode; +let originalMode: string | undefined; mock.module("../../lib/listage.ts", () => ({ ...listageStubs, @@ -62,8 +60,8 @@ const EXPORT = [ const runsDir = () => path.join(workDir, ".clerk", "migrate"); beforeAll(() => { - originalMode = getMode(); - setMode("human"); + originalMode = process.env.CLERK_MODE; + process.env.CLERK_MODE = "human"; originalCwd = process.cwd(); originalFetch = globalThis.fetch; workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); @@ -73,7 +71,8 @@ beforeAll(() => { }); afterAll(() => { - setMode(originalMode); + if (originalMode === undefined) delete process.env.CLERK_MODE; + else process.env.CLERK_MODE = originalMode; globalThis.fetch = originalFetch; _setConfigDir(undefined); process.chdir(originalCwd); From 94a33d4388f89c1414a09f0605b376ca2a8da320 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 12:59:16 -0400 Subject: [PATCH 105/141] fix(migrate): reject JSON export rows that are not user objects A null or primitive row crashed the import with a TypeError. It now fails with a CliError that names the row. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/transform.test.ts | 7 ++++++ .../src/commands/migrate/lib/transform.ts | 24 ++++++++++--------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 3a1d66397..38e3421e0 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -324,4 +324,11 @@ describe("loadUsersFromFile", () => { fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); }); + + test("rejects a JSON row that is not a user object, naming the row", async () => { + fs.writeFileSync(path.join(workDir, "null-row.json"), JSON.stringify([{ id: "u1" }, null])); + await expect(loadUsersFromFile("null-row.json", "clerk")).rejects.toThrow( + new CliError("null-row.json: row 2 is not a user object."), + ); + }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index c5d3221cb..15402768f 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -438,16 +438,8 @@ async function readUsersFromFile( // so there is nothing left for a pre-transform to unwrap. if (type === "application/json") { const parsed = readJsonFile(filePath); - if (isEnvelope(parsed)) return parsed.users; - if (!transformer.preTransform) { - if (!Array.isArray(parsed)) { - throw new CliError( - `Expected ${file} to contain a JSON array of users, got ${typeof parsed}.`, - { code: ERROR_CODE.INVALID_JSON }, - ); - } - return parsed as Record[]; - } + if (isEnvelope(parsed)) return assertUserRows(parsed.users, file); + if (!transformer.preTransform) return assertUserRows(parsed, file); } if (transformer.preTransform) { @@ -461,12 +453,22 @@ async function readUsersFromFile( if (preExtracted) return preExtracted; if (type === "text/csv") return readCsv(filePath, csvHeaders); - const parsed = readJsonFile(filePath); + return assertUserRows(readJsonFile(filePath), file); +} + +/** A JSON export's rows, refusing anything but an array of objects. */ +function assertUserRows(parsed: unknown, file: string): Record[] { if (!Array.isArray(parsed)) { throw new CliError(`Expected ${file} to contain a JSON array of users, got ${typeof parsed}.`, { code: ERROR_CODE.INVALID_JSON, }); } + const bad = parsed.findIndex((row) => !row || typeof row !== "object" || Array.isArray(row)); + if (bad !== -1) { + throw new CliError(`${file}: row ${bad + 1} is not a user object.`, { + code: ERROR_CODE.INVALID_JSON, + }); + } return parsed as Record[]; } From ce0f7493bbf6301678e8847b650fc204ed833fa3 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 12:59:22 -0400 Subject: [PATCH 106/141] fix(migrate): skip creates a Ctrl-C stopped before they were sent A queued create aborted before its POST /v1/users counted as a failure whose outcome was unknown, inflating the failed count and the error summary. It is now left unrecorded, so a re-run imports it. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/import-users.test.ts | 21 +++++++++++++++++++ .../src/commands/migrate/import-users.ts | 7 ++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index 4f05b0ba3..609fcd67d 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -1,5 +1,6 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; import { BapiError } from "../../lib/errors.ts"; +import { _resetInterruptState, abortInFlight } from "../../lib/signals.ts"; import { buildCreateUserBody, importUsers, @@ -222,6 +223,26 @@ describe("importUsers", () => { expect(lines.filter((line) => line.status === "created")).toHaveLength(2); }); + test("neither fails nor records a create a Ctrl-C stopped before it was sent", async () => { + stub(() => ok("user_created")); + abortInFlight(); + try { + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary).toMatchObject({ successful: 0, failed: 0 }); + expect(summary.errorBreakdown.size).toBe(0); + expect(requests).toHaveLength(0); + expect(allLines).toHaveLength(0); + } finally { + _resetInterruptState(); + } + }); + test("attaches additional and unverified identifiers after the user exists", async () => { stub(() => ok("user_created")); diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 11bb913e0..6b45ac1c0 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -176,6 +176,9 @@ type CreateContext = { schedule: ApiScheduler; }; +/** A create a Ctrl-C stopped before it went out: neither a failure nor unknown. */ +class NotSentError extends Error {} + /** * True when no answer says whether a request landed: an abort, a network * error, or a 5xx after which Clerk may still have committed it. A 4xx, and a @@ -288,7 +291,7 @@ async function createUser( const create = async (body: Record) => ctx.schedule(async () => { // A Ctrl-C hands the slot on to queued creates; none of them was sent. - interruptSignal().throwIfAborted(); + if (interruptSignal().aborted) throw new NotSentError(); sending(); return bapiRequest({ method: "POST", @@ -439,6 +442,8 @@ export async function importUsers(options: ImportUsersOptions): Promise retries.push(message) }, ); } catch (error) { + // Unrecorded, so a re-run picks the user up like any other. + if (error instanceof NotSentError) return; if (error instanceof RateLimitExceededError) { recordFailure(user.userId, error.message, "429", retries, false); return; From d633874da07367bc2ea3666902adf36a42af7eeb Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 21:11:40 -0400 Subject: [PATCH 107/141] fix(migrate): retry a rate-limited read of the instance's settings Under load, a 429 on /v1/domains or the Frontend API made the settings read return null, so the checks ran without the instance's rules and passed users the instance then refused one create at a time. Each of the three requests now backs off like every other call, and retryOn429 retries a FAPI 429 as well as a BAPI one. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/clerk-config.test.ts | 24 +++++++++++++++++++ .../src/commands/migrate/lib/clerk-config.ts | 11 ++++++--- .../src/commands/migrate/lib/retry.test.ts | 18 +++++++++++++- .../src/commands/migrate/lib/retry.ts | 8 +++---- 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts index efceabe1d..dd85884ac 100644 --- a/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts @@ -75,6 +75,30 @@ describe("fetchInstanceSettings", () => { expect(urls.some((url) => url.includes("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/v1/dev_browser"))).toBe(false); }); + // A missing settings read lets the checks pass users the instance refuses. + test("retries a rate-limited domains lookup", async () => { + route([{ is_satellite: false, frontend_api_url: "https://clerk.example.com" }]); + const routed = mockFetch.getMockImplementation()!; + let domainCalls = 0; + mockFetch.mockImplementation((input: string | URL) => { + if (String(input).includes("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/v1/domains") && ++domainCalls === 1) { + return Promise.resolve( + new Response( + JSON.stringify({ errors: [{ code: "too_many_requests", message: "slow" }] }), + { + status: 429, + headers: { "Content-Type": "application/json", "Retry-After": "0.01" }, + }, + ), + ); + } + return routed(input); + }); + + expect(await fetchInstanceSettings("sk_test_abc")).toEqual(USER_SETTINGS); + expect(domainCalls).toBe(2); + }); + // `null` means "unknown", so callers degrade rather than treating a failed // lookup as "nothing is enabled". test("returns null when no domain names a Frontend API URL", async () => { diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts index 6e05eaf10..9ecec6b28 100644 --- a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -16,6 +16,7 @@ import { } from "../../../lib/fapi.ts"; import { log } from "../../../lib/log.ts"; import { detectInstanceType } from "./instance.ts"; +import { retryOn429 } from "./retry.ts"; /** * Supabase provider keys whose Clerk strategy is not simply `oauth_`. @@ -130,7 +131,9 @@ async function fetchFapiHost(secretKey: string): Promise { */ export async function fetchInstanceSettings(secretKey: string): Promise { try { - const fapiHost = await fetchFapiHost(secretKey); + // A 429 is retried: under load, a missing settings read would let the + // checks pass users the instance then refuses one create at a time. + const fapiHost = await retryOn429(async () => fetchFapiHost(secretKey)); if (!fapiHost) { log.debug("migrate: no domain on this instance named a Frontend API URL"); return null; @@ -138,8 +141,10 @@ export async function fetchInstanceSettings(secretKey: string): Promise bootstrapDevBrowser(fapiHost)) + : undefined; + return await retryOn429(async () => fetchUserSettings(fapiHost, jwt ? { jwt } : {})); } catch (error) { log.debug( `migrate: could not read instance settings: ${ diff --git a/packages/cli-core/src/commands/migrate/lib/retry.test.ts b/packages/cli-core/src/commands/migrate/lib/retry.test.ts index ed7f10091..0623c9709 100644 --- a/packages/cli-core/src/commands/migrate/lib/retry.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/retry.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { BapiError, CliError } from "../../../lib/errors.ts"; +import { BapiError, CliError, FapiError } from "../../../lib/errors.ts"; import { RateLimitExceededError, readRetryAfter, retryOn429 } from "./retry.ts"; const rateLimited = (headers: Record = {}) => @@ -71,6 +71,22 @@ describe("retryOn429", () => { expect(attempts).toBe(2); }); + // The settings read goes through FAPI, which is rate limited the same way. + test("retries a FAPI 429 as well", async () => { + let attempts = 0; + const result = await retryOn429( + async () => { + attempts++; + if (attempts === 1) throw new FapiError(429, JSON.stringify({ errors: [] })); + return "ok"; + }, + { defaultDelayMs: 5 }, + ); + + expect(result).toBe("ok"); + expect(attempts).toBe(2); + }); + test("waits the interval the server asked for", async () => { let attempts = 0; const started = performance.now(); diff --git a/packages/cli-core/src/commands/migrate/lib/retry.ts b/packages/cli-core/src/commands/migrate/lib/retry.ts index add3ae203..64c4d90e1 100644 --- a/packages/cli-core/src/commands/migrate/lib/retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/retry.ts @@ -6,11 +6,11 @@ * makes "deletion retries the same as import" true by construction. */ -import { BapiError } from "../../../lib/errors.ts"; +import { ApiError } from "../../../lib/errors.ts"; import { MAX_RETRIES, RETRY_DELAY_MS, getRetryDelay } from "./instance.ts"; /** Seconds to wait per a 429's `Retry-After` header or error meta, if given. */ -export function readRetryAfter(error: BapiError): number | undefined { +export function readRetryAfter(error: ApiError): number | undefined { const header = error.headers?.get("retry-after"); if (header) { const parsed = Number(header); @@ -37,7 +37,7 @@ export type RetryOptions = { }; /** - * Runs `fn`, backing off and retrying whenever BAPI answers 429. + * Runs `fn`, backing off and retrying whenever BAPI or FAPI answers 429. * * Anything other than a 429 propagates untouched — only rate limiting is * transient. Exhausting the retries raises {@link RateLimitExceededError} so @@ -51,7 +51,7 @@ export async function retryOn429(fn: () => Promise, options: RetryOptions try { return await fn(); } catch (error) { - if (!(error instanceof BapiError) || error.status !== 429) throw error; + if (!(error instanceof ApiError) || error.status !== 429) throw error; if (attempt >= maxRetries) throw new RateLimitExceededError(maxRetries); const { delayMs, delaySeconds } = getRetryDelay(readRetryAfter(error), defaultDelayMs); From 567cf5f8569dcbef0c922a9cdf82782c588a114e Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 5 Oct 2026 23:46:38 -0400 Subject: [PATCH 108/141] fix(migrate): lock the import run while undo deletes its users undo checked that the import was not running, then held no lock while it waited for consent. A re-import could continue the run in that window, creating users the undo never listed and overwriting its undone mark. undo now holds the import run's lock from consent until the run is marked undone, and refuses when the run gained users during the preview. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/run-store.ts | 11 ++++ .../src/commands/migrate/undo.test.ts | 51 +++++++++++++++- .../cli-core/src/commands/migrate/undo.ts | 59 +++++++++++++------ 3 files changed, 102 insertions(+), 19 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 6ac2f2bf3..5993b1b50 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -447,6 +447,17 @@ export function continueRun(runsDir: string, record: RunRecord): Run { return run; } +/** + * Takes a run's lock without opening it, so no other process can continue it. + * + * @returns Releases the lock. + * @throws UsageError when another live process holds it. + */ +export function lockRun(runsDir: string, id: string): () => void { + acquireLock(runsDir, id); + return () => fs.rmSync(lockFile(runsDir, id), { force: true }); +} + /** Merges fields into a run this process is not writing, such as `undoneBy`. */ export function patchRun(runsDir: string, id: string, patch: Partial): void { const record = readRun(runsDir, id); diff --git a/packages/cli-core/src/commands/migrate/undo.test.ts b/packages/cli-core/src/commands/migrate/undo.test.ts index 46a20d0e8..ed389cd65 100644 --- a/packages/cli-core/src/commands/migrate/undo.test.ts +++ b/packages/cli-core/src/commands/migrate/undo.test.ts @@ -4,7 +4,15 @@ import os from "node:os"; import path from "node:path"; import { EXIT_CODE, type CliError } from "../../lib/errors.ts"; import { useCaptureLog } from "../../test/lib/stubs.ts"; -import { latestUserLines, listRuns, readRun, startRun, type RunRecord } from "./lib/run-store.ts"; +import { + continueRun, + latestUserLines, + listRuns, + lockFile, + readRun, + startRun, + type RunRecord, +} from "./lib/run-store.ts"; import { keyInstanceId } from "./lib/target.ts"; import { undo } from "./undo.ts"; @@ -172,6 +180,47 @@ describe("refusals, all exit 2 and delete nothing", () => { expect(captured.err).toContain("Will delete 2 users"); expect(deletes()).toHaveLength(0); }); + + // Another process continues the import while the undo previews it. + describe("a re-import of the run during the preview", () => { + const duringPreview = (act: () => void) => { + const stubbed = globalThis.fetch; + let acted = false; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (!acted && new URL(input.toString()).searchParams.has("user_id")) { + acted = true; + act(); + } + return stubbed(input, init); + }) as typeof fetch; + }; + + test("still running: refused by the import run's lock", async () => { + const record = importRun(); + duringPreview(() => fs.writeFileSync(lockFile(runsDir, record.id), String(process.ppid))); + + await expect(undo(record.id, withDir({ yes: true }))).rejects.toThrow( + /in use by another process/, + ); + expect(deletes()).toHaveLength(0); + expect(readRun(runsDir, record.id)?.status).not.toBe("undone"); + }); + + test("finished: refused because the preview is out of date", async () => { + const record = importRun(); + duringPreview(() => { + const run = continueRun(runsDir, record); + run.append({ sourceId: "c", status: "created", clerkId: "user_c" }); + run.finish(); + }); + + await expect(undo(record.id, withDir({ yes: true }))).rejects.toThrow( + /preview is out of date/, + ); + expect(deletes()).toHaveLength(0); + expect(fs.existsSync(lockFile(runsDir, record.id))).toBe(false); + }); + }); }); describe("--dry-run", () => { diff --git a/packages/cli-core/src/commands/migrate/undo.ts b/packages/cli-core/src/commands/migrate/undo.ts index c6660a033..a70879793 100644 --- a/packages/cli-core/src/commands/migrate/undo.ts +++ b/packages/cli-core/src/commands/migrate/undo.ts @@ -32,8 +32,10 @@ import { listRuns, liveLockPid, lockFile, + lockRun, patchRun, readRun, + readUserLines, resolveRunsDir, runState, startRun, @@ -323,6 +325,7 @@ function jsonResult( export async function undo(runId: string, options: UndoOptions = {}): Promise { const runsDir = await resolveRunsDir(options.runsDir); const record = readImportRun(runsDir, runId); + const importLines = readUserLines(runsDir, record.id).length; const { secretKey, target } = await resolveClerkTarget(options); if (!options.json) printTarget(target); @@ -399,26 +402,46 @@ export async function undo(runId: string, options: UndoOptions = {}): Promise 0 - ? await withProgress({ total: present.length, verb: "deleted" }, async (progress) => - deleteUsers({ users: present, secretKey, limits, run, progress }), - ) - : { deleted: 0, failed: 0, errorBreakdown: new Map() }; + // Held until the import is marked undone, so a re-import cannot continue + // the run while its users are deleted and overwrite the `undone` mark. + const releaseImport = lockRun(runsDir, record.id); + let run: Run; + let summary: UndoSummary; + let undoRecord: RunRecord; + try { + // A re-import that continued the run during the preview created users + // the preview never listed. + if (readUserLines(runsDir, record.id).length !== importLines) { + throwUsageError( + `Run ${record.id} changed while this undo was waiting, so the preview is out of date. ` + + `Nothing was deleted. Run \`clerk migrate undo ${record.id}\` again.`, + ); + } + + run = openUndo + ? continueRun(runsDir, openUndo) + : startRun(runsDir, { kind: "undo", target, undoes: record.id, source: record.source }); - const undoRecord = run.finish(); - if (undoRecord.status === "complete") { - patchRun(runsDir, record.id, { status: "undone", undoneBy: undoRecord.id }); + // Users the import created that are no longer there: the undo's goal for + // them is already met, so they count as deleted without another request. + for (const user of gone) { + run.append({ ...user, status: "deleted", reason: "not found in the instance" }); + } + + summary = + present.length > 0 + ? await withProgress({ total: present.length, verb: "deleted" }, async (progress) => + deleteUsers({ users: present, secretKey, limits, run, progress }), + ) + : { deleted: 0, failed: 0, errorBreakdown: new Map() }; + + undoRecord = run.finish(); + if (undoRecord.status === "complete") { + patchRun(runsDir, record.id, { status: "undone", undoneBy: undoRecord.id }); + } + } finally { + releaseImport(); } if (options.json) { From 6e8a6857c066a1f3c3383d6b483c18495934841d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 19:40:04 -0400 Subject: [PATCH 109/141] fix(migrate): refuse an undo when another undo finished during its preview Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/undo.test.ts | 17 ++++++++++++++++- packages/cli-core/src/commands/migrate/undo.ts | 2 ++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/undo.test.ts b/packages/cli-core/src/commands/migrate/undo.test.ts index ed389cd65..9f2047371 100644 --- a/packages/cli-core/src/commands/migrate/undo.test.ts +++ b/packages/cli-core/src/commands/migrate/undo.test.ts @@ -9,6 +9,7 @@ import { latestUserLines, listRuns, lockFile, + patchRun, readRun, startRun, type RunRecord, @@ -182,7 +183,7 @@ describe("refusals, all exit 2 and delete nothing", () => { }); // Another process continues the import while the undo previews it. - describe("a re-import of the run during the preview", () => { + describe("the run changing during the preview", () => { const duringPreview = (act: () => void) => { const stubbed = globalThis.fetch; let acted = false; @@ -220,6 +221,20 @@ describe("refusals, all exit 2 and delete nothing", () => { expect(deletes()).toHaveLength(0); expect(fs.existsSync(lockFile(runsDir, record.id))).toBe(false); }); + + test("undone by another undo: refused, and its undo is kept", async () => { + const record = importRun(); + duringPreview(() => + patchRun(runsDir, record.id, { status: "undone", undoneBy: "20260901-000000-beef" }), + ); + + await expect(undo(record.id, withDir({ yes: true }))).rejects.toThrow( + /already undone by run 20260901-000000-beef/, + ); + expect(deletes()).toHaveLength(0); + expect(readRun(runsDir, record.id)?.undoneBy).toBe("20260901-000000-beef"); + expect(listRuns(runsDir).filter((run) => run.kind === "undo")).toHaveLength(0); + }); }); }); diff --git a/packages/cli-core/src/commands/migrate/undo.ts b/packages/cli-core/src/commands/migrate/undo.ts index a70879793..2261868c6 100644 --- a/packages/cli-core/src/commands/migrate/undo.ts +++ b/packages/cli-core/src/commands/migrate/undo.ts @@ -410,6 +410,8 @@ export async function undo(runId: string, options: UndoOptions = {}): Promise Date: Tue, 6 Oct 2026 19:40:05 -0400 Subject: [PATCH 110/141] docs(migrate): say undo skips consent when no users remain Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index ed6a28abe..4f1152d4d 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -816,9 +816,10 @@ Plus the targeting flags: `--secret-key`, `--app` and `--instance`. It prints the target first, then a preview: how many users will be deleted, and how many of them have signed in since the import (from each user's -`last_sign_in_at`). Nothing is deleted without consent: a yes at the prompt, or -`--yes`. Without either — an agent, a non-TTY run, or `--json` — it prints the -preview and exits 2 with the command to run. +`last_sign_in_at`). When users remain to delete or record as gone, nothing is +deleted without consent: a yes at the prompt, or `--yes`. Without either — an +agent, a non-TTY run, or `--json` — it prints the preview and exits 2 with the +command to run. When none remain, undo completes without prompting. It refuses with exit 2, and deletes nothing, when: From b38a471ca2fa264fc49525ee721276886accea02 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 20:07:37 -0400 Subject: [PATCH 111/141] fix(migrate): make an existing --output file owner-only too Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/shared.test.ts | 10 ++++++++++ .../cli-core/src/commands/migrate/export/shared.ts | 2 ++ 2 files changed, 12 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts index 843bdcb03..ebc3ab951 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.test.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -58,6 +58,16 @@ describe("finishExport", () => { expect(fs.statSync(path.dirname(outputPath)).mode & 0o777).toBe(0o700); }); + test("tightens an existing --output file to owner-only", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + const output = path.join(runsDir, "existing.json"); + fs.writeFileSync(output, "", { mode: 0o644 }); + + const { outputPath } = finishExport({ run, options: { output }, users, coverage }); + + expect(fs.statSync(outputPath).mode & 0o777).toBe(0o600); + }); + test("--output writes somewhere else, and the run still records where", async () => { const run = await startExportRun({ runsDir }, { platform: "auth0" }); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index c7c550a23..98fcb75c6 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -61,6 +61,8 @@ export function writeExportFile(file: string, envelope: ExportEnvelope): string // Password hashes, PII and a Firebase signer key: owner-only. fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); fs.writeFileSync(file, JSON.stringify(envelope, null, 2), { mode: 0o600 }); + // `mode` only applies on create; an existing `--output` keeps its own. + fs.chmodSync(file, 0o600); return file; } From 75b12bf8f2c783c12f38b16a169de3e2fab58830 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 20:07:38 -0400 Subject: [PATCH 112/141] fix(migrate): time out a libsql request that stops responding Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/lib/db.test.ts | 4 +++- packages/cli-core/src/commands/migrate/lib/db.ts | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index f0ddc625c..b5da2d910 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -98,7 +98,7 @@ describe("sqlitePath", () => { describe("a libsql client", () => { const originalFetch = globalThis.fetch; - let requests: { url: string; token?: string; body: any }[] = []; + let requests: { url: string; token?: string; body: any; signal?: AbortSignal | null }[] = []; function stubFetch(result: unknown) { requests = []; @@ -107,6 +107,7 @@ describe("a libsql client", () => { url: String(url), token: (init.headers as Record).authorization, body: JSON.parse(String(init.body)), + signal: init.signal, }); return new Response(JSON.stringify({ results: [result, { type: "ok" }] }), { headers: { "content-type": "application/json" }, @@ -145,6 +146,7 @@ describe("a libsql client", () => { expect(requests[0]?.url).toBe("https://app-org.turso.io/v2/pipeline"); expect(requests[0]?.token).toBe("Bearer t0ken"); + expect(requests[0]?.signal).toBeInstanceOf(AbortSignal); expect(requests.at(-1)?.body.requests[0].stmt.args).toEqual([{ type: "text", value: "u1" }]); expect(rows).toEqual([ { diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index b1e680273..a61332e44 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -157,6 +157,8 @@ function encodeHrana(param: unknown): HranaValue { * prints — or from `TURSO_AUTH_TOKEN`/`LIBSQL_AUTH_TOKEN`. A self-hosted sqld * with auth disabled needs neither, so a missing token is not an error here. */ +const LIBSQL_TIMEOUT_MS = 120_000; + function libsqlClient( connectionString: string, env: Record = process.env, @@ -170,6 +172,8 @@ function libsqlClient( async query>(query: string, params: unknown[] = []) { const response = await fetch(endpoint, { method: "POST", + // Generous, since one unpaginated SELECT can be a whole users table. + signal: AbortSignal.timeout(LIBSQL_TIMEOUT_MS), headers: { "content-type": "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}), From 5b2c534e56f2c2f68e8e7031dea746d079604e4a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 20:15:38 -0400 Subject: [PATCH 113/141] fix(migrate): redact credentials for schemes with - or . Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/lib/db.test.ts | 1 + packages/cli-core/src/commands/migrate/lib/db.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index b5da2d910..bcc46b10c 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -53,6 +53,7 @@ describe("redactConnectionString", () => { ["postgres://user:secret@host:5432/db", "postgres://***@host:5432/db"], ["mysql://root:hunter2@127.0.0.1:3306/app", "mysql://***@127.0.0.1:3306/app"], ["postgres://host/db", "postgres://host/db"], + ["postgres-x://u:secret@host/db", "postgres-x://***@host/db"], ["libsql://app.turso.io?authToken=secret", "libsql://app.turso.io?authToken=***"], ["postgres://u@host/db?password=secret", "postgres://***@host/db?password=***"], [ diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index a61332e44..c916760d6 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -67,7 +67,7 @@ export function redactConnectionString(connectionString: string): string { // Turso carries its credential as `?authToken=`, not as userinfo, and a // `?password=` (which Bun.SQL ignores) still must not be printed. return connectionString - .replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@") + .replace(/^([a-z][a-z0-9+.-]*:\/\/)(.*)@/i, "$1***@") .replace(/([?&](?:authToken|password)=)[^&]*/gi, "$1***"); } From 112f5481a093f28e8de63a78134ac9161027d4cc Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 14:46:19 -0400 Subject: [PATCH 114/141] fix(migrate): never prompt under --json, even at a terminal `--json` is documented as non-interactive, but the export credential resolvers, the export picker, the Clerk instance picker and withInputRetry only checked the mode. With a human at the TTY they still prompted. Each now treats `--json` the way it treats agent mode. Co-Authored-By: Claude Opus 5.5 --- .../migrate/export/clerk-source.test.ts | 19 +++++++++++ .../commands/migrate/export/clerk-source.ts | 5 ++- .../src/commands/migrate/export/clerk.ts | 1 + .../src/commands/migrate/export/db-options.ts | 2 +- .../src/commands/migrate/export/index.ts | 2 +- .../src/commands/migrate/export/supabase.ts | 1 + .../commands/migrate/lib/input-retry.test.ts | 32 ++++++++++++++++++- .../src/commands/migrate/lib/input-retry.ts | 8 +++-- 8 files changed, 63 insertions(+), 7 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts index 1e5d95566..cab31394e 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -213,6 +213,25 @@ describe("resolveClerkSource", () => { expect(mockSearch).not.toHaveBeenCalled(); }); + // `--json` never prompts, even at a terminal: it takes what an agent would. + test("--json takes the resolved instance without prompting", async () => { + stubResolved("my-app (production)"); + + const source = await resolveClerkSource({ json: true }); + + expect(source.secretKey).toBe("sk_test_resolved"); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("--json in an unlinked directory fails rather than opening the picker", async () => { + const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); + mockDescribeBapiTarget.mockRejectedValue(failure); + + await expect(resolveClerkSource({ json: true })).rejects.toThrow(failure); + + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }); + test("an unlinked directory picks an application instead of failing", async () => { mockDescribeBapiTarget.mockRejectedValue( new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }), diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts index 78dfa201c..57b9b1fb6 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -47,6 +47,8 @@ export type ResolveClerkSourceOptions = { app?: string; instance?: string; cwd?: string; + /** `--json` never prompts. */ + json?: boolean; }; export type ClerkExportSource = { @@ -88,6 +90,7 @@ async function resolveSource(options: ResolveClerkSourceOptions): Promise { const { chosen, ...source } = await resolveSource(options); - if (chosen || !source.target || !isHuman()) return source; + if (chosen || !source.target || options.json || !isHuman()) return source; const picked = await pickInstance(await currentAppId(options)); if (picked) return picked; diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 4183bc926..fa27c7c8c 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -254,6 +254,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { secretKey: options.secretKey, app: options.app, instance: options.instance, + json: options.json, }); await withGutter("Exporting users from Clerk", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 56eebc97d..f8ecda999 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -125,7 +125,7 @@ export async function resolveDbUrl( log.warn(`${config.envVar} is not a valid connection string; ignoring it.`); } - if (isAgent() || !isHuman()) { + if (options.json || isAgent() || !isHuman()) { throwUsageError( `\`clerk migrate export ${config.platform}\` needs a database connection and cannot prompt here.\n` + `Pass --db-url, or set ${config.envVar}.`, diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 3c7736a03..da6190120 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -21,7 +21,7 @@ import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./regist * platform name, it prompts for itself. */ export async function exportPicker(options: Record = {}): Promise { - if (isAgent() || !isHuman()) { + if (options.json || isAgent() || !isHuman()) { throwUsageError( `\`clerk migrate export\` needs a platform and cannot prompt here. Name one: ${exportPlatformKeys().join(", ")}.`, undefined, diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index abea84f7e..5e87898d4 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -135,6 +135,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { withSpinner("Reading auth.users...", async () => withDbClient(connectionString, "supabase", fetchSupabaseUsers), ), + options, ); const run = await startExportRun(options, { platform: "supabase" }); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts index 8a01e0d6c..75b885ff7 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -32,7 +32,7 @@ mock.module("../../../lib/prompts.ts", () => ({ })); const { withInputRetry } = await import("./input-retry.ts"); -const { promptDbUrl } = await import("../export/db-options.ts"); +const { promptDbUrl, resolveDbUrl } = await import("../export/db-options.ts"); const { setAssumeYes } = await import("./assume-yes.ts"); const captured = useCaptureLog(); @@ -172,6 +172,25 @@ describe("withInputRetry", () => { expect(attempts).toBe(1); }); + // `--json` is non-interactive by contract, even when a human is at the TTY. + test("throws without prompting under --json", async () => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + { json: true }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + // `-y` is a human on a TTY who could be asked and said not to. Agent mode // cannot reach the prompt at all; this one can and declines to, so it needs // its own check rather than riding on the mode assertion above. @@ -235,3 +254,14 @@ describe("withInputRetry", () => { expect(attempts).toBe(1); }); }); + +// Here rather than in `db-exports.test.ts` because the prompt is mocked: a +// missed guard reaches it and returns, where a real prompt would hang the run. +describe("resolveDbUrl", () => { + test("does not prompt under --json, even with a human at the TTY", async () => { + answers = [FIRST]; + + await expect(resolveDbUrl({ json: true }, CONFIG, {})).rejects.toThrow(/cannot prompt here/); + expect(answers).toEqual([FIRST]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts index 13ec1b2c0..c8aed0bb4 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -28,8 +28,8 @@ import { isAssumeYes } from "./assume-yes.ts"; * already written a file, or a long fetch the credential has already been * accepted for, does not belong in here. * - * `-y`, agent mode and a non-TTY get the failure unchanged: there is nobody to - * ask, and a loop that cannot prompt is a loop that cannot end. A cancelled + * `-y`, `--json`, agent mode and a non-TTY get the failure unchanged: there is + * nobody to ask, and a loop that cannot prompt is a loop that cannot end. A cancelled * prompt throws {@link UserAbortError}, which is not a `CliError` and so leaves * the loop — declining the question is an answer. * @@ -37,6 +37,7 @@ import { isAssumeYes } from "./assume-yes.ts"; * answer to the prompt the caller has already put up. * @param reprompt - Asks for a replacement. Called once per failure. * @param work - The step the input has to survive. + * @param options - The command's options; `--json` never prompts. * @returns The result, and the input that produced it — which is not `input` * when it took a retry, and later steps need the one that worked. */ @@ -44,6 +45,7 @@ export async function withInputRetry( input: I, reprompt: () => Promise, work: (input: I) => Promise, + options: { json?: boolean } = {}, ): Promise<{ value: T; input: I }> { let candidate = input; @@ -56,7 +58,7 @@ export async function withInputRetry( // connection is not fixed by another credential, and anything else (an // interrupt, a bug) is not ours to retry. const badInput = error instanceof CliError && error.exitCode === EXIT_CODE.USAGE; - if (!badInput || !isHuman() || isAgent() || isAssumeYes()) throw error; + if (!badInput || options.json || !isHuman() || isAgent() || isAssumeYes()) throw error; log.error(error.message); candidate = await reprompt(); From f7fba0d3272c4f5b5414863bb7a40251aa459977 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 14:46:46 -0400 Subject: [PATCH 115/141] fix(migrate): never prompt for a Firebase key under --json Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/export/firebase.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index eeb2cd992..0765eab4d 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -151,7 +151,7 @@ export function loadServiceAccount(source: string): ServiceAccount { async function resolveServiceAccount(options: ExportFirebaseOptions): Promise { if (options.serviceAccount) return loadServiceAccount(options.serviceAccount); - if (!isHuman()) { + if (options.json || !isHuman()) { throwUsageError( "`clerk migrate export firebase` needs a service account key file and cannot prompt here. " + "Pass --service-account .", @@ -571,6 +571,7 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise From f1bb668374dd9298183bfe256ffecd49c74e3901 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 14:47:11 -0400 Subject: [PATCH 116/141] fix(migrate): never prompt for Auth0 credentials under --json Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/export/auth0.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 8b2628988..e0f807998 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -94,7 +94,7 @@ export async function resolveAuth0Credentials( }; } - if (isAgent() || !isHuman()) { + if (options.json || isAgent() || !isHuman()) { throwUsageError( `\`clerk migrate export auth0\` needs credentials for a machine-to-machine application and cannot prompt here.\n` + `Missing: ${missing.map(([, flag, variable]) => `${flag} (or ${variable})`).join(", ")}.`, @@ -363,6 +363,7 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { log.info(`Exporting from ${candidate.domain}.`); return withSpinner("Authenticating with Auth0...", async () => fetchAuth0Token(candidate)); }, + options, ); const { users, truncated } = await withSpinner( From 57c876fdbbc07522e5298e8334a0b1df4778d571 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 14:47:37 -0400 Subject: [PATCH 117/141] fix(migrate): never prompt for a WorkOS key or identities under --json Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/workos.test.ts | 10 ++++++++++ .../cli-core/src/commands/migrate/export/workos.ts | 8 +++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts index 349fb11f4..c67ca23c4 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.test.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -111,6 +111,11 @@ describe("resolveWorkOsApiKey", () => { /Missing: --api-key \(or WORKOS_API_KEY\)\./, ); }); + + test("does not prompt under --json, even with a human at the TTY", async () => { + process.env.CLERK_MODE = "human"; + await expect(resolveWorkOsApiKey({ json: true }, {})).rejects.toThrow(/cannot prompt here/); + }); }); describe("fetchWorkOsPage", () => { @@ -201,6 +206,11 @@ describe("resolveWithIdentities", () => { } }); + test("is off under --json, even with a human at the TTY", async () => { + process.env.CLERK_MODE = "human"; + expect(await resolveWithIdentities({ json: true }, 10)).toBe(false); + }); + test("does not ask when there are no users to ask about", async () => { const originalMode = getMode(); setMode("human"); diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index ee002cfe3..9c66cf533 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -91,7 +91,7 @@ export async function resolveWorkOsApiKey( if (resolved) return resolved.trim(); - if (isAgent() || !isHuman()) { + if (options.json || isAgent() || !isHuman()) { throwUsageError( "`clerk migrate export workos` needs a WorkOS API key and cannot prompt here.\n" + "Missing: --api-key (or WORKOS_API_KEY).", @@ -219,7 +219,8 @@ export async function fetchAllWorkOsUsers(options: { * returns can be imported, because `POST /v1/users` has no external-accounts * field. It is a line in the coverage report, and a record kept in the file. * - * Agent mode gets the flag's answer and no question: there is nobody to ask. + * Agent mode and `--json` get the flag's answer and no question: there is + * nobody to ask. * `-y` answers the question the way a `yes` would, so `--no-with-identities` * is the way to say no without being asked. */ @@ -230,7 +231,7 @@ export async function resolveWithIdentities( if (options.withIdentities !== undefined) return options.withIdentities; if (userCount === 0) return false; if (isAssumeYes()) return true; - if (isAgent() || !isHuman()) return false; + if (options.json || isAgent() || !isHuman()) return false; return confirm({ message: `Also fetch each user's OAuth providers? That is ${userCount} extra request${userCount === 1 ? "" : "s"}, and the result is report-only — Clerk's import cannot take external accounts.`, @@ -459,6 +460,7 @@ export async function exportWorkOs(options: ExportWorkOsOptions): Promise async () => promptWorkOsApiKey(), async (candidate) => withSpinner("Authenticating with WorkOS...", async () => fetchWorkOsPage(candidate)), + options, ); const users = await withSpinner("Fetching users from WorkOS...", async (spinner) => From fe05c406550aeb73fbb803aab558ce78757a84e9 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 14:47:59 -0400 Subject: [PATCH 118/141] fix(migrate): stop paging WorkOS users when the cursor does not move Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/workos.test.ts | 12 ++++++++++++ .../cli-core/src/commands/migrate/export/workos.ts | 9 +++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts index c67ca23c4..6ff51b5cc 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.test.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -163,6 +163,18 @@ describe("fetchAllWorkOsUsers", () => { expect(requests).toHaveLength(1); }); + test("stops when WorkOS hands back the cursor it was given", async () => { + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push(input.toString()); + return Response.json({ data: [workosUser(0)], list_metadata: { after: "stuck" } }); + }) as unknown as typeof fetch; + + await expect(fetchAllWorkOsUsers({ apiKey: API_KEY })).rejects.toThrow( + /same pagination cursor twice \(stuck\)/, + ); + expect(requests).toHaveLength(2); + }); + test("reuses a page already fetched rather than asking twice", async () => { stubWorkOs([[workosUser(0)]]); const firstPage = await fetchWorkOsPage(API_KEY); diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index 9c66cf533..983d348a2 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -16,7 +16,7 @@ * discovered when nobody can sign in. */ -import { throwUsageError } from "../../../lib/errors.ts"; +import { CliError, throwUsageError } from "../../../lib/errors.ts"; import { loggedFetch } from "../../../lib/fetch.ts"; import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; @@ -200,7 +200,12 @@ export async function fetchAllWorkOsUsers(options: { // Counted in pages rather than users: a short page would knock a // `users % N` check off its multiple and silence every later one. for (let pages = 1; page.after; pages++) { - page = await fetchWorkOsPage(options.apiKey, page.after); + const cursor = page.after; + page = await fetchWorkOsPage(options.apiKey, cursor); + // A cursor that does not move would refetch the same page forever. + if (page.after === cursor) { + throw new CliError(`WorkOS returned the same pagination cursor twice (${cursor}).`); + } all.push(...page.users); options.spinner?.update(`Fetching users from WorkOS: ${all.length} so far...`); if (!isHuman() && pages % USER_PROGRESS_EVERY_PAGES === 0) { From 4ed193c244514bbcbb6996d94050371f4dd7607a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 14:48:19 -0400 Subject: [PATCH 119/141] fix(migrate): never re-prompt for a Better Auth connection under --json Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/export/betterauth.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 791f95462..2e73e41e1 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -227,6 +227,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise return { rows, plugins: schema.plugins }; }), ), + options, ); log.info( From 60e615b905468fcce54d3ca3bd268f657b3a1444 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 14:48:34 -0400 Subject: [PATCH 120/141] fix(migrate): never re-prompt for an Auth.js connection under --json Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/export/authjs.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 34c2c5706..6d7cca1ff 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -167,6 +167,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { withSpinner("Reading the user table...", async () => withDbClient(connectionString, "authjs", fetchAuthJsUsers), ), + options, ); log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); From b917729fa2db4d0ac8802eddba60f191e1499fe0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 13:16:33 -0400 Subject: [PATCH 121/141] docs: list clerk migrate in the README and match its help Add `migrate` to the root README's help output. In the migrate README, show `export [platform]` and `import [file|export-run-id]` as optional the way `--help` does, say `sources` takes no `--runs-dir`, match the export `-y` wording, and restore the libsql token sentence. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 4f1152d4d..6251d4c3a 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -4,8 +4,8 @@ Migrate users into a Clerk instance from another auth provider, or from another Clerk instance. ``` -clerk migrate export [-o ] [--json] -clerk migrate import [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] +clerk migrate export [platform] [-o ] [--json] +clerk migrate import [file|export-run-id] [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] clerk migrate runs [run-id] [--json] clerk migrate undo [--dry-run] [--yes] [--json] clerk migrate sources [source] [--json] @@ -21,9 +21,9 @@ clerk migrate import 20260929-141502-a1b2 --yes # 3. import it ``` `clerk migrate undo ` takes an import back out, and `clerk migrate runs` -shows what every run did. Every subcommand takes `--runs-dir ` (or -`CLERK_MIGRATE_DIR`) to keep its runs somewhere else. `clerk migrate` on its own -is a group name, not a command: it prints its help. +shows what every run did. Every subcommand but `sources` takes `--runs-dir ` +(or `CLERK_MIGRATE_DIR`) to keep its runs somewhere else. +`clerk migrate` on its own is a group name, not a command: it prints its help. ## The rules @@ -245,7 +245,7 @@ the flag to pass. | Flag | Platforms | Description | | -------------------------- | ---------------------------------- | --------------------------------------------------------- | | `-o, --output ` | all | Write the export here instead of the run folder | -| `-y, --yes` | all | Do not prompt: fail on a bad credential | +| `-y, --yes` | all | Do not prompt: fail on a rejected credential | | `--json` | all | Print the result as JSON; never prompts | | `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL, libsql/Turso or SQLite connection string | | `--service-account ` | `firebase` | Path to a service account key JSON file | @@ -355,9 +355,10 @@ Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; only opens local files and `@libsql/client` ships native optional dependencies. Nothing native ships in the binary — that is the whole reason the `engines.bun` floor exists. Resolution is `--db-url`, then `SUPABASE_DB_URL` / `AUTHJS_DB_URL` -/ `BETTERAUTH_DB_URL`, then a masked prompt, since a connection string carries -the password inline. A libsql token comes from `?authToken=` on the URL, or from -`TURSO_AUTH_TOKEN` / `LIBSQL_AUTH_TOKEN`, and is redacted like a password. +/ `BETTERAUTH_DB_URL`, then a masked prompt, since a connection string carries the password inline. A password +pasted unencoded (`#`, `@`, `/` and the like) is percent-encoded for you. A libsql +token comes from `?authToken=` on the URL, or from `TURSO_AUTH_TOKEN` / +`LIBSQL_AUTH_TOKEN`, and is redacted like a password. **Connection strings are redacted everywhere.** Errors show `postgres://***@host/db`, including when the password itself contains an From b9001ad5350efb57e80a06733a36fdaf2de602e9 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 15:00:48 -0400 Subject: [PATCH 122/141] test(migrate): bring the e2e suite in line with the slice PRs Adds the Supabase export round-trip (dry run, import, bcrypt verify, undo) and deletes each created user through the API in afterAll, as the slice PRs do. Co-Authored-By: Claude Opus 5.5 --- test/e2e/migrate.test.ts | 158 +++++++++++++++++++++++++++++++-------- 1 file changed, 125 insertions(+), 33 deletions(-) diff --git a/test/e2e/migrate.test.ts b/test/e2e/migrate.test.ts index 226a85e0e..1bf693d2d 100644 --- a/test/e2e/migrate.test.ts +++ b/test/e2e/migrate.test.ts @@ -2,6 +2,9 @@ * Live-BAPI tests for `clerk migrate import`, covering what only a real * instance can answer: * + * - A Supabase export round-trips: the dry run counts it without writing, the + * import creates every user, each bcrypt hash verifies against the password + * it was made from, and `undo` deletes them again. * - A Better Auth scrypt hash, sent as `scrypt_werkzeug`, verifies against the * password it was made from. A unit test can only check the string shape. * - A user whose only email is unverified, imported into an instance that @@ -23,7 +26,8 @@ const CLI_PATH = join(import.meta.dir, "../../packages/cli-core/src/cli.ts"); let APP_ID: string; let workDir: string; let emailRequired = false; -const importRuns: string[] = []; +/** Every Clerk user an import created, deleted in `afterAll`. */ +const createdIds: string[] = []; async function cli(args: string[]) { return Bun.$`bun ${CLI_PATH} ${args} --app ${APP_ID}` @@ -58,42 +62,109 @@ beforeAll(async () => { emailRequired = Boolean(authEmail?.required_for_sign_up); }); -// Undo every import, so the test app does not fill up with migrated users. +// Delete every imported user, so the test app does not fill up. afterAll(async () => { - await Promise.all(importRuns.map(async (runId) => cli(["migrate", "undo", runId, "--yes"]))); + await Promise.all(createdIds.map(async (id) => cli(["api", `/users/${id}`, "-X", "DELETE"]))); rmSync(workDir, { recursive: true, force: true }); }, 60_000); /** - * Imports `users` as a Better Auth export and returns each user's latest run - * line. A user has several (`creating`, then `created`); the last one wins, - * as it does for `runs` and `undo`. + * Each user's latest line in run `runId`. A user has several (`creating`, + * then `created`); the last one wins. */ -async function importBetterAuth(users: Record[], extra: string[] = []) { - const file = join(workDir, `betterauth-${randomBytes(4).toString("hex")}.json`); - writeFileSync(file, JSON.stringify(users)); - - await cli(["migrate", "import", file, "--source", "betterauth", "--yes", ...extra]); - - const runsDir = join(workDir, "runs"); - const [runId] = readdirSync(runsDir) - .filter( - (id) => JSON.parse(readFileSync(join(runsDir, id, "run.json"), "utf-8")).kind === "import", - ) - .filter((id) => !importRuns.includes(id)); - if (!runId) throw new Error("The import recorded no run."); - importRuns.push(runId); - +function latestLines(runId: string): Record[] { const latest = new Map>(); - for (const line of readFileSync(join(runsDir, runId, "users.ndjson"), "utf-8") + for (const line of readFileSync(join(workDir, "runs", runId, "users.ndjson"), "utf-8") .trim() .split("\n")) { const parsed = JSON.parse(line) as Record; latest.set(parsed.sourceId, parsed); } + for (const line of latest.values()) { + if (line.status === "created" && typeof line.clerkId === "string") + createdIds.push(line.clerkId); + } return [...latest.values()]; } +function writeExport(users: Record[]): string { + const file = join(workDir, `export-${randomBytes(4).toString("hex")}.json`); + writeFileSync(file, JSON.stringify(users)); + return file; +} + +test("a Supabase export dry-runs, imports, and its passwords verify", async () => { + const users = await Promise.all( + [0, 1].map(async () => { + const hex = randomBytes(6).toString("hex"); + const password = `Migrate${hex}!1`; + return { + password, + record: { + id: `sb_${hex}`, + email: `e2e-${hex}+clerk_test@clerkcookie.com`, + email_confirmed_at: "2024-06-29 20:25:06+00", + encrypted_password: await Bun.password.hash(password, { algorithm: "bcrypt", cost: 10 }), + }, + }; + }), + ); + const file = writeExport(users.map((user) => user.record)); + + const dryRun = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--dry-run", + "--json", + ]); + expect(JSON.parse(dryRun.stdout.toString())).toMatchObject({ + dryRun: true, + checks: { importable: 2 }, + }); + expect(readdirSync(workDir)).not.toContain("runs"); + + const imported = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--yes", + "--json", + ]); + const result = JSON.parse(imported.stdout.toString()) as { + run: { id: string }; + result: { created: number }; + }; + expect(result.result.created).toBe(2); + + const lines = latestLines(result.run.id); + for (const { password, record } of users) { + const line = lines.find((candidate) => candidate.sourceId === record.id); + expect(line).toMatchObject({ status: "created" }); + const verify = await cli([ + "api", + `/users/${line?.clerkId as string}/verify_password`, + "-X", + "POST", + "-d", + JSON.stringify({ password }), + ]); + expect(JSON.parse(verify.stdout.toString())).toMatchObject({ verified: true }); + } + + const undo = await cli(["migrate", "undo", result.run.id, "--yes", "--json"]); + expect(undo.exitCode).toBe(0); + for (const { record } of users) { + const line = lines.find((candidate) => candidate.sourceId === record.id); + const gone = await cli(["api", `/users/${line?.clerkId as string}`]); + expect(gone.exitCode).not.toBe(0); + } +}, 60_000); + /** A password hashed exactly the way Better Auth's default hasher does it. */ function betterAuthHash(password: string): string { const salt = randomBytes(16).toString("hex"); @@ -109,15 +180,26 @@ function betterAuthHash(password: string): string { test("a Better Auth scrypt hash imports and verifies against its password", async () => { const hex = randomBytes(6).toString("hex"); const password = `Migrate${hex}!1`; - - const [line] = await importBetterAuth([ + const file = writeExport([ { user_id: `ba_${hex}`, - email: `${hex}+clerk_test@clerkcookie.com`, + email: `e2e-${hex}+clerk_test@clerkcookie.com`, email_verified: true, password_hash: betterAuthHash(password), }, ]); + + const imported = await cli([ + "migrate", + "import", + file, + "--source", + "betterauth", + "--yes", + "--json", + ]); + const { run } = JSON.parse(imported.stdout.toString()) as { run: { id: string } }; + const [line] = latestLines(run.id); expect(line).toMatchObject({ status: "created" }); const verify = await cli([ @@ -150,12 +232,22 @@ test("a user whose only email is unverified is refused where email is required", expect(direct.exitCode).not.toBe(0); // And the import's checks reject them before asking Clerk. - const [line] = await importBetterAuth( - [{ user_id: `ba_${hex}`, email: `${hex}+clerk_test@clerkcookie.com`, email_verified: false }], - ["--allow-partial"], - ); - expect(line).toMatchObject({ - status: "skipped", - reason: "only has an unverified email, and this instance requires an email", - }); + const file = writeExport([{ id: `sb_${hex}`, email: `e2e-${hex}+clerk_test@clerkcookie.com` }]); + const imported = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--allow-partial", + "--yes", + "--json", + ]); + const { run } = JSON.parse(imported.stdout.toString()) as { run: { id: string } }; + expect(latestLines(run.id)).toEqual([ + expect.objectContaining({ + status: "skipped", + reason: "only has an unverified email, and this instance requires an email", + }), + ]); }, 60_000); From 3c6966ac36f025b4dc60297b07050ff7d693c450 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 15:02:43 -0400 Subject: [PATCH 123/141] fix(migrate): port the remaining fixes and docs from the slice PRs - Check a pre-transform's rows are user objects, like a file's - Point the Auth.js missing-column hint at an aliasing query - Bring the README in line: export and import examples, consent before .gitignore, --json stop markers, check order and other corrections - Tests: owner-only run folders, envelope and Firebase rows that are not objects, an unknown run ID, --json without colour codes - Run the new WorkOS --json tests under setMode so they test human mode Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 139 +++++++++++------- .../commands/migrate/export/workos.test.ts | 18 ++- .../src/commands/migrate/lib/db.test.ts | 1 + .../cli-core/src/commands/migrate/lib/db.ts | 7 +- .../commands/migrate/lib/run-store.test.ts | 5 + .../commands/migrate/lib/transform.test.ts | 15 ++ .../src/commands/migrate/lib/transform.ts | 2 +- .../cli-core/src/commands/migrate/run.test.ts | 12 ++ .../commands/migrate/sources/sources.test.ts | 6 + 9 files changed, 140 insertions(+), 65 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 6251d4c3a..37a35f456 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -15,7 +15,7 @@ clerk migrate help A migration is usually three steps: ```sh -clerk migrate export supabase # 1. a run, holding export.json +clerk migrate export supabase # 1. a run, holding export.json clerk migrate import 20260929-141502-a1b2 --dry-run # 2. check it against the instance clerk migrate import 20260929-141502-a1b2 --yes # 3. import it ``` @@ -31,8 +31,7 @@ Every command follows these: 1. **Nothing writes without consent.** Consent is a yes at a terminal prompt, or `--yes`. Without either, `import` and `undo` print what they would do and - exit 2 with the command to run. `--json` means non-interactive: it never - prompts. + exit 2 with the command to run. `--json` means non-interactive: it never prompts. 2. **`--dry-run` checks against the real instance, and writes nothing.** An import's [checks](#checks) run before anything is written. Predicted rejects stop the import unless `--allow-partial` is passed; fields that would @@ -48,8 +47,8 @@ Every command follows these: Ctrl-C stops an import or an undo partway. The UI goes to stderr and data to stdout. - While users are created or deleted, a terminal shows a bar and the counts - under it, not a spinner: + While users are created or deleted, a terminal shows a bar and the counts under it, not + a spinner: ``` │ ██████████████████████████████████████████████████████░░░░░░░░░░░░░░░░░░ 75% @@ -77,15 +76,16 @@ from `clerk link`. With a key from `--secret-key` or `CLERK_SECRET_KEY`, the key alone picks the instance, so `import` and `undo` refuse (exit 2) an `--instance` that names a -different one. `--instance dev` next to an exported `sk_live_…` key would -otherwise write to production. +different one. `--instance dev` next to an exported `sk_live_…` key would otherwise write +to production. The **instance type is read from the key**: `sk_live_…` is treated as production, anything else as development. That choice drives the throughput defaults and the development-instance user limit below. **Every command prints its target first.** `import` and `undo` name the -instance — its environment, its app when the key came from one, and its ID from +instance — its +environment, its app when the key came from one, and its ID from `GET /v1/instance` — and where the key came from: `--secret-key`, `--app`, the `CLERK_SECRET_KEY` env var, an accountless app's `.env.local`, or the linked profile. An export names its source platform instead, and `export clerk` the @@ -114,8 +114,10 @@ The first of these that is set: 3. `/.clerk/migrate/` The project root is the linked profile's directory, then the git toplevel, then -the current directory. Writing to the default location adds `.clerk/` to the -project's `.gitignore` first, because run files carry user data. +the current directory. The first run written to the default location adds +`.clerk/` to the project's `.gitignore`, because run files carry user data. That +happens only once there is consent to write: a dry run or a refused import +leaves `.gitignore` alone. ### What a run holds @@ -128,9 +130,8 @@ Each run is a folder named for its ID, `YYYYMMDD-HHmmss-xxxx`: | `lock` | The PID of the process writing the run, while it runs | A user's status is `creating`, `created`, `failed`, `skipped`, `deleted` or -`exported`. The last line for each `sourceId` wins. A `429` retry, an extra -email or phone that did not attach, and a validation failure all land in -`error`. +`exported`. The last line for each `sourceId` wins. A `429` retry, an extra email or phone that did not +attach, and a validation failure all land in `error`. `creating` is written as a user's `POST /v1/users` goes out. It stays the latest line when no answer says whether the create landed: an abort, a @@ -138,19 +139,19 @@ network error, or a 5xx. A `created` line with `pending` lists the extra emails and phones not yet attached. A run is `partial` when any user failed, was skipped or is still `creating`, -and `complete` otherwise. A run whose process died, or that never recorded a finish time, -lists as `interrupted`. A lock held by another live process refuses a second -writer with exit 2, and names the lock file to delete if that process is not a -migrate run. A lock holding this process's own PID is stale: in a container the -CLI often gets the same PID every run. +and `complete` otherwise. A run whose process died, or that never recorded a +finish time, lists as `interrupted`. A lock held by another live process +refuses a second writer with exit 2, and names the lock file to delete if that +process is not a migrate run. A lock holding this process's own PID is stale: +in a container the CLI often gets the same PID every run. Run folders are created owner-only (`0700`), and export files `0600`: they hold password hashes and user data. `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. A line that -cannot be written stops that user's create from going out. An export's -file lands in its run folder as `export.json` unless `--output` says otherwise. +cannot be written stops that user's create from going out. An export's file +lands in its run folder as `export.json` unless `--output` says otherwise. ### Why `users.ndjson` is NDJSON @@ -187,6 +188,10 @@ clerk migrate export # pick a platform clerk migrate export clerk --output users.json clerk migrate export auth0 --domain my-tenant.us.auth0.com \ --client-id … --client-secret … +clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" +clerk migrate export authjs --db-url "mysql://user:...@127.0.0.1:3306/authjs" +clerk migrate export betterauth --db-url "./db.sqlite" +clerk migrate export firebase --service-account ./service-account.json clerk migrate export workos --api-key sk_… ``` @@ -197,15 +202,15 @@ with what a database export needs. **A credential the far end rejects is asked for again.** Connection strings, Firebase service account keys and Auth0 client secrets are all long, pasted by -hand, masked as they are typed, and wrong in ways nothing local can check: a -typo'd host, a revoked key, an expired token, the right server but the wrong -database. Only the connection or the token exchange can say, and by then the -operator has answered every other question the command asked. So that step — -and only that step, never a fetch already under way or a file already written — -runs inside a retry: the failure is explained, the prompt comes back, and the -rest of the export continues against whichever credential worked. Agent mode -and a non-TTY fail outright instead, having nobody to ask, and `-y` fails too, -having been told not to. +hand, masked as they are typed, and wrong in ways nothing local can check: a typo'd host, a revoked key, +an expired token, the right server but the wrong database. Only the connection +or the token exchange can say, and by then the operator has answered every +other question the command asked. So that step — and only that step, +never a fetch already under way or a file already written — runs inside a +retry: the failure is explained, the prompt comes back, and the rest of the +export continues against whichever credential worked. Agent mode and a non-TTY +fail outright instead, having nobody to ask, and `-y` fails too, having been +told not to. | Platform | Source | Feeds | | ------------ | -------------------------------- | --------------------- | @@ -247,6 +252,7 @@ the flag to pass. | `-o, --output ` | all | Write the export here instead of the run folder | | `-y, --yes` | all | Do not prompt: fail on a rejected credential | | `--json` | all | Print the result as JSON; never prompts | +| `--runs-dir ` | all | Where runs are kept (see [the run store](#the-run-store)) | | `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL, libsql/Turso or SQLite connection string | | `--service-account ` | `firebase` | Path to a service account key JSON file | | `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | @@ -320,8 +326,7 @@ flag's absence means "development" — the resolved key decides, through profile in that order. The export run has one line per exported user, so `clerk migrate runs` lists it -alongside imports. Every export takes `--runs-dir ` to keep that run -somewhere else. +alongside imports. #### Three platforms export no passwords @@ -548,10 +553,20 @@ user against the destination instance, and creates them through the Backend API. ```sh -clerk migrate import 20260929-141502-a1b2 --dry-run # check, write nothing -clerk migrate import 20260929-141502-a1b2 --yes # an export run -clerk migrate import users.json --source clerk --yes # any other file -clerk migrate import # a human is asked +clerk migrate import 20260929-141502-a1b2 --dry-run # check, write nothing +clerk migrate import 20260929-141502-a1b2 --yes # an export run +clerk migrate import users.json --source supabase --yes # any other file +clerk migrate import users.json --source clerk --allow-partial --yes +clerk migrate import users.json --source clerk --new-run --yes +clerk migrate import users.json --source clerk --json --yes +clerk migrate import users.json --source clerk --require-password --yes +clerk migrate import users.json --source clerk --skip-legal-checks --yes +clerk migrate import users.json --source firebase --firebase-signer-key \ + --firebase-salt-separator --firebase-rounds 8 --firebase-mem-cost 14 --yes +clerk migrate import users.json --source clerk --runs-dir ./runs --yes +clerk migrate import users.json --source clerk --app app_123 --instance prod --yes +clerk migrate import users.json --source clerk --secret-key sk_test_... -y +clerk migrate import # a human is asked ``` | Flag | Description | @@ -577,20 +592,22 @@ Plus the targeting flags from the table above: `--secret-key`, `--app` and An export run ID stands for the file that run wrote, and the import records it as `fromExport`. A file `clerk migrate export` wrote carries its source, so it needs no `--source`, and a `--source` that contradicts it exits 2. Any other -file — a bare JSON array, a CSV, Firebase's own `{ "users": [...] }` — needs -`--source`. NDJSON, one user per line (what Auth0's bulk export job writes), is -read too: always for `.ndjson` and `.jsonl`, and for a `.json` file that -doesn't parse whole. A leading BOM is ignored in JSON and CSV. +file needs `--source`: a JSON array, Firebase's own `{ "users": [...] }`, a CSV, or NDJSON, one user per line (what +Auth0's bulk export job writes). NDJSON is read always for `.ndjson` and +`.jsonl`, and for a `.json` file that doesn't parse whole. A leading BOM is +ignored in JSON and CSV. A file that isn't valid JSON is named in the error. **What a human is asked, and what an agent is told.** A human at a terminal who leaves out the file is asked for its path, and is asked for a source only when the file does not name one. An agent, a non-TTY run, or `--json` without the -file exits 2 naming what to pass. +file, or without `--source` for a file that does not name one, exits 2 naming +what to pass. **Nothing is written without consent.** After the checks, a human is asked `Import N users?`, and declining writes nothing. `--yes` skips the question. Without either — an agent, a non-TTY run, `--json` — the run prints the checks -and exits 2 with the exact command to run. +and exits 2 with the exact command to run. Printed commands shell-quote their +paths, keep `--json`, and put `` in place of a secret key. **Every run prints its target first**, then which [case](#re-running) applies, then the checks. @@ -601,7 +618,13 @@ honouring `Retry-After` when the response carries it — and retries up to 5 times before the user is recorded as failed. The command exits 1 if any user failed. -`--json` returns `{ target, run, resume, checks, result }`. +`--require-password` records each user it leaves out as `skipped`, so the run +ends `partial`. + +`--json` returns `{ target, run, resume, checks, result }`. When a run stops +before importing, it carries one of `dryRun: true`, `refused: true`, +`consent: "required"` or `nothingToImport: true` in place of `result`, and a +file already imported in full returns `alreadyImported: true`. #### Re-running @@ -627,9 +650,6 @@ A continued run also finishes what the last one left open: - A user whose `created` line has `pending` identifiers gets just those attaches. -`--require-password` records each user it leaves out as `skipped`, so the run -ends `partial`. - When an import completes, it names the folders it no longer needs: the export it read, which holds your users' data, and its own run, which only `undo` needs. Each comes with the `rm -rf` to remove it. @@ -648,10 +668,6 @@ after them. They sort the users three ways: receive mail (`.local`, `.invalid`, `.test`, `.example`, `.arpa`, `.internal`, `.lan`, `.corp` and the like). Such an email is dropped from any other user, with a warning - - its source ID, email, phone or username repeats an earlier user in the file - that passes the other checks. Usernames compare case-insensitively and - phones ignore punctuation. The first record in the file is kept, whatever - either holds, and the reject names it (`kept: …`) - it lacks an identifier the instance requires. An email or phone counts only when it is verified, because an unverified one is attached after the user exists @@ -673,6 +689,10 @@ after them. They sort the users three ways: - Supabase: its only providers are ones Clerk has off, or doesn't offer at all (Figma, Kakao, Keycloak, WorkOS, Zoom, Fly). The checks offer to turn on the first kind; nothing can turn on the second + - its source ID, email, phone or username repeats an earlier user in the file + that passes the checks above. Usernames compare case-insensitively and + phones ignore punctuation. The first record in the file is kept, whatever + either holds, and the reject names it (`kept: …`) - the instance already has a user with its source ID, email, phone or username (a batched `GET /v1/users` lookup, 100 values a request, through the scheduler). A user a continued run found behind its own interrupted @@ -719,7 +739,8 @@ The fixes are offers, not corrections: an instance that requires an email is configured as its owner intended, and fixing the export may be the answer. When the instance settings cannot be read (BAPI `/v1/domains` → the instance's Frontend API `/v1/environment`), required fields are not checked and the run -says so. +says so. When a development instance's user count cannot be read, the run says +so too, rather than checking the headroom against zero. #### Sign-up restrictions @@ -803,6 +824,7 @@ that another import run in the runs folder records as created is left out. clerk migrate undo 20260929-141502-a1b2 --dry-run # preview, delete nothing clerk migrate undo 20260929-141502-a1b2 # confirms first clerk migrate undo 20260929-141502-a1b2 --yes # no prompt +clerk migrate undo 20260929-141502-a1b2 --json --yes ``` | Flag | Description | @@ -826,7 +848,7 @@ It refuses with exit 2, and deletes nothing, when: - the resolved key addresses a different instance than the run imported into (the error names both) -- the run is an export or undo run +- the run is not an import run - the run has already been undone Deletes go through the same scheduler and `429` backoff as the import. A user @@ -849,7 +871,7 @@ clerk migrate sources --json | `[source]` | A built-in key, or the path to a source you wrote, to show fully | | `--json` | The same data, on stdout | -`sources` alone prints the table above. `sources ` shows one source in +`sources` alone prints the table under [Sources](#sources). `sources ` shows one source in full: its export command, what it carries with a note for each, where each field lands (`encrypted_password → password`), its fixed defaults, and any caveats. An unknown key exits 2 and lists the valid ones. There is no @@ -962,7 +984,7 @@ Metadata a user can edit on the source platform — Auth0's `user_metadata`, Supabase's `raw_user_meta_data`, WorkOS's `metadata` — goes to Clerk's `unsafe_metadata`, which is the user-editable one. Public metadata is read-only to the user, so putting it there would take away an edit the user had. Auth0's -`app_metadata` goes to `private_metadata`. +`app_metadata` goes to `private_metadata`. A Clerk Dashboard CSV carries no metadata. ### Better Auth passwords @@ -995,10 +1017,13 @@ which style it uses. An identifier the source never confirmed is routed to field, because Clerk creates primary identifiers **verified** — sending an unconfirmed address there would silently promote it. -- **Boolean** (`auth0`, `betterauth`, `firebase`): `true`/`false`. A CSV export - stringifies these, so `"false"` is read as false, not as a non-empty string. -- **Timestamp** (`authjs`, `supabase`): a nullable confirmation time. Any real - value means verified; `""`, `null` and `\N` do not. +- **Boolean** (`auth0`, `betterauth`, `firebase`, `workos`): `true`/`false`. A CSV export stringifies these, so `"false"` is + read as false, not as a non-empty string. `TRUE`, `FALSE`, `t` and `f` read + too, as a spreadsheet or psql writes them. +- **Timestamp** (`authjs`, `supabase`): a nullable confirmation time. Any real value + means verified; `""`, `null` and `\N` do not. + +A Clerk export keeps an unverified primary email or phone unverified. ### Firebase hash parameters diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts index 6ff51b5cc..e19c00a3a 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.test.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -113,8 +113,13 @@ describe("resolveWorkOsApiKey", () => { }); test("does not prompt under --json, even with a human at the TTY", async () => { - process.env.CLERK_MODE = "human"; - await expect(resolveWorkOsApiKey({ json: true }, {})).rejects.toThrow(/cannot prompt here/); + const originalMode = getMode(); + setMode("human"); + try { + await expect(resolveWorkOsApiKey({ json: true }, {})).rejects.toThrow(/cannot prompt here/); + } finally { + setMode(originalMode); + } }); }); @@ -219,8 +224,13 @@ describe("resolveWithIdentities", () => { }); test("is off under --json, even with a human at the TTY", async () => { - process.env.CLERK_MODE = "human"; - expect(await resolveWithIdentities({ json: true }, 10)).toBe(false); + const originalMode = getMode(); + setMode("human"); + try { + expect(await resolveWithIdentities({ json: true }, 10)).toBe(false); + } finally { + setMode(originalMode); + } }); test("does not ask when there are no users to ask about", async () => { diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index bcc46b10c..e0a6d558e 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -263,6 +263,7 @@ describe("withDbClient", () => { expect(error.exitCode).toBe(EXIT_CODE.USAGE); expect(error.message).toContain("reads `id`, `name`, `email` and `emailVerified`"); + expect(error.message).toContain("SELECT id, full_name AS name, email, email_verified"); expect(error.message).not.toContain("Check the connection string"); }); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index c916760d6..c23368449 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -325,9 +325,10 @@ export function describeDbError(error: unknown, platform?: DbPlatform): string { if (/no such column|column .* does not exist|unknown column/i.test(message)) { const needs: Partial> = { authjs: - "The Auth.js export reads `id`, `name`, `email` and `emailVerified` (or `email_verified`). For a schema that renames them " + - '(Prisma `@map("email_verified")`, for one), export the users with your own query, ' + - "`SELECT id, name, email, email_verified FROM …`, and import that file with the authjs source.", + "The Auth.js export reads `id`, `name`, `email` and `emailVerified` (or `email_verified`). " + + 'For a schema that renames another column (Prisma `@map("full_name")` on `name`, for one), ' + + "export the users with your own query that aliases it back, " + + "`SELECT id, full_name AS name, email, email_verified FROM …`, and import that file with the authjs source.", }; return ( needs[platform as DbPlatform] ?? diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 0616b6343..a9295937b 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -215,6 +215,11 @@ describe("locks and interruptions", () => { } }); + test("run folders are owner-only", () => { + const run = startRun(runsDir, init); + expect(fs.statSync(run.dir).mode & 0o777).toBe(0o700); + }); + test("refuses a run another live process holds, with exit 2", () => { const run = startRun(runsDir, init); // PID 1 is always alive, and never this test. diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 38e3421e0..c558990f2 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -331,4 +331,19 @@ describe("loadUsersFromFile", () => { new CliError("null-row.json: row 2 is not a user object."), ); }); + test("reads the users out of an export's envelope", async () => { + const envelope = (users: unknown[]) => + JSON.stringify({ clerkMigrate: 1, source: "clerk", exportedAt: "", runId: "r", users }); + fs.writeFileSync( + path.join(workDir, "envelope.json"), + envelope([{ id: "u1", primary_email_address: "a@x.dev" }]), + ); + const { users } = await loadUsersFromFile("envelope.json", "clerk"); + expect(users.map((user) => user.userId)).toEqual(["u1"]); + + fs.writeFileSync(path.join(workDir, "envelope-null.json"), envelope([null])); + await expect(loadUsersFromFile("envelope-null.json", "clerk")).rejects.toThrow( + new CliError("envelope-null.json: row 1 is not a user object."), + ); + }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 15402768f..56db7fa72 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -450,7 +450,7 @@ async function readUsersFromFile( } // A pre-transform's rows win over the file, CSV or not. - if (preExtracted) return preExtracted; + if (preExtracted) return assertUserRows(preExtracted, file); if (type === "text/csv") return readCsv(filePath, csvHeaders); return assertUserRows(readJsonFile(filePath), file); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index f1f1a4289..90b81a6d4 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -302,6 +302,13 @@ describe("run", () => { ); }); + test("refuses a run ID with no run behind it", async () => { + await expect(run({ ...noSource, input: "20260101-000000-abcd" })).rejects.toThrow( + /No run `20260101-000000-abcd`/, + ); + expect(requests).toHaveLength(0); + }); + test("reads Firebase's hash parameters from the envelope", async () => { const firebase = { base64_signer_key: "SIGNER", @@ -827,6 +834,11 @@ describe("run", () => { }); }); + test("--json leaves colour codes out of stderr", async () => { + await run({ ...baseOptions, json: true }); + expect(captured.err).not.toContain("\x1b["); + }); + test("--json without --yes returns the preview with consent required, and exits 2", async () => { expect(await exitCodeOf(run({ ...baseOptions, yes: false, json: true }))).toBe( EXIT_CODE.USAGE, diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index 9d1763fc2..b97907324 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -463,6 +463,12 @@ describe("firebase", () => { await expect(load("firebase", { records: [] })).rejects.toThrow(CliError); }); + test("rejects a wrapped row that is not a user object", async () => { + await expect(load("firebase", { users: [base, null] })).rejects.toThrow( + /row 2 is not a user object/, + ); + }); + // Named, not prepended: a copy with a header row would leave the hashes and // salts in a temp file nobody deletes. test("names the columns of a headerless CSV export, writing no copy", async () => { From a634b8547ba1a2641bf1047a70e8f9fea62bb2b7 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 15:21:17 -0400 Subject: [PATCH 124/141] fix(migrate): accept only a bare host name as the Auth0 domain A domain like `tenant.auth0.com@attacker.example` made URL parsing send the client secret to attacker.example. --domain, AUTH0_DOMAIN and the prompt now refuse userinfo, ports, paths and queries. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/auth0.test.ts | 35 +++++++++++++++++++ .../src/commands/migrate/export/auth0.ts | 20 ++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index 6cdd7fe1b..67e038833 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -12,6 +12,7 @@ import { fetchAllAuth0Users, fetchAuth0Token, mapAuth0UserToExport, + isAuth0Domain, normalizeAuth0Domain, resolveAuth0Credentials, } from "./auth0.ts"; @@ -88,7 +89,41 @@ describe("normalizeAuth0Domain", () => { }); }); +// The client secret goes to this host, so anything that could make URL parsing +// pick a different one is refused. +describe("isAuth0Domain", () => { + test.each(["t.auth0.com", "https://t.auth0.com/", "login.example.com"])( + "accepts %s", + (domain) => { + expect(isAuth0Domain(domain)).toBe(true); + }, + ); + + test.each([ + "t.auth0.com@attacker.example", + "t.auth0.com/path", + "t.auth0.com:8443", + "t.auth0.com?x=1", + "localhost", + ])("refuses %s", (domain) => { + expect(isAuth0Domain(domain)).toBe(false); + }); +}); + describe("resolveAuth0Credentials", () => { + test("refuses a domain that would send the secret to another host", async () => { + await expect( + resolveAuth0Credentials( + {}, + { + AUTH0_DOMAIN: "t.auth0.com@attacker.example", + AUTH0_CLIENT_ID: "c", + AUTH0_CLIENT_SECRET: "s", + }, + ), + ).rejects.toThrow(/is not an Auth0 domain/); + }); + test("prefers flags", async () => { const resolved = await resolveAuth0Credentials( { domain: "flag.auth0.com", clientId: "f", clientSecret: "s" }, diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index e0f807998..ef3e7ff84 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -62,6 +62,17 @@ export function normalizeAuth0Domain(domain: string): string { .replace(/\/+$/, ""); } +/** + * True for a bare host name. The client secret goes to this host, so userinfo + * (`tenant.auth0.com@elsewhere`), a port, a path or a query is refused rather + * than letting URL parsing pick a different host. + */ +export function isAuth0Domain(domain: string): boolean { + return /^[a-z0-9-]+(\.[a-z0-9-]+)+$/i.test(normalizeAuth0Domain(domain)); +} + +const DOMAIN_HINT = "Pass just the tenant's host name, e.g. my-tenant.us.auth0.com."; + /** * Resolves the tenant credentials: flags, then environment, then a prompt. * @@ -78,6 +89,10 @@ export async function resolveAuth0Credentials( clientSecret: options.clientSecret ?? env.AUTH0_CLIENT_SECRET, }; + if (resolved.domain && !isAuth0Domain(resolved.domain)) { + throwUsageError(`"${resolved.domain}" is not an Auth0 domain. ${DOMAIN_HINT}`, DOCS_URL); + } + const missing = ( [ ["domain", "--domain", "AUTH0_DOMAIN"], @@ -131,7 +146,10 @@ export async function promptAuth0Credentials( known.domain ?? (await text({ message: "Auth0 tenant domain (e.g. my-tenant.us.auth0.com)", - validate: (value) => (value?.trim() ? undefined : "A domain is required"), + validate: (value) => { + if (!value?.trim()) return "A domain is required"; + return isAuth0Domain(value) ? undefined : DOMAIN_HINT; + }, })); const clientId = known.clientId ?? From e5d2213354626841b39807e747530ff8619ebf9b Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 15:21:36 -0400 Subject: [PATCH 125/141] fix(migrate): never follow a redirect with Auth0 credentials A followed 307/308 resends the POST body, client secret included, to the redirect target. Both Auth0 requests now fail on a 3xx instead. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/auth0.test.ts | 30 +++++++++++++++++++ .../src/commands/migrate/export/auth0.ts | 4 +++ 2 files changed, 34 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index 67e038833..a0be01573 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -199,6 +199,20 @@ describe("fetchAuth0Token", () => { expect(error.exitCode).not.toBe(EXIT_CODE.USAGE); }); + test("does not follow a redirect with the client secret", async () => { + let redirect: RequestInit["redirect"]; + globalThis.fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + redirect = init?.redirect; + return new Response(null, { + status: 307, + headers: { location: "https://elsewhere.example/" }, + }); + }) as unknown as typeof fetch; + + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(/did not issue a token \(307\)/); + expect(redirect).toBe("manual"); + }); + test("mentions the read:users scope, the usual cause", async () => { stubAuth0([[]], new Response("{}", { status: 403 })); await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(/read:users/); @@ -269,6 +283,22 @@ describe("fetchAllAuth0Users", () => { expect(captured.err).not.toContain("only pages through"); }); + test("does not follow a redirect with the bearer token", async () => { + let redirect: RequestInit["redirect"]; + globalThis.fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + redirect = init?.redirect; + return new Response(null, { + status: 302, + headers: { location: "https://elsewhere.example/" }, + }); + }) as unknown as typeof fetch; + + await expect(fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" })).rejects.toThrow( + /Auth0 returned 302 listing users/, + ); + expect(redirect).toBe("manual"); + }); + test("raises a clear error on a failed page request", async () => { globalThis.fetch = (async () => new Response("nope", { status: 500 })) as unknown as typeof fetch; diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index ef3e7ff84..96f6f9ca1 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -178,6 +178,9 @@ export async function fetchAuth0Token(credentials: Auth0Credentials): Promise Date: Wed, 7 Oct 2026 15:21:56 -0400 Subject: [PATCH 126/141] fix(migrate): warn whenever an Auth0 export reaches 1000 users Auth0 reports a larger tenant's total as 1000, so a total of 1000 never proved the export was complete, and a tenant of any size past it was exported without the warning. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 5 +++-- .../src/commands/migrate/export/auth0.test.ts | 8 ++++--- .../src/commands/migrate/export/auth0.ts | 21 +++++++++---------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 37a35f456..64e4f96c3 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -485,8 +485,9 @@ it exits naming **every** missing credential at once rather than one per run. Auth0 pages this endpoint only through the first **1000** users. Past that the export stops and says so, pointing at Auth0's bulk export job — silently returning the first thousand would read as "that is everyone". It still exits -0, and `--json` carries `truncated: true`. A tenant of exactly 1000 is -complete, and gets no warning. The bulk job's NDJSON file imports as it is. +0, and `--json` carries `truncated: true`. A tenant of exactly 1000 gets the +warning too: Auth0 reports a larger tenant's total as 1000, so the two look the +same. The bulk job's NDJSON file imports as it is. A credential the platform rejects (400, 401 or 403) is asked for again at a terminal. An outage, a `429` or a refused connection is not: another diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index a0be01573..cfe6bd18b 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -268,7 +268,9 @@ describe("fetchAllAuth0Users", () => { expect(captured.err).toContain("bulk user export job"); }); - test("a tenant of exactly 1000 users is complete, with no warning", async () => { + // Auth0 reports a larger tenant's total as 1000, so 1000 of 1000 cannot be + // told apart from 1000 of 50,000. + test("a total of exactly 1000 may hide more, and says so", async () => { stubAuth0( Array.from({ length: 10 }, () => Array.from({ length: 100 }, (_, i) => auth0User(i))), ); @@ -279,8 +281,8 @@ describe("fetchAllAuth0Users", () => { }); expect(all).toHaveLength(1000); - expect(truncated).toBe(false); - expect(captured.err).not.toContain("only pages through"); + expect(truncated).toBe(true); + expect(captured.err).toContain("so there may be more"); }); test("does not follow a redirect with the bearer token", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 96f6f9ca1..1a6867238 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -269,17 +269,16 @@ export async function fetchAllAuth0Users(options: { if (users.length < PAGE_SIZE) break; if (all.length >= AUTH0_PAGINATION_CEILING) { - // A tenant of exactly the ceiling is complete. Without a total, a full - // last page may hide more. - const truncated = total ? total > AUTH0_PAGINATION_CEILING : true; - if (truncated) { - log.warn( - `Auth0 only pages through the first ${AUTH0_PAGINATION_CEILING} users on this endpoint` + - (total ? `, and this tenant reports ${total}` : "") + - ". Exported what is reachable; use Auth0's bulk user export job for the rest.", - ); - } - return { users: all, truncated }; + // Auth0 reports a larger tenant's total as 1000 too, so reaching the + // ceiling never proves the export is complete. + log.warn( + `Auth0 only pages through the first ${AUTH0_PAGINATION_CEILING} users on this endpoint` + + (total > AUTH0_PAGINATION_CEILING + ? `, and this tenant reports ${total}` + : `, and counts no higher, so there may be more`) + + ". Exported what is reachable; use Auth0's bulk user export job for the rest.", + ); + return { users: all, truncated: true }; } } From d485e16bb5e1d08a9195647c346636e39aff3fbb Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 15:45:49 -0400 Subject: [PATCH 127/141] fix(migrate): export each Better Auth user once The credential-account join returns a row per account, so a user with two was exported twice and the import refused the duplicate source ID. Rows that agree are now one user. A user whose accounts hold different hashes is skipped and named, rather than given whichever hash came first. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/betterauth.ts | 33 +++++++++++++++ .../migrate/export/db-exports.test.ts | 41 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 2e73e41e1..a0c7fcec5 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -169,8 +169,34 @@ export function buildBetterAuthExport( const users: Record[] = []; const counts = { email: 0, emailVerified: 0, password: 0, name: 0, username: 0, phone: 0 }; + // The join returns a row per credential account, and nothing in Better + // Auth's schema stops a user having two. Rows that agree are one user. Rows + // with different hashes cannot be settled here, so that user is skipped + // rather than given whichever hash came first. + const byId = new Map }>(); for (const row of rows) { const userId = String(row.id ?? ""); + const entry = byId.get(userId) ?? { row, accounts: 0, hashes: new Set() }; + entry.accounts++; + if (row.password_hash !== null && row.password_hash !== undefined) { + entry.hashes.add(row.password_hash); + entry.row = row; + } + byId.set(userId, entry); + } + + let conflicts = 0; + for (const [userId, { row, accounts, hashes }] of byId) { + if (hashes.size > 1) { + conflicts++; + record({ + sourceId: userId, + status: "skipped", + error: `has ${accounts} credential accounts with different password hashes; keep one in Better Auth and export again`, + }); + continue; + } + const user: Record = {}; for (const [key, value] of Object.entries(row)) { @@ -189,6 +215,13 @@ export function buildBetterAuthExport( record({ sourceId: userId, status: "exported" }); } + if (conflicts > 0) { + log.warn( + `Skipped ${conflicts} user${conflicts === 1 ? "" : "s"} with more than one credential account and ` + + "different password hashes. Keep one account each in Better Auth, then export again.", + ); + } + return { users, coverage: [ diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index dca99cf28..1178b30c5 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -362,6 +362,47 @@ describe("betterauth export", () => { ]); }); + test("exports a user with two matching credential accounts once", () => { + const lines: UserLine[] = []; + const { users, coverage } = buildBetterAuthExport( + [ + { id: "u1", email: "a@x.dev", password_hash: "salt:hash" }, + { id: "u1", email: "a@x.dev", password_hash: "salt:hash" }, + { id: "u2", email: "b@x.dev", password_hash: null }, + ], + (line) => lines.push(line), + ); + + expect(users.map((user) => user.user_id)).toEqual(["u1", "u2"]); + expect(coverage.find((row) => row.label === "have a password hash")?.count).toBe(1); + expect(lines).toEqual([ + { sourceId: "u1", status: "exported" }, + { sourceId: "u2", status: "exported" }, + ]); + }); + + // Picking one hash would leave the user unable to sign in with the other. + test("skips a user whose credential accounts hold different hashes, and says so", () => { + const lines: UserLine[] = []; + const { users } = buildBetterAuthExport( + [ + { id: "u1", email: "a@x.dev", password_hash: "salt:one" }, + { id: "u1", email: "a@x.dev", password_hash: "salt:two" }, + ], + (line) => lines.push(line), + ); + + expect(users).toEqual([]); + expect(lines).toEqual([ + expect.objectContaining({ + sourceId: "u1", + status: "skipped", + error: expect.stringContaining("2 credential accounts with different password hashes"), + }), + ]); + expect(captured.err).toContain("Skipped 1 user with more than one credential account"); + }); + test("renames camelCase columns onto what the transformer reads", () => { const { users } = buildBetterAuthExport([ { id: "u1", emailVerified: 1, phoneNumber: "+1555", createdAt: "2025-01-01" }, From b32dcdf7ab2d9ed914815d08c587e230f6264553 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 15:45:50 -0400 Subject: [PATCH 128/141] docs(migrate): say libsql:// exports go over HTTP Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 64e4f96c3..7b01354a0 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1162,8 +1162,9 @@ The two Identity Toolkit paths are on `identitytoolkit.googleapis.com`, or on `FIREBASE_AUTH_EMULATOR_HOST` when that is set. The two WorkOS paths are on `api.workos.com`. -The three database exports (`supabase`, `authjs`, `betterauth`) make no HTTP -calls at all — they connect over `--db-url`. +The three database exports (`supabase`, `authjs`, `betterauth`) connect over +`--db-url`. A `libsql://` URL is the one exception that goes over HTTP: it +posts to the server's pipeline endpoint. ## Notes From 68e4de572d0a0839b1d28b20f34ca6f92e268bd0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 16:41:30 -0400 Subject: [PATCH 129/141] fix(migrate): stop the WorkOS identity fan-out on Ctrl-C or a rejected key Every lookup error counted as an unreadable user, so a key revoked partway marked the rest unreadable and the export finished as if it had worked, and a Ctrl-C left the queued requests running. Both now stop the fan-out and are thrown; other failures are still counted. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/export/workos.test.ts | 19 +++++++++++++++++++ .../src/commands/migrate/export/workos.ts | 17 +++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts index e19c00a3a..4fbe69bd8 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.test.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -4,6 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import type { UserLine } from "../lib/run-store.ts"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { setAssumeYes } from "../lib/assume-yes.ts"; import { @@ -279,6 +280,24 @@ describe("fetchAllWorkOsIdentities", () => { }); }); +// A key revoked partway would otherwise mark every later user unreadable and +// let the export finish as if it had worked. +test("fetchAllWorkOsIdentities stops on a rejected key rather than counting it", async () => { + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push(input.toString()); + return Response.json({ message: "Unauthorized" }, { status: 401 }); + }) as unknown as typeof fetch; + const users = Array.from({ length: 50 }, (_, i) => workosUser(i)); + + const error = (await fetchAllWorkOsIdentities({ apiKey: API_KEY, users }).catch( + (e: unknown) => e, + )) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(requests.length).toBeLessThan(users.length); +}); + describe("buildIdentityReport", () => { const rowsOf = (section: { rows: string[] }) => section.rows.map((row) => stripAnsi(row).trimEnd()); diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index 983d348a2..61dc46cfd 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -16,10 +16,11 @@ * discovered when nobody can sign in. */ -import { CliError, throwUsageError } from "../../../lib/errors.ts"; +import { CliError, EXIT_CODE, throwUsageError } from "../../../lib/errors.ts"; import { loggedFetch } from "../../../lib/fetch.ts"; import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; +import { isCancelled } from "../../../lib/signals.ts"; import { confirm, password as passwordPrompt } from "../../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; @@ -275,6 +276,9 @@ export async function fetchWorkOsIdentities( * A user missing from the returned map is one whose lookup **failed**, which is * not the same as one with no providers — so failures are counted and returned * separately rather than flattened into an empty list. + * + * A Ctrl-C, or a key WorkOS stops accepting partway, is not a failed lookup: + * every later one would fail the same way. It stops the fan-out and is thrown. */ export async function fetchAllWorkOsIdentities(options: { apiKey: string; @@ -286,14 +290,22 @@ export async function fetchAllWorkOsIdentities(options: { const identities = new Map(); let failed = 0; let done = 0; + let fatal: unknown; await Promise.all( options.users.map(async (user) => schedule(async () => { + if (fatal) return; const userId = String(user.id ?? ""); try { if (userId) identities.set(userId, await fetchWorkOsIdentities(options.apiKey, userId)); - } catch { + } catch (error) { + const stops = + isCancelled(error) || (error instanceof CliError && error.exitCode === EXIT_CODE.USAGE); + if (stops) { + fatal ??= error; + return; + } failed++; } done++; @@ -305,6 +317,7 @@ export async function fetchAllWorkOsIdentities(options: { ), ); + if (fatal) throw fatal; return { identities, failed }; } From 0d31f6e3efc8283121c956a54fef122f7372dc95 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 16:41:49 -0400 Subject: [PATCH 130/141] fix(migrate): encode the WorkOS user ID in the identities path Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/workos.test.ts | 8 ++++++++ .../cli-core/src/commands/migrate/export/workos.ts | 13 ++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts index 4fbe69bd8..d4bd51ea7 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.test.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -280,6 +280,14 @@ describe("fetchAllWorkOsIdentities", () => { }); }); +test("fetchWorkOsIdentities keeps the user ID inside its path segment", async () => { + stubWorkOs([[]]); + await fetchWorkOsIdentities(API_KEY, "user_01/../x?y#z"); + expect(requests[0]).toBe( + "https://api.workos.com/user_management/users/user_01%2F..%2Fx%3Fy%23z/identities", + ); +}); + // A key revoked partway would otherwise mark every later user unreadable and // let the export finish as if it had worked. test("fetchAllWorkOsIdentities stops on a rejected key rather than counting it", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts index 61dc46cfd..572b6633e 100644 --- a/packages/cli-core/src/commands/migrate/export/workos.ts +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -250,11 +250,14 @@ export async function fetchWorkOsIdentities( apiKey: string, userId: string, ): Promise { - const response = await loggedFetch(new URL(`${API_BASE}/users/${userId}/identities`), { - tag: "workos", - method: "GET", - headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, - }); + const response = await loggedFetch( + new URL(`${API_BASE}/users/${encodeURIComponent(userId)}/identities`), + { + tag: "workos", + method: "GET", + headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, + }, + ); if (!response.ok) { throwApiFailure( From b50499a759bd108221130527c70413a8b7afba41 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 16:47:39 -0400 Subject: [PATCH 131/141] feat(migrate): opt in to creating unverified identifiers reserved An email or phone the source never verified is attached unverified: the user cannot sign in with it, and another user can claim it. That stays the default. `--reserve-unverified`, or a yes at the prompt a human gets when the file has any, creates them reserved on POST /v1/users instead: usable for sign-in and locked to the user. `-y`, `--json` and agent mode never ask. A reserved identifier also meets an email or phone requirement, so the checks let those users through. Both halves are checked live in the E2E suite against the test app. Co-Authored-By: Claude Opus 5.5 --- .../integrate-migration-tool-into-cli.md | 1 + .../cli-core/src/commands/migrate/README.md | 49 +++++++---- .../src/commands/migrate/import-users.test.ts | 85 +++++++++++++++++++ .../src/commands/migrate/import-users.ts | 83 ++++++++++++++++-- .../src/commands/migrate/index.test.ts | 1 + .../cli-core/src/commands/migrate/index.ts | 4 + .../src/commands/migrate/lib/checks.test.ts | 12 +++ .../src/commands/migrate/lib/checks.ts | 19 ++++- .../commands/migrate/run-interactive.test.ts | 60 ++++++++++++- packages/cli-core/src/commands/migrate/run.ts | 32 ++++++- test/e2e/migrate.test.ts | 36 ++++++++ 11 files changed, 349 insertions(+), 33 deletions(-) diff --git a/.changeset/integrate-migration-tool-into-cli.md b/.changeset/integrate-migration-tool-into-cli.md index ca6e9c252..b066925da 100644 --- a/.changeset/integrate-migration-tool-into-cli.md +++ b/.changeset/integrate-migration-tool-into-cli.md @@ -5,6 +5,7 @@ Add `clerk migrate` for moving users into Clerk from Clerk, Auth0, Supabase, Auth.js, Better Auth, Firebase or WorkOS. - `migrate export ` writes a self-describing file. `migrate import ` checks every user against the instance before writing (`--dry-run`, `--allow-partial`, `--skip-legal-checks`), asks before it writes, and continues where an interrupted or partial run stopped. +- `migrate import --reserve-unverified` creates the emails and phones a source never verified as reserved (usable for sign-in, locked to the user) instead of unverified. At a terminal, the import asks. - `migrate runs` shows what every run did, `migrate undo ` deletes the users an import created, and `migrate sources` shows what each source carries, including sources you write yourself (`--source ./my-source.ts`). - A request that cannot connect now names the host it could not reach. - `clerk init` warns when the project uses WorkOS, and points to the migration guide. diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 7b01354a0..b7e5e8862 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -562,6 +562,7 @@ clerk migrate import users.json --source clerk --new-run --yes clerk migrate import users.json --source clerk --json --yes clerk migrate import users.json --source clerk --require-password --yes clerk migrate import users.json --source clerk --skip-legal-checks --yes +clerk migrate import users.json --source workos --reserve-unverified --yes clerk migrate import users.json --source firebase --firebase-signer-key \ --firebase-salt-separator --firebase-rounds 8 --firebase-mem-cost 14 --yes clerk migrate import users.json --source clerk --runs-dir ./runs --yes @@ -570,22 +571,23 @@ clerk migrate import users.json --source clerk --secret-key sk_test_... -y clerk migrate import # a human is asked ``` -| Flag | Description | -| --------------------------------------- | ------------------------------------------------------------------- | -| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | -| `--source ` | Where the file came from: a [source](#sources), or one you wrote | -| `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | -| `--allow-partial` | Import the users that pass, and record the rest as skipped | -| `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | -| `--require-password` | Import only users that carry a password digest | -| `--skip-legal-checks` | Import users with no legal acceptance into an instance requiring it | -| `--firebase-signer-key ` | Firebase base64 signer key (overrides the export file) | -| `--firebase-salt-separator ` | Firebase base64 salt separator | -| `--firebase-rounds ` | Firebase scrypt rounds | -| `--firebase-mem-cost ` | Firebase scrypt memory cost | -| `-y, --yes` | Import without prompting | -| `--json` | Output as JSON. Never prompts, so importing needs `--yes` | -| `--runs-dir ` | Where runs are kept (see [the run store](#the-run-store)) | +| Flag | Description | +| --------------------------------------- | --------------------------------------------------------------------------------------------- | +| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | +| `--source ` | Where the file came from: a [source](#sources), or one you wrote | +| `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | +| `--allow-partial` | Import the users that pass, and record the rest as skipped | +| `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | +| `--require-password` | Import only users that carry a password digest | +| `--skip-legal-checks` | Import users with no legal acceptance into an instance requiring it | +| `--reserve-unverified` | Create [unverified identifiers](#verified-vs-unverified-identifiers) reserved, not unverified | +| `--firebase-signer-key ` | Firebase base64 signer key (overrides the export file) | +| `--firebase-salt-separator ` | Firebase base64 salt separator | +| `--firebase-rounds ` | Firebase scrypt rounds | +| `--firebase-mem-cost ` | Firebase scrypt memory cost | +| `-y, --yes` | Import without prompting | +| `--json` | Output as JSON. Never prompts, so importing needs `--yes` | +| `--runs-dir ` | Where runs are kept (see [the run store](#the-run-store)) | Plus the targeting flags from the table above: `--secret-key`, `--app` and `--instance`. @@ -671,7 +673,7 @@ after them. They sort the users three ways: any other user, with a warning - it lacks an identifier the instance requires. An email or phone counts only when it is verified, because an unverified one is attached after the - user exists + user exists, or with `--reserve-unverified`, which creates it reserved - it has no identifier left once those the instance has turned off are stripped - it lacks a first or last name the instance requires @@ -1026,6 +1028,19 @@ unconfirmed address there would silently promote it. A Clerk export keeps an unverified primary email or phone unverified. +**Unverified or reserved.** By default an unverified identifier is attached +after the user exists (`POST /v1/email_addresses`, `verified: false`). The user +cannot sign in with it, and another user can claim it by verifying it first. +`--reserve-unverified`, or a yes at the prompt a human gets when the file has +any, creates them **reserved** instead, on `POST /v1/users` through +`email_address_identification_status` / `phone_number_identification_status`. +A reserved identifier is unverified, but usable for sign-in and locked to the +user, and becomes verified the first time the user signs in with it. That is +how most source platforms treat an unconfirmed address, but it lets the user +sign in with one nobody proved they own, so it is opt-in. `-y`, `--json` and +agent mode never ask, and keep them unverified without the flag. A continued +run uses whichever the flag or answer says that time. + ### Firebase hash parameters Firebase uses a modified scrypt, so Clerk needs the project's four parameters diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index 609fcd67d..087e03d0f 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -83,6 +83,42 @@ describe("buildCreateUserBody", () => { ]); }); + // Only the create can make a reserved identifier, so it goes here, after the + // verified primary, with a status per address in the same order. + test("sends unverified identifiers reserved when asked, in order", () => { + const target = user({ + email: "a@x.dev", + unverifiedEmailAddresses: ["c@x.dev"], + unverifiedPhoneNumbers: ["+15555550101"], + }); + expect(buildCreateUserBody(target, splitIdentifiers(target), true, true)).toMatchObject({ + email_address: ["a@x.dev", "c@x.dev"], + email_address_identification_status: ["verified", "reserved"], + phone_number: ["+15555550101"], + phone_number_identification_status: ["reserved"], + }); + }); + + test("makes a reserved email the primary when there is no verified one", () => { + const target = user({ email: undefined, unverifiedEmailAddresses: ["c@x.dev"] }); + expect(buildCreateUserBody(target, splitIdentifiers(target), true, true)).toMatchObject({ + email_address: ["c@x.dev"], + email_address_identification_status: ["reserved"], + }); + }); + + test("sends no status arrays by default, or with nothing to reserve", () => { + const unverified = user({ unverifiedEmailAddresses: ["c@x.dev"] }); + const body = buildCreateUserBody(unverified, splitIdentifiers(unverified), true); + expect(body.email_address).toEqual(["a@x.dev"]); + expect(body).not.toHaveProperty("email_address_identification_status"); + + const verified = user(); + expect( + buildCreateUserBody(verified, splitIdentifiers(verified), true, true), + ).not.toHaveProperty("email_address_identification_status"); + }); + // Allowlists and blocklists police sign-ups; these users already signed up. test("skips the instance's sign-up restrictions", () => { expect(buildCreateUserBody(user(), splitIdentifiers(user()), true)).toMatchObject({ @@ -267,6 +303,29 @@ describe("importUsers", () => { expect(requests.filter((r) => r.url.endsWith("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/v1/phone_numbers"))).toHaveLength(1); }); + test("creates unverified identifiers reserved rather than attaching them", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"], unverifiedEmailAddresses: ["c@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(requests.find((r) => r.url.endsWith("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/v1/users"))?.body).toMatchObject({ + email_address: ["a@x.dev", "c@x.dev"], + email_address_identification_status: ["verified", "reserved"], + }); + expect( + requests.filter((r) => r.url.endsWith("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/v1/email_addresses")).map((r) => r.body), + ).toEqual([ + { user_id: "user_created", email_address: "b@x.dev", primary: false, verified: true }, + ]); + expect(lines.at(-1)).not.toHaveProperty("pending"); + }); + test("marks a user whose password the source dropped", async () => { stub(() => ok("user_created")); @@ -454,6 +513,32 @@ describe("importUsers", () => { ); }); + test("drops a refused phone's statuses with it", async () => { + stub((url, attempt) => + url.endsWith("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/v1/users") && attempt === 1 + ? new Response( + JSON.stringify({ + errors: [{ code: "x", message: "bad phone", meta: { param_name: "phone_number" } }], + }), + { status: 422 }, + ) + : ok("user_created"), + ); + + await importUsers({ + users: [user({ unverifiedPhoneNumbers: ["+31612345678"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(requests[0]?.body).toHaveProperty("phone_number_identification_status"); + expect(requests[1]?.body).not.toHaveProperty("phone_number"); + expect(requests[1]?.body).not.toHaveProperty("phone_number_identification_status"); + expect(lines.at(-1)?.error).toContain("Failed to add phone +31612345678"); + }); + test("does not retry without the phone when it is the only identifier", async () => { stub( () => diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 6b45ac1c0..0bb4f55b4 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -120,23 +120,59 @@ export function splitIdentifiers(user: User): Identifiers { }; } +/** + * The values and parallel statuses for one identifier kind on `POST /v1/users`. + * + * A reserved identifier is unverified, but usable for sign-in and locked to + * this user. Only the create can make one, so with `reserve` the unverified + * identifiers go here instead of being attached afterwards. + */ +function createIdentifiers( + verified: string | undefined, + unverified: string[], + reserve: boolean, +): { values: string[]; statuses?: string[] } { + const values = [...(verified ? [verified] : []), ...(reserve ? unverified : [])]; + if (!reserve || unverified.length === 0) return { values }; + return { + values, + statuses: values.map((_, index) => (verified && index === 0 ? "verified" : "reserved")), + }; +} + /** * Builds the `POST /v1/users` request body. * * Optional fields are omitted rather than sent as null so Clerk applies its own * defaults for anything the source platform did not record. + * + * @param reserveUnverified - Create the identifiers the source never verified + * as reserved, in this request, rather than attaching them unverified. */ export function buildCreateUserBody( user: User, identifiers: Identifiers, skipPasswordRequirement: boolean, + reserveUnverified = false, ): Record { // The instance's allowlist, blocklist, disposable-email and subaddress rules // police sign-ups. These users already signed up, on the source platform. const body: Record = { external_id: user.userId, skip_restriction_checks: true }; - if (identifiers.primaryEmail) body.email_address = [identifiers.primaryEmail]; - if (identifiers.primaryPhone) body.phone_number = [identifiers.primaryPhone]; + const emails = createIdentifiers( + identifiers.primaryEmail, + identifiers.unverifiedEmails, + reserveUnverified, + ); + if (emails.values.length > 0) body.email_address = emails.values; + if (emails.statuses) body.email_address_identification_status = emails.statuses; + const phones = createIdentifiers( + identifiers.primaryPhone, + identifiers.unverifiedPhones, + reserveUnverified, + ); + if (phones.values.length > 0) body.phone_number = phones.values; + if (phones.statuses) body.phone_number_identification_status = phones.statuses; if (user.firstName) body.first_name = user.firstName; if (user.lastName) body.last_name = user.lastName; if (user.username) body.username = user.username; @@ -174,6 +210,8 @@ export function buildCreateUserBody( type CreateContext = { secretKey: string; schedule: ApiScheduler; + /** Unverified identifiers go on the create as reserved, not attached after. */ + reserveUnverified: boolean; }; /** A create a Ctrl-C stopped before it went out: neither a failure nor unknown. */ @@ -190,8 +228,19 @@ export function outcomeUnknown(error: unknown): boolean { return true; } -/** The extra identifiers a user carries, in the order they are attached. */ -export function pendingIdentifiers(identifiers: Identifiers): PendingIdentifier[] { +/** + * The extra identifiers a user carries, in the order they are attached. + * + * @param reserveUnverified - The unverified ones went on the create as + * reserved, so there is nothing to attach for them. + */ +export function pendingIdentifiers( + identifiers: Identifiers, + reserveUnverified = false, +): PendingIdentifier[] { + if (reserveUnverified) { + identifiers = { ...identifiers, unverifiedEmails: [], unverifiedPhones: [] }; + } return [ ...identifiers.additionalEmails.map((value) => ({ kind: "email" as const, @@ -301,7 +350,12 @@ async function createUser( }); }); - const body = buildCreateUserBody(user, identifiers, skipPasswordRequirement); + const body = buildCreateUserBody( + user, + identifiers, + skipPasswordRequirement, + ctx.reserveUnverified, + ); const notes: string[] = []; let response; try { @@ -313,11 +367,15 @@ async function createUser( const phoneRefused = error instanceof BapiError && (error.code === "unsupported_country_code" || error.meta?.param_name === "phone_number"); - if (!phoneRefused || !identifiers.primaryEmail) throw error; - const { phone_number: _dropped, ...withoutPhone } = body; + if (!phoneRefused || !body.email_address) throw error; + const { + phone_number: dropped, + phone_number_identification_status: _statuses, + ...withoutPhone + } = body; response = await create(withoutPhone); notes.push( - `Failed to add phone ${identifiers.primaryPhone}: ${(error as BapiError).longMessage ?? (error as BapiError).message}`, + `Failed to add phone ${(dropped as string[]).join(", ")}: ${(error as BapiError).longMessage ?? (error as BapiError).message}`, ); } @@ -349,6 +407,8 @@ export type ImportUsersOptions = { adopted?: Map; /** Allow users that carry no password. */ skipPasswordRequirement?: boolean; + /** Create the identifiers the source never verified as reserved. */ + reserveUnverified?: boolean; /** Carried into the summary so the report covers the whole file. */ validationFailed?: number; /** Receives the counts as each user finishes. */ @@ -371,6 +431,7 @@ export async function importUsers(options: ImportUsersOptions): Promise(), skipPasswordRequirement = true, + reserveUnverified = false, validationFailed = 0, progress: report, } = options; @@ -384,6 +445,7 @@ export async function importUsers(options: ImportUsersOptions): Promise report?.({ done: processed, ok: successful, failed }); @@ -468,7 +530,10 @@ export async function importUsers(options: ImportUsersOptions): Promise { "--allow-partial", "--new-run", "--require-password", + "--reserve-unverified", "--json", "--firebase-signer-key", "--firebase-salt-separator", diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index 7e29975dc..383f06cec 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -76,6 +76,10 @@ export function registerMigrate(program: Program): void { "--skip-legal-checks", "Import users with no legal acceptance on record into an instance that requires it", ) + .option( + "--reserve-unverified", + "Create emails and phones the source never verified as reserved (usable for sign-in, locked to the user) instead of unverified", + ) .option("--firebase-signer-key ", "Firebase base64 signer key (overrides the export file)") .option("--firebase-salt-separator ", "Firebase base64 salt separator") .option("--firebase-rounds ", "Firebase scrypt rounds", (value: string) => diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts index 858a609a1..feaeca590 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -218,6 +218,18 @@ describe("rejects", () => { }); }); + // Reserved is usable for sign-in, so it meets the requirement (checked live + // against the E2E test app, which requires an email). + test("a reserved email meets an email requirement", async () => { + expect( + await reasonsOf({ + settings: EMAIL_REQUIRED, + reserveUnverified: true, + users: [user("unverified", { email: undefined, unverifiedEmailAddresses: ["u@x.dev"] })], + }), + ).toEqual({}); + }); + test("a password that is not the shape its hasher says", async () => { expect( await reasonsOf({ diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index a384588ac..f51aa4090 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -104,6 +104,8 @@ export type CheckInput = { * sending `skip_legal_checks`. Without it they are rejected. */ skipLegalChecks?: boolean; + /** Unverified identifiers are created reserved, so they meet a requirement. */ + reserveUnverified?: boolean; /** * Clerk IDs a continued run found behind its own in-flight creates: finding * them in the instance is expected. @@ -160,21 +162,32 @@ export function hashShapeProblem(password: string, hasher: string): string | und * * An email or phone counts only when it is verified: an unverified one is * attached after the user exists, so it cannot satisfy a sign-up requirement. + * With `reserveUnverified` it goes on the create as reserved, which does. */ function missingRequiredIdentifier( user: User, settings: UserSettingsJSON | null, + reserveUnverified = false, ): string | undefined { if (!settings) return undefined; const required = (attribute: AttributeName) => isRequired(settings, attribute); const identifiers = splitIdentifiers(user); + const reserved = (unverified: string[]) => reserveUnverified && unverified.length > 0; - if (required("email_address") && !identifiers.primaryEmail) { + if ( + required("email_address") && + !identifiers.primaryEmail && + !reserved(identifiers.unverifiedEmails) + ) { return identifiers.unverifiedEmails.length > 0 ? "only has an unverified email, and this instance requires an email" : "no email, which this instance requires"; } - if (required("phone_number") && !identifiers.primaryPhone) { + if ( + required("phone_number") && + !identifiers.primaryPhone && + !reserved(identifiers.unverifiedPhones) + ) { return identifiers.unverifiedPhones.length > 0 ? "only has an unverified phone number, and this instance requires one" : "no phone number, which this instance requires"; @@ -796,7 +809,7 @@ export async function checkImport(input: CheckInput): Promise { (refused.length > 0 && !hasAnyIdentifier(user) ? "only has emails Clerk refuses (malformed, or a domain that can't receive mail)" : undefined) ?? - missingRequiredIdentifier(user, input.settings) ?? + missingRequiredIdentifier(user, input.settings, input.reserveUnverified) ?? // Stripping the identifiers the instance has off can leave nothing to // sign in with; Clerk would still create the user. (!hasAnyIdentifier(dropDisabledIdentifiers(user, input.settings)) diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 082121d28..3dfb00b29 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -17,6 +17,8 @@ import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; const mockSelect = mock(async () => "clerk" as unknown); const mockText = mock(async () => "export.json" as unknown); let confirmAnswer = true; +/** The answer to the reserved-identifiers question, apart from consent. */ +let reserveAnswer = true; /** Every confirmation the run put up, in order — the wording is the assertion. */ let confirmMessages: string[] = []; let originalMode: string | undefined; @@ -31,7 +33,7 @@ mock.module("../../lib/listage.ts", () => ({ mock.module("../../lib/prompts.ts", () => ({ confirm: async ({ message }: { message: string }) => { confirmMessages.push(message); - return confirmAnswer; + return message.includes("never verified") ? reserveAnswer : confirmAnswer; }, multiselect: async () => [], text: (...args: unknown[]) => mockText(...(args as [])), @@ -50,7 +52,7 @@ let workDir: string; let configDir: string; let originalCwd: string; let originalFetch: typeof globalThis.fetch; -let requests: { method: string; url: string }[]; +let requests: { method: string; url: string; body: unknown }[]; const EXPORT = [ { id: "u1", primary_email_address: "a@x.dev" }, @@ -83,6 +85,7 @@ afterAll(() => { beforeEach(() => { requests = []; confirmAnswer = true; + reserveAnswer = true; confirmMessages = []; mockSelect.mockReset(); mockText.mockReset(); @@ -94,7 +97,11 @@ beforeEach(() => { globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { const url = new URL(input.toString()); const method = init?.method ?? "GET"; - requests.push({ method, url: url.toString() }); + requests.push({ + method, + url: url.toString(), + body: init?.body ? JSON.parse(init.body as string) : null, + }); if (url.pathname === "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/v1/instance") { return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); } @@ -211,3 +218,50 @@ describe("consent", () => { expect(created()).toHaveLength(0); }); }); + +describe("unverified identifiers", () => { + const withUnverified = [ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", primary_email_address: "b@x.dev", unverified_email_addresses: "c@x.dev" }, + ]; + const statuses = () => + created().map((r) => (r.body as Record)?.email_address_identification_status); + + beforeEach(() => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(withUnverified)); + }); + + test("asks whether to reserve them, before consent, and a yes creates them reserved", async () => { + await run(importOptions); + + expect(confirmMessages).toEqual([ + "1 user has an email or phone the source never verified. Create them reserved (usable for sign-in, locked to the user) instead of unverified?", + "Import 2 users?", + ]); + expect(statuses()).toEqual([undefined, ["verified", "reserved"]]); + }); + + test("a no keeps them unverified, attached after the user exists", async () => { + reserveAnswer = false; + + await run(importOptions); + + expect(statuses()).toEqual([undefined, undefined]); + expect(requests.some((r) => new URL(r.url).pathname === "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/v1/email_addresses")).toBe(true); + }); + + // `-y` is consent to write, not a yes to making unconfirmed addresses usable. + test("--yes does not ask, and keeps them unverified", async () => { + await run({ ...importOptions, yes: true }); + + expect(confirmMessages).toEqual([]); + expect(statuses()).toEqual([undefined, undefined]); + }); + + test("--reserve-unverified does not ask, and creates them reserved", async () => { + await run({ ...importOptions, reserveUnverified: true }); + + expect(confirmMessages).toEqual(["Import 2 users?"]); + expect(statuses()).toEqual([undefined, ["verified", "reserved"]]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 2d7a32ec0..e18b118aa 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -35,7 +35,7 @@ import { NEXT_STEPS, printAgentNextSteps } from "../../lib/next-steps.ts"; import { confirm } from "../../lib/prompts.ts"; import { withGutter, withSpinner } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; -import { importUsers } from "./import-users.ts"; +import { importUsers, splitIdentifiers } from "./import-users.ts"; import { checkImport, type ImportChecks } from "./lib/checks.ts"; import { fetchInstanceSettings, fetchUserCount } from "./lib/clerk-config.ts"; import { readEnvelope, type ExportEnvelope } from "./lib/export-file.ts"; @@ -92,6 +92,12 @@ export type MigrateRunOptions = { * Without it, a prompt asks; where nobody can be asked, they are rejected. */ skipLegalChecks?: boolean; + /** + * Create the emails and phones the source never verified as reserved, not + * unverified. Without it, a prompt asks; where nobody can be asked, they + * stay unverified. + */ + reserveUnverified?: boolean; /** Check against the instance, report, and write nothing. */ dryRun?: boolean; /** Import the users that pass, and record the rest as skipped. */ @@ -541,6 +547,7 @@ function commandFor(options: MigrateRunOptions, fromExport: string | undefined, if (options.newRun) parts.push("--new-run"); if (options.requirePassword) parts.push("--require-password"); if (options.skipLegalChecks) parts.push("--skip-legal-checks"); + if (options.reserveUnverified) parts.push("--reserve-unverified"); if (options.firebaseSignerKey) parts.push("--firebase-signer-key", ""); if (options.firebaseSaltSeparator) parts.push("--firebase-salt-separator", ""); if (options.firebaseRounds) parts.push("--firebase-rounds", ""); @@ -720,10 +727,32 @@ export async function run(rawOptions: MigrateRunOptions): Promise { }); } + // Unverified stays the default: reserved makes an address the source never + // confirmed usable for sign-in, which is the operator's call to make. + let reserveUnverified = options.reserveUnverified ?? false; + const withUnverified = users.filter((user) => { + const { unverifiedEmails, unverifiedPhones } = splitIdentifiers(user); + return unverifiedEmails.length > 0 || unverifiedPhones.length > 0; + }).length; + // `-y` is consent to write, not a yes to this, so it does not ask. + if ( + !reserveUnverified && + withUnverified > 0 && + !options.dryRun && + !options.yes && + canPrompt(options) + ) { + reserveUnverified = await confirm({ + message: `${plural(withUnverified, "user")} ${withUnverified === 1 ? "has" : "have"} an email or phone the source never verified. Create them reserved (usable for sign-in, locked to the user) instead of unverified?`, + default: false, + }); + } + const checks = await withSpinner("Checking users against the instance...", async (spinner) => checkImport({ users, skipLegalChecks, + reserveUnverified, failures, unknownFields: loaded.unknownFields, ...(supabaseRows ? { supabaseRows } : {}), @@ -855,6 +884,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { attachOnly, adopted, skipPasswordRequirement: !options.requirePassword, + reserveUnverified, progress, }), ) diff --git a/test/e2e/migrate.test.ts b/test/e2e/migrate.test.ts index 1bf693d2d..1d7ab304b 100644 --- a/test/e2e/migrate.test.ts +++ b/test/e2e/migrate.test.ts @@ -10,6 +10,9 @@ * - A user whose only email is unverified, imported into an instance that * requires an email, is refused by Clerk. The import's checks reject that * user up front on the strength of this test, so it checks both halves. + * - The same user imported with `--reserve-unverified` is created, with that + * email reserved and primary. The checks let them through on the strength + * of this test. * * Requires `CLERK_PLATFORM_API_KEY` and `CLERK_CLI_TEST_APP_ID`. Locally, run * via `bun run test:e2e:op` so 1Password resolves both in-memory. @@ -251,3 +254,36 @@ test("a user whose only email is unverified is refused where email is required", }), ]); }, 60_000); + +test("--reserve-unverified creates an unverified-only user with the email reserved", async () => { + const hex = randomBytes(6).toString("hex"); + const email = `e2e-${hex}+clerk_test@clerkcookie.com`; + const file = writeExport([{ id: `sb_${hex}`, email }]); + + const imported = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--reserve-unverified", + "--yes", + "--json", + ]); + const { run } = JSON.parse(imported.stdout.toString()) as { run: { id: string } }; + const [line] = latestLines(run.id); + expect(line).toMatchObject({ status: "created" }); + + const fetched = await cli(["api", `/users/${line?.clerkId as string}`]); + const user = JSON.parse(fetched.stdout.toString()) as { + primary_email_address_id: string; + email_addresses: { id: string; email_address: string; reserved: boolean }[]; + }; + expect(user.email_addresses).toEqual([ + expect.objectContaining({ + id: user.primary_email_address_id, + email_address: email, + reserved: true, + }), + ]); +}, 60_000); From 4e35d86421c43988be5208f5458ad7487e2186d5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 17:11:43 -0400 Subject: [PATCH 132/141] fix(migrate): read the Better Auth account table's column casing on its own Better Auth maps each model's fields separately, so a snake_case user table can sit next to a camelCase account table. The casing came from the user table alone, and the join then named account columns that do not exist. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/betterauth.ts | 22 ++++++++++--- .../migrate/export/db-exports.test.ts | 31 +++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index a0c7fcec5..e6ee76e96 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -59,6 +59,11 @@ export type BetterAuthSchema = { * and Prisma setups keep camelCase. */ column: (field: string) => string; + /** + * The same for the account table. Better Auth maps each model's fields on + * its own, so its casing is read from that table rather than assumed. + */ + accountColumn: (field: string) => string; /** The plugin columns this database has. */ plugins: Set; }; @@ -110,7 +115,15 @@ export async function detectSchema(client: DbClient): Promise const snake = !columns.has("emailVerified") && columns.has("email_verified"); const column = (field: string) => (snake ? toSnakeCase(field) : field); const plugins = new Set(PLUGIN_COLUMNS.filter((field) => columns.has(column(field)))); - return { userTable, accountTable, column, plugins }; + // No account table: keep the user table's casing, and let the query's + // "no such table" error say what is missing. + const accountColumns = await tableColumns(client, accountTable); + const accountSnake = + accountColumns.size === 0 + ? snake + : !accountColumns.has("userId") && accountColumns.has("user_id"); + const accountColumn = (field: string) => (accountSnake ? toSnakeCase(field) : field); + return { userTable, accountTable, column, accountColumn, plugins }; } // No table found: the query names the default, and its "no such table" // error carries the hint. @@ -118,6 +131,7 @@ export async function detectSchema(client: DbClient): Promise userTable: "user", accountTable: "account", column: (field) => field, + accountColumn: (field) => field, plugins: new Set(), }; } @@ -130,7 +144,7 @@ export async function detectSchema(client: DbClient): Promise */ export function buildBetterAuthQuery(client: DbClient, schema: BetterAuthSchema): string { const q = (identifier: string) => client.quote(identifier); - const { column } = schema; + const { column, accountColumn } = schema; const select = (field: string) => column(field) === field ? `u.${q(field)}` : `u.${q(column(field))} AS ${q(field)}`; const selected = [ @@ -143,8 +157,8 @@ export function buildBetterAuthQuery(client: DbClient, schema: BetterAuthSchema) return ( `SELECT ${selected.join(", ")}, a.${q("password")} AS ${q("password_hash")} ` + `FROM ${q(schema.userTable)} u ` + - `LEFT JOIN ${q(schema.accountTable)} a ON a.${q(column("userId"))} = u.${q("id")} ` + - `AND a.${q(column("providerId"))} = 'credential' ` + + `LEFT JOIN ${q(schema.accountTable)} a ON a.${q(accountColumn("userId"))} = u.${q("id")} ` + + `AND a.${q(accountColumn("providerId"))} = 'credential' ` + `ORDER BY u.${q("id")} ASC` ); } diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 1178b30c5..6a32f91e2 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -362,6 +362,37 @@ describe("betterauth export", () => { ]); }); + // Better Auth maps each model's fields on its own, so the two tables can + // disagree on casing. + test.each([ + ["snake_case user table, camelCase account table", "snake", "userId", "providerId"], + ["camelCase user table, snake_case account table", "camel", "user_id", "provider_id"], + ])("joins a %s", async (_label, userCase, userId, providerId) => { + const [verified, created, updated] = + userCase === "snake" + ? ["email_verified", "created_at", "updated_at"] + : ["emailVerified", "createdAt", "updatedAt"]; + const file = makeDb((db) => { + db.run( + `CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "${verified}" INTEGER, name TEXT, + "${created}" TEXT, "${updated}" TEXT)`, + ); + db.run( + `CREATE TABLE "account" (id TEXT, "${userId}" TEXT, "${providerId}" TEXT, password TEXT)`, + ); + db.run(`INSERT INTO "user" (id, email, "${verified}") VALUES ('u1', 'a@x.dev', 1)`); + db.run(`INSERT INTO "account" VALUES ('a1', 'u1', 'credential', 'salt:hash')`); + }); + + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, await detectSchema(client))), + ); + + expect(rows).toEqual([ + expect.objectContaining({ id: "u1", emailVerified: 1, password_hash: "salt:hash" }), + ]); + }); + test("exports a user with two matching credential accounts once", () => { const lines: UserLine[] = []; const { users, coverage } = buildBetterAuthExport( From ecb65b41cb9ea24ce6ef34230f25ae060e2a3ccb Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 17:12:13 -0400 Subject: [PATCH 133/141] fix(migrate): say Auth.js Credentials-provider passwords are not exported The warning said Auth.js users sign in with OAuth or email links. An app using the Credentials provider keeps its own passwords, which the export does not read, so the operator needs to migrate those separately or have those users reset their password. The warning, the source note and the README now say so. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 5 ++++- packages/cli-core/src/commands/migrate/export/authjs.ts | 4 +++- .../src/commands/migrate/export/db-exports.test.ts | 1 + packages/cli-core/src/commands/migrate/sources/authjs.ts | 8 ++++---- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index b7e5e8862..8bdd77f78 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -391,7 +391,10 @@ schema — Prisma capitalizes the table, Drizzle does not, and Postgres treats the difference as significant once quoted. The run reports which one it found. The verified column is read as `emailVerified`, or `email_verified` on a legacy NextAuth table. Auth.js core stores no passwords, so its users arrive without -credentials. +credentials. That is all an OAuth or email-link app has. An app that also uses +the Credentials provider keeps passwords in its own tables, which the export +does not read: migrate those separately, or have those users reset their +password. **`betterauth` reads its schema before it queries.** It finds the tables (`user` and `account`, or `users` and `accounts` under `usePlural: true`), how diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 6d7cca1ff..85558ffef 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -177,7 +177,9 @@ export async function exportAuthJs(options: DbExportOptions): Promise { if (users.length > 0) { log.warn( - "Auth.js core stores no passwords — its users sign in with OAuth or email links, so they arrive without credentials.", + "Auth.js core stores no passwords, so users arrive without one. That covers OAuth and email-link sign-in. " + + "If the app also uses the Credentials provider, its passwords live in the app's own tables and were not exported: " + + "migrate them separately, or have those users reset their password.", ); } }); diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 6a32f91e2..b0f520931 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -266,6 +266,7 @@ describe("authjs export", () => { expect(written.users).toHaveLength(2); expect(captured.err).toContain("Read 2 rows from"); expect(captured.err).toContain("stores no passwords"); + expect(captured.err).toContain("Credentials provider"); }); }); diff --git a/packages/cli-core/src/commands/migrate/sources/authjs.ts b/packages/cli-core/src/commands/migrate/sources/authjs.ts index abf7ed97f..89d814d8b 100644 --- a/packages/cli-core/src/commands/migrate/sources/authjs.ts +++ b/packages/cli-core/src/commands/migrate/sources/authjs.ts @@ -12,9 +12,9 @@ import { routeByVerification, splitName } from "./shared.ts"; * `email_verified` is a nullable timestamp rather than a boolean — any value * means verified. * - * No password default: Auth.js's core is passwordless (OAuth and email links), - * so users arrive without a digest and are imported with - * `skip_password_requirement`. + * No password default: Auth.js core stores no passwords, so users arrive + * without a digest and are imported with `skip_password_requirement`. An app + * using the Credentials provider keeps its own, which this does not read. */ const authjsSource = { key: "authjs", @@ -24,7 +24,7 @@ const authjsSource = { carries: { passwords: { level: "no", - note: "Auth.js is passwordless (OAuth and email links), so users arrive without one.", + note: "Auth.js core stores no passwords. An app using the Credentials provider keeps its own, which this does not read: migrate them separately, or have those users reset their password.", }, mfa: { level: "no", note: "Auth.js has no MFA of its own." }, metadata: { From c5b23b11ec721dea6a31ebe14376deec103f11f2 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 17:31:24 -0400 Subject: [PATCH 134/141] fix(migrate): resolve each Better Auth column on its own Better Auth maps each field separately, so one table can hold user_id next to providerId. Each column now resolves against the table's own columns, as named or in snake_case, rather than from one guess per table. A `user` table without Better Auth's verified column is also passed over for the plural one. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/betterauth.ts | 24 ++++++++++-- .../migrate/export/db-exports.test.ts | 37 +++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index e6ee76e96..b275ca995 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -103,6 +103,20 @@ const TABLE_CANDIDATES = [ ["users", "accounts"], ] as const; +/** + * A field's column: as named when the table has it, else its snake_case form + * when the table has that, else the table's general guess. Better Auth maps + * each field on its own, so one table can mix the two; a column that is truly + * missing still fails the query, which names it. + */ +function resolveColumn(columns: Set, snake: boolean): (field: string) => string { + return (field) => { + if (columns.has(field)) return field; + const snaked = toSnakeCase(field); + return columns.has(snaked) || snake ? snaked : field; + }; +} + /** * Asks the database how it names Better Auth's tables and columns, and which * plugin columns it has. Selecting a column that is not there fails the whole @@ -111,9 +125,11 @@ const TABLE_CANDIDATES = [ export async function detectSchema(client: DbClient): Promise { for (const [userTable, accountTable] of TABLE_CANDIDATES) { const columns = await tableColumns(client, userTable); - if (columns.size === 0) continue; - const snake = !columns.has("emailVerified") && columns.has("email_verified"); - const column = (field: string) => (snake ? toSnakeCase(field) : field); + // A table without Better Auth's verified column is some other `user` + // table, so the plural one is tried next. + if (!columns.has("emailVerified") && !columns.has("email_verified")) continue; + const snake = !columns.has("emailVerified"); + const column = resolveColumn(columns, snake); const plugins = new Set(PLUGIN_COLUMNS.filter((field) => columns.has(column(field)))); // No account table: keep the user table's casing, and let the query's // "no such table" error say what is missing. @@ -122,7 +138,7 @@ export async function detectSchema(client: DbClient): Promise accountColumns.size === 0 ? snake : !accountColumns.has("userId") && accountColumns.has("user_id"); - const accountColumn = (field: string) => (accountSnake ? toSnakeCase(field) : field); + const accountColumn = resolveColumn(accountColumns, accountSnake); return { userTable, accountTable, column, accountColumn, plugins }; } // No table found: the query names the default, and its "no such table" diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index b0f520931..59fdf60b1 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -394,6 +394,43 @@ describe("betterauth export", () => { ]); }); + // Each field is mapped on its own, so one table can mix the two casings. + test("resolves each column on its own when a table mixes casings", async () => { + const file = makeDb((db) => { + db.run( + `CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, email_verified INTEGER, name TEXT, + "createdAt" TEXT, updated_at TEXT)`, + ); + db.run(`CREATE TABLE "account" (id TEXT, user_id TEXT, "providerId" TEXT, password TEXT)`); + db.run(`INSERT INTO "user" (id, email, email_verified) VALUES ('u1', 'a@x.dev', 1)`); + db.run(`INSERT INTO "account" VALUES ('a1', 'u1', 'credential', 'salt:hash')`); + }); + + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, await detectSchema(client))), + ); + + expect(rows).toEqual([ + expect.objectContaining({ id: "u1", emailVerified: 1, password_hash: "salt:hash" }), + ]); + }); + + test("passes over a `user` table that is not Better Auth's for the plural one", async () => { + const file = makeDb((db) => { + db.run(`CREATE TABLE "user" (id TEXT PRIMARY KEY, handle TEXT)`); + db.run( + `CREATE TABLE "users" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER, name TEXT, + "createdAt" TEXT, "updatedAt" TEXT)`, + ); + db.run(`CREATE TABLE "accounts" (id TEXT, "userId" TEXT, "providerId" TEXT, password TEXT)`); + db.run(`INSERT INTO "users" (id, email) VALUES ('u1', 'a@x.dev')`); + }); + + const schema = await withClient(file, async (client) => detectSchema(client)); + + expect(schema).toMatchObject({ userTable: "users", accountTable: "accounts" }); + }); + test("exports a user with two matching credential accounts once", () => { const lines: UserLine[] = []; const { users, coverage } = buildBetterAuthExport( From 9df8d7b3b61c6befad411cb8de86f60f3e4d27b6 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 17:32:11 -0400 Subject: [PATCH 135/141] fix(migrate): keep looking for the Auth.js table past one without its columns Postgres keeps a quoted "User" apart from "user", so an app's own User table stopped the search before Auth.js's table was tried. A candidate without the columns is now passed over; the first column error is still the one thrown when nothing reads. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/authjs.ts | 12 ++++++++---- .../commands/migrate/export/db-exports.test.ts | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 85558ffef..79c601862 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -74,22 +74,26 @@ function isMissingTable(error: unknown): boolean { /** * Reads the user table, trying each casing until one answers. * + * A table that exists but lacks the columns is passed over too: Postgres + * keeps a quoted `"User"` apart from `"user"`, and an app's own `User` table + * must not hide Auth.js's. When nothing reads, the first column error is the + * one thrown, since it names a table that was there. + * * @returns The rows and the table they came from, so the run can say which. */ export async function fetchAuthJsUsers( client: DbClient, ): Promise<{ rows: AuthJsRow[]; table: string }> { let lastError: unknown; + let columnError: unknown; for (const table of TABLE_CANDIDATES) { - let columnError: unknown; for (const column of VERIFIED_COLUMNS) { try { const rows = await client.query(buildAuthJsQuery(client, table, column)); return { rows, table }; } catch (error) { - // The table is there: try the other name for the verified column, and - // report the first failure if neither reads. + // The table is there: try the other name for the verified column. if (isMissingColumn(error)) { columnError ??= error; continue; @@ -99,9 +103,9 @@ export async function fetchAuthJsUsers( break; } } - if (columnError) throw columnError; } + if (columnError) throw columnError; throw lastError instanceof Error ? new Error( `No Auth.js user table found. Tried ${TABLE_CANDIDATES.join(", ")}. ${lastError.message}`, diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 59fdf60b1..f2eedfd9b 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -223,6 +223,23 @@ describe("authjs export", () => { expect(rows.map((row) => row.email_verified)).toEqual([null, "2024-01-15"]); }); + // Postgres keeps a quoted "User" apart from "user"; SQLite does not, so the + // plural table stands in for the later candidate here. + test("passes over a candidate table without the columns for a later one", async () => { + const file = makeDb((db) => { + db.run(`CREATE TABLE "User" (id TEXT PRIMARY KEY, handle TEXT)`); + db.run( + `CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT, email TEXT, "emailVerified" TEXT)`, + ); + db.run(`INSERT INTO users VALUES (?,?,?,?)`, ["u1", "U", "u@x.dev", null]); + }); + + const { rows, table } = await withClient(file, fetchAuthJsUsers); + + expect(table).toBe("users"); + expect(rows).toHaveLength(1); + }); + test("a missing column is an error, not a literal", async () => { const file = makeDb((db) => { db.run(`CREATE TABLE "User" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" TEXT)`); From 6906d8af6a0c1dd6675aa6a4d9acf5c763ac464b Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 17:51:15 -0400 Subject: [PATCH 136/141] fix(migrate): redact credentials from database error messages The connection string was redacted where we print it, but the driver's or server's own message was appended as is, and a libsql server or a driver can quote the URL, the password or the token. Connection and query failures now pass that message through redactDbMessage too. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/db.test.ts | 53 +++++++++++++++ .../cli-core/src/commands/migrate/lib/db.ts | 64 +++++++++++++++++-- 2 files changed, 113 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index e0a6d558e..358aa5dd4 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -9,6 +9,7 @@ import { describeDbError, detectDbType, redactConnectionString, + redactDbMessage, sqlitePath, withDbClient, } from "./db.ts"; @@ -85,6 +86,29 @@ describe("redactConnectionString", () => { ); }); +describe("redactDbMessage", () => { + test.each([ + ["the whole URL", "failed: postgres://u:pw%40x@h/db", "failed: postgres://***@h/db"], + ["the password, raw", 'auth failed for "pw%40x"', 'auth failed for "***"'], + ["the password, decoded", 'auth failed for "pw@x"', 'auth failed for "***"'], + ])("redacts %s", (_label, message, expected) => { + expect(redactDbMessage(message, "postgres://u:pw%40x@h/db")).toBe(expected); + }); + + test("redacts a libsql token from the URL or the environment", () => { + expect(redactDbMessage("token abc is bad", "libsql://h.turso.io?authToken=abc")).toBe( + "token *** is bad", + ); + expect( + redactDbMessage("token envtok is bad", "libsql://h.turso.io", { TURSO_AUTH_TOKEN: "envtok" }), + ).toBe("token *** is bad"); + }); + + test("leaves a message with no credentials alone", () => { + expect(redactDbMessage("no such table: user", "./db.sqlite")).toBe("no such table: user"); + }); +}); + describe("sqlitePath", () => { test.each([ ["./db.sqlite", "./db.sqlite"], @@ -161,6 +185,35 @@ describe("a libsql client", () => { expect(client.dbType).toBe("sqlite"); }); + // A server that echoes the token must not get it printed. + test("redacts the token from a server error it echoes", async () => { + stubFetch({ type: "error", error: { message: "bad auth token t0ken for app-org" } }); + + const error = (await createDbClient("libsql://app-org.turso.io?authToken=t0ken").catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.message).not.toContain("t0ken"); + expect(error.message).toContain("bad auth token *** for app-org"); + }); + + test("redacts the token from a query failure after connecting", async () => { + stubFetch(okRows(["1"], [[{ type: "integer", value: "1" }]])); + + const error = (await withDbClient( + "libsql://app-org.turso.io?authToken=t0ken", + undefined, + async () => { + throw new Error("query rejected for token t0ken"); + }, + ).catch((caught: unknown) => caught)) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.message).toContain("query rejected for token ***"); + expect(error.message).not.toContain("t0ken"); + }); + test("reports a server-side error", async () => { stubFetch({ type: "error", error: { message: "no such table: user" } }); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index c23368449..0d5aa36b4 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -71,6 +71,55 @@ export function redactConnectionString(connectionString: string): string { .replace(/([?&](?:authToken|password)=)[^&]*/gi, "$1***"); } +/** + * A driver's error message with every credential from `connectionString` + * replaced by `***`. + * + * {@link redactConnectionString} covers the string we print; this covers what + * the server or driver says back, which can quote the URL, the password or a + * libsql token. A short password can blank an unrelated word too, and a + * mangled message is the better failure. + */ +export function redactDbMessage( + message: string, + connectionString: string, + env: Record = process.env, +): string { + const trimmed = connectionString.trim(); + let redacted = message.split(trimmed).join(redactConnectionString(trimmed)); + + const secrets = new Set(); + try { + const url = new URL(trimmed); + for (const value of [ + url.password, + url.searchParams.get("authToken"), + url.searchParams.get("password"), + ]) { + if (!value) continue; + secrets.add(value); + try { + secrets.add(decodeURIComponent(value)); + } catch { + // Not valid percent-encoding: the raw form is the only one there is. + } + } + } catch { + // A SQLite path is not a URL, and carries no credentials. + } + if (isLibsqlUrl(trimmed)) { + for (const value of [env.TURSO_AUTH_TOKEN, env.LIBSQL_AUTH_TOKEN]) { + if (value) secrets.add(value); + } + } + + // Longest first, so a secret that contains another is replaced whole. + for (const secret of [...secrets].sort((a, b) => b.length - a.length)) { + redacted = redacted.split(secret).join("***"); + } + return redacted; +} + /** Strips a `file:` prefix and any URL query, leaving a filesystem path. */ export function sqlitePath(connectionString: string): string { const trimmed = connectionString.trim(); @@ -358,7 +407,10 @@ function connectionError( connectionString: string, platform?: DbPlatform, ): CliError { - const message = error instanceof Error ? error.message : String(error); + const message = redactDbMessage( + error instanceof Error ? error.message : String(error), + connectionString, + ); return new CliError( `Could not connect to ${redactConnectionString(connectionString)}: ${message}\n\n${describeDbError(error, platform)}`, { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }, @@ -383,10 +435,14 @@ export async function withDbClient( // A query failure carries the same actionable hints as a connection one: // a missing table is the most common thing that goes wrong here. if (error instanceof CliError) throw error; - throw new CliError( - `${error instanceof Error ? error.message : String(error)}\n\n${describeDbError(error, platform)}`, - { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }, + const message = redactDbMessage( + error instanceof Error ? error.message : String(error), + connectionString, ); + throw new CliError(`${message}\n\n${describeDbError(error, platform)}`, { + code: ERROR_CODE.USAGE_ERROR, + exitCode: EXIT_CODE.USAGE, + }); } finally { await client.close().catch(() => {}); } From 67ac723fdaeab882950b50b202eb7dda5264de65 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 18:37:21 -0400 Subject: [PATCH 137/141] test(migrate): cover the generic missing-column advice again The Auth.js hint replaced the only test of the generic advice that the Supabase and Better Auth exports get. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/lib/db.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index 358aa5dd4..eb258391c 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -320,6 +320,18 @@ describe("withDbClient", () => { expect(error.message).not.toContain("Check the connection string"); }); + // Supabase and Better Auth get the generic advice; only Auth.js has its own. + test("gives the generic column advice outside Auth.js, and exits 2", async () => { + const error = (await withDbClient(dbPath, "supabase", (client) => + client.query(`SELECT no_such_column FROM sqlite_master`), + ).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("missing a column the export reads"); + expect(error.message).not.toContain("reads `id`, `name`, `email`"); + expect(error.message).not.toContain("Check the connection string"); + }); + test("passes a CliError through unchanged", async () => { await expect( withDbClient(dbPath, undefined, async () => { From 0e0aad1bec19767a220bb6d1cfd8749f84095b8e Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 18:51:22 -0400 Subject: [PATCH 138/141] docs(migrate): list both hook validation errors for a custom source The validation table quoted only the postTransform message, without the backticks the real one has, though preTransform is checked the same way. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 8bdd77f78..3abea2d8a 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -970,15 +970,15 @@ edited source counts as a different source. The file is code the CLI executes, so its shape is checked before use and rejected with the specific problem rather than crashing mid-pipeline: -| Problem | Message | -| ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| Path does not exist | `No source file at /abs/path.ts.` | -| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | -| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | -| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | -| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | -| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | -| A hook is not a function | `postTransform must be a function when present` | +| Problem | Message | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Path does not exist | `No source file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | +| `preTransform` or `postTransform` is not a function | `` `preTransform` must be a function when present `` (likewise `postTransform`) | The `userId` check is the load-bearing one: without it the import would run to completion and create every user with no `external_id`, which is what makes a From 71e49e610c0801cba912d7756a482fa6dbaeb680 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 19:11:36 -0400 Subject: [PATCH 139/141] fix(migrate): attach what an adopted user's create left out, in its own mode A continued run adopts a user whose earlier create got no answer, and only attaches the rest. Which unverified identifiers were left came from this run's --reserve-unverified, not the earlier create's, so switching the flag lost them or attached duplicates. The creating line now records a reserved create, and an adopted user follows it. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/import-users.test.ts | 56 +++++++++++++++++++ .../src/commands/migrate/import-users.ts | 21 ++++++- .../src/commands/migrate/lib/run-store.ts | 5 ++ .../cli-core/src/commands/migrate/run.test.ts | 32 +++++++++++ packages/cli-core/src/commands/migrate/run.ts | 3 + 5 files changed, 115 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts index 087e03d0f..895d1aa17 100644 --- a/packages/cli-core/src/commands/migrate/import-users.test.ts +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -469,6 +469,62 @@ describe("importUsers", () => { expect(lines.at(-1)).toMatchObject({ clerkId: "user_found", status: "created" }); }); + // The `creating` line says what the create did, so a continued run that + // adopts the user knows what is left to attach. + test("marks a creating line reserved only when the create reserved something", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [ + user({ userId: "u1", unverifiedEmailAddresses: ["c@x.dev"] }), + user({ userId: "u2", email: "b@x.dev" }), + ], + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(allLines.filter((line) => line.status === "creating")).toEqual([ + { sourceId: "u1", status: "creating", reserved: true }, + { sourceId: "u2", status: "creating" }, + ]); + }); + + // Adopted, the user is not created again, so its create's mode decides what + // is still missing, not this run's flag. + test("attaches an adopted user's unverified email its create left out, even with the flag", async () => { + stub(() => ok("idn_1")); + + await importUsers({ + users: [user({ unverifiedEmailAddresses: ["c@x.dev"] })], + adopted: new Map([["u1", "user_found"]]), + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(requests.map((r) => r.body)).toEqual([ + { user_id: "user_found", email_address: "c@x.dev", primary: false, verified: false }, + ]); + }); + + test("attaches nothing for an adopted user whose create reserved them, even without the flag", async () => { + stub(() => ok("idn_1")); + + await importUsers({ + users: [user({ unverifiedEmailAddresses: ["c@x.dev"] })], + adopted: new Map([["u1", "user_found"]]), + adoptedReserved: new Set(["u1"]), + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(requests).toEqual([]); + }); + // Shapes from clerk_go's apierror: the country error carries its own code // and no param_name; the E.164 error is a form error on phone_number. test.each([ diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 0bb4f55b4..038a6466a 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -405,6 +405,12 @@ export type ImportUsersOptions = { * not created again; only their extra identifiers are sent. */ adopted?: Map; + /** + * The adopted users whose create reserved their unverified identifiers, from + * its `creating` line. The rest had them attached unverified, whatever this + * run's `reserveUnverified` says. + */ + adoptedReserved?: Set; /** Allow users that carry no password. */ skipPasswordRequirement?: boolean; /** Create the identifiers the source never verified as reserved. */ @@ -430,6 +436,7 @@ export async function importUsers(options: ImportUsersOptions): Promise(), + adoptedReserved = new Set(), skipPasswordRequirement = true, reserveUnverified = false, validationFailed = 0, @@ -493,13 +500,23 @@ export async function importUsers(options: ImportUsersOptions): Promise 0 || identifiers.unverifiedPhones.length > 0); try { created = adoptedId ? { clerkUserId: adoptedId, notes: [] } : await retryOn429( async () => createUser(ctx, user, identifiers, skipPasswordRequirement, () => - record({ sourceId: user.userId, status: "creating" }), + record({ + sourceId: user.userId, + status: "creating", + ...(reserved ? { reserved: true } : {}), + }), ), { onRetry: ({ message }) => retries.push(message) }, ); @@ -530,7 +547,7 @@ export async function importUsers(options: ImportUsersOptions): Promise { expect(readRun(runsDir(), first!.id)?.status).toBe("complete"); }); + // The `creating` line records what the create did; a continued run reads it + // back rather than guessing from this run's flags. + test("an adopted user whose create reserved its unverified email gets no attach for it", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", primary_email_address: "b@x.dev", unverified_email_addresses: "c@x.dev" }, + ]), + ); + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + fs.appendFileSync( + path.join(runsDir(), first!.id, "users.ndjson"), + `${JSON.stringify({ sourceId: "u2", status: "creating", reserved: true })}\n`, + ); + interrupt(first!.id); + + requests = []; + process.exitCode = 0; + stubClerk({ existing: [{ id: "user_found", external_id: "u2" }] }); + await run(baseOptions); + + expect(created()).toEqual([]); + expect(requests.filter((r) => r.url.endsWith("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/v1/email_addresses"))).toEqual([]); + expect(latestUserLines(runsDir(), first!.id).get("u2")).toMatchObject({ + status: "created", + clerkId: "user_found", + }); + }); + test("an interrupted run creates a user whose in-flight create never landed", async () => { stubClerk({ failing: new Set(["u2"]) }); await run(baseOptions); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index e18b118aa..d14490b9f 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -641,9 +641,11 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const done = new Map(); const attachOnly: UserLine[] = []; const inFlight: string[] = []; + const inFlightReserved = new Set(); if (continued) { for (const line of latestUserLines(runsDir, continued.id).values()) { if (line.status === "creating") inFlight.push(line.sourceId); + if (line.status === "creating" && line.reserved) inFlightReserved.add(line.sourceId); if (line.status !== "created" || !line.clerkId) continue; done.set(line.sourceId, line.clerkId); if (line.pending?.length) attachOnly.push(line); @@ -883,6 +885,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { record: run.append, attachOnly, adopted, + adoptedReserved: inFlightReserved, skipPasswordRequirement: !options.requirePassword, reserveUnverified, progress, From 16b65114939236d756fb33432126352ecadbe059 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 19:12:40 -0400 Subject: [PATCH 140/141] fix(migrate): keep prompt answers in the commands a run suggests A yes to the reserve or legal-checks prompt only set a local, so the --allow-partial command printed on a refusal left the flag out, and running it would reject the users that answer let through. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/run-interactive.test.ts | 25 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 3 +++ 2 files changed, 28 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 3dfb00b29..121be0ef1 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -258,6 +258,31 @@ describe("unverified identifiers", () => { expect(statuses()).toEqual([undefined, undefined]); }); + // A yes at the prompt is a choice; the re-run the refusal suggests must + // keep it, or it would reject the users it let through. + test("a yes carries into the command a refusal suggests", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + ...withUnverified, + { + id: "u3", + primary_email_address: "d@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + const error = (await run(importOptions).catch((caught: unknown) => caught)) as { + examples?: { command: string }[]; + }; + + expect(error.examples?.map((example) => example.command)).toEqual([ + "clerk migrate import export.json --source clerk --reserve-unverified --secret-key --allow-partial --yes", + ]); + }); + test("--reserve-unverified does not ask, and creates them reserved", async () => { await run({ ...importOptions, reserveUnverified: true }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index d14490b9f..a24ef69af 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -749,6 +749,9 @@ export async function run(rawOptions: MigrateRunOptions): Promise { default: false, }); } + // The answers carry into every command printed from here on, so a + // suggested re-run does not quietly drop what the operator chose. + options = { ...options, skipLegalChecks, reserveUnverified }; const checks = await withSpinner("Checking users against the instance...", async (spinner) => checkImport({ From 6f28ac5a56445f531e6074b09f60d6df483ac8b6 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 19:13:21 -0400 Subject: [PATCH 141/141] fix(migrate): offer no email fix that --reserve-unverified already covers With reservation, a user whose only email is unverified meets an email requirement, but the checks still offered to make email optional. A user with no email at all still gets that fix. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/checks.ts | 11 ++++++---- .../cli-core/src/commands/migrate/run.test.ts | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts index f51aa4090..5c51deb86 100644 --- a/packages/cli-core/src/commands/migrate/lib/checks.ts +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -693,10 +693,13 @@ function buildFixes(input: CheckInput, users: User[]): Fix[] { // An unverified email does not satisfy a required one, so a file of only // unverified addresses flags the requirement even when every user has one. - const unverifiedOnly = users.some((user) => { - const identifiers = splitIdentifiers(user); - return !identifiers.primaryEmail && identifiers.unverifiedEmails.length > 0; - }); + // Created reserved, it does, and making email optional fixes nothing. + const unverifiedOnly = + !input.reserveUnverified && + users.some((user) => { + const identifiers = splitIdentifiers(user); + return !identifiers.primaryEmail && identifiers.unverifiedEmails.length > 0; + }); const flagged = report.blocking.slice(); if ( unverifiedOnly && diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 46aead4d7..f7cc10d90 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -749,6 +749,26 @@ describe("run", () => { ); }); + // Reserved meets the requirement, so neither the reject nor the fix applies. + test("with --reserve-unverified, an unverified-only user is neither rejected nor a fix", async () => { + stubClerk({ + settings: { attributes: { email_address: { enabled: true, required: true } } }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", unverified_email_addresses: "b@x.dev" }, + ]), + ); + + await run({ ...baseOptions, dryRun: true, reserveUnverified: true }); + + expect(captured.err).not.toContain("only has an unverified email"); + expect(captured.err).not.toContain("required_for_sign_up"); + expect(process.exitCode).toBe(0); + }); + test("legal consent: refused without --skip-legal-checks, sent with skip_legal_checks with it", async () => { stubClerk({ settings: {