diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 1ecbdff2..ec2e69a6 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.32.0", + "version": "0.33.0", "source": "./plugins/kbagent", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/CLAUDE.md b/CLAUDE.md index 7a15a561..63256cd5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -383,7 +383,7 @@ kbagent data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-publi kbagent component list [--project NAME] [--type TYPE] [--query QUERY] kbagent component detail --component-id ID [--project NAME] -kbagent config new --component-id ID [--name NAME] [--project NAME] [--output-dir DIR] +kbagent config new --component-id ID [--name NAME] [--project NAME] [--output-dir DIR] [--push --no-files --description D --configuration JSON|@file|- --configuration-file PATH --no-validate --branch ID --dry-run] kbagent encrypt values --project ALIAS --component-id ID --input JSON|@file|- [--output-file PATH] diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index a33f341d..8a09f2e9 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.32.0", + "version": "0.33.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 80e705ff..bfa6e52e 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -79,6 +79,9 @@ a critical failure. `project edit --new-alias` (cascading rename across config.json + nested sync dir; warns on lineage cache rebuild) needs 0.31.0+, `storage truncate-table` needs 0.32.0+, + `data-app *` JSON output uses key `app_id` (was bare `id`) on 0.33.0+ + -- pipe with `jq -r '.apps[].app_id'`, not `'.id'`, + `config new --push` (one-shot remote create) needs 0.33.0+, `storage retype` is a future composite), you MUST refuse the task and return a handoff message to the parent: `"Cannot proceed safely on kbagent . Missing: . @@ -101,7 +104,7 @@ a critical failure. |---|---|---|---| | Update flow (rename, description, phases) | `kbagent flow update` (partial, no `--file`) | `--file` after fetching current phases, merging locally, passing full YAML | `tool call update_flow` (strips `behavior.onError` pre-MCP v1.60); partial `--file` that drops fields | | Schedule flow | `kbagent flow schedule --cron ... [--timezone]` | `tool call create_flow_schedule` | raw REST to `/storage/configurations/keboola.scheduler` | -| Create Snowflake transformation | `kbagent config new --component-id keboola.snowflake-transformation` + `config update --set ...` | `tool call create_sql_transformation` (lower schema, avoids the component refusal) | `tool call create_config` (refuses keboola.snowflake-transformation) | +| Create Snowflake transformation | `kbagent config new --component-id keboola.snowflake-transformation --name N --project P --push --no-files` (0.33.0+; one-shot, no scaffold, body defaults to `{}` and validation auto-skips for empty shell -- then `config update --set ...` to fill in script) **or** `kbagent config new --component-id keboola.snowflake-transformation --project P --output-dir D` + `config update --set ...` (scaffold-then-patch) | `tool call create_sql_transformation` (lower schema, avoids the MCP `create_config` Snowflake refusal) | `tool call create_config` (refuses keboola.snowflake-transformation) -- note: `config new --push` does NOT inherit this refusal because it wraps the raw Storage API directly | | Update SQL transformation body (script[]) | `kbagent config update --project P --component-id keboola.snowflake-transformation --config-id K --configuration @body.json` (0.28.0+ auto-normalizes string `script` to array; SQL gets statement-level split, Python/R gets `[script]` wrap; envelope's `normalizations: [...]` records every change. 0.31.0+ also re-splits multi-statement LIST elements -- closes the #274 ODBC `statement count 2 vs desired 1` crash that survives the 0.28.0 string fix) | `kbagent --hint client config update ...` if you need to bypass the auto-normalize for some reason | `tool call update_sql_transformation` -- still vulnerable to BOTH the #245 string-vs-array AND #274 list-element runtime crashes because it pushes raw to Storage API; raw `PUT /v2/storage/components/.../configs/...` -- same trap | | Run a job (and wait) | `kbagent job run --project P --component-id C --config-id K --wait` | `tool call run_component` | `job run` without `--wait` when user expects the result | | Search items by name across projects | `kbagent search QUERY [--project P] [--type table\|bucket\|config\|flow] [--limit N]` (0.30.0+) | `tool call search_tables` / `tool call search_configurations` (one resource-type per call) | chaining multiple `tool call` for different types | @@ -118,6 +121,7 @@ a critical failure. | Ad-hoc SQL / row-count / type audit | `kbagent workspace create` + `kbagent workspace load` + `kbagent workspace query --sql "..."` | `kbagent workspace from-transformation` for existing transform debugging | querying Keboola Storage directly via Snowflake credentials outside the workspace abstraction | | Inspect dev branch | `kbagent branch list --project P`, `kbagent branch use --project P --branch ID` | `tool call get_branch` | acting on `main` when a dev branch exists | | Audit project capabilities / features | `kbagent project info --project P` (0.30.0+) -- returns project ID, name, backend, enabled features, quota limits, and metrics | `tool call verify_token` (returns less structured info; no feature list) | inspecting the UI project settings manually | +| Create a new config (one-shot remote, no scaffold to disk) | `kbagent config new --project P --component-id C --name N --push --no-files [--configuration @body.json] [--branch ID]` (0.33.0+) -- single CLI call POSTs to `/v2/storage/components/{cid}/configs`; default body is `{}` (FIIA empty-shell pattern, validation auto-skips); explicit `--configuration` body is schema-validated by default (`--no-validate` opts out); works for ALL component types incl. `keboola.snowflake-transformation` | `kbagent config new --output-dir D` then edit + `kbagent sync push` (scaffold-then-push GitOps flow) | `tool call create_config` (refuses keboola.snowflake-transformation; raw MCP envelope, no validation) | | Create a config row | `kbagent config row-create --project P --component-id C --config-id K --name NAME` (0.30.0+) | `tool call create_config_row` | `POST /v2/storage/components/C/configs/K/rows` (raw REST) | | Update a config row | `kbagent config row-update --project P --component-id C --config-id K --row-id R [--name N] [--configuration JSON]` (0.30.0+) | `tool call update_config_row` | `PUT /v2/storage/components/C/configs/K/rows/R` (raw REST) | | Delete a config row | `kbagent config row-delete --project P --component-id C --config-id K --row-id R [--yes]` (0.30.0+) -- destructive (gated behind `--allow-destructive`); branch-aware | `tool call delete_config_row` | `DELETE /v2/storage/components/C/configs/K/rows/R` (raw REST) | @@ -160,10 +164,16 @@ success, not a failure. `update_flow` strip bug, reached via a different door. - **Snowflake transformation scaffolding**: `tool call create_config` - REFUSES `keboola.snowflake-transformation`. Use - `kbagent config new --component-id keboola.snowflake-transformation` - for the local scaffold, then `kbagent config update` for the body. - Or MCP `create_sql_transformation` which uses a lower-level schema. + REFUSES `keboola.snowflake-transformation`. Three options that work: + (a) `kbagent config new --component-id keboola.snowflake-transformation + --name N --project P --push --no-files` (0.33.0+) -- one-shot remote + create wrapping the raw Storage API, then `kbagent config update --set + parameters.blocks...=...` to fill in the body. **Recommended path.** + (b) `kbagent config new --component-id keboola.snowflake-transformation + --output-dir D` for the local scaffold, then `kbagent config update` for + the body. (c) MCP `tool call create_sql_transformation` which uses a + lower-level schema. `config new --push` does NOT inherit the MCP + refusal because it calls Storage API directly. - **`script[]` string-vs-array runtime crash** (0.28.0+ auto-fix; #245): the Storage API silently accepts `parameters.blocks[].codes[].script` diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 9f6e7111..3502bc5d 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -118,7 +118,7 @@ When working inside a git repository or project directory, run `kbagent init` (o | Set or clear ``storage.output.default_bucket`` on a configuration | `kbagent config set-default-bucket --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | Rename a configuration (update name via API + rename local sync directory) | `kbagent config rename --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --name NAME` | | Delete a configuration from a project | `kbagent config delete --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | -| Generate boilerplate configuration files for a Keboola component | `kbagent config new --component-id COMPONENT-ID` | +| Generate boilerplate configuration files for a Keboola component, optionally creating the config remotely in one shot | `kbagent config new --component-id COMPONENT-ID` | | List all metadata entries on a configuration | `kbagent config metadata-list --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | Read a single metadata value by key | `kbagent config get-metadata --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --key KEY` | | Set a metadata key/value on a configuration (upsert) | `kbagent config set-metadata --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --key KEY --value VALUE` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 34be9dc9..4622e085 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -55,7 +55,7 @@ All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the - `config set-default-bucket --project NAME --component-id ID --config-id ID (--bucket BUCKET_ID | --clear) [--dry-run] [--branch ID]` -- set or clear `configuration.storage.output.default_bucket` on a configuration. Discoverable shortcut for the raw-mode workaround at https://keboola.atlassian.net/wiki/spaces/SUP/pages/3770155030/. Read-modify-write that preserves sibling keys; returns `{"changed": false}` when the value already matches the requested state. Honored by output tables that don't pin their own `destination`. - `config rename --project NAME --component-id ID --config-id ID --name "New Name" [--branch ID] [--directory DIR]` -- rename a configuration (API update + local sync directory rename with git mv support) - `config delete --project NAME --component-id ID --config-id ID [--branch ID]` -- delete a configuration -- `config new --component-id ID [--project NAME] [--name NAME] [--output-dir DIR]` -- scaffold new config from component schema +- `config new --component-id ID [--project NAME] [--name NAME] [--output-dir DIR] [--push --no-files --description D --configuration JSON|@file|- --configuration-file PATH --no-validate --branch ID --dry-run]` -- **two modes**. **Default (no `--push`)**: scaffold new config from component schema; writes files to `--output-dir` or prints to stdout. **Zero API calls.** **With `--push`** (0.33.0+, requires `--project` + non-empty `--name`): also POSTs to `/v2/storage/components/{cid}/configs` for a one-shot remote create. `--no-files` skips the filesystem step entirely (FIIA-style empty-shell pattern). `--configuration` / `--configuration-file` override the POSTed body (default is `{}`, with validation auto-skipped for the default empty shell). `--dry-run` previews the planned POST + validation result without creating. Schema validation runs by default when an explicit body is given (fail-closed: `ConfigError` exit 5 on mismatch) but skips silently if the AI Service has no schema for the component or returns an error; `--no-validate` opts out. Works for ALL component types including `keboola.snowflake-transformation` (unlike `tool call create_config`, which refuses that component). - `config variables-set --project NAME --component-id ID --config-id ID --var KEY=VALUE [--var ...] [--replace] [--variables-id ID] [--values-id ID] [--branch ID] [--dry-run] [--allow-plaintext-on-encrypt-failure] [--yes]` -- attach variable values to a config. Auto-creates a sibling `keboola.variables` config + default row on first use and links it via the parent's `runtime.variables_id` / `variables_values_id`. Defaults to merge; `--replace` drops keys not in `--var`. `#`-prefixed values encrypt via the Encryption API (fail-closed; exit non-zero on `ENCRYPTION_FAILED`). See `variables-workflow.md` - `config variables-get --project NAME --component-id ID --config-id ID [--branch ID]` -- resolve `variables_id` + `values_id` from the parent config and fetch the current KEY=VALUE map. Returns `{linked: bool, variables_id, values_id, values}`; `linked=false` means the parent has no variables attached - `config variables-clear --project NAME --component-id ID --config-id ID [--branch ID] [--yes]` -- unlink variables from the parent config (strips `variables_id` + `variables_values_id`). **Does NOT delete** the backing `keboola.variables` config -- use `config delete` explicitly if you've verified nothing else references it @@ -139,7 +139,7 @@ All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the - `workspace from-transformation --project ALIAS --component-id ID --config-id ID [--row-id ID]` -- workspace from existing transform ## Data Apps (Streamlit / Flask / Node deployments) -Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, encrypted secrets, runtime size) with Data Science API (`/apps` -- deployment record, state, URL, configVersion). The CLI encapsulates the §9 redeploy contract so callers cannot pin to the empty-shell v2; see `data-app-workflow.md` for the gotcha inventory and recipes. +Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, encrypted secrets, runtime size) with Data Science API (`/apps` -- deployment record, state, URL, configVersion). The CLI encapsulates the §9 redeploy contract so callers cannot pin to the empty-shell v2; see `data-app-workflow.md` for the gotcha inventory and recipes. Since v0.33.0 the JSON output envelope's data-app id key is `app_id` (renamed from bare `id` for symmetry with the `--app-id` input flag); `config_id` is unchanged. - `data-app list [--project NAME ...] [--branch ID]` -- list apps across projects (Data Science index merged with Storage names) - `data-app detail --project NAME --app-id ID [--branch ID]` -- merged view (state, desired, url, configVersion, slug, git block with PAT redacted) - `data-app create --project ALIAS --name NAME --slug SLUG --git-repo URL [--git-public/--no-git-public] [--git-username USER] [--git-pat-env VAR | --git-pat-file PATH | --git-pat-encrypted KBC::Project...] [--auth password|public] [--size tiny|small|medium|large] [--auto-suspend SECONDS] [--type python-js|python|streamlit|r|...] [--branch ID] [--no-deploy] [--wait] [--timeout SECONDS] [--keep-on-failure] [--dry-run]` -- POST shell + encrypt PAT + PUT Storage config (with auto-injected `parameters.id`) + PATCH deploy with the §9 trio. Cleanup-in-finally on failure unless `--keep-on-failure`. Default `--auth password` mints a 20-char hex simpleAuth password (retrievable via `data-app password`). diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 62d6fd1d..663a7a99 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -1,5 +1,75 @@ # Gotchas -- Response Parsing and Common Pitfalls +## `kbagent config new --push` is one-shot remote create; default is scaffold-only (since v0.33.0) + +- **Pre-v0.33.0**, `kbagent config new` was scaffold-only -- it wrote + boilerplate files to `--output-dir` (or stdout) and made **zero API calls**. + The intended flow was scaffold → edit → `kbagent sync push`. The agent docs + in `keboola-expert.md` and SKILL.md conflated this with "create config" + intent, which was wrong if the goal was an API mutation. +- **Since v0.33.0**, `--push` adds a one-shot remote create: + `kbagent config new --component-id C --name N --project P --push` calls + `POST /v2/storage/components/C/configs` after the scaffold step. Returns + the new config ID immediately. `--no-files` skips the filesystem step + entirely (no scaffold to disk or stdout, only the API POST) -- this is + the FIIA-style "empty shell, then patch via `config update --set ...`" + pattern. +- `--push` **requires** `--project` AND a non-empty `--name`. All other + push-gated flags (`--no-files`, `--description`, `--configuration` / + `--configuration-file`, `--no-validate`, `--branch`, `--dry-run`) are + no-ops without `--push` and exit 2 if set independently. +- `--configuration` and `--configuration-file` are mutually exclusive; + `--no-files` and `--output-dir` are mutually exclusive. +- **MCP `create_config` quirk does NOT apply**: the raw MCP tool refuses + `keboola.snowflake-transformation` and routes you to + `tool call create_sql_transformation`. `kbagent config new --push` does + NOT refuse; the typed CLI wraps the raw Storage API directly. For + Snowflake transformations: one `config new --push` call works; the + MCP-typed `create_sql_transformation` shape is only needed if you + specifically want that envelope. +- **Schema validation** runs by default whenever `--configuration` / + `--configuration-file` provide an explicit body. On mismatch the create + aborts with exit 5 and a list of error paths. If the AI Service has no + schema for the component or returns an error, validation skips silently + (the result envelope shows `validation_status: "skipped"`). Use + `--no-validate` to skip the AI Service call entirely. +- **Empty-shell exception**: when no body is provided (default `{}`), + validation auto-skips. Component schemas almost always require parameters + and would reject `{}` -- skipping is the FIIA-pattern-friendly default. + Passing `--configuration '{}'` explicitly does NOT take the skip path: + the body is treated as caller-provided and validated, which typically + fails. Use `--no-validate` to suppress validation entirely. +- **`--push --dry-run`** returns the planned POST body + validation result + without making the API call (`dry_run: true` in the envelope, exit 0 even + on validation failure -- dry-run is inspection-only). +- The result envelope on success includes the full Storage API response + plus `project_alias`, `branch_id`, `validation_status`, and + `validation_errors` (always present, even if empty). Shape-symmetric with + `config detail` single-config mode and `config row-create`. +## `data-app` JSON output: key for the app's own id is `app_id` (since v0.33.0) + +- Every `kbagent --json data-app ` envelope emits the + data-app's own identifier under the key `app_id`. Prior to v0.33.0 the + same key was named bare `id`, which did not match the `--app-id` input + flag. Affects `data-app list / detail / create / deploy / start / stop / + delete / password / secrets-set / secrets-list / secrets-get / + secrets-remove`. The companion `config_id` key is unchanged. +- Pipe-friendly chain that v0.33.0 enables: + `kbagent --json data-app list | jq -r '.apps[].app_id' | xargs -I{} kbagent data-app deploy --project P --app-id {}`. + On pre-v0.33.0 you had to read `.apps[].id` (mismatched the input flag, + surprised AI agents that templated `.app_id`). +- **What is NOT renamed:** the Storage config back-pointer at + `parameters.id` inside the configuration body sent TO Storage (writeup + §5) -- that lives in the Storage config, not in kbagent's output + envelope. The auth-provider id (`auth_providers[].id == "simpleAuth"`) + is also unchanged. +- The Data Science API on every Keboola stack we've probed (europe-west3.gcp, + us-east4.gcp; 2026-05-12) serves camelCase keys on the wire (`id`, + `configId`, `desiredState`, `configVersion`, ...). kbagent reads those + camelCase keys directly and emits its own snake_case-ish output keys. + If a future API shape change introduces snake_case wire keys, this + helper will need a defensive alias pass -- not yet warranted. + ## `project edit --new-alias` does NOT rewrite lineage caches (since v0.31.0) - `kbagent project edit --project OLD --new-alias NEW` cascades the rename diff --git a/plugins/kbagent/skills/kbagent/references/scaffold-workflow.md b/plugins/kbagent/skills/kbagent/references/scaffold-workflow.md index 35414620..daf2d213 100644 --- a/plugins/kbagent/skills/kbagent/references/scaffold-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/scaffold-workflow.md @@ -1,10 +1,22 @@ # Creating New Configurations (Scaffold Workflow) +> **Two modes, one command (since v0.33.0):** +> - `kbagent config new --output-dir DIR` (this workflow) -- generate +> scaffold files to disk, edit them, then push later with +> `kbagent sync push`. The "GitOps for configs" path. +> - `kbagent config new --push --no-files --project P --name N` -- one-shot +> remote create via Storage API, no filesystem step. The FIIA "empty +> shell, then patch via `config update --set ...`" path. See +> `gotchas.md` "`kbagent config new --push` is one-shot remote create" +> for the full side-by-side and the schema-validation behavior. + ## When to use - User wants to create a new extractor, writer, transformation, or application - User asks "how do I set up a new Snowflake extractor" or similar -- User wants to scaffold config files for a component +- User wants to scaffold config files for a component (default mode), OR + wants a single CLI call that posts to the Storage API and returns the new + config ID (use `--push --no-files`) ## Step-by-step workflow @@ -34,6 +46,14 @@ kbagent --json config new --component-id COMPONENT_ID --project ALIAS --name "Co # To disk (auto-detects kbc project structure, writes under main/ if applicable) kbagent config new --component-id COMPONENT_ID --project ALIAS --name "Config Name" --output-dir . + +# One-shot remote create (since 0.33.0) -- no filesystem, just POST + return ID +kbagent --json config new --component-id COMPONENT_ID --project ALIAS --name "Config Name" \ + --push --no-files + +# Scaffold AND remote create in one step (writes files AND POSTs) +kbagent config new --component-id COMPONENT_ID --project ALIAS --name "Config Name" \ + --output-dir . --push ``` Generated files by component type: diff --git a/pyproject.toml b/pyproject.toml index bd218ce0..f14b9a2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.32.0" +version = "0.33.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 23290c87..4fff3931 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,13 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.33.0": [ + "New: `kbagent config new --push` -- one-shot remote configuration create via the Storage API. Previously `config new` was scaffold-only (writes files to disk, zero API calls); the AI agent docs (`keboola-expert.md` tool selection matrix, SKILL.md decision row) conflated this with API-create intent. `--push` adds the actual remote create as a single CLI call. Requires `--project` and a non-empty `--name`. `--no-files` skips the filesystem step entirely for FIIA-style 'empty shell, then patch via `config update --set`' workflows. Optional `--configuration JSON|@file|-` / `--configuration-file PATH` override the POSTed body (default is `{}`). `--dry-run` previews the planned POST and validation outcome without calling the API. `--branch ID` targets a specific dev branch. `--description D` sets the description. Resolves the F2 gap in `kbagent-feature-gaps.md`. Schema validation runs by default when `--configuration` provides an explicit body (`jsonschema.Draft7Validator` against the component's AI Service `configurationSchema`, fail-closed with exit 5); graceful skip if the AI Service has no schema; `--no-validate` opts out. Default empty-shell `{}` auto-skips validation (FIIA pattern). Snowflake transformation matrix fix: `tool call create_config` refuses `keboola.snowflake-transformation`; `config new --push` wraps the raw Storage API directly and does NOT inherit the refusal -- works for ALL component types. Without `--push`, today's scaffold-only behavior is byte-for-byte preserved -- pure regression-zero addition.", + "Change: `kbagent --json data-app ` envelopes now emit the data-app's own identifier under the key `app_id` (was bare `id`). Affects 12 subcommands: `list, detail, create, deploy, start, stop, delete, password, secrets-set, secrets-list, secrets-get, secrets-remove`. The companion `config_id` key is unchanged. The Storage config back-pointer at `parameters.id` (lives INSIDE the configuration body sent TO Storage) is unchanged. The auth-provider id (`auth_providers[].id == \"simpleAuth\"`) is unchanged. The pipe-friendly chain now works as the `--app-id` input flag implies: `kbagent --json data-app list | jq -r '.apps[].app_id' | xargs -I{} kbagent data-app deploy --project P --app-id {}`. Output-key rename only; the Data Science API still serves camelCase keys on the wire (verified 2026-05-12 on europe-west3.gcp across projects 1143/2738/2959). No deprecation alias; the breakage is surface-level and trivially scriptable (`jq -r '.apps[].id'` → `jq -r '.apps[].app_id'`). No in-tree consumers of `apps[].id` were found.", + "Tests: 47 new tests across `tests/test_config_create_service.py` (15 service-level: happy paths, dry-run, schema validation ok/failed/skipped, malformed schemas, AI Service errors, client cleanup), `tests/test_config_create_cli.py` (20 CLI-level: flag-combination validation, push-mode happy paths, body parsing inline/@file/stdin, error propagation, dry-run envelopes, JSON-mode stdout correctness for `--push --output-dir`), and `tests/test_data_app_service.py` (12: `TestDataAppEnvelopesNoBareIdKey` regression class for the 11 affected subcommands + 1 list-rename coverage test). E2E flow extended with step 19b (`config new --push --dry-run` → real `--push --no-files` → `config detail` verify → `config update --set` patch → `config delete` cleanup, wrapped in try/finally + pre-registered safety-net teardown) and `test_data_app_lifecycle_public` extended with list-step round-trip `app_id` assertion. Existing `TestConfigNew` in `test_component_cli.py` continues to pass byte-for-byte (regression coverage for scaffold-only mode).", + "Plugin docs: synced across all 7 silent-drift surfaces (CLAUDE.md #17). `commands/context.py` (`config new --push` inventory + `--app-id` mention), `CLAUDE.md ## All CLI Commands` (replaced `config new` line), `keboola-expert.md` Rule 6 VERSION GATE adds `config new --push needs 0.33.0+` and `data-app *` JSON output uses key `app_id` on 0.33.0+; the Snowflake matrix row is rewritten to surface `config new --push --no-files` as first-choice; new 'Create a new config (one-shot remote)' matrix row added. `SKILL.md` auto-regenerated. `commands-reference.md` adds the two-mode `config new --push` entry and the `data-app` rename note. `gotchas.md` adds two `(since v0.33.0)` sections: `config new --push` scaffold-vs-push split + MCP refusal nuance + validation behavior; `data-app` `app_id` rename with input/output symmetry rationale. `scaffold-workflow.md` adds the dual-mode callout + push examples in step 3.", + "Review fixes: post-rebase against main (which had already shipped 0.32.0 via PR #277), `--push --output-dir` no longer emits a plain-text dim line above the JSON envelope when `--json` is set (the `_write_scaffold_to_disk` helper now honors `formatter.json_mode` instead of hardcoded `False`) -- closes a silent JSON-pipeline regression flagged in PR #278 review. The push-with-output-dir CLI test now asserts `json.loads(result.output)` parses cleanly.", + ], "0.32.0": [ 'New: `kbagent storage truncate-table --project NAME --table-id ID [--table-id ...] [--dry-run] [--yes] [--branch ID]` -- row-level truncation that drops every row from one or more storage tables while preserving the table definition (columns, types, primary key, descriptions, sharing edges, and every downstream config reference). Closes the only confirmed FIIA-migration gap (Coverage Matrix Row 25; R5 per-phase reload invariant). Wraps `DELETE /v2/storage/[branch/{id}/]tables/{id}/rows?allowTruncate=1`. Notable departure from sibling destructive endpoints: the row-delete endpoint is inherently async on every branch and rejects `async=true` as an unknown field (verified live 2026-05-11: HTTP 400 `"async: This field was not expected."`). The client therefore omits `async=true` and lets the endpoint return its natural HTTP 202 + queued `tableRowsDelete` job, which `_wait_for_storage_job` polls to completion -- same machinery as `delete_table`, just without the query-param dance. Multi-target: per-table errors accumulate without aborting the batch (one missing table does not block the rest). JSON envelope mirrors `delete-tables`\'s naming with a richer per-target receipt: `{truncated: [{table_id, rows_before, rows_after, branch_id}], failed: [{id, error}], dry_run, project_alias, would_truncate?}`. `--dry-run` captures `rows_before` via `get_table_detail` without truncating. Idempotent (truncating an empty table is a no-op success). Permission classification: `destructive` (gated behind `--allow-destructive` / `cli:destructive` policies) alongside `delete-table` / `delete-column` / `delete-bucket` / `swap-tables` -- schema preservation does not downgrade row-data deletion. Use this over `delete-table` whenever the schema contract must survive (sharing edges, aliases, dependent transformations, primary keys, column descriptions).', "Tests: `tests/test_storage_truncate.py` adds 21 unit tests across three layers -- HTTP shape (5: URL+query-params with allowTruncate=1, branch_id URL prefix, async-poll roundtrip, URL encoding of dotted table IDs, 4xx propagation via `pytest_httpx`), service business logic (10: happy path, branch_id carried into truncated[] entries, non-numeric `rowsCount` defaults to 0, missing `rowsCount` defaults to 0, batch partial failure with NOT_FOUND on second target, dry-run skips `truncate_table`, branch_id propagation to both `get_table_detail` and `truncate_table`, unknown-project `ConfigError`, try/finally `close()` on API error, empty-list short-circuit), and CLI integration (6: JSON happy path with `--yes`, `--dry-run` JSON shape, `--branch` flag override, active-branch fallback, exit 1 on `failed[]`, exit 5 on `ConfigError`). E2E coverage in `tests/test_e2e.py::TestFullE2E` adds step 11.1 `_test_truncate_table_roundtrip`: snapshots schema (columns + primary key + identity) on the 8-row test table, dry-runs the truncate (verifies `would_truncate.rows_before` matches), applies it (verifies `rows_after=0`), re-verifies schema integrity (columns + PK + identity unchanged), then restores the 5+3 CSV pair so downstream hops see the original row count. Live-API smoke against project 1143 on `connection.europe-west3.gcp.keboola.com` 2026-05-11 confirmed the full flow including the async-only endpoint discovery.", diff --git a/src/keboola_agent_cli/commands/config.py b/src/keboola_agent_cli/commands/config.py index f4dd1453..346b96fb 100644 --- a/src/keboola_agent_cli/commands/config.py +++ b/src/keboola_agent_cli/commands/config.py @@ -1102,72 +1102,257 @@ def config_new( name: str = typer.Option( "", "--name", - help="Configuration name (default: auto-generated from component)", + help="Configuration name (default: auto-generated from component; required with --push)", ), project: str | None = typer.Option( None, "--project", - help="Project alias (for AI Service auth)", + help="Project alias (for AI Service auth; required with --push)", ), output_dir: str | None = typer.Option( None, "--output-dir", help="Write scaffold files to disk instead of printing", ), + push: bool = typer.Option( + False, + "--push", + help="Also create the configuration remotely via the Storage API (one-shot; requires --project and --name)", + ), + no_files: bool = typer.Option( + False, + "--no-files", + help="With --push: skip writing/printing scaffold; only POST to API (FIIA-style one-shot)", + ), + description: str = typer.Option( + "", + "--description", + help="Configuration description (used with --push)", + ), + configuration: str | None = typer.Option( + None, + "--configuration", + help="Override the configuration body to POST (used with --push): JSON inline, @file, or - for stdin", + ), + configuration_file: Path | None = typer.Option( + None, + "--configuration-file", + help="Override the configuration body from a JSON file (used with --push)", + exists=True, + readable=True, + ), + no_validate: bool = typer.Option( + False, + "--no-validate", + help="Skip schema validation against the component's AI Service spec (used with --push)", + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Create in a specific dev branch (used with --push; defaults to active branch)", + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="With --push: show planned POST + validation result without creating", + ), ) -> None: - """Generate boilerplate configuration files for a Keboola component. + """Generate boilerplate configuration files for a Keboola component, optionally creating the config remotely in one shot. + + \b + Two modes: + * Default (no --push): scaffold only -- generates ready-to-edit files + (config YAML, code blocks, description). Writes to --output-dir or + prints to stdout. **Zero API calls.** + * With --push: scaffold step + POST to Storage API. Requires --project + and a non-empty --name. Use --no-files for FIIA-style one-shot create + with no filesystem step. + + \b + Examples: + # Scaffold-only (today's behavior, unchanged) + kbagent config new --component-id keboola.ex-http --output-dir ./scratch + + # Scaffold AND remote create: + kbagent config new --component-id keboola.ex-http --name "API ingest" \\ + --project prod --output-dir ./scratch --push + + # FIIA-style: one-shot remote create with no filesystem step: + kbagent config new --component-id keboola.ex-http --name "API ingest" \\ + --project prod --push --no-files - Produces a ready-to-edit scaffold (config YAML, SQL/Python code blocks, - description) that can be written to disk with --output-dir or printed - to stdout for inspection. + # Override the POSTed body with a pre-made config: + kbagent config new --component-id keboola.python-transformation-v2 \\ + --name "T1" --project prod --push --no-files \\ + --configuration-file ./body.json --branch 42 + + # Preview the planned POST without creating: + kbagent config new --component-id keboola.ex-http --name "smoke" \\ + --project prod --push --no-files --dry-run """ formatter = get_formatter(ctx) - service = get_service(ctx, "component_service") - - try: - scaffold = service.generate_scaffold( - alias=project, - component_id=component_id, - name=name or None, - ) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: - exit_code = map_error_to_exit_code(exc) - formatter.error( - message=exc.message, - error_code=exc.error_code, - project=project or "", - retryable=exc.retryable, - ) - raise typer.Exit(code=exit_code) from None - if output_dir: - # Detect kbc project branch prefix (e.g. "main/") - branch_prefix = _detect_branch_prefix(Path(output_dir)) - if branch_prefix: - scaffold_dir = branch_prefix + "/" + scaffold["directory"] - else: - scaffold_dir = scaffold["directory"] + # ── Flag-combination validation ────────────────────────────────────────── + # + # All --push-gated flags must be set only when --push is on; --push itself + # requires --project and a non-empty --name; --configuration vs + # --configuration-file and --output-dir vs --no-files are mutually + # exclusive. + push_gated: list[tuple[str, bool]] = [ + ("--no-files", no_files), + ("--description", bool(description)), + ("--configuration", configuration is not None), + ("--configuration-file", configuration_file is not None), + ("--no-validate", no_validate), + ("--branch", branch is not None), + ("--dry-run", dry_run), + ] + if not push: + for flag_name, flag_set in push_gated: + if flag_set: + formatter.error( + message=f"{flag_name} requires --push", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + else: + if not project: + formatter.error( + message="--push requires --project", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + if not name: + formatter.error( + message="--push requires a non-empty --name", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + if configuration is not None and configuration_file is not None: + formatter.error( + message="--configuration and --configuration-file are mutually exclusive", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + if no_files and output_dir: + formatter.error( + message="--no-files and --output-dir are mutually exclusive", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + + # ── Parse the optional body override (used only with --push) ───────────── + config_body: dict[str, Any] | None = None + if push: + if configuration is not None: + try: + config_body = _parse_json_input(configuration) + except (json.JSONDecodeError, FileNotFoundError) as exc: + formatter.error( + message=f"Invalid --configuration input: {exc}", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) from None + elif configuration_file is not None: + try: + config_body = json.loads(configuration_file.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + formatter.error( + message=f"Invalid JSON in --configuration-file {configuration_file}: {exc}", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) from None - base_path = Path(output_dir) / scaffold_dir - base_path.mkdir(parents=True, exist_ok=True) + # ── Scaffold step (runs in scaffold-only mode and in push+files modes) ─── + scaffold: dict[str, Any] | None = None + skip_scaffold = push and no_files + if not skip_scaffold: + component_service = get_service(ctx, "component_service") + try: + scaffold = component_service.generate_scaffold( + alias=project, + component_id=component_id, + name=name or None, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error( + message=exc.message, + error_code=exc.error_code, + project=project or "", + retryable=exc.retryable, + ) + raise typer.Exit(code=exit_code) from None + + # ── Push path: also create remotely via Storage API ────────────────────── + if push: + if should_hint(ctx): + emit_hint( + ctx, + "config.new", + project=project, + component_id=component_id, + name=name, + description=description, + configuration=config_body if config_body is not None else {}, + no_validate=no_validate, + branch=branch, + dry_run=dry_run, + ) - for file_entry in scaffold["files"]: - file_path = base_path / file_entry["path"] - file_path.parent.mkdir(parents=True, exist_ok=True) - file_path.write_text(file_entry["content"], encoding="utf-8") + config_service = get_service(ctx, "config_service") + try: + push_result = config_service.create_config( + alias=project, + component_id=component_id, + name=name, + description=description, + configuration=config_body, + branch_id=branch, + dry_run=dry_run, + validate=not no_validate, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error( + message=exc.message, + error_code=exc.error_code, + project=project or "", + retryable=exc.retryable, + ) + raise typer.Exit(code=exit_code) from None + + # When --push is set AND --output-dir is given AND we're not in + # dry-run mode, ALSO write the scaffold to disk in addition to the + # POST (the "scaffold + push" combo). Dry-run must NOT touch the + # filesystem -- the user expects a preview, not a side effect. + # + # ``silent=True``: the push-result envelope below is the authoritative + # output for this path. In JSON mode emitting the scaffold envelope + # too would produce two concatenated JSON documents on stdout (breaks + # ``jq``); in human mode the dim banner adds noise above the "Created + # config ..." line that already lists the same path. ``json_mode`` is + # passed as a sentinel ``False`` because ``silent=True`` short-circuits + # before it is read. + if output_dir and scaffold is not None and not dry_run: + _write_scaffold_to_disk(formatter, scaffold, output_dir, json_mode=False, silent=True) if formatter.json_mode: - formatter.output( - { - "directory": str(base_path), - "files_written": [f["path"] for f in scaffold["files"]], - } - ) + formatter.output(push_result) else: - formatter.success(f"Scaffold written to {base_path} ({len(scaffold['files'])} file(s))") + _render_push_result_human(formatter, push_result, component_id, name) + return + + # ── Scaffold-only path: today's behavior, byte-for-byte unchanged ──────── + assert scaffold is not None # narrowing for type-checker; skip_scaffold is False here. + if output_dir: + _write_scaffold_to_disk(formatter, scaffold, output_dir, json_mode=formatter.json_mode) else: # Print scaffold content if formatter.json_mode: @@ -1195,6 +1380,99 @@ def config_new( formatter.console.print() +def _write_scaffold_to_disk( + formatter: Any, + scaffold: dict[str, Any], + output_dir: str, + json_mode: bool, + silent: bool = False, +) -> None: + """Shared helper: write the generated scaffold files under ``output_dir``. + + Detects an enclosing ``main/`` branch prefix the same way the pre-push + path does, so the layout matches what ``kbagent sync push`` would expect. + + Output rules: + - ``silent=True``: write files only; emit no banner and no JSON envelope. + Used by the ``--push --output-dir`` path, where the push-result envelope + (single JSON in JSON mode; the "Created config ..." line in human mode) + is the authoritative output and the scaffold-banner side-channel would + either duplicate JSON (breaking ``jq`` pipes) or distract in human mode. + - ``silent=False`` (default; scaffold-only mode): emit a JSON envelope + ``{directory, files_written}`` when ``json_mode`` is set, otherwise a + dim-formatted "Scaffold written to ..." banner. + """ + branch_prefix = _detect_branch_prefix(Path(output_dir)) + if branch_prefix: + scaffold_dir = branch_prefix + "/" + scaffold["directory"] + else: + scaffold_dir = scaffold["directory"] + + base_path = Path(output_dir) / scaffold_dir + base_path.mkdir(parents=True, exist_ok=True) + + for file_entry in scaffold["files"]: + file_path = base_path / file_entry["path"] + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(file_entry["content"], encoding="utf-8") + + if silent: + return + + if json_mode: + formatter.output( + { + "directory": str(base_path), + "files_written": [f["path"] for f in scaffold["files"]], + } + ) + else: + formatter.console.print( + f"[dim]Scaffold written to {base_path} ({len(scaffold['files'])} file(s))[/dim]" + ) + + +def _render_push_result_human( + formatter: Any, + result: dict[str, Any], + component_id: str, + name: str, +) -> None: + """Render the human-mode banner for ``config new --push`` (dry-run + success).""" + validation_status = result.get("validation_status", "skipped") + validation_errors = result.get("validation_errors", []) or [] + branch_info = f" (branch {result['branch_id']})" if result.get("branch_id") else "" + + if result.get("dry_run"): + formatter.console.print( + f"\n[bold]Dry-run -- would POST {escape(component_id)} '{escape(name)}'{branch_info}[/bold]\n" + ) + formatter.console.print("[dim]Planned configuration body:[/dim]") + formatter.console.print(json.dumps(result.get("configuration", {}), indent=2)) + if validation_status == "ok": + formatter.console.print("\n[green]✓ Schema validation passed[/green]") + elif validation_status == "skipped": + formatter.console.print( + "\n[yellow]⚠ Schema validation skipped[/yellow] " + "[dim](empty shell, no schema available, or --no-validate)[/dim]" + ) + else: # failed + formatter.console.print("\n[red]✗ Schema validation failed:[/red]") + for err in validation_errors: + formatter.console.print(f" [red]• {escape(err)}[/red]") + return + + config_id = result.get("id", "") + formatter.success( + f"Created config '{escape(name)}' [{config_id}] in {escape(component_id)}{branch_info}" + ) + if validation_status == "skipped": + formatter.console.print( + "[dim]Note: schema validation was skipped " + "(empty shell, no schema available, or --no-validate).[/dim]" + ) + + # ── Config metadata commands ─────────────────────────────────────────── diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index ffd60909..6413414e 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -187,7 +187,14 @@ Delete a configuration. Branch-aware. kbagent config new --component-id ID [--name NAME] [--project NAME] [--output-dir DIR] - Generate boilerplate config from component schema. Use --output-dir to write files. + [--push --no-files --description D --configuration JSON|@file|- --configuration-file PATH --no-validate --branch ID --dry-run] + Default: generate boilerplate config from component schema (scaffold to --output-dir or stdout). + With --push (0.33.0+): also create the config remotely via Storage API in one shot. + --push requires --project AND a non-empty --name. --no-files skips the filesystem step + entirely for FIIA-style one-shot creates. Schema validation runs by default when an explicit + --configuration body is given (fail-closed; --no-validate opts out). Default body is {{}} + (empty shell, validation auto-skipped). Works for ALL component types including + keboola.snowflake-transformation (unlike tool call create_config which refuses it). kbagent config search --query PATTERN [--project NAME] [--component-type TYPE] [-i] [-r] [--branch ID] Search config bodies for string/regex. Reports match location in JSON tree. Branch-aware. diff --git a/src/keboola_agent_cli/commands/data_app.py b/src/keboola_agent_cli/commands/data_app.py index 08ed3d56..e3340a2e 100644 --- a/src/keboola_agent_cli/commands/data_app.py +++ b/src/keboola_agent_cli/commands/data_app.py @@ -50,7 +50,7 @@ def _print_data_app_table(formatter, result: dict) -> None: return for app in apps: formatter.console.print( - f" [bold]{app['id']}[/bold] " + f" [bold]{app['app_id']}[/bold] " f"[cyan]{app.get('name', '')}[/cyan] " f"({app.get('type', '?')}) " f"state=[yellow]{app.get('state', '?')}[/yellow] " @@ -167,7 +167,7 @@ def data_app_detail( formatter.output( result, lambda c, d: ( - c.print(f"\n[bold]Data app:[/bold] {d.get('name', '')} ({d['id']})"), + c.print(f"\n[bold]Data app:[/bold] {d.get('name', '')} ({d['app_id']})"), c.print(f" [bold]Project:[/bold] {d['project_alias']}"), c.print(f" [bold]Slug:[/bold] {d.get('slug', '')}"), c.print(f" [bold]Type:[/bold] {d.get('type', '')}"), @@ -405,7 +405,7 @@ def data_app_create( formatter.console.print( f"[bold green]Success:[/bold green] {result.get('message', '')}" ) - formatter.console.print(f" [bold]App ID:[/bold] {result['id']}") + formatter.console.print(f" [bold]App ID:[/bold] {result['app_id']}") formatter.console.print(f" [bold]Config ID:[/bold] {result['config_id']}") if result.get("url"): formatter.console.print(f" [bold]URL:[/bold] {result['url']}") @@ -926,7 +926,7 @@ def data_app_secrets_list( return formatter.console.print( f"\n[bold]{result['count']} secret(s)[/bold] on data app " - f"[cyan]{result['id']}[/cyan] in [magenta]{result['project_alias']}[/magenta]:" + f"[cyan]{result['app_id']}[/cyan] in [magenta]{result['project_alias']}[/magenta]:" ) for entry in result["secrets"]: marker = ( diff --git a/src/keboola_agent_cli/hints/definitions/config.py b/src/keboola_agent_cli/hints/definitions/config.py index 678c2b79..9bdc20ee 100644 --- a/src/keboola_agent_cli/hints/definitions/config.py +++ b/src/keboola_agent_cli/hints/definitions/config.py @@ -791,3 +791,62 @@ ], ) ) + +# ── config new --push (one-shot remote create) ──────────────────────────────── +# +# Scaffold-only mode (no --push) has no hint -- it's a local filesystem +# operation with no API mapping. The hint emits only when --push is set; the +# command layer guards the emit_hint() call accordingly. + +HintRegistry.register( + CommandHint( + cli_command="config.new", + description=( + "Create a new configuration remotely via the Storage API " + "(`config new --push`). Scaffold-only mode is a local filesystem " + "operation and not represented in this hint." + ), + steps=[ + HintStep( + comment="Create configuration via Storage API POST", + client=ClientCall( + method="create_config", + args={ + "component_id": "{component_id}", + "name": "{name}", + "configuration": "{configuration}", + "description": "{description}", + "branch_id": "{branch}", + }, + result_var="config", + result_hint="dict", + ), + service=ServiceCall( + service_class="ConfigService", + service_module="config_service", + method="create_config", + args={ + "alias": "{project}", + "component_id": "{component_id}", + "name": "{name}", + "description": "{description}", + "configuration": "{configuration}", + "branch_id": "{branch}", + "dry_run": "{dry_run}", + "validate": "not {no_validate}", + }, + ), + ), + ], + notes=[ + "configuration defaults to {} (empty shell) when omitted -- FIIA's " + "'create-then-patch' pattern. Validation auto-skips for the empty case.", + "Without --no-validate, the body is validated against the component's " + "AI Service JSON schema before POSTing (fail-closed: exit 5 on mismatch). " + "Skips gracefully when the AI Service has no schema for the component.", + "The returned dict includes the new config 'id' assigned by the API " + "plus 'project_alias', 'branch_id', and 'validation_status' annotations.", + "The client-mode snippet bypasses validation; the service-mode snippet runs it.", + ], + ) +) diff --git a/src/keboola_agent_cli/services/config_service.py b/src/keboola_agent_cli/services/config_service.py index a2a7ff29..1a2a8baa 100644 --- a/src/keboola_agent_cli/services/config_service.py +++ b/src/keboola_agent_cli/services/config_service.py @@ -10,16 +10,27 @@ import re import shutil import subprocess +from collections.abc import Callable from pathlib import Path from typing import Any +from ..ai_client import AiServiceClient +from ..config_store import ConfigStore from ..errors import ConfigError, ErrorCode, KeboolaApiError from ..json_utils import compute_diff, deep_merge, set_nested_value -from ..models import ProjectConfig +from ..models import ComponentDetail, ProjectConfig from ..sync.code_extraction import normalize_blocks_codes_script from ..sync.manifest import Manifest, load_manifest, save_manifest from ..sync.naming import sanitize_name -from .base import BaseService, sanitize_unexpected_error +from .base import BaseService, ClientFactory, sanitize_unexpected_error + +AiClientFactory = Callable[[str, str], AiServiceClient] + + +def _default_ai_client_factory(stack_url: str, token: str) -> AiServiceClient: + """Default factory: build an ``AiServiceClient`` for the given project.""" + return AiServiceClient(stack_url=stack_url, token=token) + logger = logging.getLogger(__name__) @@ -85,9 +96,20 @@ class ConfigService(BaseService): using ThreadPoolExecutor, collects results, and reports per-project errors without stopping others. - Uses dependency injection for config_store and client_factory. + Uses dependency injection for config_store, client_factory, and + ai_client_factory (the last one is only exercised by ``create_config`` + when the AI Service component-schema lookup runs). """ + def __init__( + self, + config_store: ConfigStore, + client_factory: ClientFactory | None = None, + ai_client_factory: AiClientFactory | None = None, + ) -> None: + super().__init__(config_store=config_store, client_factory=client_factory) + self._ai_client_factory = ai_client_factory or _default_ai_client_factory + def _fetch_project_configs( self, alias: str, @@ -1436,6 +1458,181 @@ def create_config_row( result["branch_id"] = effective_branch_id return result + # ── config create (one-shot remote create via `config new --push`) ───────── + + def create_config( + self, + alias: str, + component_id: str, + name: str, + description: str = "", + configuration: dict[str, Any] | None = None, + branch_id: int | None = None, + dry_run: bool = False, + validate: bool = True, + ) -> dict[str, Any]: + """Create a new configuration via the Storage API (one-shot remote). + + Backs the ``kbagent config new --push`` lifecycle path. When a body is + passed explicitly (via ``configuration``), the body is validated + against the component's AI Service JSON schema before POSTing (unless + ``validate=False``); on validation failure raises ``ConfigError``. + When no body is passed (default = ``{}``), validation is auto-skipped + because empty shells almost always fail component schemas that require + parameters -- this is FIIA's "empty shell, patch later" pattern. + + Args: + alias: Project alias. + component_id: The component ID. + name: Configuration name (required by Storage API). + description: Optional description. + configuration: Configuration body dict. ``None`` => default empty + shell ``{}`` and auto-skipped validation. + branch_id: If set, create in a specific dev branch. Falls back to + the project's active branch when None. + dry_run: If True, return the planned POST envelope (including + validation result) without calling the Storage API. + validate: If True (default), validate ``configuration`` against + the component schema when a body is explicitly provided. + + Returns: + The created configuration dict from the API (includes the new + ``id``) annotated with ``project_alias``, ``branch_id``, and + ``validation_status``. When ``dry_run`` the dict contains + ``dry_run: True`` plus the planned POST fields and validation + envelope, with no API call. + + Raises: + ConfigError: If the alias is not found, or if validation runs + and fails on a real (non-dry-run) create. + KeboolaApiError: If the Storage API call fails. + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + effective_branch_id = branch_id or project.active_branch_id + + body_was_explicit = configuration is not None + effective_config: dict[str, Any] = configuration if body_was_explicit else {} + + # Validation only runs when the caller passed an explicit body (i.e. + # ``configuration`` is not ``None``). When no body is passed at all, + # the effective body defaults to ``{}`` and validation auto-skips -- + # most component schemas require parameters and would reject ``{}``, + # which would block FIIA's "empty shell, then patch via + # ``config update``" pattern. An *explicit* ``configuration={}`` IS + # validated (and typically fails for the same reason) -- pass + # ``--no-validate`` or omit ``--configuration`` to skip. + if validate and body_was_explicit: + validation_status, validation_errors = self._validate_config_body( + project, component_id, effective_config + ) + else: + validation_status = "skipped" + validation_errors = [] + + if validation_status == "failed" and not dry_run: + joined = "\n - ".join(validation_errors) + raise ConfigError( + f"Configuration body failed schema validation for '{component_id}':\n - {joined}" + ) + + if dry_run: + return { + "dry_run": True, + "project_alias": alias, + "component_id": component_id, + "name": name, + "description": description, + "configuration": effective_config, + "branch_id": effective_branch_id, + "validation_status": validation_status, + "validation_errors": validation_errors, + } + + client = self._client_factory(project.stack_url, project.token) + try: + result = client.create_config( + component_id=component_id, + name=name, + configuration=effective_config, + description=description, + branch_id=effective_branch_id, + ) + finally: + client.close() + + result["project_alias"] = alias + result["branch_id"] = effective_branch_id + result["validation_status"] = validation_status + # Symmetric with the dry-run envelope. On a successful real create + # ``validation_status`` is always "ok" or "skipped" so the list is + # empty -- but we annotate it anyway so JSON consumers can rely on + # the key being present. + result["validation_errors"] = validation_errors + return result + + def _validate_config_body( + self, + project: ProjectConfig, + component_id: str, + body: dict[str, Any], + ) -> tuple[str, list[str]]: + """Validate a configuration body against the component's JSON schema. + + Returns: + ``("ok", [])`` when the body matches the schema. + ``("failed", [errors])`` when validation reports issues. + ``("skipped", [])`` when the AI Service has no schema for this + component or the lookup itself fails (graceful fallback so a + missing schema does not block a create). + """ + # Local import to keep ``jsonschema`` out of the cold-start path of + # commands that never call ``create_config``. + import jsonschema + + ai_client = self._ai_client_factory(project.stack_url, project.token) + try: + try: + raw = ai_client.get_component_detail(component_id) + except KeboolaApiError: + return ("skipped", []) + finally: + ai_client.close() + + try: + detail = ComponentDetail(**raw) + except (TypeError, ValueError): + return ("skipped", []) + + schema = detail.configuration_schema + if not schema: + return ("skipped", []) + + try: + validator = jsonschema.Draft7Validator(schema) + errors: list[str] = [] + for err in validator.iter_errors(body): + path = ".".join(str(p) for p in err.absolute_path) or "" + errors.append(f"{path}: {err.message}") + except jsonschema.SchemaError: + # Component schema itself is malformed -- don't block the create. + return ("skipped", []) + except Exception: + # iter_errors() can also raise late (e.g. ``UnknownType`` for an + # unknown ``type`` keyword) when the schema is invalid in a way + # the constructor accepted. Treat that as "skipped" too -- a + # broken component schema must not block a real create. + logger.warning( + "Schema validation for component %s raised during iter_errors; treating as skipped", + component_id, + exc_info=True, + ) + return ("skipped", []) + + if errors: + return ("failed", errors) + return ("ok", []) + # ── config row-update ────────────────────────────────────────────────────── def update_config_row( diff --git a/src/keboola_agent_cli/services/data_app_service.py b/src/keboola_agent_cli/services/data_app_service.py index 9e4c3372..9b7cef79 100644 --- a/src/keboola_agent_cli/services/data_app_service.py +++ b/src/keboola_agent_cli/services/data_app_service.py @@ -360,7 +360,7 @@ def worker( merged.append( { "project_alias": alias, - "id": str(app.get("id", "")), + "app_id": str(app.get("id", "")), "config_id": config_id, "name": config_names.get(config_id, app.get("name", "")), "type": app.get("type", ""), @@ -393,7 +393,7 @@ def worker( all_apps: list[dict[str, Any]] = [] for _alias, apps, _ok in successes: all_apps.extend(apps) - all_apps.sort(key=lambda a: (a["project_alias"], a.get("id", ""))) + all_apps.sort(key=lambda a: (a["project_alias"], a.get("app_id", ""))) errors.sort(key=lambda e: e.get("project_alias", "")) return {"apps": all_apps, "errors": errors} @@ -441,7 +441,7 @@ def get_data_app( return { "project_alias": alias, - "id": str(app.get("id", "")), + "app_id": str(app.get("id", "")), "config_id": config_id, "config_version_storage": str(storage_config.get("version", "") or ""), "config_version_deployed": str(app.get("configVersion", "") or ""), @@ -642,9 +642,11 @@ def create_data_app( timeout_seconds=timeout_seconds, ) + url_record = deployed_record or shell + state_record = poll_result or deployed_record or shell return { "project_alias": alias, - "id": app_id, + "app_id": app_id, "config_id": config_id, "name": name, "slug": slug, @@ -657,9 +659,9 @@ def create_data_app( "config_version": storage_version, "deployed": bool(deploy), "wait": bool(wait), - "url": (deployed_record or shell).get("url", ""), - "state": (poll_result or deployed_record or shell).get("state", ""), - "desired_state": (poll_result or deployed_record or shell).get("desiredState", ""), + "url": url_record.get("url", ""), + "state": state_record.get("state", ""), + "desired_state": state_record.get("desiredState", ""), "last_start_timestamp": (poll_result or deployed_record or {}).get( "lastStartTimestamp" ), @@ -668,7 +670,7 @@ def create_data_app( auth=auth, deployed=bool(deploy), wait=bool(wait), - state=(poll_result or deployed_record or shell).get("state", ""), + state=state_record.get("state", ""), ), } except Exception: @@ -846,7 +848,7 @@ def delete_data_app(self, alias: str, app_id: str) -> dict[str, Any]: ds_client.close() return { "project_alias": alias, - "id": str(app_id), + "app_id": str(app_id), "deleted": True, "message": ( f"Data app {app_id} deleted from project '{alias}'. " @@ -888,7 +890,7 @@ def get_data_app_password( password = payload.get("password", "") if isinstance(payload, dict) else "" return { "project_alias": alias, - "id": str(app_id), + "app_id": str(app_id), "password": password, "message": ( f"Retrieved simpleAuth password for data app {app_id}. " @@ -1045,7 +1047,7 @@ def set_data_app_secrets( return { "dry_run": True, "project_alias": alias, - "id": str(app_id), + "app_id": str(app_id), "config_id": config_id, "secrets_set": sorted(_derive_runtime_env_var_name(k) for k in validated), "secrets_unchanged": unchanged, @@ -1128,7 +1130,7 @@ def set_data_app_secrets( secrets_set = sorted(_derive_runtime_env_var_name(k) for k in validated) return { "project_alias": alias, - "id": str(app_id), + "app_id": str(app_id), "config_id": config_id, "secrets_set": secrets_set, "secrets_unchanged": unchanged, @@ -1191,7 +1193,7 @@ def list_data_app_secrets( return { "project_alias": alias, - "id": str(app_id), + "app_id": str(app_id), "config_id": config_id, "secrets": entries, "count": len(entries), @@ -1245,7 +1247,7 @@ def get_data_app_secret( env_var = _derive_runtime_env_var_name(key) return { "project_alias": alias, - "id": str(app_id), + "app_id": str(app_id), "config_id": config_id, "key": key, "env_var": env_var, @@ -1305,7 +1307,7 @@ def remove_data_app_secrets( # Idempotent: removing a non-existent key is success. return { "project_alias": alias, - "id": str(app_id), + "app_id": str(app_id), "config_id": config_id, "removed": [], "not_found": not_found, @@ -1324,7 +1326,7 @@ def remove_data_app_secrets( return { "dry_run": True, "project_alias": alias, - "id": str(app_id), + "app_id": str(app_id), "config_id": config_id, "to_remove": removed, "not_found": not_found, @@ -1352,7 +1354,7 @@ def remove_data_app_secrets( new_version = str(put_response.get("version", "") or "") return { "project_alias": alias, - "id": str(app_id), + "app_id": str(app_id), "config_id": config_id, "removed": removed, "not_found": not_found, @@ -1862,7 +1864,7 @@ def _format_lifecycle_result( record = poll_result or deployed return { "project_alias": alias, - "id": str(app_id), + "app_id": str(app_id), "action": action, "state": record.get("state", ""), "desired_state": record.get("desiredState", ""), diff --git a/tests/test_config_create_cli.py b/tests/test_config_create_cli.py new file mode 100644 index 00000000..80ef30f8 --- /dev/null +++ b/tests/test_config_create_cli.py @@ -0,0 +1,695 @@ +"""CLI tests for ``kbagent config new --push`` (one-shot remote create). + +Scope: +- Flag-combination validation (exit 2 on misuse). +- Push-mode happy paths: minimal, --no-files, --output-dir + push, --dry-run. +- Body parsing: --configuration (inline / @file / -), --configuration-file. +- Validation propagation: service-layer ConfigError surfaces as exit 5. + +Regression coverage for scaffold-only mode lives in ``test_component_cli.py`` +(TestConfigNew) -- preserved byte-for-byte. +""" + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +from click.testing import Result +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + +runner = CliRunner() + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _setup_config(config_dir: Path) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="Production", + project_id=1234, + ), + ) + return store + + +def _push_result() -> dict: + """Canonical success envelope from ConfigService.create_config.""" + return { + "id": "12345", + "name": "test-config", + "description": "", + "configuration": {}, + "version": 1, + "created": "2026-05-11T10:00:00+00:00", + "project_alias": "prod", + "branch_id": None, + "validation_status": "skipped", + } + + +def _invoke_push( + args: list[str], + *, + config_service_mock: MagicMock | None = None, + component_service_mock: MagicMock | None = None, + config_dir: Path | None = None, + input_text: str | None = None, +) -> Result: + """Invoke the CLI with both service mocks installed.""" + assert config_dir is not None, "tests must pass a config_dir" + store = _setup_config(config_dir) + + svc_config = config_service_mock or MagicMock() + svc_component = component_service_mock or MagicMock() + if not config_service_mock: + svc_config.create_config.return_value = _push_result() + if not component_service_mock: + # Default scaffold response for paths that also exercise scaffold step. + svc_component.generate_scaffold.return_value = { + "component_id": "keboola.ex-http", + "component_name": "HTTP", + "component_type": "extractor", + "directory": "extractor/keboola.ex-http/test-config", + "files": [ + {"path": "_config.yml", "content": "name: test\n"}, + ], + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ComponentService") as MockCompService, + patch("keboola_agent_cli.cli.ConfigService") as MockConfigService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCompService.return_value = svc_component + MockConfigService.return_value = svc_config + return runner.invoke(app, args, input=input_text) + + +# --------------------------------------------------------------------------- +# Flag-combination validation +# --------------------------------------------------------------------------- + + +class TestConfigNewFlagValidation: + """Push-gated flags must reject misuse with exit 2.""" + + def test_push_requires_project(self, tmp_path: Path) -> None: + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--name", + "T1", + "--push", + ], + config_dir=tmp_path / "config", + ) + assert result.exit_code == 2, result.output + envelope = json.loads(result.output) + assert envelope["status"] == "error" + assert "requires --project" in envelope["error"]["message"] + + def test_push_requires_non_empty_name(self, tmp_path: Path) -> None: + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--push", + ], + config_dir=tmp_path / "config", + ) + assert result.exit_code == 2, result.output + assert "requires a non-empty --name" in json.loads(result.output)["error"]["message"] + + def test_no_files_without_push(self, tmp_path: Path) -> None: + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--no-files", + ], + config_dir=tmp_path / "config", + ) + assert result.exit_code == 2, result.output + assert "--no-files requires --push" in json.loads(result.output)["error"]["message"] + + def test_description_without_push(self, tmp_path: Path) -> None: + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--description", + "desc", + ], + config_dir=tmp_path / "config", + ) + assert result.exit_code == 2, result.output + + def test_branch_without_push(self, tmp_path: Path) -> None: + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--branch", + "42", + ], + config_dir=tmp_path / "config", + ) + assert result.exit_code == 2, result.output + + def test_dry_run_without_push(self, tmp_path: Path) -> None: + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--dry-run", + ], + config_dir=tmp_path / "config", + ) + assert result.exit_code == 2, result.output + + def test_no_validate_without_push(self, tmp_path: Path) -> None: + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--no-validate", + ], + config_dir=tmp_path / "config", + ) + assert result.exit_code == 2, result.output + + def test_configuration_and_configuration_file_mutually_exclusive(self, tmp_path: Path) -> None: + body_file = tmp_path / "body.json" + body_file.write_text("{}") + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "T1", + "--push", + "--configuration", + "{}", + "--configuration-file", + str(body_file), + ], + config_dir=tmp_path / "config", + ) + assert result.exit_code == 2, result.output + assert "mutually exclusive" in json.loads(result.output)["error"]["message"] + + def test_no_files_and_output_dir_mutually_exclusive(self, tmp_path: Path) -> None: + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "T1", + "--push", + "--no-files", + "--output-dir", + str(tmp_path / "scaffold"), + ], + config_dir=tmp_path / "config", + ) + assert result.exit_code == 2, result.output + + +# --------------------------------------------------------------------------- +# Push-mode happy paths +# --------------------------------------------------------------------------- + + +class TestConfigNewPushHappyPath: + def test_push_no_files_json(self, tmp_path: Path) -> None: + svc_config = MagicMock() + svc_config.create_config.return_value = _push_result() + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--no-files", + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + ) + + assert result.exit_code == 0, result.output + envelope = json.loads(result.output) + assert envelope["status"] == "ok" + data = envelope["data"] + assert data["id"] == "12345" + assert data["project_alias"] == "prod" + assert data["validation_status"] == "skipped" + + # Verify the service was called with validate=True and an empty body. + svc_config.create_config.assert_called_once() + call_kwargs = svc_config.create_config.call_args.kwargs + assert call_kwargs["alias"] == "prod" + assert call_kwargs["component_id"] == "keboola.ex-http" + assert call_kwargs["name"] == "test-config" + assert call_kwargs["configuration"] is None + assert call_kwargs["validate"] is True + assert call_kwargs["dry_run"] is False + + def test_push_with_output_dir_scaffolds_and_posts(self, tmp_path: Path) -> None: + """With --push --output-dir: scaffold to disk AND POST.""" + scaffold_dir = tmp_path / "scaffold" + scaffold_dir.mkdir() + svc_config = MagicMock() + svc_config.create_config.return_value = _push_result() + svc_component = MagicMock() + svc_component.generate_scaffold.return_value = { + "component_id": "keboola.ex-http", + "component_name": "HTTP", + "component_type": "extractor", + "directory": "extractor/keboola.ex-http/test-config", + "files": [{"path": "_config.yml", "content": "name: test\n"}], + } + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--output-dir", + str(scaffold_dir), + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + component_service_mock=svc_component, + ) + + assert result.exit_code == 0, result.output + svc_config.create_config.assert_called_once() + # The scaffold file should have been written. + written = scaffold_dir / "extractor/keboola.ex-http/test-config/_config.yml" + assert written.exists(), f"Scaffold file not written at {written}" + # JSON mode must emit a SINGLE valid JSON document on stdout; the + # scaffold "Written ..." dim line must not leak above it (B-2 regression + # fix: _write_scaffold_to_disk now honors formatter.json_mode). + json.loads(result.output) + + def test_push_output_dir_dry_run_does_not_write_files(self, tmp_path: Path) -> None: + """--push --output-dir --dry-run must NOT write scaffold to disk. + + Dry-run is a preview; it must have zero filesystem side effects + regardless of which flags accompany it. Regression guard for NB-1 + from the PR #282 review: the scaffold-write branch at config.py:1338 + previously had no `and not dry_run` clause, so dry-run silently + created files alongside the preview envelope. + """ + scaffold_dir = tmp_path / "scaffold" + scaffold_dir.mkdir() + svc_config = MagicMock() + svc_config.create_config.return_value = { + "dry_run": True, + "would_post": {"name": "test-config", "configuration": {}}, + "validation_status": "skipped", + "validation_errors": [], + "project_alias": "prod", + "branch_id": None, + } + svc_component = MagicMock() + svc_component.generate_scaffold.return_value = { + "component_id": "keboola.ex-http", + "component_name": "HTTP", + "component_type": "extractor", + "directory": "extractor/keboola.ex-http/test-config", + "files": [{"path": "_config.yml", "content": "name: test\n"}], + } + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--output-dir", + str(scaffold_dir), + "--dry-run", + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + component_service_mock=svc_component, + ) + + assert result.exit_code == 0, result.output + # Envelope must reflect dry-run (the formatter wraps the payload under + # ``data``). + envelope = json.loads(result.output) + assert envelope["data"]["dry_run"] is True, envelope + # No scaffold files anywhere under output_dir. + written_files = list(scaffold_dir.rglob("*.yml")) + list(scaffold_dir.rglob("*.json")) + assert written_files == [], f"Dry-run must not write files; found {written_files}" + + def test_push_with_configuration_inline(self, tmp_path: Path) -> None: + svc_config = MagicMock() + svc_config.create_config.return_value = _push_result() + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--no-files", + "--configuration", + '{"parameters":{"url":"https://api.example.com"}}', + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + ) + + assert result.exit_code == 0, result.output + call_kwargs = svc_config.create_config.call_args.kwargs + assert call_kwargs["configuration"] == {"parameters": {"url": "https://api.example.com"}} + + def test_push_with_configuration_at_file(self, tmp_path: Path) -> None: + body_file = tmp_path / "body.json" + body_file.write_text('{"parameters": {"k": 1}}', encoding="utf-8") + svc_config = MagicMock() + svc_config.create_config.return_value = _push_result() + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--no-files", + "--configuration", + f"@{body_file}", + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + ) + + assert result.exit_code == 0, result.output + assert svc_config.create_config.call_args.kwargs["configuration"] == { + "parameters": {"k": 1} + } + + def test_push_with_configuration_stdin(self, tmp_path: Path) -> None: + svc_config = MagicMock() + svc_config.create_config.return_value = _push_result() + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--no-files", + "--configuration", + "-", + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + input_text='{"parameters":{"k":42}}', + ) + + assert result.exit_code == 0, result.output + assert svc_config.create_config.call_args.kwargs["configuration"] == { + "parameters": {"k": 42} + } + + def test_push_with_configuration_file_typed_path(self, tmp_path: Path) -> None: + body_file = tmp_path / "body.json" + body_file.write_text('{"parameters": {"k": "v"}}', encoding="utf-8") + svc_config = MagicMock() + svc_config.create_config.return_value = _push_result() + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--no-files", + "--configuration-file", + str(body_file), + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + ) + + assert result.exit_code == 0, result.output + assert svc_config.create_config.call_args.kwargs["configuration"] == { + "parameters": {"k": "v"} + } + + def test_push_no_validate_passed_through(self, tmp_path: Path) -> None: + svc_config = MagicMock() + svc_config.create_config.return_value = _push_result() + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--no-files", + "--no-validate", + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + ) + + assert result.exit_code == 0, result.output + assert svc_config.create_config.call_args.kwargs["validate"] is False + + def test_push_dry_run_returns_envelope(self, tmp_path: Path) -> None: + svc_config = MagicMock() + svc_config.create_config.return_value = { + "dry_run": True, + "project_alias": "prod", + "component_id": "keboola.ex-http", + "name": "test-config", + "description": "", + "configuration": {}, + "branch_id": None, + "validation_status": "skipped", + "validation_errors": [], + } + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--no-files", + "--dry-run", + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + ) + + assert result.exit_code == 0, result.output + envelope = json.loads(result.output) + assert envelope["data"]["dry_run"] is True + assert envelope["data"]["validation_status"] == "skipped" + assert svc_config.create_config.call_args.kwargs["dry_run"] is True + + +# --------------------------------------------------------------------------- +# Error propagation +# --------------------------------------------------------------------------- + + +class TestConfigNewPushErrors: + def test_invalid_inline_json_exits_2(self, tmp_path: Path) -> None: + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--no-files", + "--configuration", + "not-json", + ], + config_dir=tmp_path / "config", + ) + assert result.exit_code == 2, result.output + + def test_validation_failure_surfaces_as_exit_5(self, tmp_path: Path) -> None: + """ConfigError from the service => exit 5 + CONFIG_ERROR envelope.""" + svc_config = MagicMock() + svc_config.create_config.side_effect = ConfigError( + "Configuration body failed schema validation for 'X':\n - foo: bar" + ) + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--no-files", + "--configuration", + '{"bad":"body"}', + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + ) + + assert result.exit_code == 5, result.output + envelope = json.loads(result.output) + assert envelope["status"] == "error" + assert "schema validation" in envelope["error"]["message"] + + def test_api_error_surfaces_with_mapped_exit_code(self, tmp_path: Path) -> None: + svc_config = MagicMock() + svc_config.create_config.side_effect = KeboolaApiError( + message="500 boom", error_code="STORAGE_ERROR", status_code=500 + ) + + result = _invoke_push( + [ + "--json", + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + "prod", + "--name", + "test-config", + "--push", + "--no-files", + ], + config_dir=tmp_path / "config", + config_service_mock=svc_config, + ) + + # STORAGE_ERROR is not in the explicit "network/auth/timeout" mapping + # in map_error_to_exit_code(), so it falls through to the general + # error bucket (exit 1). Tighten the assertion so any future change + # to the mapping is caught. + assert result.exit_code == 1, result.output + envelope = json.loads(result.output) + assert envelope["status"] == "error" diff --git a/tests/test_config_create_service.py b/tests/test_config_create_service.py new file mode 100644 index 00000000..418ed819 --- /dev/null +++ b/tests/test_config_create_service.py @@ -0,0 +1,370 @@ +"""Tests for ConfigService.create_config (the `config new --push` lifecycle). + +Covers: +- Happy paths: minimal create, explicit body, description forwarding, + branch_id resolution. +- Dry-run: planned envelope without API call. +- Schema validation: ok / failed / skipped (no-schema, AI error, malformed + schema, validate=False). +- Empty-shell short-circuit: validation auto-skips when no body is provided. +- Cleanup: client.close() and ai_client.close() always called. +""" + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from helpers import setup_single_project +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.services.config_service import ConfigService + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +SAMPLE_CREATED = { + "id": "12345", + "name": "My Config", + "description": "", + "configuration": {}, + "version": 1, + "created": "2026-05-11T10:00:00+00:00", +} + +# A trivial JSON schema that requires a top-level "parameters" object with +# a required "table" string field. Used to drive the validation branch. +TABLE_SCHEMA = { + "type": "object", + "properties": { + "parameters": { + "type": "object", + "properties": {"table": {"type": "string"}}, + "required": ["table"], + } + }, + "required": ["parameters"], +} + +VALID_BODY = {"parameters": {"table": "orders"}} +INVALID_BODY = {"parameters": {"limit": 100}} # missing required "table" + + +def _make_service( + tmp_config_dir: Path, + *, + schema: dict | None = None, + ai_raises: Exception | None = None, +) -> tuple[ConfigService, MagicMock, MagicMock]: + """Build a ConfigService wired to a mock Storage client + mock AI client. + + Args: + tmp_config_dir: pytest fixture providing a temp directory for ConfigStore. + schema: when set, the mock AI client returns a component detail with + this ``configurationSchema``. When None, the AI client returns a + detail with no schema (the "no-schema available" branch). + ai_raises: when set, the mock AI client raises this exception on + ``get_component_detail`` to exercise the "AI Service error" branch. + + Returns: + (service, mock_storage_client, mock_ai_client) + """ + store = setup_single_project(tmp_config_dir) + + mock_storage = MagicMock() + mock_storage.create_config.return_value = dict(SAMPLE_CREATED) + + mock_ai = MagicMock() + if ai_raises is not None: + mock_ai.get_component_detail.side_effect = ai_raises + else: + # Mirror the raw AI Service response shape (camelCase). ComponentDetail + # parses this via its field aliases (componentId, componentName, ...). + mock_ai.get_component_detail.return_value = { + "componentId": "keboola.ex-db-snowflake", + "componentName": "Snowflake Extractor", + "componentType": "extractor", + "configurationSchema": schema or {}, + } + + service = ConfigService( + config_store=store, + client_factory=lambda url, token: mock_storage, + ai_client_factory=lambda url, token: mock_ai, + ) + return service, mock_storage, mock_ai + + +# --------------------------------------------------------------------------- +# Core happy-path tests +# --------------------------------------------------------------------------- + + +class TestCreateConfigCore: + def test_minimal_create_empty_shell(self, tmp_config_dir: Path) -> None: + """No body provided => POST {} and skip validation (empty-shell mode).""" + service, storage, ai = _make_service(tmp_config_dir, schema=TABLE_SCHEMA) + + result = service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + ) + + storage.create_config.assert_called_once_with( + component_id="keboola.ex-db-snowflake", + name="My Config", + configuration={}, + description="", + branch_id=None, + ) + # AI Service must NOT be consulted for the empty-shell case -- it + # would always fail and that's not useful for FIIA's pattern. + ai.get_component_detail.assert_not_called() + + assert result["id"] == "12345" + assert result["project_alias"] == "prod" + assert result["branch_id"] is None + assert result["validation_status"] == "skipped" + # validation_errors is always present (symmetric with dry-run envelope). + assert result["validation_errors"] == [] + + def test_create_with_explicit_body_validates_ok(self, tmp_config_dir: Path) -> None: + """Explicit body + valid against schema => validation_status='ok'.""" + service, storage, ai = _make_service(tmp_config_dir, schema=TABLE_SCHEMA) + + result = service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + configuration=VALID_BODY, + ) + + ai.get_component_detail.assert_called_once_with("keboola.ex-db-snowflake") + storage.create_config.assert_called_once() + assert result["validation_status"] == "ok" + + def test_create_with_description(self, tmp_config_dir: Path) -> None: + """Description forwards to the client call.""" + service, storage, _ = _make_service(tmp_config_dir) + + service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + description="A test config", + ) + + call_kwargs = storage.create_config.call_args.kwargs + assert call_kwargs["description"] == "A test config" + + def test_branch_id_override(self, tmp_config_dir: Path) -> None: + """Explicit branch_id beats the project's active_branch_id.""" + service, storage, _ = _make_service(tmp_config_dir) + + result = service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + branch_id=42, + ) + + assert storage.create_config.call_args.kwargs["branch_id"] == 42 + assert result["branch_id"] == 42 + + def test_branch_id_falls_back_to_active(self, tmp_config_dir: Path) -> None: + """When branch_id is None, the project's active_branch_id is used.""" + store = setup_single_project(tmp_config_dir) + # Mutate the registered project to have an active branch. + config = store.load() + project = config.projects["prod"] + project.active_branch_id = 99 + store.save(config) + + mock_storage = MagicMock() + mock_storage.create_config.return_value = dict(SAMPLE_CREATED) + service = ConfigService( + config_store=store, + client_factory=lambda url, token: mock_storage, + ) + + result = service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + ) + + assert mock_storage.create_config.call_args.kwargs["branch_id"] == 99 + assert result["branch_id"] == 99 + + def test_unknown_project_raises_config_error(self, tmp_config_dir: Path) -> None: + """Alias not in the config store => ConfigError, no API call.""" + service, storage, _ = _make_service(tmp_config_dir) + + with pytest.raises(ConfigError, match="not found"): + service.create_config( + alias="unknown", + component_id="keboola.ex-db-snowflake", + name="My Config", + ) + storage.create_config.assert_not_called() + + def test_api_error_propagates_and_closes_client(self, tmp_config_dir: Path) -> None: + """Storage API failure propagates KeboolaApiError; client.close() still called.""" + service, storage, _ = _make_service(tmp_config_dir) + storage.create_config.side_effect = KeboolaApiError( + message="500 boom", + error_code="STORAGE_ERROR", + status_code=500, + ) + + with pytest.raises(KeboolaApiError, match="500 boom"): + service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + ) + storage.close.assert_called_once() + + +# --------------------------------------------------------------------------- +# Dry-run tests +# --------------------------------------------------------------------------- + + +class TestCreateConfigDryRun: + def test_dry_run_returns_planned_envelope_no_api_call(self, tmp_config_dir: Path) -> None: + """Dry-run returns the planned POST body; no API call is made.""" + service, storage, _ = _make_service(tmp_config_dir, schema=TABLE_SCHEMA) + + result = service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + configuration=VALID_BODY, + dry_run=True, + ) + + storage.create_config.assert_not_called() + assert result["dry_run"] is True + assert result["project_alias"] == "prod" + assert result["component_id"] == "keboola.ex-db-snowflake" + assert result["name"] == "My Config" + assert result["configuration"] == VALID_BODY + assert result["validation_status"] == "ok" + + def test_dry_run_with_validation_failure_does_not_raise(self, tmp_config_dir: Path) -> None: + """Dry-run + invalid body: envelope reports failure but doesn't raise.""" + service, storage, _ = _make_service(tmp_config_dir, schema=TABLE_SCHEMA) + + result = service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + configuration=INVALID_BODY, + dry_run=True, + ) + + storage.create_config.assert_not_called() + assert result["dry_run"] is True + assert result["validation_status"] == "failed" + assert result["validation_errors"], "Expected non-empty validation_errors" + + +# --------------------------------------------------------------------------- +# Schema validation tests +# --------------------------------------------------------------------------- + + +class TestCreateConfigSchemaValidation: + def test_validation_failure_aborts_real_create(self, tmp_config_dir: Path) -> None: + """Real (non-dry-run) create with invalid body => ConfigError; no POST.""" + service, storage, _ = _make_service(tmp_config_dir, schema=TABLE_SCHEMA) + + with pytest.raises(ConfigError, match="failed schema validation"): + service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + configuration=INVALID_BODY, + ) + storage.create_config.assert_not_called() + + def test_no_schema_available_skips_validation(self, tmp_config_dir: Path) -> None: + """No configurationSchema on the component => skip, proceed with POST.""" + service, storage, _ = _make_service(tmp_config_dir, schema=None) + + result = service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + configuration=VALID_BODY, + ) + + storage.create_config.assert_called_once() + assert result["validation_status"] == "skipped" + + def test_ai_service_error_skips_validation_silently(self, tmp_config_dir: Path) -> None: + """AI Service raises => skip validation, do NOT block the create.""" + service, storage, ai = _make_service( + tmp_config_dir, + ai_raises=KeboolaApiError( + message="503 unavailable", error_code="UNAVAILABLE", status_code=503 + ), + ) + + result = service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + configuration=VALID_BODY, + ) + + ai.get_component_detail.assert_called_once() + ai.close.assert_called_once() + storage.create_config.assert_called_once() + assert result["validation_status"] == "skipped" + + def test_validate_false_skips_ai_client_call_entirely(self, tmp_config_dir: Path) -> None: + """validate=False => AI Service is never consulted.""" + service, storage, ai = _make_service(tmp_config_dir, schema=TABLE_SCHEMA) + + result = service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + configuration=INVALID_BODY, # would fail validation, but skipped + validate=False, + ) + + ai.get_component_detail.assert_not_called() + storage.create_config.assert_called_once() + assert result["validation_status"] == "skipped" + + def test_malformed_schema_skips_validation(self, tmp_config_dir: Path) -> None: + """A broken JSON schema => skip validation rather than block the create.""" + broken_schema = {"type": "not-a-real-type"} # invalid Draft7 schema + service, storage, _ = _make_service(tmp_config_dir, schema=broken_schema) + + result = service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + configuration=VALID_BODY, + ) + + storage.create_config.assert_called_once() + assert result["validation_status"] == "skipped" + + def test_ai_client_closed_even_when_detail_call_succeeds(self, tmp_config_dir: Path) -> None: + """AI client.close() is called after a successful detail fetch.""" + service, _, ai = _make_service(tmp_config_dir, schema=TABLE_SCHEMA) + + service.create_config( + alias="prod", + component_id="keboola.ex-db-snowflake", + name="My Config", + configuration=VALID_BODY, + ) + + ai.close.assert_called_once() diff --git a/tests/test_data_app_cli.py b/tests/test_data_app_cli.py index 6aabd279..7c94fc20 100644 --- a/tests/test_data_app_cli.py +++ b/tests/test_data_app_cli.py @@ -80,7 +80,7 @@ def test_json_success(self, tmp_path: Path) -> None: "apps": [ { "project_alias": "prod", - "id": "42", + "app_id": "42", "config_id": "ulid", "name": "App", "type": "python-js", @@ -100,7 +100,8 @@ def test_json_success(self, tmp_path: Path) -> None: assert result.exit_code == 0, result.output body = json.loads(result.output) assert body["status"] == "ok" - assert body["data"]["apps"][0]["id"] == "42" + assert body["data"]["apps"][0]["app_id"] == "42" + assert body["data"]["apps"][0]["config_id"] == "ulid" # --------------------------------------------------------------------------- @@ -225,7 +226,7 @@ def test_deploy_success(self, tmp_path: Path) -> None: mock = MagicMock() mock.deploy_data_app.return_value = { "project_alias": "prod", - "id": "42", + "app_id": "42", "action": "deploy", "state": "starting", "desired_state": "running", @@ -293,7 +294,7 @@ def test_delete_with_yes(self, tmp_path: Path) -> None: mock = MagicMock() mock.delete_data_app.return_value = { "project_alias": "prod", - "id": "42", + "app_id": "42", "deleted": True, "message": "Data app 42 deleted.", } @@ -328,7 +329,7 @@ def test_password_success(self, tmp_path: Path, monkeypatch) -> None: mock = MagicMock() mock.get_data_app_password.return_value = { "project_alias": "prod", - "id": "42", + "app_id": "42", "password": "deadbeefcafe", "message": "Retrieved.", } diff --git a/tests/test_data_app_secrets_cli.py b/tests/test_data_app_secrets_cli.py index 104c6c57..aca7c4d3 100644 --- a/tests/test_data_app_secrets_cli.py +++ b/tests/test_data_app_secrets_cli.py @@ -135,7 +135,7 @@ def test_happy_path_json_envelope(self, tmp_path: Path) -> None: mock = MagicMock() mock.set_data_app_secrets.return_value = { "project_alias": "prod", - "id": "12345", + "app_id": "12345", "config_id": "01ABC", "secrets_set": ["API_KEY"], "secrets_unchanged": [], @@ -208,7 +208,7 @@ def test_empty_list_json(self, tmp_path: Path) -> None: mock = MagicMock() mock.list_data_app_secrets.return_value = { "project_alias": "prod", - "id": "12345", + "app_id": "12345", "config_id": "01ABC", "secrets": [], "count": 0, @@ -246,7 +246,7 @@ def test_decrypted_plaintext_never_in_output(self, tmp_path: Path) -> None: # Service returns metadata only -- the public contract. mock.get_data_app_secret.return_value = { "project_alias": "prod", - "id": "12345", + "app_id": "12345", "config_id": "01ABC", "key": "#API_KEY", "env_var": "API_KEY", @@ -298,7 +298,7 @@ def test_service_leak_attempt_is_filtered(self, tmp_path: Path) -> None: # appear in result.output -- the assertion is a regression guard. mock.get_data_app_secret.return_value = { "project_alias": "prod", - "id": "12345", + "app_id": "12345", "config_id": "01ABC", "key": "#API_KEY", "env_var": "API_KEY", @@ -376,7 +376,7 @@ def test_idempotent_on_missing_key_yes(self, tmp_path: Path) -> None: mock = MagicMock() mock.remove_data_app_secrets.return_value = { "project_alias": "prod", - "id": "12345", + "app_id": "12345", "config_id": "01ABC", "removed": [], "not_found": ["MISSING"], diff --git a/tests/test_data_app_service.py b/tests/test_data_app_service.py index 6548b383..154dd5ff 100644 --- a/tests/test_data_app_service.py +++ b/tests/test_data_app_service.py @@ -468,7 +468,7 @@ def test_happy_path_private_repo(self, tmp_path: Path) -> None: restart_if_running=True, ) - assert result["id"] == "43661269" + assert result["app_id"] == "43661269" assert result["config_id"] == "01kqj88t0vktxe0vfhk6ps5kzs" assert result["url"].endswith("hub.us-east4.gcp.keboola.com") # Encrypted PAT is redacted in the returned dict for human display. @@ -804,7 +804,7 @@ def test_delete_calls_data_science_delete(self, tmp_path: Path) -> None: result = service.delete_data_app(alias="prod", app_id="42") ds_mock.delete_app.assert_called_once_with("42") assert result["deleted"] is True - assert result["id"] == "42" + assert result["app_id"] == "42" # --------------------------------------------------------------------------- @@ -851,7 +851,7 @@ def test_detail_merges_data_science_and_storage(self, tmp_path: Path) -> None: } result = service.get_data_app(alias="prod", app_id="42") - assert result["id"] == "42" + assert result["app_id"] == "42" assert result["state"] == "running" assert result["config_version_storage"] == "5" assert result["config_version_deployed"] == "3" @@ -914,3 +914,211 @@ def test_no_git_block_is_noop(self) -> None: def test_empty_dict_passes_through(self) -> None: assert _redact_storage_config({}) == {} + + +# --------------------------------------------------------------------------- +# List service — output key rename to `app_id` (v0.33.0) +# --------------------------------------------------------------------------- + + +class TestDataAppListOutputKeys: + """Lock the v0.33.0 JSON output rename ``id`` -> ``app_id``. + + Prior to v0.33.0 ``list_data_apps`` emitted the data-app's own + identifier as bare ``id``. The renamed key matches the ``--app-id`` + input flag and the rest of kbagent's CLI convention (e.g. + ``config_id``, ``bucket_id``, ``table_id``). + """ + + def test_list_emits_app_id_key(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.list_apps.return_value = [ + { + "id": "43661269", + "configId": "01kqj88t0vktxe0vfhk6ps5kzs", + "type": "python-js", + "state": "running", + "desiredState": "running", + "configVersion": "3", + "url": "https://x.hub.example.com", + "size": "tiny", + "autoSuspendAfterSeconds": 900, + "lastStartTimestamp": "2026-05-01T00:00:00Z", + } + ] + storage_mock.list_component_configs.return_value = [ + {"id": "01kqj88t0vktxe0vfhk6ps5kzs", "name": "App"} + ] + + result = service.list_data_apps(aliases=["prod"]) + + assert result["errors"] == [] + assert len(result["apps"]) == 1 + app = result["apps"][0] + assert app["app_id"] == "43661269" + # Regression guard: pre-0.33.0 callers would have read ``app["id"]``. + assert "id" not in app + assert app["config_id"] == "01kqj88t0vktxe0vfhk6ps5kzs" + assert app["name"] == "App" + + +class TestDataAppEnvelopesNoBareIdKey: + """Regression guard: NO ``DataAppService`` envelope emits the legacy bare + ``id`` key. The id key was renamed to ``app_id`` in v0.33.0; future + edits that accidentally re-add ``"id":`` to any envelope must fail here. + + Covers every method whose return dict carries the data-app identifier: + detail, create, deploy, start, stop, delete, password, secrets-* (set, + list, get, remove). ``list_data_apps`` is covered by + ``TestDataAppListOutputKeys``. + """ + + def test_detail_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.get_app.return_value = {"id": "42", "configId": "ulid", "state": "running"} + storage_mock.get_config_detail.return_value = {"version": "3", "name": "App"} + result = service.get_data_app(alias="prod", app_id="42") + assert result["app_id"] == "42" + assert "id" not in result + + def test_create_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.create_app.return_value = {"id": "42", "configId": "ulid"} + storage_mock.update_config.return_value = {"version": "2"} + result = service.create_data_app( + alias="prod", + name="App", + description="", + slug="my-app", + git_repo="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/o/r", + git_public=True, + auth="password", + size="tiny", + auto_suspend_after_seconds=900, + type_="python-js", + deploy=False, + wait=False, + dry_run=False, + ) + assert result["app_id"] == "42" + assert "id" not in result + + def test_deploy_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.get_app.return_value = {"configId": "ulid"} + storage_mock.get_config_detail.return_value = {"version": "3"} + ds_mock.patch_app.return_value = {"state": "starting", "desiredState": "running"} + result = service.deploy_data_app(alias="prod", app_id="42") + assert result["app_id"] == "42" + assert "id" not in result + + def test_start_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, _storage, _enc = _make_service(store) + ds_mock.patch_app.return_value = {"state": "starting", "desiredState": "running"} + result = service.start_data_app(alias="prod", app_id="42") + assert result["app_id"] == "42" + assert "id" not in result + + def test_stop_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, _storage, _enc = _make_service(store) + ds_mock.patch_app.return_value = {"state": "stopping", "desiredState": "stopped"} + result = service.stop_data_app(alias="prod", app_id="42") + assert result["app_id"] == "42" + assert "id" not in result + + def test_delete_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ds, _storage, _enc = _make_service(store) + result = service.delete_data_app(alias="prod", app_id="42") + assert result["app_id"] == "42" + assert "id" not in result + + def test_password_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, _storage, _enc = _make_service(store) + ds_mock.get_app_password.return_value = {"password": "deadbeefcafe"} + result = service.get_data_app_password( + alias="prod", app_id="42", manage_token=TEST_MANAGE_TOKEN + ) + assert result["app_id"] == "42" + assert "id" not in result + + def test_secrets_set_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, encrypt_mock = _make_service(store) + encrypt_mock.encrypt.return_value = {"#API_KEY": "KBC::ProjectSecureGKMS::xyz"} + ds_mock.get_app.return_value = {"configId": "ulid"} + storage_mock.get_config_detail.return_value = { + "version": "5", + "configuration": {"parameters": {"dataApp": {"slug": "x"}}}, + } + storage_mock.update_config.return_value = {"version": "6"} + result = service.set_data_app_secrets( + alias="prod", app_id="42", secrets={"#API_KEY": "plaintext"} + ) + assert result["app_id"] == "42" + assert "id" not in result + + def test_secrets_list_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.get_app.return_value = {"configId": "ulid"} + storage_mock.get_config_detail.return_value = { + "version": "5", + "configuration": { + "parameters": { + "dataApp": { + "slug": "x", + "secrets": {"#FOO": "KBC::ProjectSecureGKMS::abc"}, + } + } + }, + } + result = service.list_data_app_secrets(alias="prod", app_id="42") + assert result["app_id"] == "42" + assert "id" not in result + + def test_secrets_get_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.get_app.return_value = {"configId": "ulid"} + storage_mock.get_config_detail.return_value = { + "version": "5", + "configuration": { + "parameters": { + "dataApp": { + "slug": "x", + "secrets": {"#FOO": "KBC::ProjectSecureGKMS::abc"}, + } + } + }, + } + result = service.get_data_app_secret(alias="prod", app_id="42", key="#FOO") + assert result["app_id"] == "42" + assert "id" not in result + + def test_secrets_remove_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.get_app.return_value = {"configId": "ulid"} + storage_mock.get_config_detail.return_value = { + "version": "5", + "configuration": { + "parameters": { + "dataApp": { + "slug": "x", + "secrets": {"#FOO": "KBC::ProjectSecureGKMS::abc"}, + } + } + }, + } + storage_mock.update_config.return_value = {"version": "6"} + result = service.remove_data_app_secrets(alias="prod", app_id="42", keys=["#FOO"]) + assert result["app_id"] == "42" + assert "id" not in result diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 48e43715..b6e5a4a7 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -437,6 +437,9 @@ def test_full_cli_e2e(self) -> None: _step(19, "config new scaffold", "generate boilerplate for component") self._test_config_new_scaffold() + _step("19b", "config new --push", "one-shot remote create (0.33.0+)") + self._test_config_new_push() + # ============================================================== # PHASE 5: Component commands # ============================================================== @@ -1652,6 +1655,110 @@ def _test_config_new_scaffold(self) -> None: result = data["data"] assert "files_written" in result or "directory" in result + def _test_config_new_push(self) -> None: + """Test ``config new --push`` -- one-shot remote create (0.33.0+). + + Exercises the full lifecycle introduced in v0.33.0: + 1. ``--push --no-files --dry-run`` returns the planned POST envelope + with ``validation_status`` and no real API call. + 2. ``--push --no-files`` creates an empty-shell config and returns + ``project_alias`` / ``branch_id`` / ``validation_status="skipped"``. + 3. ``config detail`` finds the newly created config. + 4. ``config update --set`` patches it (proves create + update interop). + 5. ``config delete`` cleans up (also tracked in ``_created_config_ids`` + for the safety-net teardown). + """ + push_name = f"{RUN_ID} push-created" + + # 1) Dry-run -- envelope only, no POST. + dry = self._run_ok( + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + self.alias, + "--name", + f"{push_name} (dry)", + "--push", + "--no-files", + "--dry-run", + )["data"] + assert dry["dry_run"] is True, dry + assert dry["project_alias"] == self.alias + assert dry["component_id"] == "keboola.ex-http" + assert dry["configuration"] == {} # default empty shell + assert dry["validation_status"] in ("ok", "skipped", "failed") + + # 2) Real create. + created = self._run_ok( + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + self.alias, + "--name", + push_name, + "--push", + "--no-files", + )["data"] + new_config_id = str(created["id"]) + # Register for the safety-net cleanup loop in teardown BEFORE any + # downstream assertion can raise -- guarantees the config is reaped + # even if a verification step below fails. + self._created_config_ids.append(("keboola.ex-http", new_config_id)) + + try: + assert created["project_alias"] == self.alias + # Empty-shell creation auto-skips validation (FIIA pattern). + assert created["validation_status"] == "skipped", created + # validation_errors is always annotated (symmetric with dry-run). + assert created["validation_errors"] == [], created + + # 3) Verify via config detail. + detail = self._run_ok( + "config", + "detail", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + new_config_id, + )["data"] + assert str(detail["id"]) == new_config_id + assert detail["name"] == push_name + + # 4) Patch the freshly-pushed config. + self._run_ok( + "config", + "update", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + new_config_id, + "--set", + "parameters.smoke_test=true", + ) + finally: + # 5) Inline cleanup so failures mid-flow still tear down the + # remote config promptly; the global teardown loop is a safety + # net for the case where this delete itself fails. + # ``config delete`` has no ``--yes`` flag (CLAUDE.md inventory). + self._run_ok( + "config", + "delete", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + new_config_id, + ) + def _test_component_commands(self) -> None: """List components and get detail for one. @@ -6415,11 +6522,33 @@ def test_data_app_lifecycle_public(self) -> None: ) assert result.exit_code == 0, result.output body = _json_ok(result) - app_id = body["data"]["id"] + # v0.33.0 rename: envelope key is ``app_id`` (was bare ``id``). + app_id = body["data"]["app_id"] assert app_id, "expected a numeric app id from POST /apps" + assert body["data"]["config_id"], "expected a config_id from POST /apps" self._created_app_ids.append(app_id) - _step(2, "Detail merges Data Science + Storage") + _step(2, "List shows the created app with populated app_id + config_id") + list_result = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "list", + "--project", + self.alias, + ], + ) + )["data"] + listed = next( + (a for a in list_result["apps"] if a["app_id"] == app_id), + None, + ) + assert listed is not None, f"newly-created app {app_id} not found in data-app list output" + assert listed["config_id"], "data-app list must emit a populated config_id" + + _step(3, "Detail merges Data Science + Storage") detail = _json_ok( _invoke( self.config_dir, @@ -6434,7 +6563,7 @@ def test_data_app_lifecycle_public(self) -> None: ], ) )["data"] - assert detail["id"] == app_id + assert detail["app_id"] == app_id assert detail["slug"] == slug assert detail["config_version_storage"], ( "Storage config version should be populated after PUT" @@ -6473,7 +6602,7 @@ def test_data_app_lifecycle_private_and_redeploy(self) -> None: ) assert result.exit_code == 0, result.output body = _json_ok(result) - app_id = body["data"]["id"] + app_id = body["data"]["app_id"] self._created_app_ids.append(app_id) # The encrypted PAT must NEVER appear in the JSON output. plaintext_pat = os.environ[ENV_DATA_APP_GIT_PAT] @@ -6554,7 +6683,7 @@ def test_data_app_secrets_round_trip(self) -> None: ], ) )["data"] - app_id = create["id"] + app_id = create["app_id"] self._created_app_ids.append(app_id) _step(2, "secrets-set: encrypt and write") diff --git a/uv.lock b/uv.lock index d0fd8aa0..eaf81b19 100644 --- a/uv.lock +++ b/uv.lock @@ -439,7 +439,7 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.32.0" +version = "0.33.0" source = { editable = "." } dependencies = [ { name = "httpx" },