From 54f8d2c2bc772dd84bfc6aacdd073d51b571ec92 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:40:28 +0000 Subject: [PATCH 01/16] feat(fp): route new Cloud families in key mode and degrade older APIs --- fp-cloud-cli/fp_cli/client.py | 29 ++++++++++++ fp-cloud-cli/fp_cli/commands/_cloud.py | 55 ++++++++++++++++++++++ fp-cloud-cli/tests/test_cloud_transport.py | 33 +++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 fp-cloud-cli/fp_cli/commands/_cloud.py create mode 100644 fp-cloud-cli/tests/test_cloud_transport.py diff --git a/fp-cloud-cli/fp_cli/client.py b/fp-cloud-cli/fp_cli/client.py index 7368e2f9d..5bed12088 100644 --- a/fp-cloud-cli/fp_cli/client.py +++ b/fp-cloud-cli/fp_cli/client.py @@ -134,6 +134,14 @@ class ClientContext: "alerts", "audits", "evaluations", + "evaluator-definitions", + "evaluator-instances", + "evaluator-schema-profile", + "evaluation-jobs", + "dashboards", + "otel", + "offers", + "team-trial", "events", "issues", "keys", @@ -576,6 +584,27 @@ def _delete(ctx: ClientContext, path: str) -> Any: return _request_json(ctx, "DELETE", path) +def cloud_request( + ctx: ClientContext, method: str, path: str, *, + body: Any = None, params: Optional[Dict[str, Any]] = None, +) -> Any: + """New Cloud capabilities, with a predictable older-deployment failure. + + Existing callers retain their response and error contracts. Paths still pass + through the credential and identifier guards; this is not a second transport. + """ + try: + if method == "GET": + return _get_json(ctx, path, params) + return _request_json(ctx, method, path, json_body=body, params=params) + except NotFoundError as exc: + raise NotFoundError( + "not available on this FailproofAI Cloud yet", + hint="check the resource id or upgrade your Cloud deployment", + request_id=exc.request_id, + ) from exc + + # --- Auth / identity -------------------------------------------------------- diff --git a/fp-cloud-cli/fp_cli/commands/_cloud.py b/fp-cloud-cli/fp_cli/commands/_cloud.py new file mode 100644 index 000000000..6d72afbd6 --- /dev/null +++ b/fp-cloud-cli/fp_cli/commands/_cloud.py @@ -0,0 +1,55 @@ +"""Shared output and auth for the additive Cloud command surface.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from .. import _click_compat as click +from .. import output +from .._context import deny_in_key_mode +from ..errors import KeyModeUnsupportedError + + +def session_only(state, command: str) -> None: + try: + deny_in_key_mode(state, command, "there is no /v1 route") + except KeyModeUnsupportedError: + raise KeyModeUnsupportedError( + f"`fp {command}` needs a signed-in session: run `fp login`" + ) from None + + +def rows(data: Any, key: str) -> list: + if isinstance(data, list): + return data + if isinstance(data, dict) and isinstance(data.get(key), list): + return data[key] + raise click.UsageError(f"Cloud returned an invalid {key} collection") + + +def emit(state, data: Any, title: str) -> None: + if state.json: + output.emit_json(data) + return + if isinstance(data, list) and all(isinstance(r, dict) for r in data): + columns = list(dict.fromkeys(k for row in data for k in row)) + output.print_table(columns, [[cell(r.get(k)) for k in columns] for r in data], title=title) + elif isinstance(data, dict): + output.print_table(["Field", "Value"], [[k, cell(v)] for k, v in data.items()], title=title) + else: + output.print_table(["Value"], [[cell(data)]], title=title) + + +def cell(value: Any) -> str: + return json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) + + +def write_file(path: str, text: str) -> None: + try: + # Exclusive creation: never silently replace a user's source or export. + with Path(path).open("x", encoding="utf-8") as dest: + dest.write(text) + except OSError as exc: + raise click.BadParameter(f"cannot write '{path}': {exc}", param_hint="--out") from exc diff --git a/fp-cloud-cli/tests/test_cloud_transport.py b/fp-cloud-cli/tests/test_cloud_transport.py new file mode 100644 index 000000000..435229e97 --- /dev/null +++ b/fp-cloud-cli/tests/test_cloud_transport.py @@ -0,0 +1,33 @@ +import httpx +import pytest +import respx + +from fp_cli import client +from fp_cli.errors import NotFoundError + + +@pytest.mark.parametrize("family", [ + "evaluator-definitions", "evaluator-instances", "evaluator-schema-profile", + "evaluation-jobs", "dashboards", "otel", "offers", "team-trial", +]) +def test_added_v1_families(family): + assert client._v1_path(f"/api/{family}") == f"/v1/{family}" + + +@pytest.mark.parametrize("method", ["GET", "POST", "PATCH", "DELETE"]) +@pytest.mark.parametrize("key_mode", [False, True]) +def test_cloud_request_degrades(method, key_mode): + ctx = client.ClientContext( + "http://dash.test", token=None if key_mode else "tok", + api_key="key" if key_mode else None, + auth_mode=client.AuthMode.API_KEY if key_mode else client.AuthMode.SESSION, + ) + prefix = "v1" if key_mode else "api" + with respx.mock: + route = respx.request(method, f"http://dash.test/{prefix}/dashboards").respond(404, json={}) + with pytest.raises(NotFoundError, match="not available on this FailproofAI Cloud yet") as exc: + client.cloud_request(ctx, method, "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/api/dashboards") + assert exc.value.exit_code == 6 + headers = route.calls[0].request.headers + assert ("authorization" in headers) is key_mode + assert ("cookie" in headers) is not key_mode From 90b6b8c65f19da7ddc897d099407f104597de987 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:45:08 +0000 Subject: [PATCH 02/16] feat(fp): author and deploy code, judge and Jev evaluators --- fp-cloud-cli/fp_cli/app.py | 2 + .../fp_cli/commands/evaluators_cmds.py | 317 ++++++++++++++++++ fp-cloud-cli/fp_cli/commands/sessions_cmds.py | 15 +- fp-cloud-cli/fp_cli/evaluator_spec.py | 220 ++++++++++++ fp-cloud-cli/fp_cli/output.py | 1 + .../tests/test_evaluator_authoring.py | 217 ++++++++++++ fp-cloud-cli/tests/test_session_reevaluate.py | 17 + 7 files changed, 788 insertions(+), 1 deletion(-) create mode 100644 fp-cloud-cli/fp_cli/commands/evaluators_cmds.py create mode 100644 fp-cloud-cli/fp_cli/evaluator_spec.py create mode 100644 fp-cloud-cli/tests/test_evaluator_authoring.py create mode 100644 fp-cloud-cli/tests/test_session_reevaluate.py diff --git a/fp-cloud-cli/fp_cli/app.py b/fp-cloud-cli/fp_cli/app.py index 96ad7b745..c11d3c3ea 100644 --- a/fp-cloud-cli/fp_cli/app.py +++ b/fp-cloud-cli/fp_cli/app.py @@ -23,6 +23,7 @@ auth_cmds, errors_cmds, evals_cmds, + evaluators_cmds, events_cmds, fleet_cmds, guardrails_cmds, @@ -452,6 +453,7 @@ def help_cmd(ctx: typer.Context) -> None: events_cmds.register(app) sessions_cmds.register(app) evals_cmds.register(app) +evaluators_cmds.register(app) errors_cmds.register(app) usage_cmds.register(app) list_cmds.register(app) diff --git a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py new file mode 100644 index 000000000..b29ea40a9 --- /dev/null +++ b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py @@ -0,0 +1,317 @@ +"""Author, test and deploy managed evaluators, independently of result readers.""" + +from __future__ import annotations + +import json +import re +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +import typer + +from .. import client as api, dates, evaluator_spec as spec, output +from .._context import GLOBALS_EPILOG, require_auth +from ..errors import ApiError, NotFoundError +from . import _cloud, _write + +PATH = "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/api/evaluator-definitions" + + +def definitions(cctx) -> list: + return _cloud.rows(api.cloud_request(cctx, "GET", PATH), "definitions") + + +def select(items: list, handle: str) -> dict: + key, _, version = handle.partition("@") + matches = [r for r in items if r.get("eval_key") == key and (not version or str(r.get("eval_version")) == version)] + if not matches: + raise NotFoundError(f"no evaluator {handle}; run `fp evaluators list --all`") + if not version: + enabled = [r for r in matches if r.get("enabled", True)] + matches = enabled or matches + if len(matches) != 1: + spec.fail(f"{handle} matches multiple versions; use KEY@VERSION") + return matches[0] + + +def list_defs(ctx: typer.Context, key: Optional[str] = typer.Option(None, "--key"), + all_versions: bool = typer.Option(False, "--all")) -> None: + """List definitions (enabled only unless --all). Needs evaluations:read.""" + items = definitions(require_auth(ctx.obj)) + items = [r for r in items if (not key or r.get("eval_key") == key) and (all_versions or r.get("enabled", True))] + _cloud.emit(ctx.obj, items, "Evaluators") + + +def show(ctx: typer.Context, handle: str = typer.Argument(..., metavar="KEY[@VERSION]")) -> None: + """Show one definition, including its full source.""" + _cloud.emit(ctx.obj, select(definitions(require_auth(ctx.obj)), handle), "Evaluator") + + +def _create(state, cctx, body, sessions, disabled=False) -> dict: + if sessions: + api.cloud_request(cctx, "POST", f"{PATH}/dry-run", body=spec.dry_run_body(body, sessions)) + _write.confirm(state, f"Create evaluator {body['eval_key']}@{body['eval_version']}", assume_yes=False) + row = api.cloud_request(cctx, "POST", PATH, body=body) + if not isinstance(row, dict) or not row.get("id"): + raise ApiError("Cloud created no identifiable evaluator; check evaluators list --all before retrying") + if disabled: + api.cloud_request(cctx, "PATCH", f"{PATH}/{row['id']}", body={"enabled": False}) + row = {**row, "enabled": False} + _write.record_action("evaluator_created", resource="evaluator", mode=spec.source_kind(body), success=True) + return row + + +def create( + ctx: typer.Context, + key: Optional[str] = typer.Option(None, "--key"), + name: Optional[str] = typer.Option(None, "--name"), + version: Optional[str] = typer.Option(None, "--version"), + label: Optional[List[str]] = typer.Option(None, "--label"), + condition: Optional[str] = typer.Option(None, "--condition", help="Python expression or @file."), + code: Optional[str] = typer.Option(None, "--code", help="Restricted Python expression file, or - for stdin."), + result_kind: Optional[str] = typer.Option(None, "--result-kind"), + timeout: Optional[float] = typer.Option(None, "--timeout"), + judge: bool = typer.Option(False, "--judge"), + criteria: Optional[str] = typer.Option(None, "--criteria"), + step: Optional[List[str]] = typer.Option(None, "--step"), + threshold: Optional[float] = typer.Option(None, "--threshold"), + jev: Optional[str] = typer.Option(None, "--jev", help="noul or score."), + instructions: Optional[str] = typer.Option(None, "--instructions"), + true: Optional[str] = typer.Option(None, "--true"), + false: Optional[str] = typer.Option(None, "--false"), + level: Optional[List[str]] = typer.Option(None, "--level", help="3–5 distinct levels, worst to best."), + file: Optional[str] = typer.Option(None, "--file"), + dry_run_on: Optional[List[str]] = typer.Option(None, "--dry-run-on"), + disabled: bool = typer.Option(False, "--disabled"), +) -> None: + """Create a code, LLM judge or Jev evaluator (enabled by default). + + Needs evaluations:trigger; session dry-run also needs events:read. + + Code: `fp --json evaluators create --key concise --name Concise --code concise.py --result-kind score --dry-run-on run-001` + + Judge: `fp --json evaluators create --key relevance --name Relevance --judge --criteria "Grade relevance to the question" --step "Assess answer specificity" --threshold 0.7` + + Jev: `fp --json evaluators create --key resolved --name Resolved --jev noul --instructions "Assess whether the request was resolved" --true "Fully resolved" --false "Not resolved" --threshold 0.8` + + Spec: `fp --json evaluators create --file evaluator.json` + """ + body = spec.build( + key=key, name=name, version=version, label=label, condition=condition, + code=code, result_kind=result_kind, timeout=timeout, judge=judge, criteria=criteria, + step=step, threshold_value=threshold, jev=jev, instructions=instructions, + true=true, false=false, level=level, file=file, + ) + # Validate before any request, including the judge refusal / session bound. + if dry_run_on: + spec.dry_run_body(body, dry_run_on) + row = _create(ctx.obj, require_auth(ctx.obj), body, dry_run_on, disabled) + _cloud.emit(ctx.obj, row, "Evaluator created") + + +def deploy( + ctx: typer.Context, + key: str = typer.Argument(..., metavar="EVAL_KEY"), + name: Optional[str] = typer.Option(None, "--name"), + version: Optional[str] = typer.Option(None, "--version"), + label: Optional[List[str]] = typer.Option(None, "--label"), + condition: Optional[str] = typer.Option(None, "--condition"), + code: Optional[str] = typer.Option(None, "--code"), + result_kind: Optional[str] = typer.Option(None, "--result-kind"), + timeout: Optional[float] = typer.Option(None, "--timeout"), + judge: bool = typer.Option(False, "--judge"), + criteria: Optional[str] = typer.Option(None, "--criteria"), + step: Optional[List[str]] = typer.Option(None, "--step"), + threshold: Optional[float] = typer.Option(None, "--threshold"), + jev: Optional[str] = typer.Option(None, "--jev"), + instructions: Optional[str] = typer.Option(None, "--instructions"), + true: Optional[str] = typer.Option(None, "--true"), + false: Optional[str] = typer.Option(None, "--false"), + level: Optional[List[str]] = typer.Option(None, "--level"), + file: Optional[str] = typer.Option(None, "--file"), + dry_run_on: Optional[List[str]] = typer.Option(None, "--dry-run-on"), + backfill: Optional[str] = typer.Option(None, "--backfill", help="Code only; e.g. 7d (up to 90d)."), +) -> None: + """Publish a NEW immutable version, then disable enabled predecessors. + + Numeric versions auto-increment; for other version names pass --version. + A failed predecessor PATCH leaves the new version enabled and reports its id. + """ + body = spec.build( + key=None if file else key, name=name, version=None if file else version, label=label, + condition=condition, code=code, result_kind=result_kind, timeout=timeout, + judge=judge, criteria=criteria, step=step, threshold_value=threshold, jev=jev, + instructions=instructions, true=true, false=false, level=level, file=file, complete=False, + ) + if file and body.get("eval_key") != key: + spec.fail("The spec eval_key must match the deploy argument") + body["eval_key"] = key + if version is not None: + body["eval_version"] = version + if dry_run_on: + spec.dry_run_body(body, dry_run_on) + backfill_body = None + if backfill: + if spec.source_kind(body) != "code": + spec.fail("--backfill excludes judge/Jev evaluators; use code only") + if not re.fullmatch(r"[1-9][0-9]*d", backfill) or int(backfill[:-1]) > 90: + spec.fail("--backfill must be a duration in days from 1d to 90d") + now = datetime.now(timezone.utc) + backfill_body = {"from": (now - timedelta(days=int(backfill[:-1]))).isoformat(), "to": now.isoformat(), "eval_key": key, "dry_run": False} + cctx = require_auth(ctx.obj) + existing = [r for r in definitions(cctx) if r.get("eval_key") == key] + if "eval_version" not in body: + versions = [str(r.get("eval_version", "")) for r in existing] + if any(not v.isdigit() for v in versions): + spec.fail("Non-numeric versions exist; specify a new --version") + body["eval_version"] = str(max([int(v) for v in versions] or [0]) + 1) + if any(str(r.get("eval_version")) == body["eval_version"] for r in existing): + spec.fail(f"Version {key}@{body['eval_version']} already exists; deploy needs a NEW version") + body.setdefault("display_name", existing[-1].get("display_name", key) if existing else key) + spec.validate(body) + created = _create(ctx.obj, cctx, body, dry_run_on) + disabled_ids = [] + try: + for old in existing: + if old.get("enabled", True): + api.cloud_request(cctx, "PATCH", f"{PATH}/{old['id']}", body={"enabled": False}) + disabled_ids.append(old["id"]) + if backfill_body: + api.cloud_request(cctx, "POST", "/api/evaluations/backfill", body=backfill_body) + except Exception as exc: + output.warn(f"Published {key}@{body['eval_version']} (id {created['id']}); disabled {len(disabled_ids)} predecessors. Follow-up failed; inspect evaluators list --all before retrying.") + raise exc + _write.record_action("evaluator_deployed", resource="evaluator", success=True) + _cloud.emit(ctx.obj, {"created": created, "disabled": disabled_ids}, "Evaluator deployed") + + +def draft( + ctx: typer.Context, + intent: str = typer.Argument(...), + kind: str = typer.Option("code", "--kind"), + out: str = typer.Option("evaluator.json", "--out"), +) -> None: + """AI-compose an API spec file. Session only; review it before deployment.""" + _cloud.session_only(ctx.obj, "evaluators draft") + if kind not in ("code", "judge", "jev"): + spec.fail("--kind must be code, judge or jev") + data = api.cloud_request(require_auth(ctx.obj), "POST", "/api/agent/compose-eval", + body={"intent": f"Author a {kind} evaluator. {intent}", "resultKind": "score"}) + if not isinstance(data, dict): + spec.fail("Cloud draft was not an evaluator JSON spec") + body = spec.from_draft(data, kind) + _cloud.write_file(out, json.dumps(body, indent=2, ensure_ascii=False) + "\n") + _cloud.emit(ctx.obj, {"out": out, "spec": body}, "Evaluator draft") + + +def dry_run( + ctx: typer.Context, + code: Optional[str] = typer.Option(None, "--code"), + result_kind: Optional[str] = typer.Option(None, "--result-kind"), + timeout: Optional[float] = typer.Option(None, "--timeout"), + condition: Optional[str] = typer.Option(None, "--condition"), + judge: bool = typer.Option(False, "--judge"), + criteria: Optional[str] = typer.Option(None, "--criteria"), + step: Optional[List[str]] = typer.Option(None, "--step"), + threshold: Optional[float] = typer.Option(None, "--threshold"), + jev: Optional[str] = typer.Option(None, "--jev"), + instructions: Optional[str] = typer.Option(None, "--instructions"), + true: Optional[str] = typer.Option(None, "--true"), + false: Optional[str] = typer.Option(None, "--false"), + level: Optional[List[str]] = typer.Option(None, "--level"), + file: Optional[str] = typer.Option(None, "--file"), + session: Optional[List[str]] = typer.Option(None, "--session"), +) -> None: + """Compile only, or run on ≤10 sessions. Cloud refuses judge dry-runs. + + Jev dry-runs execute model calls and spend budget, including before deployment. + """ + body = spec.build( + code=code, result_kind=result_kind, timeout=timeout, condition=condition, + judge=judge, criteria=criteria, step=step, threshold_value=threshold, jev=jev, + instructions=instructions, true=true, false=false, level=level, file=file, complete=False, + ) + body = spec.dry_run_body(body, session) + if spec.source_kind(body) == "jev": + output.warn("Jev dry-runs spend model budget.") + _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "POST", f"{PATH}/dry-run", body=body), "Evaluator dry-run") + + +def toggle(ctx, handle, enabled): + cctx = require_auth(ctx.obj) + row = select(definitions(cctx), handle) + _write.confirm(ctx.obj, f"{'Enable' if enabled else 'Disable'} {handle}", assume_yes=False) + result = api.cloud_request(cctx, "PATCH", f"{PATH}/{row['id']}", body={"enabled": enabled}) + _write.record_action("evaluator_toggled", resource="evaluator", success=True) + _cloud.emit(ctx.obj, result or {**row, "enabled": enabled}, "Evaluator") + + +def enable(ctx: typer.Context, handle: str = typer.Argument(...)) -> None: + """Enable KEY[@VERSION].""" + toggle(ctx, handle, True) + + +def disable(ctx: typer.Context, handle: str = typer.Argument(...)) -> None: + """Disable KEY[@VERSION].""" + toggle(ctx, handle, False) + + +def label_def(ctx: typer.Context, handle: str = typer.Argument(...), + add: Optional[List[str]] = typer.Option(None, "--add"), + remove: Optional[List[str]] = typer.Option(None, "--remove")) -> None: + """Add/remove labels on KEY[@VERSION].""" + if not add and not remove: + spec.fail("Pass --add or --remove") + if set(add or []) & set(remove or []): + spec.fail("A label cannot be both added and removed") + cctx = require_auth(ctx.obj) + row = select(definitions(cctx), handle) + labels = list(dict.fromkeys([v for v in row.get("labels", []) if v not in (remove or [])] + (add or []))) + _write.confirm(ctx.obj, f"Label {handle}", assume_yes=False) + data = api.cloud_request(cctx, "PATCH", f"{PATH}/{row['id']}", body={"labels": labels}) + _cloud.emit(ctx.obj, data or {**row, "labels": labels}, "Evaluator labels") + + +def instances(ctx: typer.Context) -> None: + """List managed evaluator instances.""" + _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "GET", "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/api/evaluator-instances"), "Instances") + + +def schema(ctx: typer.Context) -> None: + """Schema profile. Needs evaluations:trigger and events:read.""" + _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "GET", "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/api/evaluator-schema-profile"), "Schema profile") + + +def jobs(ctx: typer.Context) -> None: + """List evaluation jobs.""" + _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "GET", "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/api/evaluation-jobs"), "Evaluation jobs") + + +def backfill(ctx: typer.Context, frm: str = typer.Option(..., "--from"), + to: str = typer.Option(..., "--to"), key: Optional[str] = typer.Option(None, "--key"), + dry_run: bool = typer.Option(False, "--dry-run")) -> None: + """Backfill at most 90 days. Judge/Jev evaluators are excluded by Cloud.""" + try: + dates.resolve_range(ts_from=frm, ts_to=to) + start = datetime.fromisoformat(frm.replace("Z", "+00:00")) + end = datetime.fromisoformat(to.replace("Z", "+00:00")) + except ValueError as exc: + spec.fail(str(exc)) + if end <= start or end - start > timedelta(days=90): + spec.fail("Backfill needs an increasing range of at most 90 days") + output.warn("Backfill excludes judge/Jev evaluators; only code evaluators are replayed.") + _write.confirm(ctx.obj, "Backfill evaluations", assume_yes=dry_run) + _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "POST", "/api/evaluations/backfill", + body={"from": frm, "to": to, "eval_key": key, "dry_run": dry_run}), "Evaluation backfill") + + +def register(app: typer.Typer) -> None: + group = typer.Typer(no_args_is_help=True, rich_markup_mode="markdown", help="Author, test and deploy code, LLM-judge and Jev evaluators.", context_settings={"help_option_names": ["-h", "--help"]}) + for name, fn in ( + ("list", list_defs), ("show", show), ("create", create), ("deploy", deploy), + ("draft", draft), ("dry-run", dry_run), ("enable", enable), ("disable", disable), + ("label", label_def), ("instances", instances), ("schema", schema), ("jobs", jobs), + ("backfill", backfill), + ): + group.command(name, epilog=GLOBALS_EPILOG)(fn) + app.add_typer(group, name="evaluators") diff --git a/fp-cloud-cli/fp_cli/commands/sessions_cmds.py b/fp-cloud-cli/fp_cli/commands/sessions_cmds.py index 8ec9286cb..07e39307b 100644 --- a/fp-cloud-cli/fp_cli/commands/sessions_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/sessions_cmds.py @@ -19,6 +19,7 @@ validate_limit, ) from ..models import Session +from . import _cloud, _write def sessions( @@ -70,6 +71,8 @@ def sessions( * `fp sessions --agents` — expand each multi-agent session to its full agent roster * `fp --json sessions --all --fields session_id,agents | jq '.sessions'` — every session's full agent roster as JSON """ + if ctx.invoked_subcommand: + return state: AppState = ctx.obj cctx = require_auth(state) validate_limit(limit) @@ -147,5 +150,15 @@ def fetch(cursor: Optional[str], limit: Optional[int]): output.recheck_filters_hint(active_filters) +def re_evaluate(ctx: typer.Context, session_id: str = typer.Argument(...)) -> None: + """Re-run evaluators on one session. Needs evaluations:trigger.""" + _write.confirm(ctx.obj, f"Re-evaluate session {session_id}", assume_yes=False) + _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "POST", + f"/api/sessions/{session_id}/re-evaluate"), "Session re-evaluation") + + def register(app: typer.Typer) -> None: - app.command("sessions", epilog=GLOBALS_EPILOG)(sessions) + group = typer.Typer(invoke_without_command=True, rich_markup_mode="markdown", context_settings={"help_option_names": ["-h", "--help"]}) + group.callback(epilog=GLOBALS_EPILOG)(sessions) + group.command("re-evaluate", epilog=GLOBALS_EPILOG)(re_evaluate) + app.add_typer(group, name="sessions") diff --git a/fp-cloud-cli/fp_cli/evaluator_spec.py b/fp-cloud-cli/fp_cli/evaluator_spec.py new file mode 100644 index 000000000..46b3a3a39 --- /dev/null +++ b/fp-cloud-cli/fp_cli/evaluator_spec.py @@ -0,0 +1,220 @@ +"""Evaluator API bodies: validate data, never execute customer Python locally.""" + +from __future__ import annotations + +import ast +import json +import math +import re +from typing import Any, Optional + +from . import _click_compat as click +from .commands._write import read_text_arg + +FIELDS = frozenset({ + "eval_key", "display_name", "eval_version", "result_kind", "labels", + "condition_source", "evaluator_source", "timeout_seconds", +}) + + +def fail(message: str) -> None: + raise click.UsageError(message) + + +def text(value: Any, name: str) -> str: + if not isinstance(value, str) or not value.strip(): + fail(f"{name} must be a non-empty string") + return value + + +def threshold(value: Any) -> None: + if value is not None and ( + isinstance(value, bool) or not isinstance(value, (int, float)) + or not math.isfinite(value) or not 0 <= value <= 1 + ): + fail("threshold must be between 0 and 1") + + +def source_kind(body: dict) -> str: + source = body.get("evaluator_source", "") + try: + envelope = json.loads(source) + except (ValueError, TypeError): + return "code" + return envelope.get("kind", "code") if isinstance(envelope, dict) else "code" + + +def validate(body: dict, *, complete: bool = True) -> dict: + unknown = set(body) - FIELDS + if unknown: + fail(f"Unknown spec field(s): {', '.join(sorted(unknown))}; use the evaluator API body") + if complete: + key = text(body.get("eval_key"), "eval_key") + if not re.fullmatch(r"[a-z][a-z0-9_]*", key): + fail("eval_key must match ^[a-z][a-z0-9_]*$") + text(body.get("display_name"), "display_name") + text(body.get("eval_version"), "eval_version") + elif body.get("eval_key") is not None and not re.fullmatch(r"[a-z][a-z0-9_]*", str(body["eval_key"])): + fail("eval_key must match ^[a-z][a-z0-9_]*$") + result_kind = body.get("result_kind", "score") + if result_kind not in ("score", "metric", "assertion"): + fail("result_kind must be score, metric or assertion") + labels = body.get("labels", []) + if not isinstance(labels, list) or any(not isinstance(v, str) or not v.strip() for v in labels): + fail("labels must be a list of non-empty strings") + timeout = body.get("timeout_seconds") + if timeout is not None and ( + isinstance(timeout, bool) or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) or timeout <= 0 + ): + fail("timeout_seconds must be positive and finite") + condition = body.get("condition_source") + if condition: + text(condition, "condition_source") + expression(condition, "condition_source") + source = text(body.get("evaluator_source"), "evaluator_source") + kind = source_kind(body) + if kind == "code": + expression(source, "evaluator_source") + return body + envelope = json.loads(source) + if kind not in ("judge", "jev"): + fail("evaluator_source kind must be judge or jev") + if result_kind != "score": + fail("judge/Jev evaluators require result_kind score") + threshold(envelope.get("threshold")) + if kind == "judge": + text(envelope.get("criteria"), "judge criteria") + steps = envelope.get("steps") + if steps is not None: + if not isinstance(steps, list): + fail("judge steps must be a list") + for step in steps: + text(step, "judge step") + if re.search(r"\b(yes\s*/\s*no|true\s*/\s*false|binary|pass\s*/\s*fail|score\s+0\s+if|score\s+1\s+if)\b", step, re.I): + fail("binary judge steps are not supported; use graded evaluation steps") + else: + text(envelope.get("instructions"), "Jev instructions") + criteria = envelope.get("criteria") + if envelope.get("type") == "noul": + if not isinstance(criteria, dict) or set(criteria) != {"true", "false"}: + fail("Jev noul criteria must contain true and false") + text(criteria["true"], "true criterion") + text(criteria["false"], "false criterion") + if criteria["true"].strip() == criteria["false"].strip(): + fail("Jev true and false criteria must be distinct") + elif envelope.get("type") == "score": + if not isinstance(criteria, list) or not 3 <= len(criteria) <= 5: + fail("Jev score needs 3–5 distinct levels, ordered worst to best") + for level in criteria: + text(level, "Jev level") + if len({v.strip() for v in criteria}) != len(criteria): + fail("Jev score levels must be distinct") + else: + fail("Jev type must be noul or score") + return body + + +def from_draft(data: dict, requested_kind: str) -> dict: + """Convert the dashboard's EvalDraft (UI fields) to the immutable API body.""" + draft = data.get("spec", data) + if isinstance(draft, str): + try: + draft = json.loads(draft) + except ValueError: + fail("Cloud draft was not an evaluator JSON spec") + if not isinstance(draft, dict): + fail("Cloud draft was not an evaluator JSON spec") + body = {k: v for k, v in draft.items() if k in FIELDS} + body.setdefault("eval_version", "1") + kind = draft.get("kind", source_kind(body)) + if kind != requested_kind: + fail(f"Cloud drafted {kind}, not requested {requested_kind}; retry with a more specific intent") + if kind == "judge" and source_kind(body) != "judge": + body["evaluator_source"] = json.dumps({ + "kind": "judge", "criteria": draft.get("criteria"), + "threshold": draft.get("threshold"), "steps": draft.get("steps"), + }) + elif kind == "jev" and source_kind(body) != "jev": + body["evaluator_source"] = json.dumps({ + "kind": "jev", "type": draft.get("jevType"), "instructions": draft.get("instructions"), + "criteria": draft.get("jevCriteria"), "threshold": draft.get("threshold"), + }) + return validate(body) + + +def expression(source: str, name: str) -> None: + try: + ast.parse(source.strip(), mode="eval") + except SyntaxError as exc: + fail(f"{name} must be a restricted Python expression: {exc.msg}") + + +def build( + *, key=None, name=None, version=None, label=None, condition=None, + code=None, result_kind=None, timeout=None, judge=False, criteria=None, + step=None, threshold_value=None, jev=None, instructions=None, + true=None, false=None, level=None, file=None, complete=True, +) -> dict: + kinds = sum((bool(code), bool(judge), bool(jev), bool(file))) + if kinds != 1: + fail("Choose exactly one source: --code, --judge, --jev or --file") + source_flags = (criteria, step, threshold_value, instructions, true, false, level) + if file: + if any(v is not None for v in (key, name, version, label, condition, result_kind, timeout, *source_flags)): + fail("Pass either --file or discrete spec flags, not both") + try: + body = json.loads(read_text_arg(file)) + except ValueError as exc: + fail(f"--file is not valid JSON: {exc}") + if not isinstance(body, dict): + fail("--file must contain a JSON object") + return validate(body, complete=complete) + if code: + if any(v is not None for v in source_flags): + fail("Judge/Jev flags cannot be used with --code") + source = read_text_arg(code, flag="--code") + elif judge: + if any(v is not None for v in (instructions, true, false, level)): + fail("Jev flags cannot be used with --judge") + source = json.dumps({"kind": "judge", "criteria": criteria, "threshold": threshold_value, "steps": step}) + else: + if criteria is not None or step is not None: + fail("Judge flags cannot be used with --jev") + if jev == "noul" and level is not None or jev == "score" and (true is not None or false is not None): + fail("Use --true/--false for Jev noul; --level for Jev score") + source = json.dumps({ + "kind": "jev", "type": jev, "instructions": instructions, + "criteria": {"true": true, "false": false} if jev == "noul" else level, + "threshold": threshold_value, + }) + body = { + "result_kind": result_kind or "score", "labels": label or [], + "evaluator_source": source, + } + if key is not None: + body["eval_key"] = key + if name is not None: + body["display_name"] = name + if version is not None: + body["eval_version"] = version + if condition is not None: + body["condition_source"] = read_text_arg(condition[1:], flag="--condition") if condition.startswith("@") else condition + if timeout is not None: + body["timeout_seconds"] = timeout + if complete: + body.setdefault("eval_version", "1") + return validate(body, complete=complete) + + +def dry_run_body(body: dict, sessions: Optional[list]) -> dict: + if len(sessions or []) > 10: + fail("dry-run accepts at most 10 sessions") + if source_kind(body) == "judge": + fail("Cloud refuses judge dry-runs; create/deploy without --dry-run-on and inspect evaluation results instead") + return { + **{k: v for k, v in body.items() if k in ( + "evaluator_source", "condition_source", "timeout_seconds", "result_kind", "eval_key", + )}, + "session_ids": sessions or [], + } diff --git a/fp-cloud-cli/fp_cli/output.py b/fp-cloud-cli/fp_cli/output.py index 05e6f377b..b9c02fd6e 100644 --- a/fp-cloud-cli/fp_cli/output.py +++ b/fp-cloud-cli/fp_cli/output.py @@ -485,6 +485,7 @@ def version_banner(version: str) -> None: ("events", "List the raw per-step agent event trail.", ""), ("sessions", "List agent runs — one row per run.", ""), ("evals", "List scored agent evaluations.", "--aggregate"), + ("evaluators", "Author, test and deploy managed evaluators.", "list show create deploy draft dry-run enable disable label instances schema jobs backfill"), ("errors", "List errored events.", "--aggregate"), ("usage", "Show current org usage for the metering window.", ""), ]), diff --git a/fp-cloud-cli/tests/test_evaluator_authoring.py b/fp-cloud-cli/tests/test_evaluator_authoring.py new file mode 100644 index 000000000..a8dccc4ca --- /dev/null +++ b/fp-cloud-cli/tests/test_evaluator_authoring.py @@ -0,0 +1,217 @@ +import json + +import httpx +import pytest +import respx + +from fp_cli.app import app + +BASE = "http://dash.test" +SOURCE = "EvalResult(score=Score(value=0.9), reasoning='ok')" + + +@pytest.fixture +def code(tmp_path): + path = tmp_path / "eval.py" + path.write_text(SOURCE) + return str(path) + + +KINDS = [ + (["--judge", "--criteria", "Grade relevance", "--step", "Assess specificity"], "judge"), + (["--jev", "noul", "--instructions", "Assess resolution", "--true", "Resolved", "--false", "Unresolved"], "jev"), + (["--jev", "score", "--instructions", "Grade relevance", "--level", "Poor", "--level", "Fair", "--level", "Good"], "jev"), +] + + +@pytest.mark.parametrize("key_mode", [False, True]) +@pytest.mark.parametrize("flags,kind", KINDS + [([], "code")]) +def test_create_bodies(logged_in, runner, code, key_mode, flags, kind): + prefix = "v1" if key_mode else "api" + flags = flags or ["--code", code, "--result-kind", "assertion"] + with respx.mock as mock: + route = mock.post(f"{BASE}/{prefix}/evaluator-definitions").respond(201, json={"id": "d1", "enabled": True}) + result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), + "evaluators", "create", "--key", "quality", "--name", "Quality", *flags]) + assert result.exit_code == 0, result.output + body = json.loads(route.calls[0].request.content) + assert body["eval_key"] == "quality" + assert body["display_name"] == "Quality" + assert body["eval_version"] == "1" + assert body["labels"] == [] + if kind == "code": + assert body["evaluator_source"] == SOURCE + assert body["result_kind"] == "assertion" + else: + assert body["result_kind"] == "score" + envelope = json.loads(body["evaluator_source"]) + assert envelope["kind"] == kind + assert envelope["threshold"] is None + if kind == "judge": + assert envelope["steps"] == ["Assess specificity"] + + +@pytest.mark.parametrize("flags", [ + ["--judge", "--criteria", "Grade", "--threshold", "1.5"], + ["--judge", "--criteria", "Grade", "--threshold", "nan"], + ["--judge", "--criteria", "Grade", "--result-kind", "metric"], + ["--judge", "--criteria", "Grade", "--step", "Answer yes/no"], + ["--jev", "score", "--instructions", "Grade", "--level", "Bad", "--level", "Good"], + ["--jev", "score", "--instructions", "Grade", "--level", "Bad", "--level", "Bad", "--level", "Good"], + ["--jev", "noul", "--instructions", "Grade"], + ["--judge", "--criteria", "Grade", "--dry-run-on", "s1"], + ["--judge", "--criteria", "Grade", "--code", "-"], + ["--judge", "--criteria", "Grade", "--timeout", "-1"], +]) +def test_validation_before_network(logged_in, runner, flags): + with respx.mock(assert_all_called=False) as mock: + catch_all = mock.route().respond(200, json={}) + result = runner.invoke(app, ["--json", "evaluators", "create", "--key", "quality", "--name", "Quality", *flags]) + assert result.exit_code == 2, result.output + assert catch_all.call_count == 0 + + +@pytest.mark.parametrize("key", ["Bad", "1bad", "bad-key", "../bad", ""]) +def test_key_validation(logged_in, runner, code, key): + result = runner.invoke(app, ["--json", "evaluators", "create", "--key", key, "--name", "Quality", "--code", code]) + assert result.exit_code == 2 + + +def test_deploy_order_and_dry_run(logged_in, runner, code): + old = {"id": "d1", "eval_key": "quality", "eval_version": "1", "display_name": "Quality", "enabled": True} + with respx.mock as mock: + mock.get(f"{BASE}/v1/evaluator-definitions").respond(200, json=[old]) + dry = mock.post(f"{BASE}/v1/evaluator-definitions/dry-run").respond(200, json={"ok": True}) + create = mock.post(f"{BASE}/v1/evaluator-definitions").respond(201, json={**old, "id": "d2", "eval_version": "2"}) + patch = mock.patch(f"{BASE}/v1/evaluator-definitions/d1").respond(200, json={}) + result = runner.invoke(app, ["--json", "--api-key", "key", "evaluators", "deploy", "quality", "--code", code, "--dry-run-on", "s1"]) + order = [(call.request.method, call.request.url.path) for call in mock.calls] + assert result.exit_code == 0, result.output + assert order == [("GET", "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/v1/evaluator-definitions"), ("POST", "/v1/evaluator-definitions/dry-run"), + ("POST", "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/v1/evaluator-definitions"), ("PATCH", "/v1/evaluator-definitions/d1")] + assert json.loads(create.calls[0].request.content)["eval_version"] == "2" + assert json.loads(patch.calls[0].request.content) == {"enabled": False} + assert json.loads(dry.calls[0].request.content)["session_ids"] == ["s1"] + assert json.loads(result.stdout)["disabled"] == ["d1"] + + +def test_failed_create_never_disables(logged_in, runner, code): + with respx.mock(assert_all_called=False) as mock: + mock.get(f"{BASE}/api/evaluator-definitions").respond(200, json=[{"id": "d1", "eval_key": "quality", "eval_version": "1"}]) + mock.post(f"{BASE}/api/evaluator-definitions").respond(409, json={"error": "version conflict"}) + patch = mock.patch(f"{BASE}/api/evaluator-definitions/d1").respond(200, json={}) + result = runner.invoke(app, ["--json", "evaluators", "deploy", "quality", "--code", code]) + assert result.exit_code == 1 + assert not patch.called + + +def test_partial_deploy_is_reported(logged_in, runner, code): + with respx.mock as mock: + mock.get(f"{BASE}/api/evaluator-definitions").respond(200, json=[{"id": "d1", "eval_key": "quality", "eval_version": "1"}]) + mock.post(f"{BASE}/api/evaluator-definitions").respond(201, json={"id": "d2"}) + mock.patch(f"{BASE}/api/evaluator-definitions/d1").respond(403, json={}) + result = runner.invoke(app, ["--json", "evaluators", "deploy", "quality", "--code", code]) + assert result.exit_code == 5 + assert "Published quality@2" in result.stderr + assert "d2" in result.stderr + + +def test_draft_round_trip(logged_in, runner, tmp_path): + body = {"eval_key": "quality", "display_name": "Quality", "eval_version": "1", + "result_kind": "score", "labels": [], "evaluator_source": SOURCE} + path = tmp_path / "draft.json" + with respx.mock as mock: + mock.post(f"{BASE}/api/agent/compose-eval").respond(200, json={"spec": body}) + result = runner.invoke(app, ["--json", "evaluators", "draft", "Grade quality", "--out", str(path)]) + assert result.exit_code == 0, result.output + assert json.loads(path.read_text()) == body + with respx.mock as mock: + route = mock.post(f"{BASE}/v1/evaluator-definitions").respond(201, json={"id": "d1"}) + result = runner.invoke(app, ["--json", "--api-key", "key", "evaluators", "create", "--file", str(path)]) + assert result.exit_code == 0, result.output + assert json.loads(route.calls[0].request.content) == body + + +def test_draft_denies_key(logged_in, runner): + with respx.mock(assert_all_called=False) as mock: + route = mock.route().respond(200, json={}) + result = runner.invoke(app, ["--json", "--api-key", "key", "evaluators", "draft", "Grade"]) + assert result.exit_code == 2 + assert "needs a signed-in session: run `fp login`" in result.stdout + assert not route.called + + +@pytest.mark.parametrize("count", [0, 2, 11]) +def test_dry_run_sessions(logged_in, runner, code, count): + with respx.mock(assert_all_called=False) as mock: + route = mock.post(f"{BASE}/v1/evaluator-definitions/dry-run").respond(200, json={"ok": True}) + result = runner.invoke(app, ["--json", "--api-key", "key", "evaluators", "dry-run", "--code", code, + *[v for n in range(count) for v in ("--session", str(n))]]) + assert result.exit_code == (2 if count > 10 else 0), result.output + if count <= 10: + assert json.loads(route.calls[0].request.content)["session_ids"] == [str(n) for n in range(count)] + else: + assert not route.called + + +def test_judge_refusal(logged_in, runner): + result = runner.invoke(app, ["--json", "evaluators", "dry-run", "--judge", "--criteria", "Grade"]) + assert result.exit_code == 2 + assert "Cloud refuses judge dry-runs" in result.stdout + + +@pytest.mark.parametrize("command,path", [("instances", "evaluator-instances"), ("schema", "evaluator-schema-profile"), ("jobs", "evaluation-jobs")]) +def test_auxiliary_commands(logged_in, runner, command, path): + with respx.mock as mock: + mock.get(f"{BASE}/v1/{path}").respond(200, json=[]) + result = runner.invoke(app, ["--json", "--api-key", "key", "evaluators", command]) + assert result.exit_code == 0, result.output + + +@pytest.mark.parametrize("command,body", [("enable", {"enabled": True}), ("disable", {"enabled": False}), + ("label", {"labels": ["new"]})]) +def test_patch_commands(logged_in, runner, command, body): + with respx.mock as mock: + mock.get(f"{BASE}/v1/evaluator-definitions").respond(200, json=[ + {"id": "d1", "eval_key": "quality", "eval_version": "1", "labels": ["old"]}]) + route = mock.patch(f"{BASE}/v1/evaluator-definitions/d1").respond(200, json=body) + result = runner.invoke(app, ["--json", "--api-key", "key", "evaluators", command, "quality@1", + *(["--add", "new", "--remove", "old"] if command == "label" else [])]) + assert result.exit_code == 0, result.output + assert json.loads(route.calls[0].request.content) == body + + +def test_list_show_filter(logged_in, runner): + rows = [{"id": "d1", "eval_key": "quality", "eval_version": "1", "enabled": False}, + {"id": "d2", "eval_key": "quality", "eval_version": "2", "enabled": True}] + with respx.mock as mock: + mock.get(f"{BASE}/v1/evaluator-definitions").respond(200, json=rows) + listing = runner.invoke(app, ["--json", "--api-key", "key", "evaluators", "list", "--key", "quality"]) + showing = runner.invoke(app, ["--json", "--api-key", "key", "evaluators", "show", "quality@1"]) + assert listing.exit_code == showing.exit_code == 0 + assert json.loads(listing.stdout) == [rows[1]] + assert json.loads(showing.stdout) == rows[0] + + +@pytest.mark.parametrize("frm,to,ok", [ + ("2026-10-01T00:00:00Z", "2026-10-08T00:00:00Z", True), + ("2026-10-01T00:00:00Z", "2026-09-01T00:00:00Z", False), + ("2026-01-01T00:00:00Z", "2026-10-08T00:00:00Z", False), +]) +def test_backfill(logged_in, runner, frm, to, ok): + with respx.mock(assert_all_called=False) as mock: + route = mock.post(f"{BASE}/v1/evaluations/backfill").respond(200, json={"ok": True}) + result = runner.invoke(app, ["--json", "--api-key", "key", "evaluators", "backfill", "--from", frm, "--to", to, "--dry-run"]) + assert result.exit_code == (0 if ok else 2), result.output + if ok: + assert "excludes judge/Jev" in result.stderr + assert json.loads(route.calls[0].request.content)["dry_run"] is True + + +@pytest.mark.parametrize("command", ["list", "instances", "schema", "jobs"]) +def test_evaluator_404(logged_in, runner, command): + with respx.mock as mock: + mock.route().respond(404, json={}) + result = runner.invoke(app, ["--json", "evaluators", command]) + assert result.exit_code == 6 + assert "not available on this FailproofAI Cloud yet" in result.stdout diff --git a/fp-cloud-cli/tests/test_session_reevaluate.py b/fp-cloud-cli/tests/test_session_reevaluate.py new file mode 100644 index 000000000..4d89f2aed --- /dev/null +++ b/fp-cloud-cli/tests/test_session_reevaluate.py @@ -0,0 +1,17 @@ +import pytest +import respx + +from fp_cli.app import app + + +@pytest.mark.parametrize("key_mode", [False, True]) +@pytest.mark.parametrize("status", [200, 404]) +def test_re_evaluate(logged_in, runner, key_mode, status): + prefix = "v1" if key_mode else "api" + with respx.mock: + route = respx.post(f"http://dash.test/{prefix}/sessions/run-001/re-evaluate").respond(status, json={"queued": True}) + result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), "sessions", "re-evaluate", "run-001"]) + assert route.called + assert result.exit_code == (0 if status == 200 else 6), result.output + if status == 404: + assert "not available on this FailproofAI Cloud yet" in result.stdout From c9b71cf6f4c18a6ba8cfe2c63468cf53dd21c40d Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:49:02 +0000 Subject: [PATCH 03/16] feat(fp): add dashboards, session exports, sentiment and query authoring --- fp-cloud-cli/fp_cli/app.py | 4 + fp-cloud-cli/fp_cli/client.py | 24 ++- .../fp_cli/commands/dashboards_cmds.py | 159 ++++++++++++++++++ fp-cloud-cli/fp_cli/commands/events_cmds.py | 15 +- fp-cloud-cli/fp_cli/commands/queries_cmds.py | 67 ++++++++ .../fp_cli/commands/sentiment_cmds.py | 49 ++++++ fp-cloud-cli/fp_cli/commands/sessions_cmds.py | 32 ++++ fp-cloud-cli/fp_cli/output.py | 2 + .../tests/test_cloud_observability.py | 117 +++++++++++++ 9 files changed, 467 insertions(+), 2 deletions(-) create mode 100644 fp-cloud-cli/fp_cli/commands/dashboards_cmds.py create mode 100644 fp-cloud-cli/fp_cli/commands/sentiment_cmds.py create mode 100644 fp-cloud-cli/tests/test_cloud_observability.py diff --git a/fp-cloud-cli/fp_cli/app.py b/fp-cloud-cli/fp_cli/app.py index c11d3c3ea..202e27fdb 100644 --- a/fp-cloud-cli/fp_cli/app.py +++ b/fp-cloud-cli/fp_cli/app.py @@ -21,6 +21,7 @@ alerts_cmds, audits_cmds, auth_cmds, + dashboards_cmds, errors_cmds, evals_cmds, evaluators_cmds, @@ -34,6 +35,7 @@ policies_cmds, queries_cmds, sessions_cmds, + sentiment_cmds, settings_cmds, usage_cmds, users_cmds, @@ -454,6 +456,8 @@ def help_cmd(ctx: typer.Context) -> None: sessions_cmds.register(app) evals_cmds.register(app) evaluators_cmds.register(app) +dashboards_cmds.register(app) +sentiment_cmds.register(app) errors_cmds.register(app) usage_cmds.register(app) list_cmds.register(app) diff --git a/fp-cloud-cli/fp_cli/client.py b/fp-cloud-cli/fp_cli/client.py index 5bed12088..1882f5885 100644 --- a/fp-cloud-cli/fp_cli/client.py +++ b/fp-cloud-cli/fp_cli/client.py @@ -605,6 +605,14 @@ def cloud_request( ) from exc +def cloud_stream(ctx: ClientContext, path: str, body: Dict[str, Any]) -> Iterator[Dict[str, Any]]: + """Dashboard SSE with the same additive-capability 404 contract.""" + try: + yield from _stream_sse(ctx, path, body) + except NotFoundError as exc: + raise NotFoundError("not available on this FailproofAI Cloud yet", request_id=exc.request_id) from exc + + # --- Auth / identity -------------------------------------------------------- @@ -1531,6 +1539,8 @@ def agent_chat_oneshot( conversation_id: Optional[str] = None, page_context: Optional[str] = None, model: Optional[str] = None, + on_ask_user: Optional[Callable[[Dict[str, Any]], None]] = None, + on_artifact: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> Dict[str, Any]: """POST /api/agent/chat — accumulate the streamed answer. @@ -1550,20 +1560,32 @@ def agent_chat_oneshot( body["pageContext"] = page_context if model: body["model"] = model + if on_artifact: + body["pageActions"] = [{"action": "open_artifact"}, {"action": "home"}] parts: List[str] = [] tools: List[str] = [] interrupted = False error: Optional[str] = None - for ev in _stream_sse(ctx, "/api/agent/chat", body): + for ev in cloud_stream(ctx, "/api/agent/chat", body): kind = ev.get("type") if kind == "text-delta": parts.append(str(ev.get("text", ""))) elif kind == "tool-start": tools.append(str(ev.get("tool", ""))) elif kind == "ask-user": + if on_ask_user and ev.get("callId"): + on_ask_user(ev) + continue interrupted = True error = str(ev.get("question") or "the assistant needs interactive input") break + elif kind == "ui-action" and on_artifact: + if ev.get("action") == "open_artifact" and isinstance(ev.get("artifact"), dict): + on_artifact(ev["artifact"]) + elif ev.get("action") == "home" and isinstance(ev.get("home"), dict): + for block in ev["home"].get("blocks", []): + if isinstance(block, dict): + on_artifact(block) elif kind == "error": error = str(ev.get("message", "assistant error")) elif kind == "done": diff --git a/fp-cloud-cli/fp_cli/commands/dashboards_cmds.py b/fp-cloud-cli/fp_cli/commands/dashboards_cmds.py new file mode 100644 index 000000000..50c0bd56d --- /dev/null +++ b/fp-cloud-cli/fp_cli/commands/dashboards_cmds.py @@ -0,0 +1,159 @@ +"""Org dashboards and their saved-query tiles.""" + +from __future__ import annotations + +from typing import Optional + +import typer + +from .. import _click_compat as click, client as api +from .._context import GLOBALS_EPILOG, require_auth, validate_choice +from . import _cloud, _write + +CHARTS = ("line", "area", "bar", "stacked_bar", "pie", "kpi", "table") +TILE_FIELDS = ("query_id", "title", "chart_type", "chart_config", "param_bindings", "pos_x", "pos_y", "pos_w", "pos_h") + + +def request(ctx, method, path, body=None): + if method != "GET": + _write.confirm(ctx.obj, f"{method} {path}", assume_yes=False, destructive=method == "DELETE") + data = api.cloud_request(require_auth(ctx.obj), method, path, body=body) + if method != "GET": + _write.record_action("dashboard_changed", resource="dashboard", success=True, destructive=method == "DELETE") + _cloud.emit(ctx.obj, data, "Dashboards") + + +def list_dashboards(ctx: typer.Context) -> None: + """List org dashboards.""" + request(ctx, "GET", "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/api/dashboards") + + +def show(ctx: typer.Context, id: str = typer.Argument(...)) -> None: + """Show a dashboard and its tiles.""" + request(ctx, "GET", f"/api/dashboards/{id}") + + +def create(ctx: typer.Context, name: str = typer.Option(..., "--name"), + description: str = typer.Option("", "--description"), + file: Optional[str] = typer.Option(None, "--file", help="Dashboard definition JSON.")) -> None: + """Create a dashboard (no tiles yet).""" + if not name.strip(): + raise click.UsageError("--name cannot be empty") + definition = _write.read_body(ctx.obj, file) if file else {} + request(ctx, "POST", "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/api/dashboards", {"name": name, "description": description, "definition": definition}) + + +def update(ctx: typer.Context, id: str = typer.Argument(...), + name: Optional[str] = typer.Option(None, "--name"), + description: Optional[str] = typer.Option(None, "--description"), + file: Optional[str] = typer.Option(None, "--file", help="Replacement dashboard definition JSON.")) -> None: + """Update selected fields, preserving the others.""" + if name is None and description is None and file is None: + raise click.UsageError("Pass --name, --description or --file") + if name is not None and not name.strip(): + raise click.UsageError("--name cannot be empty") + definition = _write.read_body(ctx.obj, file) if file else None + current = api.cloud_request(require_auth(ctx.obj), "GET", f"/api/dashboards/{id}") + # Show returns {dashboard, tiles}; older deployments return the row itself. + current = current.get("dashboard", current) + body = {k: current.get(k, {} if k == "definition" else "") for k in ("name", "description", "definition")} + body.update({k: v for k, v in {"name": name, "description": description, "definition": definition}.items() if v is not None}) + request(ctx, "PUT", f"/api/dashboards/{id}", body) + + +def delete(ctx: typer.Context, id: str = typer.Argument(...)) -> None: + """Delete a dashboard (saved queries are retained).""" + request(ctx, "DELETE", f"/api/dashboards/{id}") + + +def tiles(ctx: typer.Context, id: str = typer.Argument(..., metavar="DASHBOARD_ID")) -> None: + """Manage tiles: `fp dashboards tiles ID list|add|update|remove|layout`.""" + ctx.meta["dashboard_id"] = id + + +def tile_path(ctx): + return f"/api/dashboards/{ctx.meta['dashboard_id']}/tiles" + + +def tile_list(ctx: typer.Context) -> None: + """List tiles.""" + request(ctx, "GET", tile_path(ctx)) + + +def tile_add(ctx: typer.Context, query: str = typer.Option(..., "--query"), + chart: str = typer.Option(..., "--chart"), title: str = typer.Option("", "--title"), + file: Optional[str] = typer.Option(None, "--file", help="Tile config/geometry JSON.")) -> None: + """Add a saved-query tile.""" + validate_choice(chart, CHARTS, flag="--chart") + body = {"query_id": query, "title": title, "chart_type": chart, "chart_config": {}, + "param_bindings": {}, "pos_x": 0, "pos_y": 0, "pos_w": 6, "pos_h": 4} + if file: + body.update(_write.read_body(ctx.obj, file)) + validate_tile(body) + request(ctx, "POST", tile_path(ctx), body) + + +def validate_tile(body): + if set(body) - set(TILE_FIELDS): + raise click.UsageError("Unknown tile fields") + validate_choice(body.get("chart_type"), CHARTS, flag="--chart") + if not body.get("query_id") or not body.get("chart_type"): + raise click.UsageError("Tile needs query_id and chart_type") + for key in ("pos_x", "pos_y", "pos_w", "pos_h"): + value = body.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value < (1 if key in ("pos_w", "pos_h") else 0): + raise click.UsageError("Tile position must be non-negative with positive size") + for key in ("chart_config", "param_bindings"): + if not isinstance(body.get(key), dict): + raise click.UsageError(f"{key} must be a JSON object") + + +def tile_update(ctx: typer.Context, tile: str = typer.Argument(...), + query: Optional[str] = typer.Option(None, "--query"), + chart: Optional[str] = typer.Option(None, "--chart"), + title: Optional[str] = typer.Option(None, "--title"), + file: Optional[str] = typer.Option(None, "--file")) -> None: + """Update a tile, preserving unmentioned config and geometry.""" + body = _write.read_body(ctx.obj, file, {"query_id": query, "chart_type": chart, "title": title}) + if not body: + raise click.UsageError("Pass --query, --chart, --title or --file") + data = api.cloud_request(require_auth(ctx.obj), "GET", tile_path(ctx)) + matches = [r for r in _cloud.rows(data, "tiles") if str(r.get("id")) == tile] + if len(matches) != 1: + from ..errors import NotFoundError + raise NotFoundError(f"no tile {tile}; run `fp dashboards tiles {ctx.meta['dashboard_id']} list`") + merged = {k: matches[0][k] for k in TILE_FIELDS} + merged.update(body) + validate_tile(merged) + request(ctx, "PUT", f"{tile_path(ctx)}/{tile}", merged) + + +def tile_remove(ctx: typer.Context, tile: str = typer.Argument(...)) -> None: + """Remove a tile.""" + request(ctx, "DELETE", f"{tile_path(ctx)}/{tile}") + + +def tile_layout(ctx: typer.Context, file: str = typer.Option(..., "--file")) -> None: + """Apply geometry JSON: {tiles: [{id,pos_x,pos_y,pos_w,pos_h}]}.""" + body = _write.read_body(ctx.obj, file) + if not isinstance(body.get("tiles"), list): + raise click.UsageError("Layout must contain a tiles array") + for row in body["tiles"]: + if not isinstance(row, dict) or not row.get("id"): + raise click.UsageError("Layout tiles need ids") + if set(row) != {"id", "pos_x", "pos_y", "pos_w", "pos_h"}: + raise click.UsageError("Layout tiles must contain id,pos_x,pos_y,pos_w,pos_h") + validate_tile({**{k: v for k, v in row.items() if k != "id"}, "query_id": "layout", "chart_type": "table", "chart_config": {}, "param_bindings": {}, "title": ""}) + request(ctx, "PUT", f"{tile_path(ctx)}/layout", body) + + +def register(app: typer.Typer) -> None: + group = typer.Typer(no_args_is_help=True, help="Manage org dashboards and tiles.", context_settings={"help_option_names": ["-h", "--help"]}) + for name, fn in (("list", list_dashboards), ("show", show), ("create", create), ("update", update), ("delete", delete)): + group.command(name, epilog=GLOBALS_EPILOG)(fn) + tile_group = typer.Typer(no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]}) + tile_group.callback()(tiles) + for name, fn in (("list", tile_list), ("add", tile_add), ("update", tile_update), ("remove", tile_remove), ("layout", tile_layout)): + tile_group.command(name, epilog=GLOBALS_EPILOG)(fn) + group.add_typer(tile_group, name="tiles") + app.add_typer(group, name="dashboards") diff --git a/fp-cloud-cli/fp_cli/commands/events_cmds.py b/fp-cloud-cli/fp_cli/commands/events_cmds.py index e4d4b669e..44f7ccb3d 100644 --- a/fp-cloud-cli/fp_cli/commands/events_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/events_cmds.py @@ -70,6 +70,8 @@ def events( * `fp --json events --session-id run-001 --all` — a run's full timeline: event types + summaries (light, fast) * `fp --json events --full --session-id run-001 --all | jq '.events[].payload'` — that run's raw payloads (full feed, bounded to one run) """ + if ctx.invoked_subcommand: + return state: AppState = ctx.obj cctx = require_auth(state) frm, to = resolve_dates(since, ts_from, ts_to) @@ -155,5 +157,16 @@ def events( output.recheck_filters_hint(active_filters) +def latency(ctx: typer.Context, since: Optional[str] = typer.Option(None, "--since")) -> None: + """Aggregate event latency.""" + from . import _cloud + frm, to = resolve_dates(since, None, None) + _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "GET", "/api/events/latency_aggregate", + params={"ts_from": frm, "ts_to": to}), "Event latency") + + def register(app: typer.Typer) -> None: - app.command("events", epilog=GLOBALS_EPILOG)(events) + group = typer.Typer(invoke_without_command=True, rich_markup_mode="markdown", context_settings={"help_option_names": ["-h", "--help"]}) + group.callback(epilog=GLOBALS_EPILOG)(events) + group.command("latency", epilog=GLOBALS_EPILOG)(latency) + app.add_typer(group, name="events") diff --git a/fp-cloud-cli/fp_cli/commands/queries_cmds.py b/fp-cloud-cli/fp_cli/commands/queries_cmds.py index 594bb311b..efd5f42b6 100644 --- a/fp-cloud-cli/fp_cli/commands/queries_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/queries_cmds.py @@ -17,6 +17,7 @@ from ..errors import NotFoundError from ..models import SavedQuery from . import _write +from . import _cloud def _humanize_list(items: List[str]) -> str: @@ -375,4 +376,70 @@ def register(app: typer.Typer) -> None: query_app.command("delete", epilog=GLOBALS_EPILOG)(query_delete) query_app.command("run", epilog=GLOBALS_EPILOG)(query_run) query_app.command("schema", epilog=GLOBALS_EPILOG)(query_schema) + query_app.command("compose", epilog=GLOBALS_EPILOG)(query_compose) + query_app.command("export", epilog=GLOBALS_EPILOG)(query_export) app.add_typer(query_app, name="query") + + +def query_compose(ctx: typer.Context, question: str = typer.Argument(...), + run: bool = typer.Option(False, "--run")) -> None: + """AI-compose SQL (session only); optionally run it.""" + _cloud.session_only(ctx.obj, "query compose") + cctx = require_auth(ctx.obj) + draft = None + for event in api.cloud_stream(cctx, "/api/agent/compose-sql", {"intent": question}): + if event.get("type") == "error": + from ..errors import ApiError + raise ApiError(str(event.get("error") or event.get("reason") or "SQL composer failed")) + if event.get("type") == "result": + draft = event.get("result", event) + if not isinstance(draft, dict): + raise click.UsageError("Cloud did not return composed SQL") + sql = draft.get("sql") or draft.get("sql_text") + if not isinstance(sql, str) or not sql.strip(): + raise click.UsageError("Cloud did not return composed SQL") + if run: + data = api.cloud_request(cctx, "POST", "/api/queries/run", body={"sql": sql, "params": []}) + _cloud.emit(ctx.obj, {"sql": sql, "result": data}, "Composed query") + elif ctx.obj.json: + output.emit_json(draft) + else: + print(sql) + + +def query_export(ctx: typer.Context, id: Optional[str] = typer.Argument(None), + sql: Optional[str] = typer.Option(None, "--sql"), + format: str = typer.Option("csv", "--format"), + out: Optional[str] = typer.Option(None, "--out")) -> None: + """Export a saved or ad-hoc query as CSV/JSON (all returned rows). + + --json wraps CSV in {format,content} unless --out writes the file. + """ + import csv + import io + import json + if (id is None) == (sql is None): + raise click.UsageError("Pass a saved query id or --sql, exactly one") + if format not in ("csv", "json"): + raise click.UsageError("--format must be csv or json") + data = api.cloud_request(require_auth(ctx.obj), "POST", "/api/queries/run", + body={"query_id": id, "sql": _read_sql(sql) if sql is not None else None, "params": []}) + if format == "json": + content = json.dumps(data, indent=2, ensure_ascii=False) + "\n" + else: + buf = io.StringIO() + writer = csv.writer(buf) + writer.writerow([c.get("name", "") if isinstance(c, dict) else c for c in data.get("columns", [])]) + # Spreadsheet-safe export of untrusted event cells. + writer.writerows([ + ["'" + v if isinstance(v, str) and v.startswith(("=", "+", "-", "@", "\t", "\r")) else v for v in row] + for row in data.get("rows", []) + ]) + content = buf.getvalue() + if out: + _cloud.write_file(out, content) + _cloud.emit(ctx.obj, {"out": out, "format": format}, "Query exported") + elif ctx.obj.json: + output.emit_json(data if format == "json" else {"format": format, "content": content}) + else: + print(content, end="") diff --git a/fp-cloud-cli/fp_cli/commands/sentiment_cmds.py b/fp-cloud-cli/fp_cli/commands/sentiment_cmds.py new file mode 100644 index 000000000..f77c9c7d0 --- /dev/null +++ b/fp-cloud-cli/fp_cli/commands/sentiment_cmds.py @@ -0,0 +1,49 @@ +"""Conversation sentiment summary, messages and sessions.""" + +from __future__ import annotations + +from typing import Optional + +import typer + +from .. import client as api +from .._context import GLOBALS_EPILOG, require_auth, resolve_dates, validate_limit +from . import _cloud + + +def read(ctx, suffix, since=None, limit=None, session_id=None): + validate_limit(limit) + frm, to = resolve_dates(since, None, None) + data = api.cloud_request(require_auth(ctx.obj), "GET", f"/api/events/sentiment{suffix}", + params={"ts_from": frm, "ts_to": to, "limit": limit, "session_id": session_id}) + _cloud.emit(ctx.obj, data, "Sentiment") + + +def summary(ctx: typer.Context, since: Optional[str] = typer.Option(None, "--since")) -> None: + """Summarize conversation sentiment.""" + read(ctx, "", since) + + +def messages(ctx: typer.Context, since: Optional[str] = typer.Option(None, "--since"), + limit: int = typer.Option(50, "--limit"), + session_id: Optional[str] = typer.Option(None, "--session-id")) -> None: + """List sentiment-scored messages.""" + read(ctx, "/messages", since, limit, session_id) + + +def sessions(ctx: typer.Context, since: Optional[str] = typer.Option(None, "--since"), + limit: int = typer.Option(50, "--limit")) -> None: + """List sentiment-scored sessions.""" + read(ctx, "/sessions", since, limit) + + +def session(ctx: typer.Context, id: str = typer.Argument(...)) -> None: + """Show one session's sentiment.""" + read(ctx, f"/sessions/{id}") + + +def register(app: typer.Typer) -> None: + group = typer.Typer(no_args_is_help=True, help="Inspect conversation sentiment.", context_settings={"help_option_names": ["-h", "--help"]}) + for name, fn in (("summary", summary), ("messages", messages), ("sessions", sessions), ("session", session)): + group.command(name, epilog=GLOBALS_EPILOG)(fn) + app.add_typer(group, name="sentiment") diff --git a/fp-cloud-cli/fp_cli/commands/sessions_cmds.py b/fp-cloud-cli/fp_cli/commands/sessions_cmds.py index 07e39307b..3796590ad 100644 --- a/fp-cloud-cli/fp_cli/commands/sessions_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/sessions_cmds.py @@ -157,8 +157,40 @@ def re_evaluate(ctx: typer.Context, session_id: str = typer.Argument(...)) -> No f"/api/sessions/{session_id}/re-evaluate"), "Session re-evaluation") +def show(ctx: typer.Context, id: str = typer.Argument(...)) -> None: + """Show a session overview and its event timeline.""" + cctx = require_auth(ctx.obj) + overview = api.cloud_request(cctx, "GET", "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/api/sessions", params={"session_id": id, "limit": 1}) + items = _cloud.rows(overview, "sessions") + if not items: + from ..errors import NotFoundError + raise NotFoundError(f"no session {id}") + timeline = api.cloud_request(cctx, "GET", "/api/events/summary", params={"session_id": id, "limit": 200, "order": "asc"}) + _cloud.emit(ctx.obj, {"overview": items[0], "timeline": timeline}, "Session") + + +def export(ctx: typer.Context, id: str = typer.Argument(...), + out: Optional[str] = typer.Option(None, "--out")) -> None: + """Export a full session as JSON.""" + import json + data = api.cloud_request(require_auth(ctx.obj), "GET", f"/api/sessions/{id}/export") + if out: + _cloud.write_file(out, json.dumps(data, indent=2, ensure_ascii=False) + "\n") + _cloud.emit(ctx.obj, {"out": out}, "Session exported") + else: + output.emit_json(data) + + +def history(ctx: typer.Context, id: str = typer.Argument(...)) -> None: + """Show a session's evaluation history.""" + _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "GET", f"/api/sessions/{id}/evaluation-history"), "Evaluation history") + + def register(app: typer.Typer) -> None: group = typer.Typer(invoke_without_command=True, rich_markup_mode="markdown", context_settings={"help_option_names": ["-h", "--help"]}) group.callback(epilog=GLOBALS_EPILOG)(sessions) group.command("re-evaluate", epilog=GLOBALS_EPILOG)(re_evaluate) + group.command("show", epilog=GLOBALS_EPILOG)(show) + group.command("export", epilog=GLOBALS_EPILOG)(export) + group.command("history", epilog=GLOBALS_EPILOG)(history) app.add_typer(group, name="sessions") diff --git a/fp-cloud-cli/fp_cli/output.py b/fp-cloud-cli/fp_cli/output.py index b9c02fd6e..762d2ac22 100644 --- a/fp-cloud-cli/fp_cli/output.py +++ b/fp-cloud-cli/fp_cli/output.py @@ -486,6 +486,8 @@ def version_banner(version: str) -> None: ("sessions", "List agent runs — one row per run.", ""), ("evals", "List scored agent evaluations.", "--aggregate"), ("evaluators", "Author, test and deploy managed evaluators.", "list show create deploy draft dry-run enable disable label instances schema jobs backfill"), + ("sentiment", "Inspect conversation sentiment.", "summary messages sessions session"), + ("dashboards", "Manage dashboards and saved-query tiles.", "list show create update delete tiles"), ("errors", "List errored events.", "--aggregate"), ("usage", "Show current org usage for the metering window.", ""), ]), diff --git a/fp-cloud-cli/tests/test_cloud_observability.py b/fp-cloud-cli/tests/test_cloud_observability.py new file mode 100644 index 000000000..a966208d1 --- /dev/null +++ b/fp-cloud-cli/tests/test_cloud_observability.py @@ -0,0 +1,117 @@ +import json + +import httpx +import pytest +import respx + +from fp_cli.app import app + +BASE = "http://dash.test" +DASHBOARD = {"name": "Quality", "description": "Keep", "definition": {"filters": {"env": "prod"}}} +TILE = {"id": "t1", "query_id": "q1", "title": "Before", "chart_type": "table", + "chart_config": {}, "param_bindings": {}, "pos_x": 0, "pos_y": 0, "pos_w": 6, "pos_h": 4} +RESULT = {"columns": [{"name": "agent", "type": "String"}, {"name": "n", "type": "Int64"}], + "rows": [["agent-001", 42]], "truncated": False, "elapsed_ms": 1} +CASES = [ + (["dashboards", "list"], "GET", "/dashboards", {"dashboards": []}), + (["dashboards", "show", "d1"], "GET", "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/dashboards/d1", {"dashboard": DASHBOARD, "tiles": []}), + (["dashboards", "create", "--name", "Quality"], "POST", "/dashboards", {"id": "d1"}), + (["dashboards", "update", "d1", "--name", "New"], "PUT", "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/dashboards/d1", {"id": "d1"}), + (["dashboards", "delete", "d1"], "DELETE", "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/dashboards/d1", {"deleted": True}), + (["dashboards", "tiles", "d1", "list"], "GET", "/dashboards/d1/tiles", {"tiles": [TILE]}), + (["dashboards", "tiles", "d1", "add", "--query", "q1", "--chart", "bar"], "POST", "/dashboards/d1/tiles", {"id": "t1"}), + (["dashboards", "tiles", "d1", "update", "t1", "--title", "After"], "PUT", "/dashboards/d1/tiles/t1", {"id": "t1"}), + (["dashboards", "tiles", "d1", "remove", "t1"], "DELETE", "/dashboards/d1/tiles/t1", {"deleted": True}), + (["dashboards", "tiles", "d1", "layout", "--file", "{layout}"], "PUT", "/dashboards/d1/tiles/layout", {"updated": 1}), + (["sessions", "show", "s1"], "GET", "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/events/summary", {"events": [], "next_cursor": None}), + (["sessions", "export", "s1"], "GET", "/sessions/s1/export", {"session_id": "s1", "events": []}), + (["sessions", "history", "s1"], "GET", "/sessions/s1/evaluation-history", []), + (["events", "latency"], "GET", "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/events/latency_aggregate", {"p95": 42}), + (["sentiment", "summary"], "GET", "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/events/sentiment", {"positive": 1}), + (["sentiment", "messages"], "GET", "/events/sentiment/messages", []), + (["sentiment", "sessions"], "GET", "/events/sentiment/sessions", []), + (["sentiment", "session", "s1"], "GET", "/events/sentiment/sessions/s1", {}), + (["query", "export", "q1", "--format", "json"], "POST", "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/queries/run", RESULT), + (["query", "export", "--sql", "SELECT 1", "--format", "csv"], "POST", "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/queries/run", RESULT), +] + + +@pytest.fixture +def files(tmp_path): + path = tmp_path / "layout.json" + path.write_text(json.dumps({"tiles": [{k: v for k, v in TILE.items() if k in ("id", "pos_x", "pos_y", "pos_w", "pos_h")}]})) + return {"layout": str(path)} + + +@pytest.mark.parametrize("key_mode", [False, True]) +@pytest.mark.parametrize("argv,method,path,response", CASES, ids=lambda v: " ".join(v) if isinstance(v, list) else str(v)) +def test_observability_commands(logged_in, runner, files, key_mode, argv, method, path, response): + prefix = "/v1" if key_mode else "/api" + with respx.mock(assert_all_called=False) as mock: + mock.get(f"{BASE}{prefix}/dashboards/d1").respond(200, json={"dashboard": DASHBOARD, "tiles": []}) + mock.get(f"{BASE}{prefix}/dashboards/d1/tiles").respond(200, json={"tiles": [TILE]}) + mock.get(f"{BASE}{prefix}/sessions").respond(200, json={"sessions": [{"session_id": "s1"}]}) + route = mock.request(method, f"{BASE}{prefix}{path}").respond(200, json=response) + result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), *[v.format(**files) for v in argv]]) + assert result.exit_code == 0, result.output + assert route.called + json.loads(result.stdout) + if argv[:2] == ["dashboards", "create"]: + assert json.loads(route.calls[0].request.content) == {"name": "Quality", "description": "", "definition": {}} + if argv[:2] == ["dashboards", "update"]: + assert json.loads(route.calls[0].request.content) == {**DASHBOARD, "name": "New"} + if argv[:4] == ["dashboards", "tiles", "d1", "update"]: + assert json.loads(route.calls[0].request.content) == {k: ("After" if k == "title" else v) for k, v in TILE.items() if k != "id"} + + +@pytest.mark.parametrize("argv,method,path,response", CASES) +@pytest.mark.parametrize("key_mode", [False, True]) +def test_observability_404(logged_in, runner, files, key_mode, argv, method, path, response): + with respx.mock as mock: + mock.route().respond(404, json={}) + result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), *[v.format(**files) for v in argv]]) + assert result.exit_code == 6, result.output + assert "not available on this FailproofAI Cloud yet" in result.stdout + + +def test_query_compose(logged_in, runner): + stream = 'data: {"type":"result","result":{"sql":"SELECT 1"}}\n\n' + with respx.mock as mock: + draft = mock.post(f"{BASE}/api/agent/compose-sql").respond(200, text=stream, headers={"content-type": "text/event-stream"}) + run = mock.post(f"{BASE}/api/queries/run").respond(200, json=RESULT) + result = runner.invoke(app, ["--json", "query", "compose", "Count events", "--run"]) + assert result.exit_code == 0, result.output + assert json.loads(draft.calls[0].request.content) == {"intent": "Count events"} + assert json.loads(run.calls[0].request.content) == {"sql": "SELECT 1", "params": []} + + +def test_query_compose_key_denial(logged_in, runner): + with respx.mock as mock: + result = runner.invoke(app, ["--json", "--api-key", "key", "query", "compose", "Count events"]) + assert not mock.calls + assert result.exit_code == 2 + assert "fp login" in result.stdout + + +def test_exports_write_files(logged_in, runner, tmp_path): + path = tmp_path / "data.csv" + with respx.mock as mock: + mock.post(f"{BASE}/api/queries/run").respond(200, json=RESULT) + result = runner.invoke(app, ["--json", "query", "export", "--sql", "SELECT 1", "--out", str(path)]) + assert result.exit_code == 0 + assert path.read_text().splitlines() == ["agent,n", "agent-001,42"] + + +@pytest.mark.parametrize("argv", [ + ["dashboards", "create", "--name", ""], + ["dashboards", "update", "d1"], + ["dashboards", "tiles", "d1", "add", "--query", "q1", "--chart", "invalid"], + ["sentiment", "messages", "--limit", "0"], + ["query", "export", "--sql", "SELECT 1", "--format", "xml"], + ["query", "export", "q1", "--sql", "SELECT 1"], +]) +def test_validation(logged_in, runner, argv): + with respx.mock as mock: + result = runner.invoke(app, ["--json", *argv]) + assert not mock.calls + assert result.exit_code == 2, result.output From 3fcf03b08f351b5ea73f447000df9aa0b5be56e8 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:56:09 +0000 Subject: [PATCH 04/16] feat(fp): render assistant artifacts and handle explicit write approvals --- fp-cloud-cli/fp_cli/commands/agent_cmds.py | 79 +++++++++- fp-cloud-cli/fp_cli/commands/policies_cmds.py | 44 +++++- fp-cloud-cli/tests/test_cloud_agent.py | 141 ++++++++++++++++++ 3 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 fp-cloud-cli/tests/test_cloud_agent.py diff --git a/fp-cloud-cli/fp_cli/commands/agent_cmds.py b/fp-cloud-cli/fp_cli/commands/agent_cmds.py index f5bcbd951..b0287a49a 100644 --- a/fp-cloud-cli/fp_cli/commands/agent_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/agent_cmds.py @@ -24,7 +24,7 @@ from .. import output from .._context import GLOBALS_EPILOG, AppState, deny_in_key_mode, require_auth from ..errors import NotFoundError -from . import _write +from . import _cloud, _write # The assistant has NO server route at all: `/agent/chat` and `/agent/health` are # implemented by the dashboard itself, and the conversation routes are per-human @@ -279,6 +279,7 @@ def agent_ask( chat: Optional[str] = typer.Option(None, "--chat", help="Continue this chat — its short id from `agent chats` (a prefix) or full id. Omit to start a NEW chat (its id is printed)."), model: Optional[str] = typer.Option(None, "--model", help="Model to use (see `agent models`)."), page_context: Optional[str] = typer.Option(None, "--page-context", help="Optional context string to ground the answer."), + approve: bool = typer.Option(False, "--approve", help="Explicitly approve requested writes. Without this, scripts/--json decline."), ) -> None: """Ask the assistant a question — every ask is saved to a chat. @@ -323,11 +324,36 @@ def agent_ask( ] thread.append({"role": "user", "content": {"text": message}}) + artifacts = [] + + def on_artifact(block): + rendered = run_artifact(cctx, block) + artifacts.append(rendered) + if not state.json: + render_artifact(rendered) + + def on_ask_user(event): + decision = "reject" + if event.get("kind") == "approval": + if approve: + decision = "approve" + elif not state.json and sys.stdin.isatty(): + output.info(str(event.get("question", "Approve this write?"))) + if event.get("details"): + output.info(str(event["details"])) + if typer.confirm("Approve this change", default=False, err=True): + decision = "approve" + answer = {"callId": event["callId"], "decision": decision} + api.cloud_request(cctx, "POST", "/api/agent/answer", body=answer) + if decision == "reject" and not state.json: + output.hint("Declined assistant request; no write approved.") + # The reply streams server-side (an LLM turn + tool calls can take many seconds); show a # themed spinner on stderr so the CLI doesn't look hung. No-op under --json / non-TTY. with output.thinking("thinking…", enabled=not state.json): result = api.agent_chat_oneshot( - cctx, messages=thread, conversation_id=full_chat_id, page_context=page_context, model=model + cctx, messages=thread, conversation_id=full_chat_id, page_context=page_context, model=model, + on_ask_user=on_ask_user, on_artifact=on_artifact, ) chat_id = full_chat_id @@ -337,7 +363,10 @@ def agent_ask( # create-with-empty-title then a follow-up rename PATCH. chat_id = (api.create_conversation(cctx, title=_title_from_question(message)) or {}).get("id") if chat_id: - full = thread + [{"role": "assistant", "content": {"text": result.get("answer", "")}}] + content = {"text": result.get("answer", "")} + if artifacts: + content["artifacts"] = [{k: v for k, v in a.items() if k != "result"} for a in artifacts] + full = thread + [{"role": "assistant", "content": content}] api.replace_messages(cctx, chat_id, full) if full_chat_id is not None and not prior: # Continuing a chat that had no messages yet → title it from this first question @@ -350,7 +379,7 @@ def agent_ask( mode="continue" if chat else "new", ) if state.json: - output.emit_json({**result, "chat_id": chat_id}) + output.emit_json({**result, "chat_id": chat_id, **({"artifacts": artifacts} if artifacts else {})}) else: for tool in result.get("tools", []): output.agent_tool_used(tool) @@ -404,4 +433,46 @@ def register(app: typer.Typer) -> None: agent.command("show", epilog=GLOBALS_EPILOG)(agent_show) agent.command("rename", epilog=GLOBALS_EPILOG)(agent_rename) agent.command("delete", epilog=GLOBALS_EPILOG)(agent_delete) + agent.command("brief", epilog=GLOBALS_EPILOG)(agent_brief) app.add_typer(agent, name="agent") + + +def run_artifact(cctx, block): + """Only execute through queries:run; never follow arbitrary URLs or actions.""" + rendered = dict(block) + sql = block.get("sql") + if sql: + import re + if not isinstance(sql, str) or not re.match(r"^\s*(select|with)\b", sql, re.I): + raise click.UsageError("Assistant artifact SQL must be SELECT or WITH") + rendered["result"] = api.cloud_request(cctx, "POST", "/api/queries/run", body={"sql": sql, "params": []}) + return rendered + + +def render_artifact(block): + data = block.get("result") + if isinstance(data, dict): + output._stdout.print(str(block.get("title", "Artifact")), markup=False) + output.print_table( + [c.get("name", "") if isinstance(c, dict) else str(c) for c in data.get("columns", [])], + data.get("rows", []), + ) + else: + _cloud.emit(type("HumanState", (), {"json": False})(), block, block.get("title", "Home")) + + +def agent_brief(ctx: typer.Context, range: str = typer.Option("24h", "--range")) -> None: + """Print the agent-curated home brief (24h or 7d). Session only.""" + _cloud.session_only(ctx.obj, "agent brief") + if range not in ("24h", "7d"): + raise click.UsageError("--range must be 24h or 7d") + cctx = require_auth(ctx.obj) + data = api.cloud_request(cctx, "POST", "/api/agent/home-brief", body={"range": range}) + blocks = [run_artifact(cctx, block) for block in data.get("blocks", [])] + if ctx.obj.json: + output.emit_json({**data, "blocks": blocks}) + else: + output.info(str(data.get("greeting", ""))) + output.info(str(data.get("summary", ""))) + for block in blocks: + render_artifact(block) diff --git a/fp-cloud-cli/fp_cli/commands/policies_cmds.py b/fp-cloud-cli/fp_cli/commands/policies_cmds.py index bc13b4161..c054879a9 100644 --- a/fp-cloud-cli/fp_cli/commands/policies_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/policies_cmds.py @@ -21,7 +21,7 @@ from ..enforcement import RefError, RefUsageError, read_source from ..policy_check import check_syntax, run_policy from ..errors import ApiError, NotFoundError -from . import _write +from . import _cloud, _write #: Every command here is session-only. These endpoints are ROOT-ONLY on the #: server — deliberately absent from `/v1`, because `/v1` is internet-facing and @@ -499,4 +499,46 @@ def register(app: typer.Typer) -> None: policies_app.command("delete", epilog=GLOBALS_EPILOG)(policies_delete) policies_app.command("test", epilog=GLOBALS_EPILOG)(policies_test) policies_app.command("compose", epilog=GLOBALS_EPILOG)(policies_compose) + policies_app.command("author", epilog=GLOBALS_EPILOG)(policies_author) + policies_app.command("backtest", epilog=GLOBALS_EPILOG)(policies_backtest) + policies_app.command("candidacy", epilog=GLOBALS_EPILOG)(policies_candidacy) app.add_typer(policies_app, name="policies") + + +def policies_author(ctx: typer.Context, intent: str = typer.Argument(...)) -> None: + """Author a vetted policy from an intent (session; does not publish).""" + _cloud.session_only(ctx.obj, "policies author") + events = list(api.cloud_stream(require_auth(ctx.obj), "/api/agent/policy-author", + {"issue": {"title": intent, "source": "manual"}})) + for event in events: + if event.get("type") == "error": + raise ApiError(str(event.get("message") or event.get("error") or "Policy author failed")) + result = next((e.get("result", e) for e in reversed(events) if e.get("type") == "result"), None) + if result is None: + raise ApiError("Policy author returned no result; nothing was published") + _cloud.emit(ctx.obj, result, "Policy draft") + + +def policies_backtest(ctx: typer.Context, file_or_id: str = typer.Argument(..., metavar="FILE_OR_ID")) -> None: + """Replay local or published policy source against stored events (session).""" + from pathlib import Path + _cloud.session_only(ctx.obj, "policies backtest") + cctx = require_auth(ctx.obj) + if file_or_id == "-" or file_or_id.startswith("@") or Path(file_or_id).is_file(): + source = _write.read_text_arg(file_or_id.removeprefix("@")) + else: + row = api.cloud_request(cctx, "GET", f"/api/enforcement/policies/{file_or_id}") + source = row.get("source", "") + if not isinstance(source, str) or not source.strip(): + raise click.UsageError("Policy source is empty") + _cloud.emit(ctx.obj, api.cloud_request(cctx, "POST", "/api/agent/policy-backtest", + body={"source": source}), "Policy backtest") + + +def policies_candidacy(ctx: typer.Context, issue: str = typer.Option(..., "--issue")) -> None: + """Ask whether a policy could address an issue (session; read-only).""" + _cloud.session_only(ctx.obj, "policies candidacy") + cctx = require_auth(ctx.obj) + row = api.cloud_request(cctx, "GET", f"/api/issues/{issue}") + _cloud.emit(ctx.obj, api.cloud_request(cctx, "POST", "/api/agent/policy-candidacy", + body={"issue": row}), "Policy candidacy") diff --git a/fp-cloud-cli/tests/test_cloud_agent.py b/fp-cloud-cli/tests/test_cloud_agent.py new file mode 100644 index 000000000..e074dd0e7 --- /dev/null +++ b/fp-cloud-cli/tests/test_cloud_agent.py @@ -0,0 +1,141 @@ +import json + +import pytest +import respx + +from fp_cli.app import app + +BASE = "http://dash.test" +RESULT = {"columns": [{"name": "total", "type": "UInt64"}], "rows": [[42]]} + + +def sse(events): + return "".join("data: " + json.dumps(e) + "\n\n" for e in events) + + +@pytest.mark.parametrize("json_mode,approve,tty,input_text,decision", [ + (True, False, False, "", "reject"), + (True, True, False, "", "approve"), + (False, False, False, "", "reject"), + (False, False, True, "y\n", "approve"), + (False, False, True, "n\n", "reject"), +]) +def test_ask_artifacts_and_approvals(logged_in, runner, monkeypatch, json_mode, approve, tty, input_text, decision): + events = [ + {"type": "ui-action", "action": "open_artifact", "artifact": {"title": "Event total", "sql": "SELECT count()"}}, + {"type": "ask-user", "callId": "ask_1", "kind": "approval", "question": "Disable alert?", "details": "alert a1"}, + {"type": "ask-user-answered", "callId": "ask_1", "decision": decision}, + {"type": "text-delta", "text": "Here is the result."}, + {"type": "done"}, + ] + # Patch the exact guard used by the command; CliRunner's stdin is otherwise non-TTY. + import sys + monkeypatch.setattr(sys.stdin, "isatty", lambda: tty) + # Runner replaces stdin during invocation, so patch the module's sys proxy instead. + from fp_cli.commands import agent_cmds + class Stdin: + def isatty(self): + return tty + class Sys: + stdin = Stdin() + stdout = sys.stdout + monkeypatch.setattr(agent_cmds, "sys", Sys()) + with respx.mock as mock: + stream = mock.post(f"{BASE}/api/agent/chat").respond(200, text=sse(events), headers={"content-type": "text/event-stream"}) + query = mock.post(f"{BASE}/api/queries/run").respond(200, json=RESULT) + answer = mock.post(f"{BASE}/api/agent/answer").respond(204) + mock.post(f"{BASE}/api/agent/conversations").respond(200, json={"id": "c1"}) + saved = mock.put(f"{BASE}/api/agent/conversations/c1/messages").respond(204) + result = runner.invoke(app, [*(["--json"] if json_mode else []), "agent", "ask", "Count events", + *(["--approve"] if approve else [])], input=input_text) + assert result.exit_code == 0, result.output + assert json.loads(answer.calls[0].request.content) == {"callId": "ask_1", "decision": decision} + assert query.called + assert json.loads(stream.calls[0].request.content)["pageActions"] == [{"action": "open_artifact"}, {"action": "home"}] + assert json.loads(saved.calls[0].request.content)["messages"][-1]["content"]["artifacts"][0]["sql"] == "SELECT count()" + if json_mode: + data = json.loads(result.stdout) + assert data["answer"] == "Here is the result." + assert data["artifacts"][0]["result"] == RESULT + else: + assert "Event total" in result.output + assert "42" in result.output + if tty: + assert "alert a1" in result.output + assert "Approve this change" in result.output + + +def test_home_blocks(logged_in, runner): + stream = sse([{"type": "ui-action", "action": "home", "home": {"op": "replace", "blocks": [ + {"key": "n", "kind": "stats", "title": "Count", "sql": "SELECT 42"}, + {"key": "note", "kind": "note", "title": "Summary", "text": "All good"}, + ]}}, {"type": "done"}]) + with respx.mock as mock: + mock.post(f"{BASE}/api/agent/chat").respond(200, text=stream) + query = mock.post(f"{BASE}/api/queries/run").respond(200, json=RESULT) + mock.post(f"{BASE}/api/agent/conversations").respond(200, json={"id": "c1"}) + mock.put(f"{BASE}/api/agent/conversations/c1/messages").respond(204) + result = runner.invoke(app, ["--json", "agent", "ask", "Summarize"]) + assert result.exit_code == 0, result.output + assert query.call_count == 1 + assert len(json.loads(result.stdout)["artifacts"]) == 2 + + +@pytest.mark.parametrize("json_mode", [False, True]) +def test_brief(logged_in, runner, json_mode): + with respx.mock as mock: + route = mock.post(f"{BASE}/api/agent/home-brief").respond(200, json={ + "greeting": "Hello", "summary": "Healthy", "blocks": [{"title": "Events", "sql": "SELECT count()"}]}) + mock.post(f"{BASE}/api/queries/run").respond(200, json=RESULT) + result = runner.invoke(app, [*(["--json"] if json_mode else []), "agent", "brief", "--range", "7d"]) + assert result.exit_code == 0, result.output + assert "Healthy" in result.output + assert json.loads(route.calls[0].request.content) == {"range": "7d"} + + +@pytest.mark.parametrize("argv,path,response", [ + (["policies", "author", "Prevent force push"], "/api/agent/policy-author", "stream"), + (["policies", "backtest", "{file}"], "/api/agent/policy-backtest", {"matched": 3}), + (["policies", "candidacy", "--issue", "i1"], "/api/agent/policy-candidacy", {"candidate": True}), +]) +def test_policy_commands(logged_in, runner, tmp_path, argv, path, response): + file = tmp_path / "policy.mjs" + file.write_text("export default {}") + with respx.mock(assert_all_called=False) as mock: + mock.get(f"{BASE}/api/issues/i1").respond(200, json={"id": "i1", "title": "Force pushes"}) + if response == "stream": + route = mock.post(f"{BASE}{path}").respond(200, text=sse([{"type": "result", "result": {"source": "policy"}}])) + else: + route = mock.post(f"{BASE}{path}").respond(200, json=response) + result = runner.invoke(app, ["--json", *[v.format(file=file) for v in argv]]) + assert result.exit_code == 0, result.output + assert route.called + json.loads(result.stdout) + if argv[1] == "author": + assert json.loads(route.calls[0].request.content)["issue"]["title"] == "Prevent force push" + elif argv[1] == "backtest": + assert json.loads(route.calls[0].request.content) == {"source": "export default {}"} + + +@pytest.mark.parametrize("argv", [ + ["policies", "author", "Prevent force push"], ["policies", "backtest", "p1"], + ["policies", "candidacy", "--issue", "i1"], ["agent", "brief"], +]) +def test_session_only(logged_in, runner, argv): + with respx.mock as mock: + result = runner.invoke(app, ["--json", "--api-key", "key", *argv]) + assert not mock.calls + assert result.exit_code == 2 + assert "needs a signed-in session" in result.stdout + + +@pytest.mark.parametrize("argv", [ + ["policies", "author", "Prevent force push"], ["policies", "backtest", "p1"], + ["policies", "candidacy", "--issue", "i1"], ["agent", "brief"], ["agent", "ask", "Hello"], +]) +def test_404(logged_in, runner, argv): + with respx.mock as mock: + mock.route().respond(404, json={}) + result = runner.invoke(app, ["--json", *argv]) + assert result.exit_code == 6, result.output + assert "not available on this FailproofAI Cloud yet" in result.stdout From 6cc54b593844054bca658a12f9d1c2eab2f60452 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:56:09 +0000 Subject: [PATCH 05/16] feat(fp): add account, integration, alert and audit contract controls --- fp-cloud-cli/fp_cli/app.py | 2 + fp-cloud-cli/fp_cli/client.py | 2 + fp-cloud-cli/fp_cli/commands/account_cmds.py | 169 ++++++++++++++++++ fp-cloud-cli/fp_cli/commands/alerts_cmds.py | 23 +++ fp-cloud-cli/fp_cli/commands/audits_cmds.py | 2 + .../fp_cli/commands/contracts_cmds.py | 67 +++++++ fp-cloud-cli/fp_cli/commands/orgs_cmds.py | 17 ++ fp-cloud-cli/fp_cli/commands/queries_cmds.py | 2 +- fp-cloud-cli/fp_cli/commands/settings_cmds.py | 2 + fp-cloud-cli/fp_cli/commands/usage_cmds.py | 13 +- fp-cloud-cli/fp_cli/commands/users_cmds.py | 14 ++ fp-cloud-cli/tests/test_cloud_account.py | 110 ++++++++++++ .../tests/test_cloud_observability.py | 2 +- 13 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 fp-cloud-cli/fp_cli/commands/account_cmds.py create mode 100644 fp-cloud-cli/fp_cli/commands/contracts_cmds.py create mode 100644 fp-cloud-cli/tests/test_cloud_account.py diff --git a/fp-cloud-cli/fp_cli/app.py b/fp-cloud-cli/fp_cli/app.py index 202e27fdb..82ce78c9d 100644 --- a/fp-cloud-cli/fp_cli/app.py +++ b/fp-cloud-cli/fp_cli/app.py @@ -17,6 +17,7 @@ from ._context import AppState from ._version import __version__ from .commands import ( + account_cmds, agent_cmds, alerts_cmds, audits_cmds, @@ -472,6 +473,7 @@ def help_cmd(ctx: typer.Context) -> None: policies_cmds.register(app) fleet_cmds.register(app) guardrails_cmds.register(app) +account_cmds.register(app) def _elapsed_ms(start: float) -> int: diff --git a/fp-cloud-cli/fp_cli/client.py b/fp-cloud-cli/fp_cli/client.py index 1882f5885..4cea22366 100644 --- a/fp-cloud-cli/fp_cli/client.py +++ b/fp-cloud-cli/fp_cli/client.py @@ -181,6 +181,8 @@ class ClientContext: "cloud-managed policies are an operator surface — the fleet routes are " "deliberately absent from /v1, which is internet-facing" ), + "integrations": "dashboard integration routes need a signed-in session", + "orgs": "organization creation needs a signed-in user", } diff --git a/fp-cloud-cli/fp_cli/commands/account_cmds.py b/fp-cloud-cli/fp_cli/commands/account_cmds.py new file mode 100644 index 000000000..61e5bbbe7 --- /dev/null +++ b/fp-cloud-cli/fp_cli/commands/account_cmds.py @@ -0,0 +1,169 @@ +"""Additive account, billing and integration commands.""" + +from __future__ import annotations + +from typing import List, Optional + +import typer + +from .. import _click_compat as click, client as api, permissions +from .._context import GLOBALS_EPILOG, require_auth +from . import _cloud, _write + + +def request(ctx, method, path, body=None, params=None): + if method != "GET": + _write.confirm(ctx.obj, f"{method} {path}", assume_yes=False, destructive=method == "DELETE") + result = api.cloud_request(require_auth(ctx.obj), method, path, body=body, params=params) + if method != "GET": + _write.record_action("cloud_resource_changed", resource=path.split("/")[2], success=True, destructive=method == "DELETE") + _cloud.emit(ctx.obj, result, "Cloud") + + +def roles_list(ctx: typer.Context) -> None: + """List named permission sets. Needs users:read.""" + request(ctx, "GET", "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/api/permission-sets") + + +def roles_show(ctx: typer.Context, name: str = typer.Argument(...)) -> None: + """Show one permission set by name.""" + request(ctx, "GET", f"/api/permission-sets/{name}") + + +def grants(values): + if not values: + return [] + try: + return permissions.parse_permission_tokens(values) + except permissions.PermissionTokenError as exc: + raise click.UsageError(str(exc)) from exc + + +def roles_create(ctx: typer.Context, name: str = typer.Option(..., "--name"), + permission: Optional[List[str]] = typer.Option(None, "--permission", "--add"), + description: Optional[str] = typer.Option(None, "--description"), + file: Optional[str] = typer.Option(None, "--file")) -> None: + """Create a custom permission set. Needs settings:write.""" + if not 1 <= len(name.strip()) <= 64: + raise click.UsageError("Role name must be 1–64 characters") + body = _write.read_body(ctx.obj, file, {"permissions": grants(permission) if permission is not None else None, "description": description}) + body["name"] = name.strip() + body.setdefault("permissions", []) + if not isinstance(body["permissions"], list): + raise click.UsageError("permissions must be an array") + body["permissions"] = grants(body["permissions"]) + request(ctx, "POST", "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/api/permission-sets", body) + + +def roles_update(ctx: typer.Context, name: str = typer.Argument(...), + permission: Optional[List[str]] = typer.Option(None, "--permission", help="Full replacement grants."), + add: Optional[List[str]] = typer.Option(None, "--add"), + remove: Optional[List[str]] = typer.Option(None, "--remove"), + description: Optional[str] = typer.Option(None, "--description"), + file: Optional[str] = typer.Option(None, "--file")) -> None: + """Update a custom set; affected members receive the new grants.""" + if permission is not None and (add or remove): + raise click.UsageError("Use --permission or --add/--remove, not both") + added, removed = grants(add or []), grants(remove or []) + body = _write.read_body(ctx.obj, file, {"permissions": grants(permission) if permission is not None else None, "description": description}) + if file and (add or remove): + raise click.UsageError("Use --file or discrete flags") + if not body and not added and not removed: + raise click.UsageError("Pass --permission, --add, --remove, --description or --file") + current = api.cloud_request(require_auth(ctx.obj), "GET", f"/api/permission-sets/{name}") + if current.get("is_builtin"): + raise click.UsageError("Builtin permission sets are immutable") + body.setdefault("permissions", list(dict.fromkeys([v for v in current.get("permissions", []) if v not in removed] + added))) + body["permissions"] = grants(body["permissions"]) + body.setdefault("description", current.get("description")) + request(ctx, "PUT", f"/api/permission-sets/{name}", body) + + +def roles_delete(ctx: typer.Context, name: str = typer.Argument(...)) -> None: + """Delete a custom permission set; members keep materialized permissions.""" + request(ctx, "DELETE", f"/api/permission-sets/{name}") + + +def plan(ctx: typer.Context) -> None: + """Show current offers and team-trial state.""" + if ctx.invoked_subcommand: + return + cctx = require_auth(ctx.obj) + data = {"offers": api.cloud_request(cctx, "GET", "/api/offers/me"), + "team_trial": api.cloud_request(cctx, "GET", "/api/team-trial/me")} + _cloud.emit(ctx.obj, data, "Plan") + + +def claim(ctx: typer.Context, offer: str = typer.Argument(...)) -> None: + """Claim a named offer.""" + request(ctx, "POST", "/api/offers/claim", {"offer_key": offer}) + + +def otel_status(ctx: typer.Context) -> None: + """Show Cloud OTEL ingest status.""" + request(ctx, "GET", "/api/otel/status") + + +def otel_enable(ctx: typer.Context) -> None: + """Enable OTEL ingestion for the Cloud org.""" + request(ctx, "PUT", "/api/settings/otel.ingest_enabled", {"value": True}) + + +def otel_disable(ctx: typer.Context) -> None: + """Disable OTEL ingestion for the Cloud org.""" + request(ctx, "PUT", "/api/settings/otel.ingest_enabled", {"value": False}) + + +def slack_status(ctx: typer.Context) -> None: + """Show Slack connection status (session only).""" + _cloud.session_only(ctx.obj, "slack status") + request(ctx, "GET", "/api/integrations/slack") + + +def slack_disconnect(ctx: typer.Context) -> None: + """Disconnect Slack (session only).""" + _cloud.session_only(ctx.obj, "slack disconnect") + request(ctx, "DELETE", "/api/integrations/slack") + + +def context_list(ctx: typer.Context) -> None: + """List model context-window overrides.""" + request(ctx, "GET", "/api/settings/model-context-windows") + + +def context_set(ctx: typer.Context, model: str = typer.Argument(...), + tokens: int = typer.Argument(...)) -> None: + """Set an override; zero means unknown.""" + if not 1 <= len(model.strip()) <= 256 or not 0 <= tokens <= 1_000_000: + raise click.UsageError("Model must be 1–256 characters; tokens must be 0–1000000") + request(ctx, "PUT", "/api/settings/model-context-windows", {"model_id": model.strip(), "context_window": tokens}) + + +def context_delete(ctx: typer.Context, model: str = typer.Argument(...)) -> None: + """Remove a model override.""" + if not model.strip(): + raise click.UsageError("Model cannot be empty") + request(ctx, "DELETE", "/api/settings/model-context-windows", params={"model": model}) + + +def register_context_windows(settings_app): + group = typer.Typer(no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]}) + for name, fn in (("list", context_list), ("set", context_set), ("delete", context_delete)): + group.command(name, epilog=GLOBALS_EPILOG)(fn) + settings_app.add_typer(group, name="context-windows") + + +def register(app): + for name, help_text, commands in ( + ("roles", "Manage permission sets.", (("list", roles_list), ("show", roles_show), ("create", roles_create), ("update", roles_update), ("delete", roles_delete))), + ("otel", "Control Cloud OTEL ingest.", (("status", otel_status), ("enable", otel_enable), ("disable", otel_disable))), + ("slack", "Manage the Slack integration (session only).", (("status", slack_status), ("disconnect", slack_disconnect))), + ): + group = typer.Typer(no_args_is_help=True, help=help_text, context_settings={"help_option_names": ["-h", "--help"]}) + for command, fn in commands: + group.command(command, epilog=GLOBALS_EPILOG)(fn) + app.add_typer(group, name=name) + group = typer.Typer(invoke_without_command=True, context_settings={"help_option_names": ["-h", "--help"]}) + group.callback(epilog=GLOBALS_EPILOG)(plan) + group.command("claim", epilog=GLOBALS_EPILOG)(claim) + app.add_typer(group, name="plan") diff --git a/fp-cloud-cli/fp_cli/commands/alerts_cmds.py b/fp-cloud-cli/fp_cli/commands/alerts_cmds.py index 379a87d28..d4edeff82 100644 --- a/fp-cloud-cli/fp_cli/commands/alerts_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/alerts_cmds.py @@ -442,4 +442,27 @@ def register(app: typer.Typer) -> None: alerts_app.command("update", epilog=GLOBALS_EPILOG)(alerts_update) alerts_app.command("delete", epilog=GLOBALS_EPILOG)(alerts_delete) alerts_app.command("test", epilog=GLOBALS_EPILOG)(alerts_test) + alerts_app.command("recipients", epilog=GLOBALS_EPILOG)(alerts_recipients) + alerts_app.command("issues", epilog=GLOBALS_EPILOG)(alerts_issues) app.add_typer(alerts_app, name="alerts") + + +def alerts_recipients(ctx: typer.Context) -> None: + """List available alert recipients.""" + from .account_cmds import request + request(ctx, "GET", "/api/alerts/recipients") + + +def alerts_issues(ctx: typer.Context, alert_id: str = typer.Argument(...), + create: bool = typer.Option(False, "--create"), + title: Optional[str] = typer.Option(None, "--title"), + summary: Optional[str] = typer.Option(None, "--summary"), + file: Optional[str] = typer.Option(None, "--file")) -> None: + """List alert issues, or --create one with --title/--summary or --file.""" + from .account_cmds import request + if not create and any(v is not None for v in (title, summary, file)): + raise click.UsageError("Pass --create to use issue body flags") + body = _write.read_body(ctx.obj, file, {"title": title, "summary": summary}) if create else None + if create and (not isinstance(body.get("title"), str) or not body["title"].strip()): + raise click.UsageError("Creating an issue needs --title or a title in --file") + request(ctx, "POST" if create else "GET", f"/api/alerts/{alert_id}/issues", body) diff --git a/fp-cloud-cli/fp_cli/commands/audits_cmds.py b/fp-cloud-cli/fp_cli/commands/audits_cmds.py index 9a642cfb3..ce4d76b44 100644 --- a/fp-cloud-cli/fp_cli/commands/audits_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/audits_cmds.py @@ -1051,4 +1051,6 @@ def register(app: typer.Typer) -> None: audits_app.command("resolve", epilog=GLOBALS_EPILOG)(findings_resolve) audits_app.command("reopen", epilog=GLOBALS_EPILOG)(findings_reopen) audits_app.command("assign", epilog=GLOBALS_EPILOG)(findings_assign) + from . import contracts_cmds + contracts_cmds.register(audits_app) app.add_typer(audits_app, name="audits") diff --git a/fp-cloud-cli/fp_cli/commands/contracts_cmds.py b/fp-cloud-cli/fp_cli/commands/contracts_cmds.py new file mode 100644 index 000000000..adfc5a419 --- /dev/null +++ b/fp-cloud-cli/fp_cli/commands/contracts_cmds.py @@ -0,0 +1,67 @@ +"""Org-wide agent contracts, alongside scheduled audits.""" + +from __future__ import annotations + +from typing import Optional + +import typer + +from .. import _click_compat as click, client as api +from .._context import GLOBALS_EPILOG, require_auth +from . import _write +from .account_cmds import request + +PATH = "/api/audits/contracts" + + +def list_contracts(ctx: typer.Context) -> None: + """List agent contracts.""" + request(ctx, "GET", PATH) + + +def show(ctx: typer.Context, id: str = typer.Argument(...)) -> None: + """Show one contract, including its Markdown.""" + request(ctx, "GET", f"{PATH}/{id}") + + +def validate(body): + if set(body) - {"agent_id", "body", "enabled"}: + raise click.UsageError("Contracts accept agent_id, body and enabled") + if not isinstance(body.get("agent_id"), str) or not body["agent_id"].strip(): + raise click.UsageError("Contract needs an agent_id from your telemetry") + if not isinstance(body.get("body"), str) or not body["body"].strip() or len(body["body"]) > 5000 or "\0" in body["body"]: + raise click.UsageError("Contract body must be 1–5000 characters without NUL") + if "enabled" in body and not isinstance(body["enabled"], bool): + raise click.UsageError("Contract enabled must be boolean") + + +def create(ctx: typer.Context, agent_id: Optional[str] = typer.Option(None, "--agent-id"), + body: Optional[str] = typer.Option(None, "--body", help="Markdown or @file."), + file: Optional[str] = typer.Option(None, "--file", help="Contract API body JSON.")) -> None: + """Write an agent contract.""" + data = _write.read_body(ctx.obj, file, {"agent_id": agent_id, "body": _write.read_text_arg(body[1:]) if body and body.startswith("@") else body}) + validate(data) + request(ctx, "POST", PATH, data) + + +def update(ctx: typer.Context, id: str = typer.Argument(...), + agent_id: Optional[str] = typer.Option(None, "--agent-id"), + body: Optional[str] = typer.Option(None, "--body"), + enabled: Optional[bool] = typer.Option(None, "--enabled/--disabled"), + file: Optional[str] = typer.Option(None, "--file")) -> None: + """Update selected fields, preserving the rest.""" + changes = _write.read_body(ctx.obj, file, {"agent_id": agent_id, "body": _write.read_text_arg(body[1:]) if body and body.startswith("@") else body, "enabled": enabled}) + if not changes: + raise click.UsageError("Pass --agent-id, --body, --enabled/--disabled or --file") + current = api.cloud_request(require_auth(ctx.obj), "GET", f"{PATH}/{id}") + data = {k: current[k] for k in ("agent_id", "body", "enabled") if k in current} + data.update(changes) + validate(data) + request(ctx, "PUT", f"{PATH}/{id}", data) + + +def register(audits_app): + group = typer.Typer(no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]}) + for name, fn in (("list", list_contracts), ("show", show), ("create", create), ("update", update)): + group.command(name, epilog=GLOBALS_EPILOG)(fn) + audits_app.add_typer(group, name="contracts") diff --git a/fp-cloud-cli/fp_cli/commands/orgs_cmds.py b/fp-cloud-cli/fp_cli/commands/orgs_cmds.py index 683b92739..a994c4a4a 100644 --- a/fp-cloud-cli/fp_cli/commands/orgs_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/orgs_cmds.py @@ -316,4 +316,21 @@ def register(app: typer.Typer) -> None: orgs_app.command("switch", epilog=GLOBALS_EPILOG)(org_switch) orgs_app.command("current", epilog=GLOBALS_EPILOG)(org_current) orgs_app.command("perms", epilog=GLOBALS_EPILOG)(org_perms) + orgs_app.command("create", epilog=GLOBALS_EPILOG)(org_create) app.add_typer(orgs_app, name="orgs") + + +def org_create(ctx: typer.Context, name: str = typer.Option(..., "--name"), + slug: Optional[str] = typer.Option(None, "--slug")) -> None: + """Create an organization (session only; does not switch the saved org).""" + from . import _cloud + from .account_cmds import request + from .. import _click_compat as click + _cloud.session_only(ctx.obj, "orgs create") + if not name.strip(): + raise click.UsageError("--name cannot be empty") + import re + slug = slug or re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", slug): + raise click.UsageError("--slug must be lowercase letters/digits separated by hyphens") + request(ctx, "POST", "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/api/orgs", {"name": name, "slug": slug}) diff --git a/fp-cloud-cli/fp_cli/commands/queries_cmds.py b/fp-cloud-cli/fp_cli/commands/queries_cmds.py index efd5f42b6..91941a85e 100644 --- a/fp-cloud-cli/fp_cli/commands/queries_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/queries_cmds.py @@ -391,7 +391,7 @@ def query_compose(ctx: typer.Context, question: str = typer.Argument(...), if event.get("type") == "error": from ..errors import ApiError raise ApiError(str(event.get("error") or event.get("reason") or "SQL composer failed")) - if event.get("type") == "result": + if event.get("type") in ("result", "done"): draft = event.get("result", event) if not isinstance(draft, dict): raise click.UsageError("Cloud did not return composed SQL") diff --git a/fp-cloud-cli/fp_cli/commands/settings_cmds.py b/fp-cloud-cli/fp_cli/commands/settings_cmds.py index dd65ea4a4..cc2257f41 100644 --- a/fp-cloud-cli/fp_cli/commands/settings_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/settings_cmds.py @@ -147,4 +147,6 @@ def register(app: typer.Typer) -> None: settings_app.command("list", epilog=GLOBALS_EPILOG)(settings_list) settings_app.command("schema", epilog=GLOBALS_EPILOG)(settings_schema) settings_app.command("set", epilog=GLOBALS_EPILOG)(settings_set) + from .account_cmds import register_context_windows + register_context_windows(settings_app) app.add_typer(settings_app, name="settings") diff --git a/fp-cloud-cli/fp_cli/commands/usage_cmds.py b/fp-cloud-cli/fp_cli/commands/usage_cmds.py index d3a946456..0217a027a 100644 --- a/fp-cloud-cli/fp_cli/commands/usage_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/usage_cmds.py @@ -23,6 +23,8 @@ def usage(ctx: typer.Context) -> None: * `fp usage` * `fp --json usage | jq '.usage.events_ingested'` """ + if ctx.invoked_subcommand: + return state: AppState = ctx.obj data = api.get_usage(require_auth(state)) if state.json: @@ -32,4 +34,13 @@ def usage(ctx: typer.Context) -> None: def register(app: typer.Typer) -> None: - app.command("usage", epilog=GLOBALS_EPILOG)(usage) + group = typer.Typer(invoke_without_command=True, rich_markup_mode="markdown", context_settings={"help_option_names": ["-h", "--help"]}) + group.callback(epilog=GLOBALS_EPILOG)(usage) + group.command("windows", epilog=GLOBALS_EPILOG)(usage_windows) + app.add_typer(group, name="usage") + + +def usage_windows(ctx: typer.Context) -> None: + """List usage metering windows.""" + from .account_cmds import request + request(ctx, "GET", "/api/usage/windows") diff --git a/fp-cloud-cli/fp_cli/commands/users_cmds.py b/fp-cloud-cli/fp_cli/commands/users_cmds.py index a51b1e0db..bc4aa7f55 100644 --- a/fp-cloud-cli/fp_cli/commands/users_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/users_cmds.py @@ -401,4 +401,18 @@ def register(app: typer.Typer) -> None: users_app.command("update", epilog=GLOBALS_EPILOG)(users_update) users_app.command("disable", epilog=GLOBALS_EPILOG)(users_disable) users_app.command("enable", epilog=GLOBALS_EPILOG)(users_enable) + users_app.command("delete", epilog=GLOBALS_EPILOG)(users_delete) + users_app.command("defaults", epilog=GLOBALS_EPILOG)(users_defaults) app.add_typer(users_app, name="users") + + +def users_delete(ctx: typer.Context, id: str = typer.Argument(...)) -> None: + """Delete a member by id (not email).""" + from .account_cmds import request + request(ctx, "DELETE", f"/api/users/{id}") + + +def users_defaults(ctx: typer.Context) -> None: + """Show defaults for new members.""" + from .account_cmds import request + request(ctx, "GET", "/api/users/defaults") diff --git a/fp-cloud-cli/tests/test_cloud_account.py b/fp-cloud-cli/tests/test_cloud_account.py new file mode 100644 index 000000000..dc27c2961 --- /dev/null +++ b/fp-cloud-cli/tests/test_cloud_account.py @@ -0,0 +1,110 @@ +import json + +import pytest +import respx + +from fp_cli.app import app + +BASE = "http://dash.test" +CONTRACT = {"agent_id": "agent-1", "body": "# Purpose\nResolve requests", "enabled": True} +CASES = [ + (["roles", "list"], "GET", "/permission-sets", []), + (["roles", "show", "reviewer"], "GET", "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/permission-sets/reviewer", {"name": "reviewer", "permissions": ["events:read"]}), + (["roles", "create", "--name", "reviewer", "--permission", "events:read"], "POST", "/permission-sets", {"name": "reviewer"}), + (["roles", "update", "reviewer", "--add", "queries:run"], "PUT", "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/permission-sets/reviewer", {"name": "reviewer"}), + (["roles", "delete", "reviewer"], "DELETE", "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/permission-sets/reviewer", {"deleted": True}), + (["users", "delete", "u1"], "DELETE", "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/users/u1", {"deleted": True}), + (["users", "defaults"], "GET", "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/users/defaults", {}), + (["usage", "windows"], "GET", "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/usage/windows", []), + (["plan"], "GET", "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/offers/me", []), + (["plan", "claim", "team_offer"], "POST", "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/offers/claim", {"claimed": True}), + (["settings", "context-windows", "list"], "GET", "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/settings/model-context-windows", {}), + (["settings", "context-windows", "set", "provider/model", "64000"], "PUT", "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/settings/model-context-windows", {}), + (["settings", "context-windows", "delete", "provider/model"], "DELETE", "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/settings/model-context-windows", {}), + (["otel", "status"], "GET", "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/otel/status", {}), + (["otel", "enable"], "PUT", "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/settings/otel.ingest_enabled", {"value": True}), + (["otel", "disable"], "PUT", "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/settings/otel.ingest_enabled", {"value": False}), + (["alerts", "recipients"], "GET", "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/alerts/recipients", []), + (["alerts", "issues", "a1"], "GET", "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/alerts/a1/issues", []), + (["alerts", "issues", "a1", "--create", "--title", "Failure"], "POST", "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/alerts/a1/issues", {"id": "i1"}), + (["audits", "contracts", "list"], "GET", "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/audits/contracts", []), + (["audits", "contracts", "show", "c1"], "GET", "/audits/contracts/c1", CONTRACT), + (["audits", "contracts", "create", "--agent-id", "agent-1", "--body", "# Purpose\nResolve requests"], "POST", "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/audits/contracts", CONTRACT), + (["audits", "contracts", "update", "c1", "--body", "# New purpose"], "PUT", "/audits/contracts/c1", CONTRACT), +] +SESSION = [ + (["orgs", "create", "--name", "My Team"], "POST", "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/orgs", {"id": "o1"}), + (["slack", "status"], "GET", "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/integrations/slack", {"connected": True}), + (["slack", "disconnect"], "DELETE", "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/integrations/slack", {"disconnected": True}), +] + + +@pytest.mark.parametrize("key_mode", [False, True]) +@pytest.mark.parametrize("argv,method,path,response", CASES) +def test_commands(logged_in, runner, key_mode, argv, method, path, response): + prefix = "/v1" if key_mode else "/api" + with respx.mock(assert_all_called=False) as mock: + mock.get(f"{BASE}{prefix}/permission-sets/reviewer").respond(200, json={"name": "reviewer", "permissions": ["events:read"], "description": "Keep"}) + mock.get(f"{BASE}{prefix}/audits/contracts/c1").respond(200, json=CONTRACT) + mock.get(f"{BASE}{prefix}/team-trial/me").respond(200, json={}) + route = mock.request(method, f"{BASE}{prefix}{path}").respond(200, json=response) + result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), *argv]) + assert result.exit_code == 0, result.output + assert route.called + json.loads(result.stdout) + if argv[:2] == ["roles", "create"]: + assert json.loads(route.calls[0].request.content) == {"name": "reviewer", "permissions": ["events:read"]} + if argv[:2] == ["roles", "update"]: + assert json.loads(route.calls[0].request.content) == {"permissions": ["events:read", "queries:run"], "description": "Keep"} + if argv[:2] == ["plan", "claim"]: + assert json.loads(route.calls[0].request.content) == {"offer_key": "team_offer"} + if argv[:3] == ["settings", "context-windows", "set"]: + assert json.loads(route.calls[0].request.content) == {"model_id": "provider/model", "context_window": 64000} + if argv[:3] == ["settings", "context-windows", "delete"]: + assert route.calls[0].request.url.params["model"] == "provider/model" + if argv[:3] == ["audits", "contracts", "update"]: + assert json.loads(route.calls[0].request.content) == {**CONTRACT, "body": "# New purpose"} + + +@pytest.mark.parametrize("argv,method,path,response", SESSION) +def test_session_commands(logged_in, runner, argv, method, path, response): + with respx.mock as mock: + route = mock.request(method, f"{BASE}/api{path}").respond(200, json=response) + result = runner.invoke(app, ["--json", *argv]) + assert result.exit_code == 0, result.output + if path == "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/orgs": + assert json.loads(route.calls[0].request.content) == {"name": "My Team", "slug": "my-team"} + with respx.mock as mock: + result = runner.invoke(app, ["--json", "--api-key", "key", *argv]) + assert not mock.calls + assert result.exit_code == 2 + assert "needs a signed-in session" in result.stdout + + +@pytest.mark.parametrize("argv,method,path,response", CASES + SESSION) +def test_404(logged_in, runner, argv, method, path, response): + with respx.mock as mock: + mock.route().respond(404, json={}) + result = runner.invoke(app, ["--json", *argv]) + assert result.exit_code == 6, result.output + assert "not available on this FailproofAI Cloud yet" in result.stdout + + +@pytest.mark.parametrize("argv", [ + ["roles", "create", "--name", "", "--permission", "events:read"], + ["roles", "create", "--name", "x", "--permission", "unknown:read"], + ["roles", "update", "x"], + ["roles", "update", "x", "--permission", "events:read", "--add", "queries:run"], + ["settings", "context-windows", "set", "model", "-1"], + ["settings", "context-windows", "set", "model", "1000001"], + ["alerts", "issues", "a1", "--create"], + ["alerts", "issues", "a1", "--title", "No create"], + ["audits", "contracts", "create", "--agent-id", "a", "--body", ""], + ["audits", "contracts", "update", "c1"], + ["orgs", "create", "--name", ""], +]) +def test_validation(logged_in, runner, argv): + with respx.mock as mock: + result = runner.invoke(app, ["--json", *argv]) + assert not mock.calls + assert result.exit_code == 2, result.output diff --git a/fp-cloud-cli/tests/test_cloud_observability.py b/fp-cloud-cli/tests/test_cloud_observability.py index a966208d1..fb1e78b4f 100644 --- a/fp-cloud-cli/tests/test_cloud_observability.py +++ b/fp-cloud-cli/tests/test_cloud_observability.py @@ -75,7 +75,7 @@ def test_observability_404(logged_in, runner, files, key_mode, argv, method, pat def test_query_compose(logged_in, runner): - stream = 'data: {"type":"result","result":{"sql":"SELECT 1"}}\n\n' + stream = 'data: {"type":"done","sql":"SELECT 1"}\n\n' with respx.mock as mock: draft = mock.post(f"{BASE}/api/agent/compose-sql").respond(200, text=stream, headers={"content-type": "text/event-stream"}) run = mock.post(f"{BASE}/api/queries/run").respond(200, json=RESULT) From f302e6fa2b43558dd94fc97d450dfce193d6f0f5 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:17:19 +0000 Subject: [PATCH 06/16] fix(fp): align dashboard draft and policy payloads and reject failed previews --- fp-cloud-cli/fp_cli/client.py | 8 +++- fp-cloud-cli/fp_cli/commands/_cloud.py | 2 +- fp-cloud-cli/fp_cli/commands/_write.py | 4 +- fp-cloud-cli/fp_cli/commands/agent_cmds.py | 2 +- .../fp_cli/commands/evaluators_cmds.py | 27 ++++++++++- fp-cloud-cli/fp_cli/commands/policies_cmds.py | 11 +++-- fp-cloud-cli/fp_cli/evaluator_spec.py | 37 +++++++++++---- fp-cloud-cli/tests/test_cloud_agent.py | 14 +++++- .../tests/test_cloud_observability.py | 1 - fp-cloud-cli/tests/test_cloud_transport.py | 10 +++- .../tests/test_evaluator_authoring.py | 47 ++++++++++++++++++- 11 files changed, 138 insertions(+), 25 deletions(-) diff --git a/fp-cloud-cli/fp_cli/client.py b/fp-cloud-cli/fp_cli/client.py index 4cea22366..2e934deef 100644 --- a/fp-cloud-cli/fp_cli/client.py +++ b/fp-cloud-cli/fp_cli/client.py @@ -599,7 +599,9 @@ def cloud_request( if method == "GET": return _get_json(ctx, path, params) return _request_json(ctx, method, path, json_body=body, params=params) - except NotFoundError as exc: + except (NotFoundError, ApiError) as exc: + if isinstance(exc, ApiError) and exc.status != 404: + raise raise NotFoundError( "not available on this FailproofAI Cloud yet", hint="check the resource id or upgrade your Cloud deployment", @@ -611,7 +613,9 @@ def cloud_stream(ctx: ClientContext, path: str, body: Dict[str, Any]) -> Iterato """Dashboard SSE with the same additive-capability 404 contract.""" try: yield from _stream_sse(ctx, path, body) - except NotFoundError as exc: + except (NotFoundError, ApiError) as exc: + if isinstance(exc, ApiError) and exc.status != 404: + raise raise NotFoundError("not available on this FailproofAI Cloud yet", request_id=exc.request_id) from exc diff --git a/fp-cloud-cli/fp_cli/commands/_cloud.py b/fp-cloud-cli/fp_cli/commands/_cloud.py index 6d72afbd6..1de691fd6 100644 --- a/fp-cloud-cli/fp_cli/commands/_cloud.py +++ b/fp-cloud-cli/fp_cli/commands/_cloud.py @@ -30,7 +30,7 @@ def rows(data: Any, key: str) -> list: def emit(state, data: Any, title: str) -> None: - if state.json: + if state is not None and state.json: output.emit_json(data) return if isinstance(data, list) and all(isinstance(r, dict) for r in data): diff --git a/fp-cloud-cli/fp_cli/commands/_write.py b/fp-cloud-cli/fp_cli/commands/_write.py index c8e5fe579..cf8eeb871 100644 --- a/fp-cloud-cli/fp_cli/commands/_write.py +++ b/fp-cloud-cli/fp_cli/commands/_write.py @@ -89,7 +89,9 @@ def read_text_arg(path: str, *, flag: str = "--file") -> str: if path == "-": return sys.stdin.read() try: - return Path(path).read_text() + return Path(path).read_text(encoding="utf-8") + except UnicodeError: + raise click.BadParameter(f"cannot read '{path}': expected UTF-8 text", param_hint=flag) from None except OSError as exc: raise click.BadParameter( f"cannot read '{path}': {exc.strerror or exc}", param_hint=flag diff --git a/fp-cloud-cli/fp_cli/commands/agent_cmds.py b/fp-cloud-cli/fp_cli/commands/agent_cmds.py index b0287a49a..b748a9495 100644 --- a/fp-cloud-cli/fp_cli/commands/agent_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/agent_cmds.py @@ -458,7 +458,7 @@ def render_artifact(block): data.get("rows", []), ) else: - _cloud.emit(type("HumanState", (), {"json": False})(), block, block.get("title", "Home")) + _cloud.emit(None, block, block.get("title", "Home")) def agent_brief(ctx: typer.Context, range: str = typer.Option("24h", "--range")) -> None: diff --git a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py index b29ea40a9..d6450c868 100644 --- a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py @@ -49,7 +49,10 @@ def show(ctx: typer.Context, handle: str = typer.Argument(..., metavar="KEY[@VER def _create(state, cctx, body, sessions, disabled=False) -> dict: if sessions: - api.cloud_request(cctx, "POST", f"{PATH}/dry-run", body=spec.dry_run_body(body, sessions)) + if spec.source_kind(body) == "jev": + output.warn("Jev dry-runs spend model budget.") + preview = api.cloud_request(cctx, "POST", f"{PATH}/dry-run", body=spec.dry_run_body(body, sessions)) + check_dry_run(preview) _write.confirm(state, f"Create evaluator {body['eval_key']}@{body['eval_version']}", assume_yes=False) row = api.cloud_request(cctx, "POST", PATH, body=body) if not isinstance(row, dict) or not row.get("id"): @@ -234,7 +237,26 @@ def dry_run( body = spec.dry_run_body(body, session) if spec.source_kind(body) == "jev": output.warn("Jev dry-runs spend model budget.") - _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "POST", f"{PATH}/dry-run", body=body), "Evaluator dry-run") + result = api.cloud_request(require_auth(ctx.obj), "POST", f"{PATH}/dry-run", body=body) + _cloud.emit(ctx.obj, result, "Evaluator dry-run") + if dry_run_failed(result): + # Preserve the full dry-run result in --json, but do not signal success. + raise typer.Exit(code=1) + + +def dry_run_failed(result): + return isinstance(result, dict) and ( + result.get("ok") is False + or any(r.get("ok") is False for r in result.get("runs", []) if isinstance(r, dict)) + or result.get("summary", {}).get("failed", 0) > 0 + ) + + +def check_dry_run(result): + if dry_run_failed(result): + failures = [r for r in result.get("runs", []) if r.get("ok") is False] + detail = "; ".join(str(r.get("error_message") or r.get("error_kind") or "run failed") for r in failures) + raise ApiError(f"Dry-run failed; evaluator was not created. {detail}".strip()) def toggle(ctx, handle, enabled): @@ -267,6 +289,7 @@ def label_def(ctx: typer.Context, handle: str = typer.Argument(...), cctx = require_auth(ctx.obj) row = select(definitions(cctx), handle) labels = list(dict.fromkeys([v for v in row.get("labels", []) if v not in (remove or [])] + (add or []))) + spec.validate_labels(labels) _write.confirm(ctx.obj, f"Label {handle}", assume_yes=False) data = api.cloud_request(cctx, "PATCH", f"{PATH}/{row['id']}", body={"labels": labels}) _cloud.emit(ctx.obj, data or {**row, "labels": labels}, "Evaluator labels") diff --git a/fp-cloud-cli/fp_cli/commands/policies_cmds.py b/fp-cloud-cli/fp_cli/commands/policies_cmds.py index c054879a9..0b5b89fae 100644 --- a/fp-cloud-cli/fp_cli/commands/policies_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/policies_cmds.py @@ -512,8 +512,8 @@ def policies_author(ctx: typer.Context, intent: str = typer.Argument(...)) -> No {"issue": {"title": intent, "source": "manual"}})) for event in events: if event.get("type") == "error": - raise ApiError(str(event.get("message") or event.get("error") or "Policy author failed")) - result = next((e.get("result", e) for e in reversed(events) if e.get("type") == "result"), None) + raise ApiError(str(event.get("message") or event.get("error") or event.get("reason") or "Policy author failed")) + result = next((e.get("result", e) for e in reversed(events) if e.get("type") in ("result", "done")), None) if result is None: raise ApiError("Policy author returned no result; nothing was published") _cloud.emit(ctx.obj, result, "Policy draft") @@ -527,8 +527,11 @@ def policies_backtest(ctx: typer.Context, file_or_id: str = typer.Argument(..., if file_or_id == "-" or file_or_id.startswith("@") or Path(file_or_id).is_file(): source = _write.read_text_arg(file_or_id.removeprefix("@")) else: - row = api.cloud_request(cctx, "GET", f"/api/enforcement/policies/{file_or_id}") - source = row.get("source", "") + data = api.cloud_request(cctx, "GET", "/api/enforcement/policies") + versions = [p for p in _cloud.rows(data, "policies") if p.get("id") == file_or_id] + if not versions: + raise NotFoundError(f"no policy named {file_or_id}") + source = max(versions, key=lambda p: p.get("version", 0)).get("source", "") if not isinstance(source, str) or not source.strip(): raise click.UsageError("Policy source is empty") _cloud.emit(ctx.obj, api.cloud_request(cctx, "POST", "/api/agent/policy-backtest", diff --git a/fp-cloud-cli/fp_cli/evaluator_spec.py b/fp-cloud-cli/fp_cli/evaluator_spec.py index 46b3a3a39..723d43c8f 100644 --- a/fp-cloud-cli/fp_cli/evaluator_spec.py +++ b/fp-cloud-cli/fp_cli/evaluator_spec.py @@ -35,6 +35,14 @@ def threshold(value: Any) -> None: fail("threshold must be between 0 and 1") +def validate_labels(labels: Any) -> None: + if not isinstance(labels, list) or len(labels) > 20 or any( + not isinstance(v, str) or not v.strip() or len(v.encode()) > 64 + or any(ord(c) < 32 or 127 <= ord(c) <= 159 for c in v) for v in labels + ): + fail("labels must contain at most 20 non-empty strings, up to 64 bytes each, without control characters") + + def source_kind(body: dict) -> str: source = body.get("evaluator_source", "") try: @@ -50,29 +58,34 @@ def validate(body: dict, *, complete: bool = True) -> dict: fail(f"Unknown spec field(s): {', '.join(sorted(unknown))}; use the evaluator API body") if complete: key = text(body.get("eval_key"), "eval_key") - if not re.fullmatch(r"[a-z][a-z0-9_]*", key): + if not re.fullmatch(r"[a-z][a-z0-9_]*", key) or len(key.encode()) > 128: fail("eval_key must match ^[a-z][a-z0-9_]*$") - text(body.get("display_name"), "display_name") - text(body.get("eval_version"), "eval_version") + name = text(body.get("display_name"), "display_name") + version = text(body.get("eval_version"), "eval_version") + if len(name.encode()) > 128 or len(version.encode()) > 128 or any(c.isspace() for c in version): + fail("display_name and eval_version are limited to 128 bytes; versions cannot contain whitespace") elif body.get("eval_key") is not None and not re.fullmatch(r"[a-z][a-z0-9_]*", str(body["eval_key"])): fail("eval_key must match ^[a-z][a-z0-9_]*$") result_kind = body.get("result_kind", "score") if result_kind not in ("score", "metric", "assertion"): fail("result_kind must be score, metric or assertion") labels = body.get("labels", []) - if not isinstance(labels, list) or any(not isinstance(v, str) or not v.strip() for v in labels): - fail("labels must be a list of non-empty strings") + validate_labels(labels) timeout = body.get("timeout_seconds") if timeout is not None and ( isinstance(timeout, bool) or not isinstance(timeout, (int, float)) - or not math.isfinite(timeout) or timeout <= 0 + or not math.isfinite(timeout) or not 0.001 <= timeout <= 3600 ): - fail("timeout_seconds must be positive and finite") + fail("timeout_seconds must be finite and between 0.001 and 3600") condition = body.get("condition_source") - if condition: + if condition is not None: text(condition, "condition_source") + if len(condition.encode()) > 16 * 1024: + fail("condition_source is limited to 16 KiB") expression(condition, "condition_source") source = text(body.get("evaluator_source"), "evaluator_source") + if len(source.encode()) > 128 * 1024: + fail("evaluator_source is limited to 128 KiB") kind = source_kind(body) if kind == "code": expression(source, "evaluator_source") @@ -91,7 +104,7 @@ def validate(body: dict, *, complete: bool = True) -> dict: fail("judge steps must be a list") for step in steps: text(step, "judge step") - if re.search(r"\b(yes\s*/\s*no|true\s*/\s*false|binary|pass\s*/\s*fail|score\s+0\s+if|score\s+1\s+if)\b", step, re.I): + if re.search(r"\b(yes\s*/\s*no|true\s*/\s*false|binary|pass\s*/\s*fail|score(?: of|:)? [01])\b", step, re.I): fail("binary judge steps are not supported; use graded evaluation steps") else: text(envelope.get("instructions"), "Jev instructions") @@ -108,7 +121,7 @@ def validate(body: dict, *, complete: bool = True) -> dict: fail("Jev score needs 3–5 distinct levels, ordered worst to best") for level in criteria: text(level, "Jev level") - if len({v.strip() for v in criteria}) != len(criteria): + if len({v.strip().lower() for v in criteria}) != len(criteria): fail("Jev score levels must be distinct") else: fail("Jev type must be noul or score") @@ -126,6 +139,8 @@ def from_draft(data: dict, requested_kind: str) -> dict: if not isinstance(draft, dict): fail("Cloud draft was not an evaluator JSON spec") body = {k: v for k, v in draft.items() if k in FIELDS} + if not body.get("condition_source"): + body.pop("condition_source", None) body.setdefault("eval_version", "1") kind = draft.get("kind", source_kind(body)) if kind != requested_kind: @@ -210,6 +225,8 @@ def build( def dry_run_body(body: dict, sessions: Optional[list]) -> dict: if len(sessions or []) > 10: fail("dry-run accepts at most 10 sessions") + if any(not isinstance(s, str) or not s.strip() or len(s.encode()) > 256 for s in (sessions or [])): + fail("dry-run session ids must be non-empty, up to 256 bytes") if source_kind(body) == "judge": fail("Cloud refuses judge dry-runs; create/deploy without --dry-run-on and inspect evaluation results instead") return { diff --git a/fp-cloud-cli/tests/test_cloud_agent.py b/fp-cloud-cli/tests/test_cloud_agent.py index e074dd0e7..2f51f9011 100644 --- a/fp-cloud-cli/tests/test_cloud_agent.py +++ b/fp-cloud-cli/tests/test_cloud_agent.py @@ -104,7 +104,7 @@ def test_policy_commands(logged_in, runner, tmp_path, argv, path, response): with respx.mock(assert_all_called=False) as mock: mock.get(f"{BASE}/api/issues/i1").respond(200, json={"id": "i1", "title": "Force pushes"}) if response == "stream": - route = mock.post(f"{BASE}{path}").respond(200, text=sse([{"type": "result", "result": {"source": "policy"}}])) + route = mock.post(f"{BASE}{path}").respond(200, text=sse([{"type": "done", "result": {"source": "policy"}}])) else: route = mock.post(f"{BASE}{path}").respond(200, json=response) result = runner.invoke(app, ["--json", *[v.format(file=file) for v in argv]]) @@ -139,3 +139,15 @@ def test_404(logged_in, runner, argv): result = runner.invoke(app, ["--json", *argv]) assert result.exit_code == 6, result.output assert "not available on this FailproofAI Cloud yet" in result.stdout + + +def test_backtest_published_policy_uses_latest_version(logged_in, runner): + with respx.mock as mock: + mock.get(f"{BASE}/api/enforcement/policies").respond(200, json=[ + {"id": "p1", "version": 1, "source": "old"}, + {"id": "p1", "version": 2, "source": "new"}, + ]) + route = mock.post(f"{BASE}/api/agent/policy-backtest").respond(200, json={"matched": 1}) + result = runner.invoke(app, ["--json", "policies", "backtest", "p1"]) + assert result.exit_code == 0, result.output + assert json.loads(route.calls[0].request.content) == {"source": "new"} diff --git a/fp-cloud-cli/tests/test_cloud_observability.py b/fp-cloud-cli/tests/test_cloud_observability.py index fb1e78b4f..b0bda4d57 100644 --- a/fp-cloud-cli/tests/test_cloud_observability.py +++ b/fp-cloud-cli/tests/test_cloud_observability.py @@ -1,6 +1,5 @@ import json -import httpx import pytest import respx diff --git a/fp-cloud-cli/tests/test_cloud_transport.py b/fp-cloud-cli/tests/test_cloud_transport.py index 435229e97..a073bca22 100644 --- a/fp-cloud-cli/tests/test_cloud_transport.py +++ b/fp-cloud-cli/tests/test_cloud_transport.py @@ -1,4 +1,3 @@ -import httpx import pytest import respx @@ -31,3 +30,12 @@ def test_cloud_request_degrades(method, key_mode): headers = route.calls[0].request.headers assert ("authorization" in headers) is key_mode assert ("cookie" in headers) is not key_mode + + +def test_new_capability_html_404_degrades(): + ctx = client.ClientContext("http://dash.test", api_key="key", auth_mode=client.AuthMode.API_KEY) + with respx.mock as mock: + mock.get("http://dash.test/v1/dashboards").respond(404, text="old Cloud", headers={"content-type": "text/html"}) + with pytest.raises(NotFoundError, match="not available on this FailproofAI Cloud yet") as exc: + client.cloud_request(ctx, "GET", "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/api/dashboards") + assert exc.value.exit_code == 6 diff --git a/fp-cloud-cli/tests/test_evaluator_authoring.py b/fp-cloud-cli/tests/test_evaluator_authoring.py index a8dccc4ca..66a3ecfc6 100644 --- a/fp-cloud-cli/tests/test_evaluator_authoring.py +++ b/fp-cloud-cli/tests/test_evaluator_authoring.py @@ -1,6 +1,5 @@ import json -import httpx import pytest import respx @@ -215,3 +214,49 @@ def test_evaluator_404(logged_in, runner, command): result = runner.invoke(app, ["--json", "evaluators", command]) assert result.exit_code == 6 assert "not available on this FailproofAI Cloud yet" in result.stdout + + +def test_failed_dry_run_blocks_creation(logged_in, runner, code): + with respx.mock(assert_all_called=False) as mock: + mock.post(f"{BASE}/api/evaluator-definitions/dry-run").respond(200, json={ + "runs": [{"ok": False, "error_message": "missing result"}], "summary": {"failed": 1}}) + create = mock.post(f"{BASE}/api/evaluator-definitions").respond(201, json={"id": "d1"}) + result = runner.invoke(app, ["--json", "evaluators", "create", "--key", "quality", "--name", "Quality", "--code", code, "--dry-run-on", "s1"]) + assert result.exit_code == 1 + assert not create.called + assert "was not created" in result.stdout + + +@pytest.mark.parametrize("draft_kind,fields", [ + ("code", {"evaluator_source": SOURCE}), + ("judge", {"criteria": "Grade relevance", "steps": ["Assess specificity"], "threshold": 0.7, "evaluator_source": ""}), + ("jev", {"jevType": "noul", "instructions": "Check resolution", "jevCriteria": {"true": "Resolved", "false": "Unresolved"}, "evaluator_source": ""}), +]) +def test_real_dashboard_draft_shape(logged_in, runner, tmp_path, draft_kind, fields): + draft = {"eval_key": "quality", "display_name": "Quality", "result_kind": "score", "labels": [], + "condition_source": "", "reasoning": "Draft explanation", "kind": draft_kind, **fields} + path = tmp_path / "draft.json" + with respx.mock as mock: + mock.post(f"{BASE}/api/agent/compose-eval").respond(200, json=draft) + result = runner.invoke(app, ["--json", "evaluators", "draft", "Grade", "--kind", draft_kind, "--out", str(path)]) + assert result.exit_code == 0, result.output + body = json.loads(path.read_text()) + assert body["eval_version"] == "1" + assert "reasoning" not in body and "kind" not in body and "condition_source" not in body + if draft_kind != "code": + assert json.loads(body["evaluator_source"])["kind"] == draft_kind + + +@pytest.mark.parametrize("flags", [ + ["--judge", "--criteria", "Grade", "--step", "Score of 0 on failure"], + ["--judge", "--criteria", "Grade", "--step", "score: 1 on success"], + ["--jev", "score", "--instructions", "Grade", "--level", "Poor", "--level", "poor", "--level", "Good"], + ["--judge", "--criteria", "Grade", "--version", "bad version"], + ["--judge", "--criteria", "Grade", "--label", "bad\nlabel"], + ["--judge", "--criteria", "Grade", "--timeout", "3601"], +]) +def test_server_validation_edges(logged_in, runner, flags): + with respx.mock as mock: + result = runner.invoke(app, ["--json", "evaluators", "create", "--key", "quality", "--name", "Quality", *flags]) + assert not mock.calls + assert result.exit_code == 2, result.output From df262348fccdf7dc33f4287922a8e7679f29c0d0 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:17:19 +0000 Subject: [PATCH 07/16] feat(fp): adopt mint and pink open-screen primitives without changing JSON --- fp-cloud-cli/fp_cli/app.py | 8 +- fp-cloud-cli/fp_cli/output.py | 196 +++++++++++++++----- fp-cloud-cli/fp_cli/theme.py | 42 ++--- fp-cloud-cli/tests/test_commands.py | 6 +- fp-cloud-cli/tests/test_dashboards_agent.py | 6 +- fp-cloud-cli/tests/test_facets.py | 2 +- fp-cloud-cli/tests/test_operator.py | 2 +- fp-cloud-cli/tests/test_orgs.py | 2 +- fp-cloud-cli/tests/test_output.py | 74 ++++---- fp-cloud-cli/tests/test_screen_language.py | 56 ++++++ fp-cloud-cli/tests/test_whoami.py | 2 +- 11 files changed, 277 insertions(+), 119 deletions(-) create mode 100644 fp-cloud-cli/tests/test_screen_language.py diff --git a/fp-cloud-cli/fp_cli/app.py b/fp-cloud-cli/fp_cli/app.py index 82ce78c9d..7a438a9c7 100644 --- a/fp-cloud-cli/fp_cli/app.py +++ b/fp-cloud-cli/fp_cli/app.py @@ -132,6 +132,12 @@ # function Typer calls for this; falls back silently if its internals change. try: # pragma: no cover - exercised via real command output, not unit assertions import typer.rich_utils as _rich_utils + # Keep generated subcommand help on the same shared screen primitives. + _rich_utils.Panel = output.Panel + _rich_utils.Table = output.Table + _rich_utils.STYLE_OPTION = output.theme.PINK + _rich_utils.STYLE_COMMANDS_TABLE_FIRST_COLUMN = output.theme.PINK + _rich_utils.STYLE_OPTION_ENVVAR = output.theme.LABEL import re as _re @@ -331,7 +337,7 @@ def main( "must be a positive number of seconds.", param_hint="--timeout" ) cfg = cfgmod.load_config() - output.configure(no_color=no_color, quiet=quiet, json=json_output) + output.configure(no_color=no_color, quiet=quiet, json=json_output, context=ctx.invoked_subcommand or "Cloud") # Resolve the credential BEFORE telemetry starts: the distinct-id depends on it. # `get_parameter_source` is what separates a flag from its env var — the values # alone cannot, and the precedence rules differ between the two (see resolve_auth). diff --git a/fp-cloud-cli/fp_cli/output.py b/fp-cloud-cli/fp_cli/output.py index 762d2ac22..c2a9fea1d 100644 --- a/fp-cloud-cli/fp_cli/output.py +++ b/fp-cloud-cli/fp_cli/output.py @@ -13,18 +13,99 @@ import math import re from datetime import datetime, timezone -from typing import Any, List, Optional, Sequence +from typing import Any, Dict, List, Optional, Sequence import typer from rich.box import ROUNDED, SIMPLE_HEAD from rich.console import Console, Group from rich.padding import Padding -from rich.panel import Panel -from rich.rule import Rule -from rich.table import Table +from rich.panel import Panel as RichPanel +from rich.rule import Rule as RichRule +from rich.table import Table as RichTable +from rich.measure import Measurement from rich.text import Text from . import theme +from ._version import __version__ + +_screen_started = False +_screen_context = "Cloud" + + +def wordmark(context: Optional[str] = None) -> Text: + line = Text("fa", style="bold") + line.append("il", style=f"bold {theme.PINK}") + line.append("proof ai", style="bold") + line.append(f" v{__version__} · {context or _screen_context}", style=theme.LABEL) + return line + + +def heading(value: Any) -> Text: + text = value.copy() if isinstance(value, Text) else Text.from_markup(str(value)) + # Change chrome, never the case of a resource name/id in a title. + name, sep, rest = text.plain.partition(" · ") + known = {"events", "sessions", "evals", "errors", "keys", "users", "queries", "settings", + "issues", "models", "chats", "audits", "findings", "commands", "version", + "permissions", "orgs", "assistant", "usage", "guardrails", "policies", "fleet", + "options", "arguments"} + if name.lower() in known: + text = Text(name.upper(), style=f"bold {theme.TEXT}") + Text(sep + rest, style=theme.LABEL) + return text + + +def screen_header(console=None) -> None: + global _screen_started + if not _screen_started and not _quiet and not _json_out: + _screen_started = True + (console or _stdout).print(wordmark()) + (console or _stdout).print() + + +class Panel(RichPanel): + """Legacy cards rendered through the open-screen primitive, without boxes. + + Keep RichPanel's constructor/attributes so existing renderers need no rewrite. + """ + def __rich_console__(self, console, options): + global _screen_started + if not _screen_started and not _quiet and not _json_out: + _screen_started = True + yield wordmark() + yield Text("") + if self.title: + yield heading(self.title) + yield self.renderable + if self.subtitle: + yield heading(self.subtitle) + + def __rich_measure__(self, console, options): + parts = [self.renderable] + if self.title: + parts.insert(0, heading(self.title)) + return Measurement.get(console, options, Group(*parts)) + + +class Rule(RichRule): + """Legacy separators are intentionally invisible in the 2026-10 language.""" + def __rich_console__(self, console, options): + return iter(()) + + def __rich_measure__(self, console, options): + return Measurement(0, 0) + + +class Table(RichTable): + """Shared tables: grey labels, no borders/rules, fold instead of cut.""" + def __init__(self, *args, **kwargs): + kwargs["box"] = None + kwargs["show_edge"] = False + kwargs["header_style"] = theme.LABEL + super().__init__(*args, **kwargs) + + def add_column(self, *args, **kwargs): + kwargs["overflow"] = "fold" + kwargs["no_wrap"] = False + return super().add_column(*args, **kwargs) _stdout = Console() _stderr = Console(stderr=True) @@ -33,14 +114,16 @@ _json_out = False -def configure(*, no_color: bool = False, quiet: bool = False, json: bool = False) -> None: +def configure(*, no_color: bool = False, quiet: bool = False, json: bool = False, context: str = "Cloud") -> None: """Reconfigure the consoles from the resolved global flags.""" - global _stdout, _stderr, _quiet, _no_color, _json_out + global _stdout, _stderr, _quiet, _no_color, _json_out, _screen_started, _screen_context _stdout = Console(no_color=no_color) _stderr = Console(stderr=True, no_color=no_color) _quiet = quiet _no_color = no_color _json_out = json + _screen_started = False + _screen_context = context def is_json() -> bool: @@ -70,6 +153,11 @@ def _ansi(text: str, **style: Any) -> str: """A click-styled string for use in input prompts, or plain when colour is off.""" if _no_color: return text + # Click accepts RGB tuples, not Rich's hex colour strings. + for key in ("fg", "bg"): + color = style.get(key) + if isinstance(color, str) and re.fullmatch(r"#[0-9a-fA-F]{6}", color): + style[key] = tuple(int(color[i:i + 2], 16) for i in (1, 3, 5)) return typer.style(text, **style) @@ -126,9 +214,20 @@ def print_table( title: Optional[str] = None, show_header: bool = True, ) -> None: - table = Table( - title=title, title_justify="left", header_style="bold", show_header=show_header - ) + screen_header() + if title: + _stdout.print(Text(str(title).upper(), style="bold")) + widths = [max([len(str(column))] + [len(_cell(row[i])) for row in rows if i < len(row)]) for i, column in enumerate(columns)] + if len(columns) > 1 and sum(widths) + 3 * len(columns) > _stdout.width: + # On a narrow terminal, wrapped adjacent columns interleave copyable + # ids and URLs. Stack the fields so each value remains a whole line. + for row in rows: + for column, value in zip(columns, row): + _stdout.print(Text(str(column).upper(), style=theme.LABEL)) + _stdout.print(Text(" " + _cell(value)), soft_wrap=True) + _stdout.print() + return + table = Table(show_header=show_header) for column in columns: table.add_column(str(column), overflow="fold") for row in rows: @@ -152,7 +251,7 @@ def info(message: str) -> None: def success(message: str) -> None: if not _quiet: - _stderr.print(message, style="green") + _stderr.print(Text("✓ ", style=theme.SUCCESS) + Text(message)) def hint(message: str) -> None: @@ -161,12 +260,12 @@ def hint(message: str) -> None: def error(message: str) -> None: - _stderr.print(message, style="bold red") + _stderr.print(Text("✕ ", style=theme.ERROR) + Text(message)) def warn(message: str) -> None: if not _quiet: - _stderr.print(message, style="yellow") + _stderr.print(Text("▲ ", style=theme.PINK) + Text(message)) def cli_warn(message: str) -> None: @@ -187,7 +286,7 @@ def cli_warn(message: str) -> None: # here is stderr status chrome (never the --json data on stdout); the accent is # the project's pink/magenta. All gated on `_quiet`. -_ACCENT = "magenta" +_ACCENT = theme.PINK _NEUTRAL = "grey62" # gray for 'already in that state' no-op boxes (neutral, not red/green) @@ -196,7 +295,8 @@ def auth_header() -> None: if _quiet: return _stderr.print() - _stderr.print(f" [bold {_ACCENT}]◆ fp[/]") + screen_header(_stderr) + _stderr.print(Text("SIGN IN", style="bold")) _stderr.print(" [dim]welcome — let's get you signed in with a one-time code[/]") _stderr.print() @@ -403,6 +503,7 @@ def _panel(mark: str, color: str, title: str, body: Text, hint: Optional[str] = highlighted. Built from a `rich.Text` (literal body — no markup injection). The colour carries the meaning: **green** = success, **brand-accent** = neutral 'already-in-state' no-op, **red** = failure. One shape so they all read as one family.""" + color = theme.SUCCESS if color == "green" else theme.PINK if color in ("red", "yellow", "cyan", "magenta") else color text = Text() indent = "" # A box may be mark-less (the colour/title carry the meaning, e.g. the red @@ -415,7 +516,7 @@ def _panel(mark: str, color: str, title: str, body: Text, hint: Optional[str] = text.append(f"\n{indent}") text.append(hint, style="dim") text.highlight_words( - ["fp login", "fp logout", "login --force"], style="bold cyan" + ["fp login", "fp logout", "login --force"], style=f"bold {theme.PINK}" ) panel = Panel( text, @@ -433,8 +534,12 @@ def _panel(mark: str, color: str, title: str, body: Text, hint: Optional[str] = def _error_box(message: str, hint: Optional[str] = None) -> None: - """A red `✗` failure box — the chokepoint for every CLI error.""" - _panel("✗", "red", "error", Text(message), hint) + """One failure line, followed by a grey recovery hint when present.""" + _stderr.print(Text("✕ ", style=theme.ERROR) + Text(message)) + if hint: + line = Text(" " + hint, style=theme.LABEL) + line.highlight_regex(r"(?:fp|failproofai)\s+[^`]+", style=theme.PINK) + _stderr.print(line) def _status_box(title: str, body: Text, hint: Optional[str] = None) -> None: @@ -450,21 +555,8 @@ def version_banner(version: str) -> None: if _quiet: _stdout.print(version) return - body = Text() - body.append("◆ ", style=f"bold {_ACCENT}") - body.append("fp", style="bold") - body.append(" ") - body.append(f"v{version}", style=f"bold {_ACCENT}") - panel = Panel( - body, - border_style=_ACCENT, - title=f"[bold {_ACCENT}]version[/]", - title_align="left", - expand=False, - padding=(0, 1), - ) - _stdout.print() - _stdout.print(Padding(panel, (0, 0, 0, 2))) + screen_header() + _stdout.print(Text(f"fp v{version}", style=theme.LABEL)) # ── top-level help (the grouped `fp` / `fp help` / `fp --help` screen) ── @@ -510,6 +602,10 @@ def version_banner(version: str) -> None: ("audits", "Schedule audits and triage their findings.", "list show create edit delete run runs findings context-*"), ("issues", "Triage and resolve issues.", "list count show ack assign resolve comment-* subscribe open"), ("settings", "View and change org settings.", "list schema set"), + ("roles", "Manage permission sets.", "list show create update delete"), + ("plan", "Inspect offers and team trial.", "claim"), + ("otel", "Control Cloud OTEL ingestion.", "status enable disable"), + ("slack", "Manage Slack integration.", "status disconnect"), ]), ("TOOLS", [ ("list", "List distinct values behind the filter dropdowns.", ""), @@ -533,6 +629,7 @@ def render_top_level_help() -> None: bold-white headings, BLUE command names, white one-line descriptions, FAINT subcommand hints, a FAINT footer), then aligned EXAMPLES. Presentation only — same routing, same commands. NO_COLOR keeps the box + bold headings + alignment, drops colour.""" + global _screen_started total = sum(len(cmds) for _, cmds in _TOP_LEVEL_GROUPS) def _bullet(*parts) -> Text: @@ -543,7 +640,8 @@ def _bullet(*parts) -> Text: # ── brand mark ── _stdout.print() - _stdout.print(Text(" ◆ fp", style=f"bold {theme.ACCENT}")) + _screen_started = True + _stdout.print(wordmark()) # ── getting started ── _stdout.print() @@ -627,7 +725,7 @@ def _bullet(*parts) -> Text: if hint: cell.append(" · ", style=theme.FAINT) cell.append(hint, style=theme.FAINT) - table.add_row(Text(name, style=theme.BLUE), cell) + table.add_row(Text(name, style=theme.PINK), cell) footer = Text() footer.append("run ", style=theme.FAINT) footer.append("fp --help", style=theme.TEXT_DIM) @@ -648,7 +746,7 @@ def _bullet(*parts) -> Text: ex.add_column(no_wrap=True) ex.add_column(no_wrap=True, overflow="ellipsis", max_width=max(SCORES_MIN_WIDTH, min(_stdout.width, 110) - 56)) for cmd, why in _TOP_LEVEL_EXAMPLES: - ex.add_row(Text(cmd, style=theme.ACCENT), Text(why, style=theme.TEXT_DIM)) + ex.add_row(Text(cmd, style=theme.PINK), Text(why, style=theme.TEXT_DIM)) _stdout.print(Padding(ex, (0, 0, 0, 2))) _stdout.print() @@ -733,7 +831,11 @@ def _perm_res_key(r: str): def _perm_act_key(a: str): """Action sort key: risk order (read → modify → invoke → destroy), then alphabetical.""" - return (theme.PERM_RANK.get(theme.perm_color(a), 99), a) + # Risk ordering is semantic, not a function of the two accent colours. + rank = {"read": 0, "add": 1, "create": 1, "write": 1, "update": 1, + "use": 2, "trigger": 2, "run": 2, "ack": 2, + "delete": 3, "disable": 3, "regenerate": 3} + return (rank.get(a, 99), a) def _group_permissions(perms: Sequence[str]): @@ -913,6 +1015,7 @@ def render_whoami( """The logged-in ``whoami`` view (stdout): an identity header, then the shared permissions + orgs panels. Presentation only; no legend.""" c = _stdout + screen_header(c) # ── identity header (no box) ── c.print() @@ -1322,12 +1425,6 @@ def _short_session(sid: str, *, full: bool = False) -> str: """Truncate a long session id for the table (full id always kept in ``--json``): ``sess-20260615-fcf97e01`` → ``sess-…fcf97e01`` (prefix + last 8); other long ids → first 6 + ``…`` + last 6; short ids are left intact.""" - if full or not sid or sid == "-": - return sid - if sid.startswith("sess-") and len(sid) > 14: - return "sess-…" + sid[-8:] - if len(sid) > 18: - return sid[:6] + "…" + sid[-6:] return sid @@ -1806,6 +1903,8 @@ def render_eval_aggregate(data: dict, *, show_bar: Optional[bool] = None) -> Non def _truncate(s: str, n: int = 80) -> str: + if re.search(r"(?:https?://|\bfp\s|[0-9a-f]{8}-[0-9a-f-]{8,})", s, re.I): + return s return s[: n - 1] + "…" if len(s) > n else s @@ -2013,7 +2112,7 @@ def render_value_list(kind: str, values: Sequence[str], *, description: str = "" def _short_id(value: str) -> str: """A long id → ``1f58…9826`` (first 4 + ``…`` + last 4); short ids unchanged.""" - return value[:4] + "…" + value[-4:] if len(value) > 9 else value + return value def _short_chat_id(value: str) -> str: @@ -2021,10 +2120,7 @@ def _short_chat_id(value: str) -> str: 8 hex of a UUID, e.g. ``07854990-dade-…`` → ``07854990``), or the first 8 chars when there's no ``-``. The CLI resolves this prefix back to the full id against the chat list, while the server keeps the full id intact.""" - s = str(value or "") - if "-" in s: - return s.split("-", 1)[0] - return s[:8] if len(s) > 8 else s + return str(value or "") def _fmt_key_created(iso: str) -> str: @@ -2656,13 +2752,13 @@ def _sql_syntax_theme(): class _FpSqlStyle(Style): background_color = "#0e0c12" # overridden by Syntax(background_color="default") styles = { - Token: theme.TEXT, + Token: "", Comment: f"italic {theme.FAINT}", Keyword: f"bold {theme.ACCENT}", Keyword.Type: theme.SCORE_HIGH, Operator: theme.TEXT_DIM, Operator.Word: f"bold {theme.ACCENT}", # AND / OR / NOT - Name: theme.TEXT, + Name: "", Name.Builtin: theme.SUCCESS, Name.Function: theme.SUCCESS, Name.Variable: theme.PERM_WRITE, # :name / @var params @@ -4612,7 +4708,7 @@ def confirm_issues_clear(scope_label: str, issues: int, findings: int) -> bool: h.append(f"{issues} issue{'' if issues == 1 else 's'}", style=f"bold {theme.ACCENT}") h.append(f" ({scope_label})", style=theme.LABEL) h.append("?", style=theme.TEXT) - tail = f"resolves them" + tail = "resolves them" if findings: tail += f" and {findings} audit finding{'' if findings == 1 else 's'}" tail += " — nothing is deleted or suppressed" diff --git a/fp-cloud-cli/fp_cli/theme.py b/fp-cloud-cli/fp_cli/theme.py index a46965764..3ace03420 100644 --- a/fp-cloud-cli/fp_cli/theme.py +++ b/fp-cloud-cli/fp_cli/theme.py @@ -6,35 +6,35 @@ from __future__ import annotations -ACCENT = "#9d7bff" # brand mark, box borders, active org, command names in hints -TEXT = "#d8d2dd" # primary values -TEXT_DIM = "#7d7488" # ids, resource names, org names -LABEL = "#6b6478" # field labels, header cells, hints -FAINT = "#5a5266" # separators, inactive markers -THIN_RULE = "#2e2435" # the faint rule beneath a list-panel header -BAR_EMPTY = "#2a2530" # the unfilled cells of a mini-bar (e.g. the aggregate avg bar) -INSET_BG = "#231e2d" # a hair-lighter fill for a nested inset box (the login org picker) +ACCENT = "#66d1b5" # mint: active and successful states +TEXT = "default" # the terminal's foreground (also works on light themes) +TEXT_DIM = "#767f8b" +LABEL = "#767f8b" # grey labels and metadata +FAINT = "#767f8b" +THIN_RULE = "#3e434c" +BAR_EMPTY = "#3e434c" +INSET_BG = "default" # Errors theme — the one per-command border deviation (the `errors` list + aggregate card). # Always red, regardless of error count, so the two views read as one consistent family. -BORDER_ERROR = "#e2564a" # red → the errors panel border (same as ERROR; consistent, not count-dependent) -TITLE_ERROR_DIM = "#6e3530" # dim red → the errors title's non-name part + card separators -RULE_ERROR = "#3a2d2d" # dim red → the errors list header rule (vs the neutral THIN_RULE) +BORDER_ERROR = "#e4587d" +TITLE_ERROR_DIM = LABEL +RULE_ERROR = THIN_RULE # Semantic value colors — for run/job states and score thresholds. (They coincide with # the perm risk colors below; named separately so the two uses can diverge later.) -SUCCESS = "#5dcaa5" # green → run status: done/passed -AMBER = "#ef9f27" # amber → run status: running/pending · score band .50–.80 -ERROR = "#e2564a" # red → run status: failed/error · score band < .50 -SCORE_HIGH = "#3ddbb8" # cyan-green → score band ≥ .80 (distinct from status green) -BLUE = "#6b86d8" # blue → schema uuid/timestamp type category -PINK = "#d4537e" # pink → numeric values (query run cells / scalar card) + numeric type category +SUCCESS = ACCENT +PINK = "#e4587d" # pink: commands, selection and attention +AMBER = PINK +ERROR = PINK +SCORE_HIGH = ACCENT +BLUE = ACCENT # Permission verb colors — by ACTION (the part after `:`), never by resource. -PERM_READ = "#5dcaa5" # green → read -PERM_WRITE = "#d4537e" # pink → add, create, write, update -PERM_ACTION = "#ef9f27" # amber → use, trigger, run, ack -PERM_DANGER = "#e2564a" # red → delete, disable, regenerate +PERM_READ = ACCENT +PERM_WRITE = PINK +PERM_ACTION = PINK +PERM_DANGER = PINK PERM_COLORS = { "read": PERM_READ, diff --git a/fp-cloud-cli/tests/test_commands.py b/fp-cloud-cli/tests/test_commands.py index 092551efe..bc8f9a284 100644 --- a/fp-cloud-cli/tests/test_commands.py +++ b/fp-cloud-cli/tests/test_commands.py @@ -522,7 +522,7 @@ def test_sessions_table(logged_in, runner): ) result = runner.invoke(app, ["sessions"]) assert result.exit_code == 0, result.output - assert "sessions" in result.stdout # the boxed panel title + assert "SESSIONS" in result.stdout assert "done" in result.stdout # status flattened up from latest_evaluation @@ -679,7 +679,7 @@ def test_help_command(home, runner): result = runner.invoke(app, ["help"]) assert result.exit_code == 0 out = result.stdout - assert "Commands" in out # the grouped top-level panel + assert "COMMANDS" in out # the four purpose groups, in order for g in ("ESSENTIALS", "OBSERVE", "MANAGE", "TOOLS"): assert g in out @@ -744,7 +744,7 @@ def test_fields_unknown_is_usage_error(logged_in, runner): def test_dash_h_alias_shows_help(home, runner): result = runner.invoke(app, ["-h"]) assert result.exit_code == 0 - assert "Commands" in result.stdout and "sessions" in result.stdout + assert "COMMANDS" in result.stdout and "sessions" in result.stdout assert "ESSENTIALS" in result.stdout # the grouped top-level help, via the rich_format_help override diff --git a/fp-cloud-cli/tests/test_dashboards_agent.py b/fp-cloud-cli/tests/test_dashboards_agent.py index 2bd87bbee..be8baea07 100644 --- a/fp-cloud-cli/tests/test_dashboards_agent.py +++ b/fp-cloud-cli/tests/test_dashboards_agent.py @@ -171,7 +171,7 @@ def test_agent_health_renders_configured_card(logged_in, runner): result = runner.invoke(app, ["agent", "health"]) assert result.exit_code == 0, result.output out = result.stdout - assert "assistant" in out and "configured" in out and "claude-x" in out + assert "ASSISTANT" in out and "configured" in out and "claude-x" in out @respx.mock @@ -192,7 +192,7 @@ def test_agent_models_renders_default_marker(logged_in, runner): result = runner.invoke(app, ["agent", "models"]) assert result.exit_code == 0, result.output out = result.stdout - assert "models" in out and "m-default" in out and "m-fast" in out and "default" in out + assert "MODELS" in out and "m-default" in out and "m-fast" in out and "default" in out @respx.mock @@ -213,7 +213,7 @@ def test_agent_chats_renders_table(logged_in, runner): ) result = runner.invoke(app, ["agent", "chats"]) assert result.exit_code == 0, result.output - assert "chats" in result.stdout and "perf review" in result.stdout + assert "CHATS" in result.stdout and "perf review" in result.stdout assert "chat-id" in result.stdout and "c1" in result.stdout # full chat-id column by default assert "open one with" not in (result.stderr or "") # footer removed diff --git a/fp-cloud-cli/tests/test_facets.py b/fp-cloud-cli/tests/test_facets.py index 31ae8c8df..86d0500c1 100644 --- a/fp-cloud-cli/tests/test_facets.py +++ b/fp-cloud-cli/tests/test_facets.py @@ -87,7 +87,7 @@ def test_errors_list_table(logged_in, runner): result = runner.invoke(app, ["errors", "--since", "24h"]) assert result.exit_code == 0, result.output # the 80-col CliRunner truncates cells; just assert the error-themed box rendered - assert "errors" in result.stdout and "newest first" in result.stdout + assert "ERRORS" in result.stdout and "newest first" in result.stdout @respx.mock diff --git a/fp-cloud-cli/tests/test_operator.py b/fp-cloud-cli/tests/test_operator.py index 2cbdc8bb6..659162842 100644 --- a/fp-cloud-cli/tests/test_operator.py +++ b/fp-cloud-cli/tests/test_operator.py @@ -388,7 +388,7 @@ def test_settings_list_human_boxed(logged_in, runner): result = runner.invoke(app, ["settings", "list"]) assert result.exit_code == 0, result.output out = result.stdout + result.stderr - assert "settings · 3" in out + assert "SETTINGS · 3" in out for c in ("key", "value", "type", "updated"): assert c in out assert "session_ttl_secs" in out and "86400" in out and "integer" in out diff --git a/fp-cloud-cli/tests/test_orgs.py b/fp-cloud-cli/tests/test_orgs.py index 1b78be434..f65caea07 100644 --- a/fp-cloud-cli/tests/test_orgs.py +++ b/fp-cloud-cli/tests/test_orgs.py @@ -393,7 +393,7 @@ def test_orgs_perms_table(logged_in, runner): # identity header (name · slug · role) + the shared grouped permissions panel, with the # org NAME in the panel's border title. assert "Globex" in result.stdout and "globex" in result.stdout and "role admin" in result.stdout - assert "permissions · 2 · Globex" in result.stdout + assert "PERMISSIONS · 2 · Globex" in result.stdout assert "events" in result.stdout and "keys" in result.stdout diff --git a/fp-cloud-cli/tests/test_output.py b/fp-cloud-cli/tests/test_output.py index f3744215c..f0957e270 100644 --- a/fp-cloud-cli/tests/test_output.py +++ b/fp-cloud-cli/tests/test_output.py @@ -51,7 +51,7 @@ def test_print_table_writes_to_stdout(capsys): output.configure(no_color=True, quiet=False) output.print_table(["a", "b"], [["1", "2"]], title="Demo") out = capsys.readouterr().out - assert "Demo" in out + assert "DEMO" in out assert "1" in out and "2" in out @@ -91,7 +91,7 @@ def test_render_events_box_to_stdout(capsys): output.render_events([_event("2026-06-22T12:21:54Z")], order="desc") captured = capsys.readouterr() # The box (human data view) goes to STDOUT; the title carries count, direction and date. - assert "events" in captured.out + assert "EVENTS" in captured.out assert "newest first" in captured.out assert "2026-06-22" in captured.out assert captured.err == "" @@ -182,7 +182,7 @@ def test_status_color_enum(): def test_short_session(): - assert output._short_session("sess-20260615-fcf97e01") == "sess-…fcf97e01" + assert output._short_session("sess-20260615-fcf97e01") == "sess-20260615-fcf97e01" assert output._short_session("sess-20260615-fcf97e01", full=True) == "sess-20260615-fcf97e01" assert output._short_session("nonnum-3-b9d9de") == "nonnum-3-b9d9de" # short → intact assert output._short_session("short") == "short" @@ -208,9 +208,9 @@ def test_render_sessions_box_has_no_scores(capsys): _wide_stdout() output.render_sessions([_sample_eval()]) out = capsys.readouterr().out - assert "sessions" in out and "newest first" in out and "2026-06-22" in out + assert "SESSIONS" in out and "newest first" in out and "2026-06-22" in out assert "agent-codegen" in out and "done" in out - assert "12:56:31" in out and "sess-…fcf97e01" in out + assert "12:56:31" in out and "sess-20260615-fcf97e01" in out assert "coherence" not in out and "scores" not in out # scores moved to evals output.configure(no_color=True, quiet=False) @@ -219,8 +219,8 @@ def test_render_evals_box_has_scores(capsys): _wide_stdout() output.render_evals([_sample_eval()]) out = capsys.readouterr().out - assert "evals" in out and "scores" in out and "coherence" in out - assert "agent-codegen" in out and "sess-…fcf97e01" in out + assert "EVALS" in out and "scores" in out and "coherence" in out + assert "agent-codegen" in out and "sess-20260615-fcf97e01" in out output.configure(no_color=True, quiet=False) @@ -296,7 +296,7 @@ def test_render_sessions_expanded_lists_full_roster(capsys): assert "agent-testgen" in out and "9 ev" in out assert "●" not in out # uniform: no special "root" marker assert "├" in out and "└" in out # uniform tree glyphs, └ closes the list - assert "sessions · 1" in out # panel count = sessions, not rows + assert "SESSIONS · 1" in out # count = sessions, not rows output.configure(no_color=True, quiet=False) @@ -403,8 +403,8 @@ def test_render_errors_box(capsys): _wide_stdout() output.render_errors([_errevent("error", "RateLimitError: upstream timed out")]) out = capsys.readouterr().out - assert "errors" in out and "newest first" in out - assert "agent-orderbot" in out and "sess-…4b90b240" in out # truncated session (shared last-8) + assert "ERRORS" in out and "newest first" in out + assert "agent-orderbot" in out and "sess-20260622-4b90b240" in out assert "RateLimitError: upstream timed out" in out # server-computed summary field assert "17:51:34" in out output.configure(no_color=True, quiet=False) @@ -479,7 +479,7 @@ def test_render_org_perms(capsys): out = capsys.readouterr().out # header leads with the org name + slug + role; the count moved into the box title assert "Globex Corp" in out and "globex" in out and "role admin" in out - assert "permissions · 5 · Globex Corp" in out + assert "PERMISSIONS · 5 · Globex Corp" in out assert "dashboards" in out and "keys" in out and "agent" in out output.configure(no_color=True, quiet=False) @@ -549,7 +549,7 @@ def test_render_value_list_narrow_caps_columns(capsys): def test_short_id(): - assert output._short_id("1f58376d-7947-9826") == "1f58…9826" + assert output._short_id("1f58376d-7947-9826") == "1f58376d-7947-9826" assert output._short_id("short") == "short" @@ -585,7 +585,7 @@ def test_render_keys_box_and_footer(capsys): def test_render_keys_show_id(capsys): _wide_stdout() output.render_keys([ApiKey(id="1f58376d-7947-9826", name="admin", created_at="2026-06-18T05:14:00Z")], show_id=True) - assert "1f58…9826" in capsys.readouterr().out # short id column when --show-id + assert "1f58376d-7947-9826" in capsys.readouterr().out output.configure(no_color=True, quiet=False) @@ -659,7 +659,7 @@ def test_render_users_box_and_footer(capsys): output.users_footer(users) cap = capsys.readouterr() out, err = cap.out, cap.err - assert "users · 3" in out + assert "USERS · 3" in out for c in ("email", "access", "perms", "joined", "status"): assert c in out assert "root@test" in out and "admin" in out and "06-25" in out @@ -695,7 +695,7 @@ def test_render_user_show_card_and_perms(capsys): out = capsys.readouterr().out # show is a read view → stdout assert "user" in out and "dev@test" in out and "protected" in out assert "access admin" in out and "3 permissions" in out and "active" in out - assert "permissions · 3" in out and "keys" in out and "dashboards" in out + assert "PERMISSIONS · 3" in out and "keys" in out and "dashboards" in out output.configure(no_color=True, quiet=False) @@ -706,7 +706,7 @@ def test_render_user_created_green_card(capsys): err = capsys.readouterr().err # write result → stderr chrome assert "user created" in err and "new@test" in err assert "access read-only" in err and "2 permissions" in err and "active" in err - assert "permissions · 2 · read-only" in err # suffix shows the set + assert "PERMISSIONS · 2 · read-only" in err # suffix shows the set output.configure(no_color=True, quiet=False) @@ -718,7 +718,7 @@ def test_render_user_updated_diff(capsys): err = capsys.readouterr().err assert "permissions updated · dev@test" in err assert "now 3" in err and "+1 added" in err and "−1 removed" in err - assert "permissions · 3" in err # NEW set size (struck removal not counted) + assert "PERMISSIONS · 3" in err # NEW set size (struck removal not counted) assert "+create" in err and "-write" in err # added chip / removed ghost (no-color prefixes) assert "unchanged" in err # legend output.configure(no_color=True, quiet=False) @@ -1146,7 +1146,7 @@ def test_render_settings_box(capsys): output.render_settings(rows) cap = capsys.readouterr() out, err = cap.out, cap.err - assert "settings · 3" in out # the count glows in the title + assert "SETTINGS · 3" in out for c in ("key", "value", "type", "updated"): assert c in out assert "session_ttl_secs" in out and "86400" in out and "integer" in out @@ -1231,10 +1231,10 @@ def test_render_incidents_box_and_footer(capsys): output.incidents_footer(incs) cap = capsys.readouterr() out, err = cap.out, cap.err - assert "issues · 3" in out + assert "ISSUES · 3" in out for c in ("id", "alert", "severity", "state", "opened", "assignees"): assert c in out - assert "1f58…9826" in out # short id (the handle) + assert "1f5803aaaaaabbbbcccc000000009826" in out assert "—" in out # manual incident (no alert_name) + no assignees assert "firing" in out and "acknowledged" in out and "resolved" in out assert "3 issues" in err and "1 firing" in err and "1 resolved" in err # footer distribution @@ -1271,7 +1271,7 @@ def test_render_incident_show_sections(capsys): ) output.render_incident_show(inc) out = capsys.readouterr().out - assert "p95 latency" in out and "1f58…9826" in out + assert "p95 latency" in out and "1f5803aaaaaabbbbcccc000000009826" in out assert "acknowledged by ops@example.com" in out and "assigned to a@example.com" in out assert "breach" in out and "1240ms" in out assert "comments · 2" in out and "on it" in out and "(deleted)" in out @@ -1357,7 +1357,7 @@ def test_confirm_incident_resolve_headline(capsys, monkeypatch): monkeypatch.setattr(output.typer, "confirm", lambda *a, **k: False) assert output.confirm_incident_resolve("1f5803aaaaaabbbbcccc000000009826", "p95 latency") is False err = capsys.readouterr().err - assert "resolve issue" in err and "1f58…9826" in err and "(p95 latency)" in err and "this closes it" in err + assert "resolve issue" in err and "1f5803aaaaaabbbbcccc000000009826" in err and "(p95 latency)" in err and "this closes it" in err def test_incident_plain_feedback_lines(capsys): @@ -1374,14 +1374,14 @@ def test_incident_plain_feedback_lines(capsys): output.incident_comment_not_found("c0ffee001111") output.incident_failed("a@x.com is not an operator") err = capsys.readouterr().err - assert "acknowledged issue 1f58…9826" in err - assert "resolved issue 1f58…9826" in err - assert "assigned 1f58…9826" in err and "a@example.com, b@example.com" in err - assert "cleared assignees on 1f58…9826" in err + assert f"acknowledged issue {iid}" in err + assert f"resolved issue {iid}" in err + assert f"assigned {iid}" in err and "a@example.com, b@example.com" in err + assert f"cleared assignees on {iid}" in err assert "subscribed you to issue" in err and "unsubscribed x@y.z from issue" in err assert "deleted comment" in err - assert "no issue 1f58…9826" in err and "fp issues list" in err - assert "no comment" in err and "c0ff…1111" in err + assert f"no issue {iid}" in err and "fp issues list" in err + assert "no comment" in err and "c0ffee001111" in err assert "a@x.com is not an operator" in err @@ -1399,7 +1399,7 @@ def test_render_agent_health_configured(capsys): _wide_stdout() output.render_agent_health(configured=True, default_model="claude-x", model_count=3) out = capsys.readouterr().out # data view → stdout - assert "assistant" in out and "configured" in out + assert "ASSISTANT" in out and "configured" in out assert "default model claude-x" in out and "3 models available" in out output.configure(no_color=True, quiet=False) @@ -1417,7 +1417,7 @@ def test_render_agent_models_marks_default(capsys): _wide_stdout() output.render_agent_models(["m-default", "m-fast"], default_model="m-default") out = capsys.readouterr().out - assert "models · 2" in out and "m-default" in out and "m-fast" in out and "default" in out + assert "MODELS · 2" in out and "m-default" in out and "m-fast" in out and "default" in out output.configure(no_color=True, quiet=False) @@ -1425,7 +1425,7 @@ def test_render_agent_models_empty(capsys): _wide_stdout() output.render_agent_models([], default_model=None) out = capsys.readouterr().out - assert "models · 0" in out and "no models reported" in out + assert "MODELS · 0" in out and "no models reported" in out output.configure(no_color=True, quiet=False) @@ -1438,12 +1438,12 @@ def test_render_agent_chats_box(capsys): output.render_agent_chats(chats) cap = capsys.readouterr() out, err = cap.out, cap.err - assert "chats · 2" in out # the count glows in the title + assert "CHATS · 2" in out for c in ("chat-id", "title", "messages", "updated"): assert c in out assert "older" in out and "newer" in out assert "07854990" in out and "17bf35c3" in out # short copy-friendly handle (first 8) - assert "dade" not in out # the rest of the id is not shown + assert "07854990-dade-4dea-aaaa" in out assert out.index("newer") < out.index("older") # newest first assert "open one with" not in err # the footer hint was removed output.configure(no_color=True, quiet=False) @@ -1492,7 +1492,7 @@ def test_render_agent_delete_preview_and_feedback(capsys): output.print_cancelled("nothing deleted") # boxed cancel, like the command uses err = capsys.readouterr().err assert "delete chat" in err and "perf review" in err and "4 messages" in err - assert "07854990" in err and "dade" not in err # short id in the preview + assert "07854990-dade" in err # full id in the preview assert "deleted chat" in err and "perf review" in err # boxed deleted assert "cancelled" in err and "nothing deleted" in err # boxed cancel output.configure(no_color=True, quiet=False) @@ -1517,7 +1517,7 @@ def test_agent_ask_chrome_lines(capsys): output.agent_unconfigured_note() err = capsys.readouterr().err assert "used tool: run_query" in err - assert "new chat" in err and 'fp agent ask --chat 07854990 "…"' in err # short id, positional msg + assert "new chat" in err and 'fp agent ask --chat 07854990-dade-4dea "…"' in err assert "assistant error: boom" in err assert "chat not found" in err and "fp agent chats" in err # boxed not-found assert "isn't configured" in err and "fp agent health" in err @@ -1619,7 +1619,7 @@ def pv(pid, version): _wide_stdout() output.render_policies([pv("a", 1), pv("b", 2), pv("b", 1)]) out = capsys.readouterr().out - assert "policies · 2 · 3 versions" in out + assert "POLICIES · 2 · 3 versions" in out # newest version of each policy first, rather than server order b_rows = [ln for ln in out.splitlines() if "b" in ln and "v" in ln and ("v1" in ln or "v2" in ln)] @@ -1634,7 +1634,7 @@ def test_policy_list_omits_the_caption_when_each_policy_has_one_version(capsys): source=None, created_at="", created_by=None, disabled=False, archived=False)]) out = capsys.readouterr().out - assert "policies · 1" in out and "versions" not in out + assert "POLICIES · 1" in out and "versions" not in out # ───────────────────────────────────────────────────────────────────────────── diff --git a/fp-cloud-cli/tests/test_screen_language.py b/fp-cloud-cli/tests/test_screen_language.py new file mode 100644 index 000000000..2c518b2a2 --- /dev/null +++ b/fp-cloud-cli/tests/test_screen_language.py @@ -0,0 +1,56 @@ +import json + +from rich.console import Console + +from fp_cli import output, theme +from fp_cli.models import AgentEvent + + +def test_two_accent_palette(): + assert theme.ACCENT == "#66d1b5" + assert theme.PINK == "#e4587d" + assert {theme.SUCCESS, theme.ERROR, theme.AMBER, theme.BLUE, theme.SCORE_HIGH} <= {theme.ACCENT, theme.PINK} + + +def test_open_table_and_header(capsys): + output.configure(no_color=True) + output.print_table(["Id", "URL"], [["run-001", "https://cloud.example.test"]], title="Resources") + text = capsys.readouterr().out + assert "failproof ai v" in text + assert "RESOURCES" in text + assert not any(c in text for c in "╭╮╰╯┌┐└┘─━│┃") + + +def test_legacy_panels_use_open_screen(capsys): + output.configure(no_color=True) + output.render_events([AgentEvent(id=1, session_id="run-001", agent_id="agent", event_type="tool_use", ts="2026-10-13T00:00:00Z")]) + text = capsys.readouterr().out + assert "EVENTS" in text + assert "failproof ai v" in text + assert not any(c in text for c in "╭╮╰╯┌┐└┘─━│┃") + + +def test_narrow_ids_and_urls_are_not_cut(capsys): + output.configure(no_color=True) + output._stdout = Console(width=45, no_color=True) + id = "session-001-abcdefghijklmnopqrstuvwxyz-123456789" + url = "https://cloud.example.test/organizations/reliability/sessions/" + id + output.print_table(["Id", "URL"], [[id, url]]) + compact = "".join(capsys.readouterr().out.split()) + assert id in compact and url in compact + assert "…" not in compact + + +def test_json_bytes_unchanged(capsys): + output.configure(json=True) + data = {"session_id": "run-001", "scores": {"quality": 0.9}} + output.emit_json(data) + assert capsys.readouterr().out == json.dumps(data, indent=2, ensure_ascii=False) + "\n" + + +def test_generated_help_has_no_boxes(logged_in, runner): + from fp_cli.app import app + result = runner.invoke(app, ["evaluators", "create", "--help"]) + assert result.exit_code == 0 + assert "Code:" in result.stdout and "Judge:" in result.stdout and "Jev:" in result.stdout + assert not any(c in result.stdout for c in "╭╮╰╯┌┐└┘─━│┃") diff --git a/fp-cloud-cli/tests/test_whoami.py b/fp-cloud-cli/tests/test_whoami.py index b03e354fc..c3c47879f 100644 --- a/fp-cloud-cli/tests/test_whoami.py +++ b/fp-cloud-cli/tests/test_whoami.py @@ -66,7 +66,7 @@ def test_whoami_human_renders_header_and_panels(logged_in, runner): assert "62230791-4811-4f04-b388-ae57bdcb422e" in out assert "instance admin" in out # permissions panel - assert "permissions" in out + assert "PERMISSIONS" in out assert "dashboards" in out and "read" in out and "write" in out and "delete" in out # orgs panel + switch hint to the non-active org assert "your orgs" in out From 3f73e853ac65ba238d9e6cad4211bfb4ac9be552 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:17:19 +0000 Subject: [PATCH 08/16] docs(fp): document evaluator workflows and add installed CLI smoke coverage --- CHANGELOG.md | 2 + docs/reference/cloud-cli.mdx | 112 +++++++++ fp-cloud-cli/CHANGELOG.md | 15 ++ fp-cloud-cli/README.md | 109 ++++++++- fp-cloud-cli/skill/SKILL.md | 79 ++++++- fp-cloud-cli/skill/references/commands.md | 109 ++++++++- fp-cloud-cli/tests/fake_cloud_smoke.py | 263 ++++++++++++++++++++++ 7 files changed, 682 insertions(+), 7 deletions(-) create mode 100644 fp-cloud-cli/tests/fake_cloud_smoke.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f09f212b2..ee4f3dabd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features +- Cloud CLI parity: hosted code, LLM-judge and Jev evaluator authoring/deployment, dashboards, sentiment, assistant artifacts/approvals, org controls and the shared mint/pink screen language. + - New terminal look for the CLI, step one: the palette gains the redesign's label grey, progress-track grey and a red for failures. Descriptions keep the terminal's own colour, and the greys fall back to the dim attribute below 24-bit, so every screen stays readable on light themes (#881) - Pickers in the new look: an UPPERCASE heading with a live "n of m selected", a `›` cursor and `■ □` boxes, twelve rows before scrolling, plain `a` to toggle all, and the key hints at the foot. The `policies -i` agent menu now uses the shared picker (#881) - Every `--help` page in the new look, cut to usage, options and examples, with every real flag and its real default. The index opens with three getting-started steps, then lists every command by what it is for. `policies add|remove|show --help` now prints the one policies page (#881) diff --git a/docs/reference/cloud-cli.mdx b/docs/reference/cloud-cli.mdx index 3bab0efb5..23947d47a 100644 --- a/docs/reference/cloud-cli.mdx +++ b/docs/reference/cloud-cli.mdx @@ -6,6 +6,9 @@ icon: "cloud-cog" Use `fp` to inspect Cloud telemetry, manage cloud-managed enforcement (policies, fleet deployments, guardrail decisions), and manage audits, findings, issues, alerts, keys, users, queries, and settings. Use [`failproofai`](/reference/failproof-cli) for local hooks, policies, capture, and machine enrollment. +You can also author and deploy hosted code, LLM-judge and Jev evaluators, manage +dashboards, inspect sentiment, and use the dashboard assistant from the terminal. + Install the released Cloud CLI as an isolated tool: ```bash @@ -36,6 +39,115 @@ Run `fp COMMAND --help` or `fp COMMAND SUBCOMMAND --help` for terminal help. ## CLI commands +### Author an evaluator + +`fp evaluators` authors definitions; `fp evals` reads results. An enabled managed +instance is required (`fp evaluators instances`). Authoring needs +`evaluations:trigger`; session dry-runs and schema profiling also need `events:read`. + +Code is one restricted Python expression. Save this as `error_free.py`: + +```python +EvalResult(score=Score(1.0 if session.count("error") == 0 else 0.0), reasoning="Error-free session") +``` + +```bash +fp --json evaluators create --key error_free --name "Error free" --code error_free.py --dry-run-on run-001 +fp --json evaluators dry-run --code error_free.py --session run-001 +fp --json evaluators deploy error_free --code error_free.py --version 2 --dry-run-on run-001 +fp --json evaluators list --key error_free --all +``` + +LLM judge: + +```bash +fp --json evaluators create --key relevance --name Relevance --judge --criteria "Grade relevance to the request" --threshold 0.7 +fp --json evaluators deploy relevance --version 2 --judge --criteria "Grade relevance and completeness" --threshold 0.8 +fp --json sessions re-evaluate run-001 +fp --json sessions history run-001 +``` + +Jev binary and graded rubric: + +```bash +fp --json evaluators create --key resolution --name Resolution --jev noul --instructions "Was the request resolved?" --true "Resolved" --false "Unresolved" +fp --json evaluators dry-run --jev noul --instructions "Was the request resolved?" --true "Resolved" --false "Unresolved" --session run-001 +fp --json evaluators deploy resolution --version 2 --jev score --instructions "Grade resolution quality" --level "Unresolved" --level "Partly resolved" --level "Fully resolved" +``` + + +Creation enables the evaluator. Deploy creates a new immutable version, then +disables older enabled versions. A failed follow-up leaves the new version +published: inspect `evaluators list --all` before retrying. +Judge dry-runs are refused by Cloud. Jev dry-runs spend model budget. +Backfill excludes judge/Jev and accepts at most 90 days. + + +AI drafting requires `fp login` and writes a spec accepted by create/deploy: + +```bash +fp --json evaluators draft "Measure error-free sessions with code" --kind code --out evaluator.json +# Review the draft's key, source and condition before enabling it. +fp --json evaluators create --file evaluator.json --dry-run-on run-001 +fp --json evaluators dry-run --file evaluator.json +``` + +| Command | Options / behavior | +| --- | --- | +| `fp evaluators list` | `--key`, `--all` | +| `fp evaluators show KEY[@VERSION]` | Full source/envelope | +| `fp evaluators create` | `--key`, `--name`, `--version`, repeated `--label`, `--condition EXPR\|@file`, one source, repeated `--dry-run-on`, `--disabled` | +| `fp evaluators deploy KEY` | Same source flags; `--version`, code-only `--backfill 7d`; numeric versions auto-increment | +| `fp evaluators draft "intent"` | Session; `--kind code\|judge\|jev`, `--out` | +| `fp evaluators dry-run` | Source flags or `--file`, repeated `--session` (maximum 10); none = compile only | +| `fp evaluators enable\|disable KEY[@VERSION]` | Toggle | +| `fp evaluators label KEY[@VERSION]` | Repeated `--add`, `--remove` | +| `fp evaluators instances\|schema\|jobs` | Inspect managed fleet/schema/jobs | +| `fp evaluators backfill` | `--from`, `--to`, `--key`, `--dry-run` | + +Code flags are `--code FILE|-`, `--result-kind score|metric|assertion`, +`--timeout S`. Judge flags are `--judge --criteria TEXT`, repeated `--step` and +`--threshold`. Jev flags are `--jev noul --instructions --true --false` or +`--jev score --instructions` with 3–5 distinct `--level` values, worst first. +Judge/Jev require a score; thresholds are 0–1. Binary judge steps are rejected. + +Spec files contain exactly `eval_key`, `display_name`, `eval_version`, +`result_kind`, `labels`, `condition_source`, `evaluator_source`, `timeout_seconds`. +Judge/Jev `evaluator_source` is a JSON envelope string. Do not include row ids, +enabled state, or AI reasoning metadata. `--file` and discrete spec flags are +exclusive; deploy can override the file's version. Output files are never overwritten. + +### New dashboard controls + +All commands accept global `--json`. New 404s exit 6 with “not available on this +FailproofAI Cloud yet”. Session-only commands deny key mode before HTTP (exit 2). + +| Commands | Options / notes | +| --- | --- | +| `fp dashboards list\|show ID\|create\|update ID\|delete ID` | Create `--name`, `--description`, `--file` definition; update preserves unmentioned fields | +| `fp dashboards tiles ID list\|add\|update TILE\|remove TILE\|layout` | Add `--query ID --chart TYPE --title`; config/geometry or layout via `--file` | +| `fp sessions show\|export\|history\|re-evaluate ID` | Export `--out`; history is evaluation history | +| `fp events latency` | `--since` | +| `fp sentiment summary\|messages\|sessions\|session ID` | `--since`; list `--limit`; messages `--session-id` | +| `fp query compose "question"` | Session; `--run` | +| `fp query export [ID]` | Or `--sql`; `--format csv\|json`, `--out` | +| `fp alerts recipients` / `fp alerts issues ID` | Issues can `--create --title --summary` or `--create --file` | +| `fp audits contracts list\|show ID\|create\|update ID` | `--agent-id`, `--body TEXT\|@file`, `--file` JSON; update `--enabled/--disabled` | +| `fp policies author "intent"\|backtest FILE_OR_ID\|candidacy --issue ID` | Session; never auto-publishes | +| `fp agent brief` | Session; `--range 24h\|7d` | +| `fp orgs create --name N [--slug S]` | Session; derives omitted slug, does not switch org | +| `fp roles list\|show NAME\|create\|update NAME\|delete NAME` | Permission sets; create `--name --permission`; update replace `--permission` or merge `--add/--remove`; `--description`, `--file` | +| `fp users delete ID` / `fp users defaults` | Deletion by id | +| `fp usage windows` / `fp plan` / `fp plan claim OFFER` | Offers and trial; Cloud requires actor identity for offers | +| `fp settings context-windows list\|set MODEL TOKENS\|delete MODEL` | 0–1000000 tokens | +| `fp otel status\|enable\|disable` | Cloud org ingest switch, not local capture | +| `fp slack status\|disconnect` | Session | + +`fp agent ask` renders artifact SQL as tables/stats through `queries:run`. +Write approvals show the proposed change and prompt on a terminal. Scripts and +`--json` decline unless `--approve` is explicit. Existing JSON shapes stay +unchanged when no artifact is returned. + ### Authentication | Command | Purpose | Options | diff --git a/fp-cloud-cli/CHANGELOG.md b/fp-cloud-cli/CHANGELOG.md index 6c273f990..5652c20bf 100644 --- a/fp-cloud-cli/CHANGELOG.md +++ b/fp-cloud-cli/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog — `fp` CLI +## 0.0.2b1 — 2026-10-13 + +### Added + +- Hosted evaluator authoring for code, LLM judge and Jev: create/deploy, AI draft specs, dry-run, labels, toggles, instances, schema, jobs and code-only backfill. +- Dashboards and tiles; session overview/export/history/re-evaluate; latency and sentiment; query composition/export; alert recipients/issues and agent contracts. +- Policy author/backtest/candidacy, assistant artifact tables, fail-closed interactive write approvals, and agent-curated home briefs. +- Organization creation, permission-set roles, member deletion/defaults, usage windows, offers/trial, model context windows, OTEL and Slack controls. + +### Changed + +- New Cloud API families work in key mode; unsupported session-only commands deny before HTTP and older-Cloud 404s exit 6 with an availability message. +- Shared screens use the mint/pink wordmark and grey labels, without boxes/rules or shortened ids; existing JSON contracts are unchanged. +- The agent skill, command reference, README and Cloud CLI docs include end-to-end evaluator workflows. No version bump or release is performed. + ## 0.0.2b0 — 2026-09-22 ### Added diff --git a/fp-cloud-cli/README.md b/fp-cloud-cli/README.md index f07ec157c..9be580909 100644 --- a/fp-cloud-cli/README.md +++ b/fp-cloud-cli/README.md @@ -171,6 +171,114 @@ These commands are **session-only** (`fp login`). They are absent from the versioned API an API key authenticates against, so `--api-key` exits 2 with the reason rather than failing at the request. +## Author an evaluator + +`fp evaluators` authors hosted definitions; `fp evals` continues to read results. +Creation enables a definition by default. Check `fp evaluators instances` first: +Cloud needs an enabled managed instance. Writes need `evaluations:trigger`; +session dry-runs and `evaluators schema` also need `events:read`. + +Write `error_free.py` as a restricted Python expression (no imports or functions): + +```python +EvalResult(score=Score(1.0 if session.count("error") == 0 else 0.0), reasoning="Error-free session") +``` + +```bash +fp --json evaluators create --key error_free --name "Error free" --code error_free.py \ + --result-kind score --dry-run-on run-001 +fp --json evaluators dry-run --code error_free.py --session run-001 +fp --json evaluators deploy error_free --code error_free.py --version 2 --dry-run-on run-001 +fp --json evaluators list --key error_free --all +``` + +LLM judge (Cloud selects the model; judge dry-runs are refused): + +```bash +fp --json evaluators create --key relevance --name Relevance --judge \ + --criteria "Grade how directly the answer addresses the request" --threshold 0.7 +fp --json evaluators deploy relevance --version 2 --judge \ + --criteria "Grade relevance and completeness" --threshold 0.8 +fp --json sessions re-evaluate run-001 +fp --json sessions history run-001 +``` + +Jev binary or graded rubric: + +```bash +fp --json evaluators create --key resolved --name Resolved --jev noul \ + --instructions "Was the request resolved?" --true "Resolved" --false "Unresolved" +fp --json evaluators dry-run --jev noul --instructions "Was the request resolved?" \ + --true "Resolved" --false "Unresolved" --session run-001 +fp --json evaluators deploy resolved --version 2 --jev score \ + --instructions "Grade resolution quality" \ + --level "Unresolved" --level "Partly resolved" --level "Fully resolved" +``` + +Jev dry-runs spend model budget. Rubrics need 3–5 distinct levels, worst first. +Thresholds are 0–1; judge/Jev always return a score. `deploy` creates the new +immutable version before disabling older enabled versions. If disabling fails, +the CLI names the published id; inspect `list --all` and recover explicitly. +`--backfill 7d` is code-only; `evaluators backfill --from T --to T` accepts ≤90 days. + +AI drafting is session-only and writes an API spec that create/deploy accept: + +```bash +fp login +fp --json evaluators draft "Measure error-free sessions with code" --kind code --out evaluator.json +# Review the spec, especially eval_key and evaluator_source, before enabling it. +fp --json evaluators create --file evaluator.json --dry-run-on run-001 +fp --json evaluators dry-run --file evaluator.json +``` + +Spec fields: `eval_key`, `display_name`, `eval_version`, `result_kind`, `labels`, +`condition_source`, `evaluator_source`, `timeout_seconds`. Judge/Jev sources are +JSON envelope **strings**, not nested objects. `--file` is exclusive with discrete +spec flags; deploy may override a file's version. Files written by the CLI are +created exclusively, so existing files are never silently replaced. +See [the command reference](skill/references/commands.md#author-an-evaluator). + +## Dashboard parity + +```text +fp dashboards list|show|create|update|delete +fp dashboards tiles list|add|update|remove|layout +fp sessions show|export|history|re-evaluate +fp events latency +fp sentiment summary|messages|sessions|session +fp query compose "question" [--run] # session +fp query export [] [--sql SQL] --format csv|json [--out FILE] +fp alerts recipients +fp alerts issues [--create --title ... --summary ...] +fp audits contracts list|show|create|update +fp policies author "intent"|backtest |candidacy --issue # session +fp agent brief [--range 24h|7d] # session +fp orgs create --name N [--slug S] # session +fp roles list|show|create|update|delete +fp users delete +fp users defaults +fp usage windows +fp plan +fp plan claim +fp settings context-windows list|set |delete +fp otel status|enable|disable +fp slack status|disconnect # session +``` + +`fp agent ask` runs artifact SQL through the permission-gated query runner and +renders tables/stats. It prompts before approving a write on a terminal. +Scripts and `--json` **decline** unless you explicitly pass `--approve`. +Normal create/update commands keep their existing non-interactive confirmation +behavior; assistant approvals are a separate safety boundary. + +All new commands accept the global `--json` before the command. Session-only +commands fail before HTTP in API-key mode, with exit 2 and a `fp login` hint. +Older-Cloud 404s exit 6 with “not available on this FailproofAI Cloud yet”. +`plan` routes are mapped for keys, but the Cloud also requires an actor email; +an API key alone cannot impersonate a person to claim an offer. +Human screens use the mint/pink wordmark, grey labels, open tables and full ids; +existing JSON shapes are unchanged. + ## Configuration | Setting | Flag | Env var | Default | @@ -256,4 +364,3 @@ working as an opt-out if it is ever switched back on. See cd fp-cloud-cli uv run --extra dev pytest ``` - diff --git a/fp-cloud-cli/skill/SKILL.md b/fp-cloud-cli/skill/SKILL.md index 1cb5392fd..614225e82 100644 --- a/fp-cloud-cli/skill/SKILL.md +++ b/fp-cloud-cli/skill/SKILL.md @@ -9,7 +9,9 @@ description: |- Served by the `fp` CLI against FailproofAI Cloud. - NOT for writing or designing an evaluator service / scoring logic (that's `agenteye-evaluator`), adding SDK/instrumentation to your app (that's `failproofai-sdk`, imported as `failproofai_sdk`), debugging the collector/daemon, or unrelated dev work (why a build/CI run failed, rotating non-FailproofAI Cloud secrets). + Also use for authoring, testing and deploying hosted code, LLM-judge or Jev evaluators with fp evaluators, and inspecting dashboards, sentiment and OTEL status. + + NOT for self-hosted evaluator services, SDK instrumentation, collector/daemon debugging, or unrelated dev work. --- # FailproofAI Cloud CLI @@ -77,7 +79,7 @@ Throughout this skill, `fp` means "whichever form you resolved." a saved session (Click treats an empty env var as unset, so it falls to rung 4). - **Some commands need a signed-in user.** `login`, `logout`, `orgs *`, the whole - `agent` group, `keys update`, and all of `policies *` / `fleet *` / + `agent` group, `evaluators draft`, `query compose`, `slack *`, `keys update`, and all of `policies *` / `fleet *` / `guardrails *` refuse a key with a usage error (**exit 2**) and make **no network call at all** — there is no user to sign in, no saved active org to switch, no private assistant thread to own, and the enforcement write routes @@ -86,8 +88,7 @@ Throughout this skill, `fp` means "whichever form you resolved." is refused up front like the rest. The key is not the problem to fix — plan around them rather than retrying or hunting for a flag. - **Default to `--json` and parse it.** It prints clean JSON to stdout and - nothing else. The plain output is a boxed Rich UI meant for human eyes — it - burns context with box-drawing characters and is awkward to parse. Use the + nothing else. The plain output uses open Rich tables for human eyes. Use the rendered output only when the user explicitly wants to *look* at something. - **Data → stdout, status/errors/prompts → stderr.** So a `--json` stdout capture is pure JSON even when a status line is shown. @@ -101,7 +102,7 @@ Throughout this skill, `fp` means "whichever form you resolved." | 3 | can't reach the dashboard | check base-url / connectivity | | 4 | no usable credential — not signed in, session expired, **or the API key was rejected** | session: user must run `fp login`. Key: it's missing, mistyped, disabled, or belongs to another deployment — don't retry, and don't fall back to a session | | 5 | authenticated but missing permission | message names the exact permission | - | 6 | resource not found | the named resource doesn't exist | + | 6 | resource not found / newer capability unavailable | check the id or Cloud version; do not retry blindly | ## 3. First call: confirm you're connected @@ -315,3 +316,71 @@ Two things about the flow matter when driving it from the CLI: Typical end-to-end: `audits list` → `audits run ` → poll `audits runs ` → `audits findings --audit ` (highest priority first) → `audits finding ` for the full write-up → **confirm with the user** → `audits resolve `. + +## Author an evaluator + +Use `fp evaluators` for hosted authoring; `fp evals` remains the results reader. +Read [the evaluator command reference](references/commands.md#author-an-evaluator) +for the full flags and API spec shape. Creation enables a definition by default; +confirm the key, version, kind and budget before creating/deploying on a live org. + +Discover with `fp --json evaluators instances` and `fp --json evaluators schema`. +The instance must be enabled. Authoring needs `evaluations:trigger`; session +dry-runs and schema profiling also need `events:read`. + +Code workflow (write `error_free.py` as a single restricted Python expression, +not a module with imports or a function): + +```python +EvalResult(score=Score(1.0 if session.count("error") == 0 else 0.0), reasoning="Error-free session") +``` + +```bash +fp --json evaluators draft "Measure the absence of error events with code" --kind code --out error-free.json +fp --json evaluators create --file error-free.json --dry-run-on run-001 +fp --json evaluators dry-run --code error_free.py --session run-001 +fp --json evaluators deploy error_free --code error_free.py --version 2 --dry-run-on run-001 +fp --json evaluators list --key error_free --all +fp --json evals --since 24h +``` + +Review the draft before creating. It may choose a different `eval_key`: read +the file and use its key later; the filename does not determine the key. + +Judge workflow (server selects the model; judge dry-runs are refused): + +```bash +fp --json evaluators create --key relevance --name Relevance --judge \ + --criteria "Grade how directly the response answers the request" \ + --step "Assess relevance and specificity on a continuous scale" --threshold 0.7 +fp --json evaluators deploy relevance --version 2 --judge \ + --criteria "Grade relevance, accuracy and completeness" --threshold 0.8 +fp --json sessions re-evaluate run-001 +fp --json sessions history run-001 +``` + +Jev workflow (dry-runs spend model budget): + +```bash +fp --json evaluators create --key resolution --name Resolution --jev noul \ + --instructions "Did the assistant resolve the request?" \ + --true "Resolved with supporting evidence" --false "Unresolved" --threshold 0.8 +fp --json evaluators dry-run --jev noul --instructions "Was the request resolved?" \ + --true "Resolved" --false "Unresolved" --session run-001 +fp --json evaluators deploy resolution --version 2 --jev noul \ + --instructions "Was the request resolved without unsupported claims?" \ + --true "Resolved with evidence" --false "Unresolved or unsupported" +fp --json evaluators show resolution@2 +``` + +Use `--jev score` with 3–5 distinct `--level` values, ordered worst to best, for +a rubric. `deploy` creates the new immutable version **before** disabling older +enabled versions. A partial failure names the new id: inspect `list --all`, +then disable any remaining old version; do not retry creation blindly. +Backfill excludes judge/Jev and accepts at most 90 days. + +AI drafting needs a signed-in session (`fp login`); evaluator API operations can +use a scoped key. New capability 404s exit 6 with “not available on this +FailproofAI Cloud yet”. An assistant approval is different from ordinary +mutation confirmation: `agent ask` declines writes in scripts/`--json` unless +`--approve` was explicitly authorized by the user. diff --git a/fp-cloud-cli/skill/references/commands.md b/fp-cloud-cli/skill/references/commands.md index b498bc9ac..1f7612456 100644 --- a/fp-cloud-cli/skill/references/commands.md +++ b/fp-cloud-cli/skill/references/commands.md @@ -17,6 +17,8 @@ command**, **`--json` to parse**, **branch on exit codes**. - [issues](#issues) - [query](#query) - [agent](#agent) +- [Author an evaluator](#author-an-evaluator) +- [Cloud dashboard parity](#cloud-dashboard-parity) ## Global options Set on the CLI, **before** the subcommand. Precedence: flag > env var > config file (`~/.failproofai/fpcli/cli-auth.json`, mode 0600). @@ -117,7 +119,7 @@ scale. `fp usage` — the active org's current fixed 30-day metering window, grouped for human reading. Needs `usage:read`. `fp --json usage` returns the dashboard contract unchanged: `org_id`, `billing_anchor`, `window`, `usage`, `calculated_at`, and `stale_after`. It has no -filters or subcommands and is read-only; limits and enforcement are not part of this command. +filters and is read-only; `usage windows` lists historical windows. ### list `fp list ` — discover valid filter values. Kinds: `envs agents event_types score_filters models hooks tools error_types`. JSON `{kind, values}`. Run this before filtering by a value you're unsure of. @@ -255,3 +257,108 @@ Cloud-managed enforcement, split the way the dashboard splits it: `policies` wri - `guardrails timeline [--since …] [--machine ID]` — one row per time bucket: a bar scaled to the busiest bucket with the blocked share in red, plus total / denied / instructed counts. Answers *when* enforcement bit, which the summary's sparkline only sketches. - A `(no policy)` row is **normal**: most evaluations are allows nothing objected to, and the row keeps the denominator visible. - Coverage comes from the control plane, decision counts from reported telemetry — a machine can be deployed-to and silent, or reporting and undeployed. + +## Author an evaluator + +Every invocation accepts global `--json` **before** the command. `evals` reads +results; `evaluators` authors definitions. Code is one restricted Python expression. +Judge/Jev use JSON envelopes inside `evaluator_source`. Never execute downloaded +or model-authored source locally merely to inspect it. + +| Command | Flags / behavior | +| --- | --- | +| `evaluators list` | `--key K`, `--all` (include disabled versions) | +| `evaluators show K[@V]` | full source; specify version when ambiguous | +| `evaluators create` | `--key K --name N`, optional `--version V` (default `1`), repeated `--label`, `--condition EXPR\|@file`, exactly one source below, repeated `--dry-run-on SESSION`, `--disabled` | +| `evaluators deploy K` | same sources/metadata, optional `--version`; numeric versions auto-increment; creates first, then disables predecessors; `--backfill 7d` code only | +| `evaluators draft "intent"` | session only; `--kind code\|judge\|jev`, `--out spec.json` (refuses to overwrite) | +| `evaluators dry-run` | source flags or `--file`; repeated `--session` (max 10), none = compile only; judge refused, Jev spends budget | +| `evaluators enable\|disable K[@V]` | toggle definition | +| `evaluators label K[@V]` | repeated `--add` / `--remove` | +| `evaluators instances\|schema\|jobs` | managed instances, event schema profile, jobs | +| `evaluators backfill` | `--from T --to T`, optional `--key K`, `--dry-run`; increasing range ≤90 days; excludes judge/Jev | + +Sources: + +- Code: `--code path.py` (or `-` for stdin), `--result-kind score|metric|assertion`, + optional `--timeout S` (0.001–3600 seconds). +- Judge: `--judge --criteria TEXT`, optional repeated `--step TEXT`, `--threshold X`. + Steps must describe continuous grading, not “score 0/1”. +- Jev binary: `--jev noul --instructions TEXT --true TEXT --false TEXT`, optional threshold. +- Jev rubric: `--jev score --instructions TEXT --level Worst --level Middle --level Best`. + 3–5 distinct, case-insensitive levels, **worst first**. +- `--file spec.json`: the API body below; do not combine with discrete spec flags. + `deploy KEY --file spec.json --version 2` may override the version. + +Spec file (exact API fields; no `id`, `enabled`, `kind` or draft `reasoning`): + +```json +{ + "eval_key": "error_free", + "display_name": "Error free", + "eval_version": "1", + "result_kind": "score", + "labels": ["reliability"], + "condition_source": null, + "evaluator_source": "EvalResult(score=Score(1.0 if session.count(\"error\") == 0 else 0.0), reasoning=\"Error-free session\")", + "timeout_seconds": 5 +} +``` + +Judge source string: `{"kind":"judge","criteria":"Grade relevance","threshold":0.7,"steps":["Assess specificity on a continuous scale"]}`. +Jev source string: `{"kind":"jev","type":"noul","instructions":"Was the request resolved?","criteria":{"true":"Resolved","false":"Unresolved"},"threshold":0.8}`. +Judge/Jev require `result_kind: "score"`; thresholds are 0–1 or null. + +Code, judge and Jev end to end: + +```bash +fp --json evaluators draft "Measure error-free sessions with code" --kind code --out error-free.json +# Read the draft's eval_key, review its source, then create. +fp --json evaluators create --file error-free.json --dry-run-on run-001 +fp --json evaluators dry-run --file error-free.json --session run-001 +fp --json evaluators deploy error_free --file error-free.json --version 2 + +fp --json evaluators create --key relevance --name Relevance --judge --criteria "Grade relevance" --threshold 0.7 +fp --json evaluators deploy relevance --version 2 --judge --criteria "Grade relevance and completeness" --threshold 0.8 + +fp --json evaluators create --key resolution --name Resolution --jev noul --instructions "Was the request resolved?" --true "Resolved" --false "Unresolved" +fp --json evaluators dry-run --jev noul --instructions "Was the request resolved?" --true "Resolved" --false "Unresolved" --session run-001 +fp --json evaluators deploy resolution --version 2 --jev score --instructions "Grade resolution quality" --level "Unresolved" --level "Partly resolved" --level "Fully resolved" +fp --json sessions re-evaluate run-001 +fp --json sessions history run-001 +fp --json evaluators list --all +``` + +Check draft keys before copying these examples. A failed `--dry-run-on` prevents +creation. A failed predecessor disable leaves the new version published: inspect +`list --all` and recover with explicit `disable KEY@VERSION`, not another create. +API 409 can mean no enabled managed instance or an existing version; 422 can mean +the org has 100 enabled keys. Read the error, do not rename/retry silently. + +## Cloud dashboard parity + +All new commands degrade on API 404 with exit 6 and “not available on this +FailproofAI Cloud yet”. Session-only commands exit 2 before HTTP in key mode. +Human output uses mint/pink open screens and full ids; legacy `--full-ids` +remains accepted. Exports refuse to overwrite existing files. + +| Commands | Options / notes | +| --- | --- | +| `dashboards list\|show ID\|create\|update ID\|delete ID` | create `--name`, `--description`, optional `--file` definition; update selected fields | +| `dashboards tiles ID list\|add\|update TILE\|remove TILE\|layout` | add `--query ID --chart line\|area\|bar\|stacked_bar\|pie\|kpi\|table`, `--title`; update supports same flags or `--file`; layout `--file` with `{tiles:[{id,pos_x,pos_y,pos_w,pos_h}]}` | +| `sessions show\|export\|history\|re-evaluate ID` | export `--out`; show includes overview + event timeline | +| `events latency` | `--since` | +| `sentiment summary\|messages\|sessions\|session ID` | list reads accept `--since`; messages/sessions `--limit`; messages `--session-id` | +| `query compose "question"` | session; `--run` | +| `query export [ID]` | or `--sql TEXT\|@file`; `--format csv\|json`, `--out`; with `--json`, CSV is `{format,content}` | +| `alerts recipients` / `alerts issues ID` | issues: optional `--create --title --summary` or `--create --file` | +| `audits contracts list\|show ID\|create\|update ID` | `--agent-id`, `--body TEXT\|@file`, or `--file` API JSON; update `--enabled/--disabled` | +| `policies author "intent"\|backtest FILE_OR_ID\|candidacy --issue ID` | session; drafts/replays only, never auto-publishes | +| `agent ask "question"` / `agent brief` | session; ask renders artifacts and prompts for approval on a terminal; scripts/JSON decline unless `--approve`; brief `--range 24h\|7d` | +| `orgs create --name N [--slug S]` | session; omitted slug derives from name; never auto-switches | +| `roles list\|show NAME\|create\|update NAME\|delete NAME` | permission sets; create `--name --permission`; update `--permission` replaces or `--add/--remove` merges; `--description`, `--file`; builtins immutable | +| `users delete ID` / `users defaults` | deletion is by id, not email | +| `usage windows` / `plan` / `plan claim OFFER` | offers + trial; offers still require an actor identity on the Cloud | +| `settings context-windows list\|set MODEL TOKENS\|delete MODEL` | tokens 0–1000000; model names may contain `/` | +| `otel status\|enable\|disable` | Cloud org switch `otel.ingest_enabled`, not local collector configuration | +| `slack status\|disconnect` | session only | diff --git a/fp-cloud-cli/tests/fake_cloud_smoke.py b/fp-cloud-cli/tests/fake_cloud_smoke.py new file mode 100644 index 000000000..b6b13479d --- /dev/null +++ b/fp-cloud-cli/tests/fake_cloud_smoke.py @@ -0,0 +1,263 @@ +"""User-level CLI smoke against a local Cloud fake, including paused SSE. + +Run after building/installing the wheel: + python tests/fake_cloud_smoke.py --fp /tmp/fp-smoke/bin/fp --artifacts /job/artifacts + +The fake is intentionally server-shaped and does not import the CLI's client. +It exercises actual subprocesses, cookie login, bearer routing and an interactive +approval through a pseudo-terminal. Never connect this script to a live Cloud. +""" + +from __future__ import annotations + +import argparse +import json +import os +import pty +import re +import select +import subprocess +import tempfile +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + + +class Cloud: + definitions = [] + dashboards = [] + tiles = [] + requests = [] + answers = [] + pending = {} + unavailable = False + source = 'EvalResult(score=Score(1.0 if session.count("error") == 0 else 0.0), reasoning="Error-free session")' + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def json_response(self, data, status=200, headers=None): + raw = json.dumps(data).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + for key, value in (headers or {}).items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(raw) + + def stream(self, events): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Connection", "close") + self.end_headers() + for event in events: + self.wfile.write(("data: " + json.dumps(event) + "\n\n").encode()) + self.wfile.flush() + if event["type"] == "ask-user": + if not Cloud.pending[event["callId"]].wait(10): + raise RuntimeError("CLI did not answer the pending approval") + self.close_connection = True + + def do_GET(self): + self.dispatch("GET") + + def do_POST(self): + self.dispatch("POST") + + def do_PATCH(self): + self.dispatch("PATCH") + + def do_PUT(self): + self.dispatch("PUT") + + def dispatch(self, method): + raw = self.rfile.read(int(self.headers.get("Content-Length", 0))) + body = json.loads(raw) if raw else None + path = self.path.split("?", 1)[0] + key_mode = path.startswith("/v1/") + route = re.sub(r"^/(api|v1)", "", path) + Cloud.requests.append({"method": method, "path": path, "body": body, "auth": "key" if key_mode else "session"}) + if route == "/auth/otp/request": + return self.json_response({"ok": True}) + if route == "/auth/otp/verify": + assert body["code"] == "123456" + return self.json_response({"user": {"id": "u1", "email": "smoke@example.test"}, "expires_in_secs": 3600}, + headers={"Set-Cookie": "ae_session=fake-session; Path=/"}) + if key_mode: + assert self.headers.get("Authorization") == "Bearer fake-key" + assert not self.headers.get("Cookie") + else: + assert "ae_session=fake-session" in self.headers.get("Cookie", "") + assert not self.headers.get("Authorization") + if Cloud.unavailable: + return self.json_response({"error": "old Cloud route absent"}, 404) + if route == "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/auth/session": + return self.json_response({"id": "u1", "email": "smoke@example.test", "memberships": [ + {"org_id": "o1", "org_slug": "smoke", "org_name": "Smoke", "permissions": ["evaluations:trigger", "events:read", "queries:run", "agent:use"], "permission_set": "admin"}]}) + if route == "/access-granters": + return self.json_response([]) + if route == "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/agent/compose-eval": + return self.json_response({"eval_key": "error_free", "display_name": "Error free", "result_kind": "score", "labels": ["reliability"], + "condition_source": "", "evaluator_source": Cloud.source, "reasoning": "Drafted from intent", "kind": "code"}) + if route == "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/evaluator-definitions/dry-run": + assert len(body["session_ids"]) <= 10 + return self.json_response({"runs": [{"ok": True, "session_id": sid, "mode": "run", "result": {"score": {"value": 1.0}}} for sid in body["session_ids"]], + "summary": {"failed": 0, "succeeded": len(body["session_ids"]), "skipped": 0}}) + if route == "/evaluator-definitions": + if method == "GET": + return self.json_response(Cloud.definitions) + assert body["eval_key"] and body["display_name"] and body["eval_version"] + assert isinstance(body["evaluator_source"], str) + if body["evaluator_source"].startswith("{"): + envelope = json.loads(body["evaluator_source"]) + assert envelope["kind"] in ("judge", "jev") + assert body["result_kind"] == "score" + row = {**body, "id": f"d{len(Cloud.definitions) + 1}", "enabled": True} + Cloud.definitions.append(row) + return self.json_response(row, 201) + if route.startswith("/evaluator-definitions/") and method == "PATCH": + row = next(r for r in Cloud.definitions if r["id"] == route.rsplit("/", 1)[1]) + assert set(body) <= {"enabled", "labels", "display_name"} + row.update(body) + return self.json_response(row) + if route == "/dashboards": + if method == "GET": + return self.json_response({"dashboards": Cloud.dashboards}) + assert isinstance(body["definition"], dict) + row = {**body, "id": "board1"} + Cloud.dashboards.append(row) + return self.json_response(row, 201) + if route == "/dashboards/board1/tiles": + if method == "GET": + return self.json_response({"tiles": Cloud.tiles}) + assert body["query_id"] == "q1" and body["chart_type"] == "bar" + assert all(k in body for k in ("pos_x", "pos_y", "pos_w", "pos_h")) + row = {**body, "id": "tile1"} + Cloud.tiles.append(row) + return self.json_response(row, 201) + if route.startswith("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/events/sentiment"): + return self.json_response({"positive": 4, "negative": 1, "neutral": 2}) + if route == "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/queries/run": + assert body["sql"].lower().startswith("select") + return self.json_response({"columns": [{"name": "agent", "type": "String"}, {"name": "events", "type": "UInt64"}], + "rows": [["agent-smoke", 42]], "truncated": False, "elapsed_ms": 1}) + if route == "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/agent/chat": + call_id = f"ask_{len(Cloud.answers) + 1}" + Cloud.pending[call_id] = threading.Event() + return self.stream([ + {"type": "ui-action", "action": "open_artifact", "artifact": {"title": "Events by agent", "sql": "SELECT agent_id, count() FROM events GROUP BY agent_id"}}, + {"type": "ask-user", "kind": "approval", "callId": call_id, "question": "Approve disabling the test alert?", "details": "Change: alert test-alert enabled=false", "allowText": False}, + {"type": "text-delta", "text": "The requested summary is ready."}, + {"type": "done"}, + ]) + if route == "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/agent/answer": + assert body["decision"] in ("approve", "reject") + Cloud.answers.append(body) + Cloud.pending[body["callId"]].set() + return self.json_response({}, 200) + if route == "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/agent/conversations" and method == "POST": + return self.json_response({"id": "chat-smoke-001"}) + if route == "/agent/conversations/chat-smoke-001/messages": + assert body["messages"][-1]["content"]["artifacts"] + return self.json_response({}) + if route == "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/agent/home-brief": + return self.json_response({"greeting": "Hello", "summary": "One agent needs attention.", + "blocks": [{"key": "traffic", "kind": "table", "title": "Events by agent", "sql": "SELECT 42"}]}) + return self.json_response({"error": "unexpected fake route"}, 404) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--fp", required=True) + parser.add_argument("--artifacts", required=True) + args = parser.parse_args() + artifacts = Path(args.artifacts) + artifacts.mkdir(parents=True, exist_ok=True) + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + base = f"http://127.0.0.1:{server.server_port}" + transcript = [] + with tempfile.TemporaryDirectory(prefix="fp-user-smoke-") as tmp: + env = {**os.environ, "FP_HOME": tmp, "FP_ANALYTICS_DISABLED": "1", "COLUMNS": "140", "NO_COLOR": "1"} + for var in ("FP_TOKEN", "FP_API_KEY", "FP_ORG", "FP_JSON", "FP_DASHBOARD_URL"): + env.pop(var, None) + fp = [args.fp, "--base-url", base] + + def run(argv, *, input=None, key=False, json_mode=True, expected=0): + cmd = fp + (["--api-key", "fake-key"] if key else []) + (["--json"] if json_mode else []) + argv + completed = subprocess.run(cmd, input=input, text=True, capture_output=True, env=env, cwd=tmp, timeout=20) + transcript.append("$ fp " + " ".join(argv) + (" [key mode]" if key else " [session mode]") + "\n" + completed.stdout + completed.stderr + f"\nexit {completed.returncode}\n") + assert completed.returncode == expected, transcript[-1] + return json.loads(completed.stdout) if json_mode and completed.stdout else completed.stdout + + run(["login", "--email", "smoke@example.test"], input="123456\n", json_mode=False) + spec = str(Path(tmp) / "evaluator.json") + run(["evaluators", "draft", "Measure error-free sessions with code", "--kind", "code", "--out", spec]) + run(["evaluators", "create", "--file", spec, "--dry-run-on", "run-001"]) + run(["evaluators", "dry-run", "--file", spec, "--session", "run-001"], key=True) + run(["evaluators", "deploy", "error_free", "--file", spec, "--version", "2"], key=True) + assert not Cloud.definitions[0]["enabled"] and Cloud.definitions[1]["enabled"] + run(["evaluators", "list", "--all"], key=True) + run(["evaluators", "disable", "error_free@2"], key=True) + run(["evaluators", "create", "--key", "relevance", "--name", "Relevance", "--judge", "--criteria", "Grade relevance"], key=True) + run(["evaluators", "dry-run", "--judge", "--criteria", "Grade relevance"], key=True, expected=2) + run(["evaluators", "create", "--key", "resolution", "--name", "Resolution", "--jev", "noul", "--instructions", "Was the request resolved?", "--true", "Resolved", "--false", "Unresolved"], key=True) + run(["dashboards", "create", "--name", "Quality"], key=True) + run(["dashboards", "tiles", "board1", "add", "--query", "q1", "--chart", "bar", "--title", "Traffic"], key=True) + run(["dashboards", "tiles", "board1", "list"]) + run(["sentiment", "summary"], key=True) + answer = run(["agent", "ask", "Count events"]) + assert answer["artifacts"][0]["result"]["rows"] == [["agent-smoke", 42]] + assert Cloud.answers[-1]["decision"] == "reject" + run(["agent", "ask", "Count events", "--approve"]) + assert Cloud.answers[-1]["decision"] == "approve" + run(["agent", "brief", "--range", "7d"], json_mode=False) + run(["evaluators", "draft", "Grade quality"], key=True, expected=2) + Cloud.unavailable = True + unavailable = run(["sentiment", "summary"], key=True, expected=6) + assert "not available on this FailproofAI Cloud yet" in unavailable["error"] + Cloud.unavailable = False + + # Real terminal approval, with the stream paused until /agent/answer. + master, slave = pty.openpty() + proc = subprocess.Popen(fp + ["agent", "ask", "Count events"], stdin=slave, stdout=slave, stderr=slave, env=env, cwd=tmp) + os.close(slave) + output = b"" + sent = False + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + ready, _, _ = select.select([master], [], [], 0.1) + if ready: + try: + data = os.read(master, 65536) + except OSError: + break + if not data: + break + output += data + if b"Approve this change" in output and not sent: + os.write(master, b"y\n") + sent = True + if proc.poll() is not None and not ready: + break + if proc.poll() is None: + proc.kill() + proc.wait(timeout=5) + os.close(master) + transcript.append("$ fp agent ask 'Count events' [interactive approval]\n" + output.decode(errors="replace") + f"\nexit {proc.returncode}\n") + assert proc.returncode == 0 and sent, transcript[-1] + assert Cloud.answers[-1]["decision"] == "approve" + server.shutdown() + (artifacts / "fp-user-smoke.txt").write_text("\n".join(transcript), encoding="utf-8") + (artifacts / "fp-fake-cloud-requests.json").write_text(json.dumps(Cloud.requests, indent=2), encoding="utf-8") + print(f"User smoke passed: {len(transcript)} CLI invocations; cookie/key auth and paused SSE approvals verified.") + + +if __name__ == "__main__": + main() From 5e1d9b23b7ccc1b582fba4356ff9c48d71cde7d7 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:23:53 +0000 Subject: [PATCH 09/16] ci(fp): verify stacked Cloud CLI without overlapping the OTEL workflow --- .github/workflows/fp-cli-parity.yml | 75 ++++++++++++++++++++++ fp-cloud-cli/tests/test_parity_workflow.py | 17 +++++ 2 files changed, 92 insertions(+) create mode 100644 .github/workflows/fp-cli-parity.yml create mode 100644 fp-cloud-cli/tests/test_parity_workflow.py diff --git a/.github/workflows/fp-cli-parity.yml b/.github/workflows/fp-cli-parity.yml new file mode 100644 index 000000000..3100e38b0 --- /dev/null +++ b/.github/workflows/fp-cli-parity.yml @@ -0,0 +1,75 @@ +name: Cloud CLI stacked parity + +# ci.yml does not yet target feat/tui-ux. Its trigger change belongs to #880; +# keep this feature's verification disjoint from that active workflow edit. +on: + pull_request: + branches: [feat/tui-ux] + paths: + - "fp-cloud-cli/**" + - "docs/reference/cloud-cli.mdx" + - ".github/workflows/fp-cli-parity.yml" + +permissions: + contents: read + +concurrency: + group: fp-parity-${{ github.ref }} + cancel-in-progress: true + +jobs: + cloud-cli: + name: Cloud CLI parity (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + defaults: + run: + working-directory: fp-cloud-cli + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + cache-dependency-glob: fp-cloud-cli/uv.lock + - name: Install locked dependencies + run: uv sync --locked --extra dev --python ${{ matrix.python-version }} + - name: Test with filesystem contract required + env: + FP_CLI_REQUIRE_CONTRACT: "1" + run: uv run --extra dev pytest -q + - name: Build distribution + run: uv build > /tmp/fp-build.log 2>&1 + - name: Clean wheel install and user-level Cloud smoke + run: | + uv venv /tmp/fp-parity-smoke --python ${{ matrix.python-version }} + uv pip install --python /tmp/fp-parity-smoke/bin/python dist/*.whl + /tmp/fp-parity-smoke/bin/fp --version + /tmp/fp-parity-smoke/bin/python tests/fake_cloud_smoke.py \ + --fp /tmp/fp-parity-smoke/bin/fp --artifacts /tmp/fp-parity-artifacts + + docs: + name: Cloud CLI parity docs + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + - uses: actions/setup-node@v7 + with: + node-version: 22 + - name: Install docs dependencies + run: bun install --frozen-lockfile --ignore-scripts + - name: Parse documentation and resolve images + run: bun run validate:mdx + - name: Validate next-beta changelog + run: python3 scripts/changelog-section.py fp-cloud-cli/CHANGELOG.md 0.0.2b1 diff --git a/fp-cloud-cli/tests/test_parity_workflow.py b/fp-cloud-cli/tests/test_parity_workflow.py new file mode 100644 index 000000000..2da93b24a --- /dev/null +++ b/fp-cloud-cli/tests/test_parity_workflow.py @@ -0,0 +1,17 @@ +"""Stacked feature verification must run against its actual PR base.""" + +from pathlib import Path + + +def test_stacked_parity_workflow_is_read_only_and_runs_real_checks(): + workflow = Path(__file__).resolve().parents[2] / ".github/workflows/fp-cli-parity.yml" + text = workflow.read_text() + assert "branches: [feat/tui-ux]" in text + assert '["3.10", "3.11", "3.12", "3.13"]' in text + assert 'FP_CLI_REQUIRE_CONTRACT: "1"' in text + assert "uv run --extra dev pytest -q" in text + assert "tests/fake_cloud_smoke.py" in text + assert "contents: read" in text + assert "persist-credentials: false" in text + assert "workflow_dispatch" not in text + assert "publish" not in text From 09e32bf2a62daa0b9c762b1223b5daf64122d549 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:31:16 +0000 Subject: [PATCH 10/16] fix(fp): keep shared labels grey and help commands pink --- CHANGELOG.md | 2 +- fp-cloud-cli/fp_cli/output.py | 15 +++++++++++---- fp-cloud-cli/tests/test_screen_language.py | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee4f3dabd..8284cf57c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -- Cloud CLI parity: hosted code, LLM-judge and Jev evaluator authoring/deployment, dashboards, sentiment, assistant artifacts/approvals, org controls and the shared mint/pink screen language. +- Cloud CLI parity: hosted code, LLM-judge and Jev evaluator authoring/deployment, dashboards, sentiment, assistant artifacts/approvals, org controls and the shared mint/pink screen language (#885) - New terminal look for the CLI, step one: the palette gains the redesign's label grey, progress-track grey and a red for failures. Descriptions keep the terminal's own colour, and the greys fall back to the dim attribute below 24-bit, so every screen stays readable on light themes (#881) - Pickers in the new look: an UPPERCASE heading with a live "n of m selected", a `›` cursor and `■ □` boxes, twelve rows before scrolling, plain `a` to toggle all, and the key hints at the foot. The `policies -i` agent menu now uses the shared picker (#881) diff --git a/fp-cloud-cli/fp_cli/output.py b/fp-cloud-cli/fp_cli/output.py index c2a9fea1d..44371dc34 100644 --- a/fp-cloud-cli/fp_cli/output.py +++ b/fp-cloud-cli/fp_cli/output.py @@ -234,7 +234,10 @@ def print_table( # Wrap each cell in Text so a value containing Rich markup (e.g. "[/]" or # "[red]" in a server-provided field) renders literally instead of raising # MarkupError mid-render (which would abort with a raw traceback). - table.add_row(*[Text(_cell(value)) for value in row]) + table.add_row(*[ + Text(_cell(value), style=theme.LABEL if list(columns) == ["Field", "Value"] and i == 0 else "") + for i, value in enumerate(row) + ]) _stdout.print(table) @@ -312,7 +315,7 @@ def code_sent(email: str) -> None: """Confirm the one-time code was emailed (the completed `request_otp` step).""" if _quiet: return - _stderr.print(f" [green]✓[/] code sent to [bold {_ACCENT}]{email}[/]") + _stderr.print(f" [{theme.SUCCESS}]✓[/] code sent to [bold {_ACCENT}]{email}[/]") def org_picker(slugs: Sequence[str], current: Optional[str] = None) -> None: @@ -635,6 +638,8 @@ def render_top_level_help() -> None: def _bullet(*parts) -> Text: line = Text(" · ", style=theme.FAINT) for text, style in parts: + if text.startswith("fp "): + style = theme.PINK line.append(text, style=style) return line @@ -673,9 +678,9 @@ def _bullet(*parts) -> Text: glob.append(g, style=theme.TEXT_DIM) _stdout.print(glob) eg = Text(" e.g. ") - eg.append("fp --json events ", style=theme.TEXT_DIM) + eg.append("fp --json events ", style=theme.PINK) eg.append("✓", style=theme.SUCCESS) - eg.append(" fp events --json ", style=theme.TEXT_DIM) + eg.append(" fp events --json ", style=theme.PINK) eg.append("✗", style=theme.ERROR) _stdout.print(eg) mt = Text(" multi-tenant: set the org at login (", style=theme.FAINT) @@ -2890,6 +2895,8 @@ def _plain(*parts, console=None) -> None: ``--quiet`` for non-error chrome; callers that must always show pass ``console``).""" line = Text(" ") for text, style in parts: + if text.startswith("fp "): + style = theme.PINK line.append(text, style=style) (console or _stderr).print(line) diff --git a/fp-cloud-cli/tests/test_screen_language.py b/fp-cloud-cli/tests/test_screen_language.py index 2c518b2a2..86bb013ef 100644 --- a/fp-cloud-cli/tests/test_screen_language.py +++ b/fp-cloud-cli/tests/test_screen_language.py @@ -1,4 +1,5 @@ import json +from io import StringIO from rich.console import Console @@ -54,3 +55,19 @@ def test_generated_help_has_no_boxes(logged_in, runner): assert result.exit_code == 0 assert "Code:" in result.stdout and "Judge:" in result.stdout and "Jev:" in result.stdout assert not any(c in result.stdout for c in "╭╮╰╯┌┐└┘─━│┃") + + +def test_shared_key_value_labels_are_grey(): + capture = StringIO() + output.configure() + output._stdout = Console(file=capture, width=140, force_terminal=True, color_system="truecolor") + output.print_table(["Field", "Value"], [["enabled", "true"]]) + assert "\x1b[38;2;118;127;139menabled" in capture.getvalue() + + +def test_help_typeable_commands_are_pink(): + capture = StringIO() + output.configure() + output._stdout = Console(file=capture, width=140, force_terminal=True, color_system="truecolor") + output.render_top_level_help() + assert "\x1b[38;2;228;88;125mfp login" in capture.getvalue() From 54c199106c462cc888b6c1ac93d4956fa2f94e85 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:08:40 +0000 Subject: [PATCH 11/16] fix(fp): address parity review timestamp and approval failure cases --- fp-cloud-cli/CHANGELOG.md | 4 ++ fp-cloud-cli/fp_cli/commands/agent_cmds.py | 17 ++++- .../fp_cli/commands/evaluators_cmds.py | 7 +- fp-cloud-cli/fp_cli/output.py | 30 ++++---- fp-cloud-cli/fp_cli/theme.py | 3 - fp-cloud-cli/tests/test_cloud_agent.py | 72 +++++++++++++++++++ .../tests/test_evaluator_authoring.py | 38 ++++++++++ fp-cloud-cli/tests/test_screen_language.py | 7 ++ 8 files changed, 157 insertions(+), 21 deletions(-) diff --git a/fp-cloud-cli/CHANGELOG.md b/fp-cloud-cli/CHANGELOG.md index 5652c20bf..11f865222 100644 --- a/fp-cloud-cli/CHANGELOG.md +++ b/fp-cloud-cli/CHANGELOG.md @@ -15,6 +15,10 @@ - Shared screens use the mint/pink wordmark and grey labels, without boxes/rules or shortened ids; existing JSON contracts are unchanged. - The agent skill, command reference, README and Cloud CLI docs include end-to-end evaluator workflows. No version bump or release is performed. +### Fixes + +- Remove the unused colour-keyed permission rank, document full-id rendering, emit UTC `Z` timestamps for deploy backfills, and report approval-answer failures without a traceback while restoring the terminal (#885). + ## 0.0.2b0 — 2026-09-22 ### Added diff --git a/fp-cloud-cli/fp_cli/commands/agent_cmds.py b/fp-cloud-cli/fp_cli/commands/agent_cmds.py index b748a9495..813989c17 100644 --- a/fp-cloud-cli/fp_cli/commands/agent_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/agent_cmds.py @@ -23,7 +23,7 @@ from .. import client as api from .. import output from .._context import GLOBALS_EPILOG, AppState, deny_in_key_mode, require_auth -from ..errors import NotFoundError +from ..errors import FpCliError, NetworkError, NotFoundError from . import _cloud, _write # The assistant has NO server route at all: `/agent/chat` and `/agent/health` are @@ -344,7 +344,20 @@ def on_ask_user(event): if typer.confirm("Approve this change", default=False, err=True): decision = "approve" answer = {"callId": event["callId"], "decision": decision} - api.cloud_request(cctx, "POST", "/api/agent/answer", body=answer) + try: + api.cloud_request(cctx, "POST", "/api/agent/answer", body=answer) + except FpCliError as exc: + # Unwind the stream and Rich status before the central error renderer + # prints one line. Preserve the transport's exit code and request id. + reason = ( + "cannot reach FailproofAI Cloud" if isinstance(exc, NetworkError) + else " ".join(exc.message.split()).rstrip(".") + ) + exc.message = ( + f"Assistant approval answer failed: {reason}; delivery unconfirmed." + ) + exc.hint = None + raise exc from None if decision == "reject" and not state.json: output.hint("Declined assistant request; no write approved.") diff --git a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py index d6450c868..d2af4f9ef 100644 --- a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py @@ -160,7 +160,12 @@ def deploy( if not re.fullmatch(r"[1-9][0-9]*d", backfill) or int(backfill[:-1]) > 90: spec.fail("--backfill must be a duration in days from 1d to 90d") now = datetime.now(timezone.utc) - backfill_body = {"from": (now - timedelta(days=int(backfill[:-1]))).isoformat(), "to": now.isoformat(), "eval_key": key, "dry_run": False} + backfill_body = { + "from": dates._iso(now - timedelta(days=int(backfill[:-1]))), + "to": dates._iso(now), + "eval_key": key, + "dry_run": False, + } cctx = require_auth(ctx.obj) existing = [r for r in definitions(cctx) if r.get("eval_key") == key] if "eval_version" not in body: diff --git a/fp-cloud-cli/fp_cli/output.py b/fp-cloud-cli/fp_cli/output.py index 44371dc34..e082a00ef 100644 --- a/fp-cloud-cli/fp_cli/output.py +++ b/fp-cloud-cli/fp_cli/output.py @@ -1427,9 +1427,10 @@ def render_events(items: Sequence[Any], *, order: Optional[str] = None, def _short_session(sid: str, *, full: bool = False) -> str: - """Truncate a long session id for the table (full id always kept in ``--json``): - ``sess-20260615-fcf97e01`` → ``sess-…fcf97e01`` (prefix + last 8); other long ids → - first 6 + ``…`` + last 6; short ids are left intact.""" + """Return the full session id unchanged for copyable terminal output. + + The legacy ``full`` argument remains accepted for compatibility. + """ return sid @@ -2116,15 +2117,15 @@ def render_value_list(kind: str, values: Sequence[str], *, description: str = "" def _short_id(value: str) -> str: - """A long id → ``1f58…9826`` (first 4 + ``…`` + last 4); short ids unchanged.""" + """Return the full id unchanged; the legacy helper name is retained.""" return value def _short_chat_id(value: str) -> str: - """A chat id → its short, copy-friendly handle: the segment before the first ``-`` (the first - 8 hex of a UUID, e.g. ``07854990-dade-…`` → ``07854990``), or the first 8 chars when there's no - ``-``. The CLI resolves this prefix back to the full id against the chat list, while the server - keeps the full id intact.""" + """Return the full chat id without shortening it (empty when absent). + + Commands still accept unambiguous prefixes, but rendered ids stay complete. + """ return str(value or "") @@ -4921,13 +4922,12 @@ def render_agent_models(models: Sequence[str], *, default_model: Optional[str] = def render_agent_chats(chats: Sequence[dict]) -> None: - """The ``agent chats`` view (stdout): a boxed ``chats · {n}`` table with columns ``chat-id · - title · messages · updated`` (newest first). ``chat-id`` is the SHORT, copy-friendly handle - (the first 8 hex before the UUID's first ``-``) that ``agent show``/``rename``/``delete``/``ask - --chat`` accept as a prefix; the ``title`` absorbs any width squeeze (``…``-truncated on a - budget). The count glows. Empty → ``no chats``.""" + """The ``agent chats`` view (stdout): an open ``chats · {n}`` table with columns + ``chat-id · title · messages · updated`` (newest first). Chat ids are shown in full; + commands also accept unambiguous prefixes. The title absorbs width pressure + (``…``-truncated on a budget). Empty → ``no chats``.""" items = sorted(chats, key=lambda c: c.get("updated_at") or "", reverse=True) # newest first - ids = [_short_chat_id(str(c.get("id") or "-")) for c in items] # short copy-friendly handle + ids = [_short_chat_id(str(c.get("id") or "-")) for c in items] # full copyable ids titles = [str(c.get("title") or "untitled") for c in items] mcs = [str(c.get("message_count")) if c.get("message_count") is not None else "-" for c in items] ages = [_age_compact(c.get("updated_at")) for c in items] @@ -4946,7 +4946,7 @@ def _w(label, vals): t = _truncate(titles[i], budget) if budget is not None else titles[i] updated = Text(ages[i], style=theme.TEXT_DIM) if ages[i] else Text("—", style=theme.FAINT) rows.append([ - Text(ids[i], style=theme.TEXT_DIM), # the short copy-friendly handle + Text(ids[i], style=theme.TEXT_DIM), # the full copyable id Text(t, style=theme.TEXT), Text(mcs[i], style=theme.TEXT_DIM), updated, diff --git a/fp-cloud-cli/fp_cli/theme.py b/fp-cloud-cli/fp_cli/theme.py index 3ace03420..7da050919 100644 --- a/fp-cloud-cli/fp_cli/theme.py +++ b/fp-cloud-cli/fp_cli/theme.py @@ -44,9 +44,6 @@ } DEFAULT_PERM_COLOR = LABEL # any unmapped action → neutral dim (never crash / guess a risk) -# Risk rank for ordering actions within a row: read → modify → invoke → destroy. -PERM_RANK = {PERM_READ: 0, PERM_WRITE: 1, PERM_ACTION: 2, PERM_DANGER: 3} - def perm_color(action: str) -> str: """The color for a permission action, or the neutral default if it's unmapped.""" diff --git a/fp-cloud-cli/tests/test_cloud_agent.py b/fp-cloud-cli/tests/test_cloud_agent.py index 2f51f9011..c07584db5 100644 --- a/fp-cloud-cli/tests/test_cloud_agent.py +++ b/fp-cloud-cli/tests/test_cloud_agent.py @@ -1,8 +1,13 @@ import json +import re +from io import StringIO +import httpx import pytest import respx +from rich.console import Console +from fp_cli import output from fp_cli.app import app BASE = "http://dash.test" @@ -151,3 +156,70 @@ def test_backtest_published_policy_uses_latest_version(logged_in, runner): result = runner.invoke(app, ["--json", "policies", "backtest", "p1"]) assert result.exit_code == 0, result.output assert json.loads(route.calls[0].request.content) == {"source": "new"} + + +@pytest.mark.parametrize("json_mode", [False, True]) +@pytest.mark.parametrize("failure,exit_code", [("404", 6), ("network", 3)]) +def test_approval_answer_failure_is_clean(logged_in, runner, json_mode, failure, exit_code): + with respx.mock(assert_all_called=False) as mock: + mock.post(f"{BASE}/api/agent/chat").respond(200, text=sse([ + {"type": "ask-user", "kind": "approval", "callId": "ask_1", "question": "Disable alert?"}, + {"type": "text-delta", "text": "Must not reach this answer."}, + {"type": "done"}, + ])) + answer = mock.post(f"{BASE}/api/agent/answer") + if failure == "404": + answer.respond(404, json={}) + else: + answer.mock(side_effect=httpx.ConnectError("connection refused")) + saved = mock.post(f"{BASE}/api/agent/conversations").respond(200, json={"id": "c1"}) + result = runner.invoke(app, [*(["--json"] if json_mode else []), "agent", "ask", "Hello", "--approve"]) + assert result.exit_code == exit_code, result.output + assert answer.called and not saved.called + assert "Traceback" not in result.output + assert "Must not reach this answer" not in result.output + if json_mode: + data = json.loads(result.stdout) + assert data["exit_code"] == exit_code + message = data["error"] + else: + assert len(result.stderr.strip().splitlines()) == 1 + message = result.stderr + assert "Assistant approval answer failed" in message + assert "delivery unconfirmed" in message + if failure == "404": + assert "not available on this FailproofAI Cloud yet" in message + + +@pytest.mark.parametrize("failure,exit_code", [("404", 6), ("network", 3)]) +def test_approval_failure_restores_terminal_status(logged_in, runner, monkeypatch, failure, exit_code): + # The suite defaults to TERM=dumb, which suppresses live cursor control. + monkeypatch.setenv("TERM", "xterm-256color") + capture = StringIO() + terminal = Console(file=capture, force_terminal=True, color_system="truecolor", width=200) + original_configure = output.configure + + def configure_terminal(**kwargs): + original_configure(**kwargs) + monkeypatch.setattr(output, "_stderr", terminal) + + monkeypatch.setattr(output, "configure", configure_terminal) + with respx.mock as mock: + mock.post(f"{BASE}/api/agent/chat").respond(200, text=sse([ + {"type": "ask-user", "kind": "approval", "callId": "ask_1", "question": "Disable alert?"}, + ])) + answer = mock.post(f"{BASE}/api/agent/answer") + if failure == "404": + answer.respond(404, json={}) + else: + answer.mock(side_effect=httpx.ConnectError("connection refused")) + result = runner.invoke(app, ["agent", "ask", "Hello", "--approve"]) + assert result.exit_code == exit_code, result.output + rendered = capture.getvalue() + assert terminal._live_stack == [] + assert "\x1b[?25l" in rendered + assert rendered.rfind("\x1b[?25h") > rendered.rfind("\x1b[?25l") + assert rendered.index("Assistant approval answer failed") > rendered.rfind("\x1b[?25h") + plain = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", rendered) + assert plain.count("Assistant approval answer failed") == 1 + assert "Traceback" not in plain diff --git a/fp-cloud-cli/tests/test_evaluator_authoring.py b/fp-cloud-cli/tests/test_evaluator_authoring.py index 66a3ecfc6..3bb6b3b13 100644 --- a/fp-cloud-cli/tests/test_evaluator_authoring.py +++ b/fp-cloud-cli/tests/test_evaluator_authoring.py @@ -1,4 +1,5 @@ import json +from datetime import datetime, timezone import pytest import respx @@ -94,6 +95,43 @@ def test_deploy_order_and_dry_run(logged_in, runner, code): assert json.loads(result.stdout)["disabled"] == ["d1"] +@pytest.mark.parametrize("key_mode", [False, True]) +def test_deploy_backfill_uses_utc_z_timestamps(logged_in, runner, code, monkeypatch, key_mode): + from fp_cli.commands import evaluators_cmds + + class FixedDateTime(datetime): + @classmethod + def now(cls, tz=None): + assert tz == timezone.utc + return cls(2026, 10, 14, 12, 34, 56, 123456, tzinfo=tz) + + monkeypatch.setattr(evaluators_cmds, "datetime", FixedDateTime) + prefix = "v1" if key_mode else "api" + with respx.mock as mock: + mock.get(f"{BASE}/{prefix}/evaluator-definitions").respond(200, json=[ + {"id": "d1", "eval_key": "quality", "eval_version": "1", "enabled": True}, + ]) + mock.post(f"{BASE}/{prefix}/evaluator-definitions").respond(201, json={"id": "d2"}) + mock.patch(f"{BASE}/{prefix}/evaluator-definitions/d1").respond(200, json={}) + backfill = mock.post(f"{BASE}/{prefix}/evaluations/backfill").respond(200, json={"queued": True}) + result = runner.invoke(app, [ + "--json", *(["--api-key", "key"] if key_mode else []), + "evaluators", "deploy", "quality", "--code", code, "--backfill", "7d", + ]) + order = [(call.request.method, call.request.url.path) for call in mock.calls] + assert result.exit_code == 0, result.output + assert json.loads(backfill.calls[0].request.content) == { + "from": "2026-10-07T12:34:56Z", + "to": "2026-10-14T12:34:56Z", + "eval_key": "quality", + "dry_run": False, + } + assert order[-2:] == [ + ("PATCH", f"/{prefix}/evaluator-definitions/d1"), + ("POST", f"/{prefix}/evaluations/backfill"), + ] + + def test_failed_create_never_disables(logged_in, runner, code): with respx.mock(assert_all_called=False) as mock: mock.get(f"{BASE}/api/evaluator-definitions").respond(200, json=[{"id": "d1", "eval_key": "quality", "eval_version": "1"}]) diff --git a/fp-cloud-cli/tests/test_screen_language.py b/fp-cloud-cli/tests/test_screen_language.py index 86bb013ef..2f5c4fff8 100644 --- a/fp-cloud-cli/tests/test_screen_language.py +++ b/fp-cloud-cli/tests/test_screen_language.py @@ -13,6 +13,13 @@ def test_two_accent_palette(): assert {theme.SUCCESS, theme.ERROR, theme.AMBER, theme.BLUE, theme.SCORE_HIGH} <= {theme.ACCENT, theme.PINK} +def test_permission_risk_order_is_independent_of_shared_colours(): + assert theme.PERM_WRITE == theme.PERM_ACTION == theme.PERM_DANGER + assert sorted(["delete", "run", "write", "read"], key=output._perm_act_key) == [ + "read", "write", "run", "delete", + ] + + def test_open_table_and_header(capsys): output.configure(no_color=True) output.print_table(["Id", "URL"], [["run-001", "https://cloud.example.test"]], title="Resources") From 7ab89dc28c081d3380c9af405c2bf740e74b3ede Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:49:44 +0000 Subject: [PATCH 12/16] fix(fp): close final QA gaps in judge validation and Cloud error semantics --- docs/reference/cloud-cli.mdx | 5 +- fp-cloud-cli/CHANGELOG.md | 3 +- fp-cloud-cli/README.md | 3 +- fp-cloud-cli/fp_cli/client.py | 97 ++++++++++++++- fp-cloud-cli/fp_cli/commands/agent_cmds.py | 3 +- .../fp_cli/commands/evaluators_cmds.py | 5 +- fp-cloud-cli/fp_cli/errors.py | 3 + fp-cloud-cli/fp_cli/evaluator_spec.py | 31 ++++- fp-cloud-cli/fp_cli/output.py | 26 ++-- fp-cloud-cli/skill/SKILL.md | 5 +- fp-cloud-cli/skill/references/commands.md | 6 +- fp-cloud-cli/tests/fake_cloud_smoke.py | 25 +++- fp-cloud-cli/tests/test_cloud_account.py | 2 +- fp-cloud-cli/tests/test_cloud_agent.py | 34 ++++- .../tests/test_cloud_observability.py | 2 +- fp-cloud-cli/tests/test_cloud_transport.py | 86 +++++++++++++ .../tests/test_evaluator_authoring.py | 117 ++++++++++++++++++ .../tests/test_help_table_coverage.py | 15 +++ fp-cloud-cli/tests/test_session_reevaluate.py | 18 ++- 19 files changed, 449 insertions(+), 37 deletions(-) diff --git a/docs/reference/cloud-cli.mdx b/docs/reference/cloud-cli.mdx index 23947d47a..65d66ec46 100644 --- a/docs/reference/cloud-cli.mdx +++ b/docs/reference/cloud-cli.mdx @@ -119,8 +119,9 @@ exclusive; deploy can override the file's version. Output files are never overwr ### New dashboard controls -All commands accept global `--json`. New 404s exit 6 with “not available on this -FailproofAI Cloud yet”. Session-only commands deny key mode before HTTP (exit 2). +All commands accept global `--json`. Unknown-route 404s exit 6 with “not available +on this FailproofAI Cloud yet”. Missing-record 404s also exit 6 but report that the +record was not found. Session-only commands deny key mode before HTTP (exit 2). | Commands | Options / notes | | --- | --- | diff --git a/fp-cloud-cli/CHANGELOG.md b/fp-cloud-cli/CHANGELOG.md index 11f865222..3d4bc74e6 100644 --- a/fp-cloud-cli/CHANGELOG.md +++ b/fp-cloud-cli/CHANGELOG.md @@ -11,13 +11,14 @@ ### Changed -- New Cloud API families work in key mode; unsupported session-only commands deny before HTTP and older-Cloud 404s exit 6 with an availability message. +- New Cloud API families work in key mode; unsupported session-only commands deny before HTTP and unknown-route 404s exit 6 with an availability message. - Shared screens use the mint/pink wordmark and grey labels, without boxes/rules or shortened ids; existing JSON contracts are unchanged. - The agent skill, command reference, README and Cloud CLI docs include end-to-end evaluator workflows. No version bump or release is performed. ### Fixes - Remove the unused colour-keyed permission rank, document full-id rendering, emit UTC `Z` timestamps for deploy backfills, and report approval-answer failures without a traceback while restoring the terminal (#885). +- Reject natural-language binary judge steps, distinguish unknown routes from missing records on 404, normalize standalone backfill ranges to UTC `Z`, expose all subcommands in help, and render artifact headers before titles with consistent session-only wording (#885). ## 0.0.2b0 — 2026-09-22 diff --git a/fp-cloud-cli/README.md b/fp-cloud-cli/README.md index 9be580909..6a5c349ad 100644 --- a/fp-cloud-cli/README.md +++ b/fp-cloud-cli/README.md @@ -273,7 +273,8 @@ behavior; assistant approvals are a separate safety boundary. All new commands accept the global `--json` before the command. Session-only commands fail before HTTP in API-key mode, with exit 2 and a `fp login` hint. -Older-Cloud 404s exit 6 with “not available on this FailproofAI Cloud yet”. +Unknown-route 404s exit 6 with “not available on this FailproofAI Cloud yet”. +Missing-record 404s also exit 6 but report that the record was not found. `plan` routes are mapped for keys, but the Cloud also requires an actor email; an API key alone cannot impersonate a person to claim an offer. Human screens use the mint/pink wordmark, grey labels, open tables and full ids; diff --git a/fp-cloud-cli/fp_cli/client.py b/fp-cloud-cli/fp_cli/client.py index 2e934deef..6e48d5b7c 100644 --- a/fp-cloud-cli/fp_cli/client.py +++ b/fp-cloud-cli/fp_cli/client.py @@ -31,6 +31,7 @@ ApiError, AuthError, ForbiddenError, + FpCliError, KeyModeUnsupportedError, NetworkError, NotFoundError, @@ -385,6 +386,40 @@ def _required_permission(response: httpx.Response) -> Optional[str]: return None +def _unknown_route_response(response: httpx.Response) -> Optional[bool]: + """Use explicit 404 details first; an empty/generic reply is ambiguous.""" + try: + data = response.json() + except ValueError: + return None + if not isinstance(data, dict): + return None + code = data.get("code") or data.get("error_code") or data.get("error") or "" + if isinstance(code, dict): + code = code.get("code", "") + code = str(code).lower() + if code in ("unknown_route", "route_not_found", "unknown_endpoint", "endpoint_not_found"): + return True + if code in ("record_not_found", "resource_not_found", "entity_not_found", "session_not_found", "query_not_found"): + return False + message = _extract_error(response) + if not message: + return None + lower = message.lower().strip().rstrip(".") + if re.search( + r"\b(?:unknown|unrecognized|unsupported|missing|no such)\s+(?:api\s+)?(?:route|endpoint)\b" + r"|\b(?:route|endpoint)\b.*\b(?:not found|not available|does not exist|absent|unknown)\b" + r"|\bcannot (?:get|post|put|patch|delete)\s+/", + lower, + ): + return True + if lower in ("not found", "not_found", "404", "404 not found"): + return None + # A specific response ("session not found", "offer expired", etc.) came + # from a handler, rather than the router. Do not replace its useful detail. + return False + + def _raise_for_status(response: httpx.Response, ctx: ClientContext) -> None: key_mode = ctx.auth_mode is AuthMode.API_KEY if response.status_code < 400: @@ -484,7 +519,9 @@ def _raise_for_status(response: httpx.Response, ctx: ClientContext) -> None: request_id=request_id, hint="point --base-url at the server itself, e.g. http://localhost:8080", ) - raise NotFoundError(message or "Not found.", request_id=request_id) + error = NotFoundError(message or "Not found.", request_id=request_id) + error.route_unknown = _unknown_route_response(response) + raise error if response.status_code == 429: retry_after = response.headers.get("retry-after") wait = f" Retry after {retry_after}s." if retry_after else " Please wait a moment and try again." @@ -602,11 +639,61 @@ def cloud_request( except (NotFoundError, ApiError) as exc: if isinstance(exc, ApiError) and exc.status != 404: raise - raise NotFoundError( + raise _cloud_not_found(exc, path, body=body, params=params) from exc + + +def _cloud_not_found(exc: FpCliError, path: str, *, body=None, params=None) -> NotFoundError: + """Classify ambiguous 404s from the request; never claim a record is a route. + + Top-level collections and named capability endpoints have no record segment. + Scoped collections (/dashboards/ID/tiles) and actions on records default to a + missing record unless the response explicitly says the route is unknown. + """ + route_unknown = getattr(exc, "route_unknown", None) + if isinstance(exc, ApiError): + # The transport only raises HTTP 404 ApiError for an HTML front door. + route_unknown = True + if route_unknown is None: + segments = path.removeprefix("/api/").split("/") + family = segments[0] + capability = len(segments) == 1 or ( + family in ("agent", "otel", "offers", "team-trial", "integrations") + or family == "events" and segments in ( + ["events", "summary"], ["events", "latency_aggregate"], + ["events", "sentiment"], ["events", "sentiment", "messages"], + ["events", "sentiment", "sessions"], + ) + or family == "evaluator-definitions" and segments == ["evaluator-definitions", "dry-run"] + or family == "evaluations" and segments == ["evaluations", "backfill"] + or family == "queries" and segments[1:] in (["run"], ["schema"]) + or family == "audits" and segments == ["audits", "contracts"] + or family == "alerts" and segments == ["alerts", "recipients"] + or family == "users" and segments == ["users", "defaults"] + or family == "usage" and segments == ["usage", "windows"] + or family == "settings" and segments == ["settings", "model-context-windows"] + ) + # Some collection operations name their target in a query or body. + request_body = body if isinstance(body, dict) else {} + record_selected = ( + (params or {}).get("session_id") or (params or {}).get("model") + or request_body.get("query_id") or request_body.get("callId") + or request_body.get("session_ids") or request_body.get("offer_key") + or request_body.get("conversationId") + ) + route_unknown = capability and not record_selected + if route_unknown: + return NotFoundError( "not available on this FailproofAI Cloud yet", - hint="check the resource id or upgrade your Cloud deployment", + hint="upgrade your Cloud deployment to use this route", request_id=exc.request_id, - ) from exc + ) + message = exc.message + if message.lower().strip().rstrip(".") in ( + "not found", "not_found", "record_not_found", "resource_not_found", + "entity_not_found", "session_not_found", "query_not_found", + ): + message = f"Record not found at {path}" + return NotFoundError(message, request_id=exc.request_id) def cloud_stream(ctx: ClientContext, path: str, body: Dict[str, Any]) -> Iterator[Dict[str, Any]]: @@ -616,7 +703,7 @@ def cloud_stream(ctx: ClientContext, path: str, body: Dict[str, Any]) -> Iterato except (NotFoundError, ApiError) as exc: if isinstance(exc, ApiError) and exc.status != 404: raise - raise NotFoundError("not available on this FailproofAI Cloud yet", request_id=exc.request_id) from exc + raise _cloud_not_found(exc, path, body=body) from exc # --- Auth / identity -------------------------------------------------------- diff --git a/fp-cloud-cli/fp_cli/commands/agent_cmds.py b/fp-cloud-cli/fp_cli/commands/agent_cmds.py index 813989c17..55dc9609b 100644 --- a/fp-cloud-cli/fp_cli/commands/agent_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/agent_cmds.py @@ -302,7 +302,7 @@ def agent_ask( * `fp agent ask "summarize p95 latency" --model claude-opus-4-7` """ state: AppState = ctx.obj - deny_in_key_mode(state, "agent", _KEY_MODE_REASON) + _cloud.session_only(state, "agent ask") if not message: message = sys.stdin.read().strip() if not sys.stdin.isatty() else typer.prompt("question", err=True) if not message: @@ -463,6 +463,7 @@ def run_artifact(cctx, block): def render_artifact(block): + output.screen_header() data = block.get("result") if isinstance(data, dict): output._stdout.print(str(block.get("title", "Artifact")), markup=False) diff --git a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py index d2af4f9ef..fc1cfed1f 100644 --- a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py @@ -320,7 +320,8 @@ def backfill(ctx: typer.Context, frm: str = typer.Option(..., "--from"), dry_run: bool = typer.Option(False, "--dry-run")) -> None: """Backfill at most 90 days. Judge/Jev evaluators are excluded by Cloud.""" try: - dates.resolve_range(ts_from=frm, ts_to=to) + frm = dates._validate_iso(frm, "--from") + to = dates._validate_iso(to, "--to") start = datetime.fromisoformat(frm.replace("Z", "+00:00")) end = datetime.fromisoformat(to.replace("Z", "+00:00")) except ValueError as exc: @@ -330,7 +331,7 @@ def backfill(ctx: typer.Context, frm: str = typer.Option(..., "--from"), output.warn("Backfill excludes judge/Jev evaluators; only code evaluators are replayed.") _write.confirm(ctx.obj, "Backfill evaluations", assume_yes=dry_run) _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "POST", "/api/evaluations/backfill", - body={"from": frm, "to": to, "eval_key": key, "dry_run": dry_run}), "Evaluation backfill") + body={"from": dates._iso(start), "to": dates._iso(end), "eval_key": key, "dry_run": dry_run}), "Evaluation backfill") def register(app: typer.Typer) -> None: diff --git a/fp-cloud-cli/fp_cli/errors.py b/fp-cloud-cli/fp_cli/errors.py index 7162fedad..924e5d194 100644 --- a/fp-cloud-cli/fp_cli/errors.py +++ b/fp-cloud-cli/fp_cli/errors.py @@ -85,6 +85,9 @@ class NotFoundError(FpCliError): """The requested resource does not exist.""" exit_code = 6 + # The HTTP response may explicitly distinguish an absent endpoint from a + # missing record. None means callers must fall back to the request shape. + route_unknown: Optional[bool] = None class ApiError(FpCliError): diff --git a/fp-cloud-cli/fp_cli/evaluator_spec.py b/fp-cloud-cli/fp_cli/evaluator_spec.py index 723d43c8f..94827ab15 100644 --- a/fp-cloud-cli/fp_cli/evaluator_spec.py +++ b/fp-cloud-cli/fp_cli/evaluator_spec.py @@ -52,6 +52,35 @@ def source_kind(body: dict) -> str: return envelope.get("kind", "code") if isinstance(envelope, dict) else "code" +def binary_judge_step(step: str) -> bool: + """Mirror the server's literal scoring check, plus natural-language pairs. + + The six score phrases match evaluator_definitions.rs::steps_are_binary_scored. + Whitespace normalization and equivalent directives catch binary rubrics without + rejecting ordinary continuous wording such as "score between 0 and 1". + """ + lowered = " ".join(step.lower().split()) + if any(phrase in lowered for phrase in ( + "score 0", "score 1", "score of 0", "score of 1", "score: 0", "score: 1", + )): + return True + return bool(re.search( + r"\bbinary\b" + r"|\bscore(?:\s+of)?\s*[:=]?\s*(?:0|1|zero|one)\b" + r"|\b(?:yes\s*(?:/|or|and|versus|vs\.?|-)\s*no" + r"|no\s*(?:/|or|and|versus|vs\.?|-)\s*yes" + r"|true\s*(?:/|or|and|versus|vs\.?|-)\s*false" + r"|false\s*(?:/|or|and|versus|vs\.?|-)\s*true" + r"|pass\s*(?:/|or|and|versus|vs\.?|-)\s*fail" + r"|fail\s*(?:/|or|and|versus|vs\.?|-)\s*pass)\b" + r"|\b(?:give|assign|award|return|output)\s+" + r"(?:(?:a\s+)?(?:score|value)(?:\s+of)?\s*[:=]?\s*)?" + r"(?:0(?:\.0+)?|1(?:\.0+)?|zero|one|true|false|yes|no|pass|fail)" + r"\s+(?:if|when|otherwise|for)\b", + lowered, + )) + + def validate(body: dict, *, complete: bool = True) -> dict: unknown = set(body) - FIELDS if unknown: @@ -104,7 +133,7 @@ def validate(body: dict, *, complete: bool = True) -> dict: fail("judge steps must be a list") for step in steps: text(step, "judge step") - if re.search(r"\b(yes\s*/\s*no|true\s*/\s*false|binary|pass\s*/\s*fail|score(?: of|:)? [01])\b", step, re.I): + if binary_judge_step(step): fail("binary judge steps are not supported; use graded evaluation steps") else: text(envelope.get("instructions"), "Jev instructions") diff --git a/fp-cloud-cli/fp_cli/output.py b/fp-cloud-cli/fp_cli/output.py index e082a00ef..623ad12b2 100644 --- a/fp-cloud-cli/fp_cli/output.py +++ b/fp-cloud-cli/fp_cli/output.py @@ -577,42 +577,42 @@ def version_banner(version: str) -> None: ("whoami", "Show the current user, active org, and perms.", ""), ]), ("OBSERVE", [ - ("events", "List the raw per-step agent event trail.", ""), - ("sessions", "List agent runs — one row per run.", ""), + ("events", "List the raw per-step agent event trail.", "latency"), + ("sessions", "List agent runs — one row per run.", "show export history re-evaluate"), ("evals", "List scored agent evaluations.", "--aggregate"), ("evaluators", "Author, test and deploy managed evaluators.", "list show create deploy draft dry-run enable disable label instances schema jobs backfill"), ("sentiment", "Inspect conversation sentiment.", "summary messages sessions session"), ("dashboards", "Manage dashboards and saved-query tiles.", "list show create update delete tiles"), ("errors", "List errored events.", "--aggregate"), - ("usage", "Show current org usage for the metering window.", ""), + ("usage", "Show current org usage for the metering window.", "windows"), ]), ("ENFORCE", [ - ("policies", "Write cloud-managed policies.", "list show publish test compose enable disable delete"), + ("policies", "Write cloud-managed policies.", "list show publish test compose author backtest candidacy enable disable delete"), ("fleet", "Deploy policies to machines.", "list show deploy diff history rollback rename"), ("guardrails", "What enforcement actually blocked.", "summary timeline"), ]), ("MANAGE", [ - ("orgs", "Switch and inspect the active org.", "list switch current perms"), + ("orgs", "Switch and inspect the active org.", "list switch current perms create"), ("keys", "Provision and manage API keys.", "list show create update disable regenerate"), - ("users", "Manage org members and their permissions.", "list show create update disable enable"), - ("query", "Run and manage saved SQL queries.", "list show create update delete run schema"), - ("alerts", "Define, edit, and test alerts.", "list show create update delete test"), + ("users", "Manage org members and their permissions.", "list show create update disable enable delete defaults"), + ("query", "Run and manage saved SQL queries.", "list show create update delete run schema compose export"), + ("alerts", "Define, edit, and test alerts.", "list show create update delete test recipients issues"), # `context-*` rather than the three names spelled out: this hint column # ellipsis-truncates at ~110 cols (see `desc_max` below) and this row was # already overflowing, so the long form would never render. The full list # is one level down, in `fp audits --help`, which walks the real # Click tree. - ("audits", "Schedule audits and triage their findings.", "list show create edit delete run runs findings context-*"), - ("issues", "Triage and resolve issues.", "list count show ack assign resolve comment-* subscribe open"), - ("settings", "View and change org settings.", "list schema set"), + ("audits", "Schedule audits and triage their findings.", "list show create edit delete run runs findings finding ack mute dismiss resolve reopen assign context-* contracts"), + ("issues", "Triage and resolve issues.", "list count show ack assign resolve close archive unarchive clear comment-* subscribe subscribers unsubscribe open"), + ("settings", "View and change org settings.", "list schema set context-windows"), ("roles", "Manage permission sets.", "list show create update delete"), ("plan", "Inspect offers and team trial.", "claim"), ("otel", "Control Cloud OTEL ingestion.", "status enable disable"), ("slack", "Manage Slack integration.", "status disconnect"), ]), ("TOOLS", [ - ("list", "List distinct values behind the filter dropdowns.", ""), - ("agent", "Chat with the FailproofAI Cloud assistant.", "health models chats ask show rename delete"), + ("list", "List distinct values behind the filter dropdowns.", "envs agents event_types score_filters models hooks tools error_types"), + ("agent", "Chat with the FailproofAI Cloud assistant.", "health models chats ask show rename delete brief"), ]), ] diff --git a/fp-cloud-cli/skill/SKILL.md b/fp-cloud-cli/skill/SKILL.md index 614225e82..dfc48db26 100644 --- a/fp-cloud-cli/skill/SKILL.md +++ b/fp-cloud-cli/skill/SKILL.md @@ -380,7 +380,8 @@ then disable any remaining old version; do not retry creation blindly. Backfill excludes judge/Jev and accepts at most 90 days. AI drafting needs a signed-in session (`fp login`); evaluator API operations can -use a scoped key. New capability 404s exit 6 with “not available on this -FailproofAI Cloud yet”. An assistant approval is different from ordinary +use a scoped key. Unknown-route 404s exit 6 with “not available on this +FailproofAI Cloud yet”; missing-record 404s report that the record was not found +(also exit 6). An assistant approval is different from ordinary mutation confirmation: `agent ask` declines writes in scripts/`--json` unless `--approve` was explicitly authorized by the user. diff --git a/fp-cloud-cli/skill/references/commands.md b/fp-cloud-cli/skill/references/commands.md index 1f7612456..b2dce7f68 100644 --- a/fp-cloud-cli/skill/references/commands.md +++ b/fp-cloud-cli/skill/references/commands.md @@ -337,8 +337,10 @@ the org has 100 enabled keys. Read the error, do not rename/retry silently. ## Cloud dashboard parity -All new commands degrade on API 404 with exit 6 and “not available on this -FailproofAI Cloud yet”. Session-only commands exit 2 before HTTP in key mode. +Unknown-route 404s exit 6 with “not available on this FailproofAI Cloud yet”. +Missing-record 404s also exit 6 but preserve the server's record-not-found message; +an empty response on a record-scoped request identifies the missing record path. +Session-only commands exit 2 before HTTP in key mode. Human output uses mint/pink open screens and full ids; legacy `--full-ids` remains accepted. Exports refuse to overwrite existing files. diff --git a/fp-cloud-cli/tests/fake_cloud_smoke.py b/fp-cloud-cli/tests/fake_cloud_smoke.py index b6b13479d..fe2ae06f6 100644 --- a/fp-cloud-cli/tests/fake_cloud_smoke.py +++ b/fp-cloud-cli/tests/fake_cloud_smoke.py @@ -96,7 +96,7 @@ def dispatch(self, method): assert "ae_session=fake-session" in self.headers.get("Cookie", "") assert not self.headers.get("Authorization") if Cloud.unavailable: - return self.json_response({"error": "old Cloud route absent"}, 404) + return self.json_response({"code": "unknown_route", "error": "old Cloud route absent"}, 404) if route == "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/auth/session": return self.json_response({"id": "u1", "email": "smoke@example.test", "memberships": [ {"org_id": "o1", "org_slug": "smoke", "org_name": "Smoke", "permissions": ["evaluations:trigger", "events:read", "queries:run", "agent:use"], "permission_set": "admin"}]}) @@ -109,6 +109,14 @@ def dispatch(self, method): assert len(body["session_ids"]) <= 10 return self.json_response({"runs": [{"ok": True, "session_id": sid, "mode": "run", "result": {"score": {"value": 1.0}}} for sid in body["session_ids"]], "summary": {"failed": 0, "succeeded": len(body["session_ids"]), "skipped": 0}}) + if route == "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/evaluations/backfill": + assert body == { + "from": "2026-10-01T00:00:00Z", "to": "2026-10-08T00:00:00Z", + "eval_key": "error_free", "dry_run": True, + } + return self.json_response({"queued": False, "dry_run": True}) + if route == "/sessions/missing/re-evaluate": + return self.json_response({"error": "session not found"}, 404) if route == "/evaluator-definitions": if method == "GET": return self.json_response(Cloud.definitions) @@ -205,8 +213,13 @@ def run(argv, *, input=None, key=False, json_mode=True, expected=0): assert not Cloud.definitions[0]["enabled"] and Cloud.definitions[1]["enabled"] run(["evaluators", "list", "--all"], key=True) run(["evaluators", "disable", "error_free@2"], key=True) + run(["evaluators", "backfill", "--from", "2026-10-01T02:00:00+02:00", + "--to", "2026-10-07T20:00:00-04:00", "--key", "error_free", "--dry-run"], key=True) run(["evaluators", "create", "--key", "relevance", "--name", "Relevance", "--judge", "--criteria", "Grade relevance"], key=True) run(["evaluators", "dry-run", "--judge", "--criteria", "Grade relevance"], key=True, expected=2) + for step in ("score 1 if correct", "give 0 when incorrect", "pass/fail", "yes or no"): + run(["evaluators", "create", "--key", "binary", "--name", "Binary", "--judge", + "--criteria", "Grade relevance", "--step", step], key=True, expected=2) run(["evaluators", "create", "--key", "resolution", "--name", "Resolution", "--jev", "noul", "--instructions", "Was the request resolved?", "--true", "Resolved", "--false", "Unresolved"], key=True) run(["dashboards", "create", "--name", "Quality"], key=True) run(["dashboards", "tiles", "board1", "add", "--query", "q1", "--chart", "bar", "--title", "Traffic"], key=True) @@ -219,6 +232,14 @@ def run(argv, *, input=None, key=False, json_mode=True, expected=0): assert Cloud.answers[-1]["decision"] == "approve" run(["agent", "brief", "--range", "7d"], json_mode=False) run(["evaluators", "draft", "Grade quality"], key=True, expected=2) + denial = run(["agent", "ask", "Hello"], key=True, expected=2) + assert denial["error"] == "`fp agent ask` needs a signed-in session: run `fp login`" + missing = run(["sessions", "re-evaluate", "missing"], key=True, expected=6) + assert missing["error"] == "session not found" + assert "not available" not in missing["error"] + help_text = run(["help"], json_mode=False) + for group in ("evaluators", "dashboards", "sentiment", "roles", "plan", "otel", "slack"): + assert group in help_text Cloud.unavailable = True unavailable = run(["sentiment", "summary"], key=True, expected=6) assert "not available on this FailproofAI Cloud yet" in unavailable["error"] @@ -253,6 +274,8 @@ def run(argv, *, input=None, key=False, json_mode=True, expected=0): transcript.append("$ fp agent ask 'Count events' [interactive approval]\n" + output.decode(errors="replace") + f"\nexit {proc.returncode}\n") assert proc.returncode == 0 and sent, transcript[-1] assert Cloud.answers[-1]["decision"] == "approve" + rendered = output.decode(errors="replace") + assert rendered.index("failproof ai") < rendered.index("Events by agent") server.shutdown() (artifacts / "fp-user-smoke.txt").write_text("\n".join(transcript), encoding="utf-8") (artifacts / "fp-fake-cloud-requests.json").write_text(json.dumps(Cloud.requests, indent=2), encoding="utf-8") diff --git a/fp-cloud-cli/tests/test_cloud_account.py b/fp-cloud-cli/tests/test_cloud_account.py index dc27c2961..fc0abdb5b 100644 --- a/fp-cloud-cli/tests/test_cloud_account.py +++ b/fp-cloud-cli/tests/test_cloud_account.py @@ -84,7 +84,7 @@ def test_session_commands(logged_in, runner, argv, method, path, response): @pytest.mark.parametrize("argv,method,path,response", CASES + SESSION) def test_404(logged_in, runner, argv, method, path, response): with respx.mock as mock: - mock.route().respond(404, json={}) + mock.route().respond(404, json={"error": "unknown route"}) result = runner.invoke(app, ["--json", *argv]) assert result.exit_code == 6, result.output assert "not available on this FailproofAI Cloud yet" in result.stdout diff --git a/fp-cloud-cli/tests/test_cloud_agent.py b/fp-cloud-cli/tests/test_cloud_agent.py index c07584db5..6654e2349 100644 --- a/fp-cloud-cli/tests/test_cloud_agent.py +++ b/fp-cloud-cli/tests/test_cloud_agent.py @@ -65,6 +65,7 @@ class Sys: else: assert "Event total" in result.output assert "42" in result.output + assert result.stdout.index("failproof ai v") < result.stdout.index("Event total") if tty: assert "alert a1" in result.output assert "Approve this change" in result.output @@ -140,7 +141,7 @@ def test_session_only(logged_in, runner, argv): ]) def test_404(logged_in, runner, argv): with respx.mock as mock: - mock.route().respond(404, json={}) + mock.route().respond(404, json={"error": "unknown route"}) result = runner.invoke(app, ["--json", *argv]) assert result.exit_code == 6, result.output assert "not available on this FailproofAI Cloud yet" in result.stdout @@ -169,7 +170,7 @@ def test_approval_answer_failure_is_clean(logged_in, runner, json_mode, failure, ])) answer = mock.post(f"{BASE}/api/agent/answer") if failure == "404": - answer.respond(404, json={}) + answer.respond(404, json={"error": "unknown route"}) else: answer.mock(side_effect=httpx.ConnectError("connection refused")) saved = mock.post(f"{BASE}/api/agent/conversations").respond(200, json={"id": "c1"}) @@ -210,7 +211,7 @@ def configure_terminal(**kwargs): ])) answer = mock.post(f"{BASE}/api/agent/answer") if failure == "404": - answer.respond(404, json={}) + answer.respond(404, json={"error": "unknown route"}) else: answer.mock(side_effect=httpx.ConnectError("connection refused")) result = runner.invoke(app, ["agent", "ask", "Hello", "--approve"]) @@ -223,3 +224,30 @@ def configure_terminal(**kwargs): plain = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", rendered) assert plain.count("Assistant approval answer failed") == 1 assert "Traceback" not in plain + + +@pytest.mark.parametrize("json_mode", [False, True]) +def test_ask_api_key_uses_session_only_wording(logged_in, runner, json_mode): + with respx.mock as mock: + result = runner.invoke(app, [ + "--api-key", "key", *(["--json"] if json_mode else []), + "agent", "ask", "Hello", + ]) + assert not mock.calls + assert result.exit_code == 2, result.output + expected = "`fp agent ask` needs a signed-in session: run `fp login`" + if json_mode: + assert json.loads(result.stdout)["error"] == expected + else: + assert expected in result.stderr + assert len(result.stderr.strip().splitlines()) == 1 + + +def test_artifact_header_precedes_title_once(capsys): + from fp_cli.commands.agent_cmds import render_artifact + output.configure(no_color=True) + render_artifact({"title": "First artifact", "result": RESULT}) + render_artifact({"title": "Second artifact", "result": RESULT}) + rendered = capsys.readouterr().out + assert rendered.count("failproof ai v") == 1 + assert rendered.index("failproof ai v") < rendered.index("First artifact") < rendered.index("Second artifact") diff --git a/fp-cloud-cli/tests/test_cloud_observability.py b/fp-cloud-cli/tests/test_cloud_observability.py index b0bda4d57..70564b1c3 100644 --- a/fp-cloud-cli/tests/test_cloud_observability.py +++ b/fp-cloud-cli/tests/test_cloud_observability.py @@ -67,7 +67,7 @@ def test_observability_commands(logged_in, runner, files, key_mode, argv, method @pytest.mark.parametrize("key_mode", [False, True]) def test_observability_404(logged_in, runner, files, key_mode, argv, method, path, response): with respx.mock as mock: - mock.route().respond(404, json={}) + mock.route().respond(404, json={"error": "unknown route"}) result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), *[v.format(**files) for v in argv]]) assert result.exit_code == 6, result.output assert "not available on this FailproofAI Cloud yet" in result.stdout diff --git a/fp-cloud-cli/tests/test_cloud_transport.py b/fp-cloud-cli/tests/test_cloud_transport.py index a073bca22..92af7c4b7 100644 --- a/fp-cloud-cli/tests/test_cloud_transport.py +++ b/fp-cloud-cli/tests/test_cloud_transport.py @@ -39,3 +39,89 @@ def test_new_capability_html_404_degrades(): with pytest.raises(NotFoundError, match="not available on this FailproofAI Cloud yet") as exc: client.cloud_request(ctx, "GET", "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/api/dashboards") assert exc.value.exit_code == 6 + + +@pytest.mark.parametrize("key_mode", [False, True]) +@pytest.mark.parametrize("method,path,body,params", [ + ("PATCH", "/api/evaluator-definitions/stale-id", {"enabled": False}, None), + ("GET", "/api/sessions/missing/export", None, None), + ("POST", "/api/sessions/missing/re-evaluate", None, None), + ("GET", "/api/dashboards/missing/tiles", None, None), + ("DELETE", "/api/dashboards/board/tiles/stale-tile", None, None), + ("GET", "/api/events/sentiment/sessions/missing", None, None), + ("GET", "/api/audits/contracts/stale-id", None, None), + ("POST", "/api/queries/run", {"query_id": "missing"}, None), + ("POST", "/api/agent/answer", {"callId": "missing", "decision": "reject"}, None), + ("DELETE", "/api/settings/model-context-windows", None, {"model": "missing"}), +]) +def test_record_404_is_not_route_unavailable(key_mode, method, path, body, params): + ctx = client.ClientContext( + "http://dash.test", token=None if key_mode else "tok", + api_key="key" if key_mode else None, + auth_mode=client.AuthMode.API_KEY if key_mode else client.AuthMode.SESSION, + ) + if key_mode and path.startswith("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/api/agent/"): + pytest.skip("the agent has no key route; tested via session-only command guards") + wire_path = client._v1_path(path) if key_mode else path + with respx.mock as mock: + mock.request(method, f"http://dash.test{wire_path}").respond(404, json={}) + with pytest.raises(NotFoundError) as exc: + client.cloud_request(ctx, method, path, body=body, params=params) + assert exc.value.exit_code == 6 + assert "Record not found" in exc.value.message + assert "not available on this FailproofAI Cloud yet" not in exc.value.message + + +@pytest.mark.parametrize("response,route_unknown", [ + ({"code": "route_not_found"}, True), + ({"error": {"code": "route_not_found"}}, True), + ({"error": "unknown endpoint"}, True), + ({"message": "Cannot PATCH /evaluator-definitions/stale-id"}, True), + ({"error": "route does not exist"}, True), + ({"error": "old Cloud route absent"}, True), + ({"error": "evaluator definition not found"}, False), + ({"error": "session not found"}, False), + ({"error": "query not found"}, False), + ({"error": "not available in demo"}, False), +]) +def test_404_response_overrides_request_shape(response, route_unknown): + ctx = client.ClientContext("http://dash.test", token="tok") + with respx.mock as mock: + mock.patch("http://dash.test/api/evaluator-definitions/stale-id").respond(404, json=response) + with pytest.raises(NotFoundError) as exc: + client.cloud_request(ctx, "PATCH", "/api/evaluator-definitions/stale-id", body={"enabled": False}) + assert exc.value.exit_code == 6 + if route_unknown: + assert exc.value.message == "not available on this FailproofAI Cloud yet" + else: + assert exc.value.message == response.get("error", response.get("message")) + + +@pytest.mark.parametrize("body", [{"code": "record_not_found"}, {"error": "resource_not_found"}]) +def test_record_error_code_on_collection_is_not_unavailable(body): + ctx = client.ClientContext("http://dash.test", token="tok") + with respx.mock as mock: + mock.post("http://dash.test/api/queries/run").respond(404, json=body) + with pytest.raises(NotFoundError, match="Record not found") as exc: + client.cloud_request(ctx, "POST", "/api/queries/run", body={"sql": "SELECT 1"}) + assert "not available" not in exc.value.message + + +def test_collection_with_missing_saved_query_preserves_server_error(): + ctx = client.ClientContext("http://dash.test", token="tok") + with respx.mock as mock: + mock.post("http://dash.test/api/queries/run").respond(404, json={"error": "saved query not found"}) + with pytest.raises(NotFoundError, match="saved query not found"): + client.cloud_request(ctx, "POST", "/api/queries/run", body={"query_id": "stale"}) + + +def test_stream_404_distinguishes_route_and_record(): + ctx = client.ClientContext("http://dash.test", token="tok") + for error, expected in [ + ({"code": "unknown_route"}, "not available on this FailproofAI Cloud yet"), + ({"error": "conversation not found"}, "conversation not found"), + ]: + with respx.mock as mock: + mock.post("http://dash.test/api/agent/chat").respond(404, json=error) + with pytest.raises(NotFoundError, match=expected): + list(client.cloud_stream(ctx, "/api/agent/chat", {"conversationId": "missing"})) diff --git a/fp-cloud-cli/tests/test_evaluator_authoring.py b/fp-cloud-cli/tests/test_evaluator_authoring.py index 3bb6b3b13..03e85c91c 100644 --- a/fp-cloud-cli/tests/test_evaluator_authoring.py +++ b/fp-cloud-cli/tests/test_evaluator_authoring.py @@ -298,3 +298,120 @@ def test_server_validation_edges(logged_in, runner, flags): result = runner.invoke(app, ["--json", "evaluators", "create", "--key", "quality", "--name", "Quality", *flags]) assert not mock.calls assert result.exit_code == 2, result.output + + +@pytest.mark.parametrize("step", [ + "Score 0 if incorrect; score 1 otherwise.", + "score 1 if the answer is correct", + "give a score of 0 for a violation", + "SCORE OF 1 when compliant", + "score: 0 on failure", + "score: 1 on success", + "score\n1 if correct", + "score : 1 if correct", + "score=0 when it fails", + "score one if compliant", + "Give 0 when the request is unresolved", + "Give 1 if the answer is correct", + "Assign a value of 0 when evidence is absent", + "Award a score of one if it succeeds", + "Return zero otherwise", + "Grade pass/fail", + "Return pass or fail", + "Answer yes/no", + "Answer yes or no", + "Respond true/false", + "Return true or false", + "Return false or true", + "Answer no or yes", + "Grade fail/pass", + "Give 0.0 when incorrect", + "Give 1.0 if correct", + "Return true if correct, false otherwise", + "Output yes when the answer is complete", +]) +@pytest.mark.parametrize("file_input", [False, True]) +def test_binary_judge_phrasing_is_rejected_before_http(logged_in, runner, tmp_path, step, file_input): + if file_input: + path = tmp_path / "judge.json" + path.write_text(json.dumps({ + "eval_key": "quality", "display_name": "Quality", "eval_version": "1", + "result_kind": "score", + "evaluator_source": json.dumps({"kind": "judge", "criteria": "Grade quality", "steps": [step]}), + })) + flags = ["--file", str(path)] + else: + flags = ["--key", "quality", "--name", "Quality", "--judge", "--criteria", "Grade quality", "--step", step] + with respx.mock as mock: + result = runner.invoke(app, ["--json", "evaluators", "create", *flags]) + assert not mock.calls + assert result.exit_code == 2, result.output + assert "binary judge steps" in result.stdout + + +@pytest.mark.parametrize("step", [ + "Return a score between 0 and 1 reflecting how well it did.", + "The score should reflect 0 to 1 confidence.", + "Award a high score when evidence supports the answer.", + "Award a low score when a refund was promised without verification.", + "Determine whether a policy-verification tool was called before the refund.", +]) +def test_continuous_judge_steps_are_accepted(logged_in, runner, step): + with respx.mock as mock: + route = mock.post(f"{BASE}/api/evaluator-definitions").respond(201, json={"id": "d1"}) + result = runner.invoke(app, [ + "--json", "evaluators", "create", "--key", "quality", "--name", "Quality", + "--judge", "--criteria", "Grade quality", "--step", step, + ]) + assert result.exit_code == 0, result.output + assert json.loads(json.loads(route.calls[0].request.content)["evaluator_source"])["steps"] == [step] + + +@pytest.mark.parametrize("server_body", [None, {"error": "binary-phrased steps are invalid"}]) +def test_judge_server_422_is_visible(logged_in, runner, server_body): + with respx.mock as mock: + mock.post(f"{BASE}/api/evaluator-definitions").respond(422, json=server_body) + result = runner.invoke(app, [ + "--json", "evaluators", "create", "--key", "quality", "--name", "Quality", + "--judge", "--criteria", "Grade quality", "--step", "Assess relevance continuously", + ]) + assert result.exit_code == 1, result.output + payload = json.loads(result.stdout) + assert payload["status"] == 422 + assert payload["error"] == (server_body["error"] if server_body else "Request failed with status 422.") + + +@pytest.mark.parametrize("key_mode", [False, True]) +def test_stale_evaluator_patch_is_missing_record(logged_in, runner, key_mode): + prefix = "v1" if key_mode else "api" + with respx.mock as mock: + mock.get(f"{BASE}/{prefix}/evaluator-definitions").respond(200, json=[ + {"id": "stale", "eval_key": "quality", "eval_version": "1", "enabled": True}, + ]) + mock.patch(f"{BASE}/{prefix}/evaluator-definitions/stale").respond(404, json={}) + result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), "evaluators", "disable", "quality@1"]) + assert result.exit_code == 6 + assert "Record not found" in result.stdout + assert "stale" in result.stdout + assert "not available on this FailproofAI Cloud yet" not in result.stdout + + +@pytest.mark.parametrize("frm,to", [ + ("2026-10-01T02:00:00+02:00", "2026-10-08T05:30:00+05:30"), + ("2026-09-30T20:00:00-04:00", "2026-10-07T20:00:00-04:00"), + (" 2026-10-01T00:00:00.123456Z ", " 2026-10-08T00:00:00.654321Z "), +]) +@pytest.mark.parametrize("key_mode", [False, True]) +def test_standalone_backfill_normalizes_utc_z(logged_in, runner, frm, to, key_mode): + prefix = "v1" if key_mode else "api" + with respx.mock as mock: + route = mock.post(f"{BASE}/{prefix}/evaluations/backfill").respond(200, json={}) + result = runner.invoke(app, [ + "--json", *(["--api-key", "key"] if key_mode else []), "evaluators", "backfill", + "--from", frm, "--to", to, "--key", "quality", "--dry-run", + ]) + assert result.exit_code == 0, result.output + assert json.loads(route.calls[0].request.content) == { + "from": "2026-10-01T00:00:00Z", "to": "2026-10-08T00:00:00Z", + "eval_key": "quality", "dry_run": True, + } diff --git a/fp-cloud-cli/tests/test_help_table_coverage.py b/fp-cloud-cli/tests/test_help_table_coverage.py index a9326c723..955037efa 100644 --- a/fp-cloud-cli/tests/test_help_table_coverage.py +++ b/fp-cloud-cli/tests/test_help_table_coverage.py @@ -125,3 +125,18 @@ def test_every_subcommand_the_hint_column_advertises_actually_exists(): bad.append(f"{name}: '{token}' is not a subcommand of `fp {name}`") assert not bad, "the help screen advertises commands that do not exist:\n " + "\n ".join(bad) + + +def test_group_hints_cover_all_registered_subcommands(): + """New commands must not silently disappear from the hand-maintained index.""" + cli = get_command(app) + missing = [] + for _section, entries in output._TOP_LEVEL_GROUPS: + for name, _description, hint in entries: + registered = set(getattr(cli.commands[name], "commands", {}) or {}) + advertised = { + sub for sub in registered + if any(sub == token or token.endswith("*") and sub.startswith(token[:-1]) for token in hint.split()) + } + missing.extend(f"{name} {sub}" for sub in sorted(registered - advertised)) + assert not missing, f"subcommands missing from the top-level help hints: {missing}" diff --git a/fp-cloud-cli/tests/test_session_reevaluate.py b/fp-cloud-cli/tests/test_session_reevaluate.py index 4d89f2aed..2bcb80b92 100644 --- a/fp-cloud-cli/tests/test_session_reevaluate.py +++ b/fp-cloud-cli/tests/test_session_reevaluate.py @@ -9,9 +9,25 @@ def test_re_evaluate(logged_in, runner, key_mode, status): prefix = "v1" if key_mode else "api" with respx.mock: - route = respx.post(f"http://dash.test/{prefix}/sessions/run-001/re-evaluate").respond(status, json={"queued": True}) + route = respx.post(f"http://dash.test/{prefix}/sessions/run-001/re-evaluate").respond( + status, json={"queued": True} if status == 200 else {"error": "unknown route"}, + ) result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), "sessions", "re-evaluate", "run-001"]) assert route.called assert result.exit_code == (0 if status == 200 else 6), result.output if status == 404: assert "not available on this FailproofAI Cloud yet" in result.stdout + + +@pytest.mark.parametrize("key_mode", [False, True]) +@pytest.mark.parametrize("error_body", [{}, {"error": "session not found"}]) +def test_re_evaluate_missing_session(logged_in, runner, key_mode, error_body): + prefix = "v1" if key_mode else "api" + with respx.mock as mock: + mock.post(f"http://dash.test/{prefix}/sessions/missing/re-evaluate").respond(404, json=error_body) + result = runner.invoke(app, [ + "--json", *(["--api-key", "key"] if key_mode else []), "sessions", "re-evaluate", "missing", + ]) + assert result.exit_code == 6, result.output + assert "not found" in result.stdout + assert "not available on this FailproofAI Cloud yet" not in result.stdout From 3ab2e093a3d28607cc4d6cc2099c748ec9041d1a Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:52:21 +0000 Subject: [PATCH 13/16] fix(fp): keep named capability families distinct from record paths --- fp-cloud-cli/fp_cli/client.py | 6 +++++- fp-cloud-cli/tests/test_cloud_transport.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/fp-cloud-cli/fp_cli/client.py b/fp-cloud-cli/fp_cli/client.py index 6e48d5b7c..738f46338 100644 --- a/fp-cloud-cli/fp_cli/client.py +++ b/fp-cloud-cli/fp_cli/client.py @@ -657,7 +657,11 @@ def _cloud_not_found(exc: FpCliError, path: str, *, body=None, params=None) -> N segments = path.removeprefix("/api/").split("/") family = segments[0] capability = len(segments) == 1 or ( - family in ("agent", "otel", "offers", "team-trial", "integrations") + family == "agent" and len(segments) == 2 + or family == "otel" and segments == ["otel", "status"] + or family == "offers" and segments[1:] in (["me"], ["claim"]) + or family == "team-trial" and segments == ["team-trial", "me"] + or family == "integrations" and segments == ["integrations", "slack"] or family == "events" and segments in ( ["events", "summary"], ["events", "latency_aggregate"], ["events", "sentiment"], ["events", "sentiment", "messages"], diff --git a/fp-cloud-cli/tests/test_cloud_transport.py b/fp-cloud-cli/tests/test_cloud_transport.py index 92af7c4b7..d44f39023 100644 --- a/fp-cloud-cli/tests/test_cloud_transport.py +++ b/fp-cloud-cli/tests/test_cloud_transport.py @@ -125,3 +125,17 @@ def test_stream_404_distinguishes_route_and_record(): mock.post("http://dash.test/api/agent/chat").respond(404, json=error) with pytest.raises(NotFoundError, match=expected): list(client.cloud_stream(ctx, "/api/agent/chat", {"conversationId": "missing"})) + + +@pytest.mark.parametrize("path", [ + "/api/agent/conversations/stale", + "/api/offers/stale", + "/api/integrations/slack/stale", +]) +def test_named_endpoint_families_do_not_hide_record_404s(path): + ctx = client.ClientContext("http://dash.test", token="tok") + with respx.mock as mock: + mock.get(f"http://dash.test{path}").respond(404, json={}) + with pytest.raises(NotFoundError, match="Record not found") as exc: + client.cloud_request(ctx, "GET", path) + assert "not available" not in exc.value.message From 79d0a16cc5e9c6109f0765f529f5a1d6c20f504e Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:57:05 +0000 Subject: [PATCH 14/16] test(fp): compare visible artifact ordering under forced ANSI styling --- fp-cloud-cli/tests/fake_cloud_smoke.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fp-cloud-cli/tests/fake_cloud_smoke.py b/fp-cloud-cli/tests/fake_cloud_smoke.py index fe2ae06f6..b1bc96848 100644 --- a/fp-cloud-cli/tests/fake_cloud_smoke.py +++ b/fp-cloud-cli/tests/fake_cloud_smoke.py @@ -274,7 +274,9 @@ def run(argv, *, input=None, key=False, json_mode=True, expected=0): transcript.append("$ fp agent ask 'Count events' [interactive approval]\n" + output.decode(errors="replace") + f"\nexit {proc.returncode}\n") assert proc.returncode == 0 and sent, transcript[-1] assert Cloud.answers[-1]["decision"] == "approve" - rendered = output.decode(errors="replace") + # CI may force colour even with NO_COLOR, inserting ANSI spans inside + # the wordmark. Compare visible text without weakening the order check. + rendered = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", output.decode(errors="replace")) assert rendered.index("failproof ai") < rendered.index("Events by agent") server.shutdown() (artifacts / "fp-user-smoke.txt").write_text("\n".join(transcript), encoding="utf-8") From 29872166cf69df63eb635b974049bbfae41af629 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:38:28 +0000 Subject: [PATCH 15/16] feat(fp): render deliberate human Cloud views and visible deployment previews --- docs/reference/cloud-cli.mdx | 12 +- fp-cloud-cli/CHANGELOG.md | 1 + fp-cloud-cli/README.md | 19 +- fp-cloud-cli/fp_cli/client.py | 1 + fp-cloud-cli/fp_cli/cloud_output.py | 523 ++++++++++++++++++ fp-cloud-cli/fp_cli/commands/_cloud.py | 16 +- fp-cloud-cli/fp_cli/commands/account_cmds.py | 4 +- fp-cloud-cli/fp_cli/commands/agent_cmds.py | 18 +- .../fp_cli/commands/dashboards_cmds.py | 2 +- .../fp_cli/commands/evaluators_cmds.py | 39 +- fp-cloud-cli/fp_cli/commands/queries_cmds.py | 5 +- .../fp_cli/commands/sentiment_cmds.py | 2 +- fp-cloud-cli/fp_cli/commands/sessions_cmds.py | 4 +- fp-cloud-cli/skill/SKILL.md | 6 +- fp-cloud-cli/skill/references/commands.md | 16 +- fp-cloud-cli/tests/fake_cloud_smoke.py | 32 +- fp-cloud-cli/tests/test_cloud_account.py | 7 + fp-cloud-cli/tests/test_cloud_agent.py | 6 +- fp-cloud-cli/tests/test_cloud_human_views.py | 288 ++++++++++ fp-cloud-cli/tests/test_round4_behaviour.py | 144 +++++ 20 files changed, 1081 insertions(+), 64 deletions(-) create mode 100644 fp-cloud-cli/fp_cli/cloud_output.py create mode 100644 fp-cloud-cli/tests/test_cloud_human_views.py create mode 100644 fp-cloud-cli/tests/test_round4_behaviour.py diff --git a/docs/reference/cloud-cli.mdx b/docs/reference/cloud-cli.mdx index 65d66ec46..85c711531 100644 --- a/docs/reference/cloud-cli.mdx +++ b/docs/reference/cloud-cli.mdx @@ -81,6 +81,8 @@ disables older enabled versions. A failed follow-up leaves the new version published: inspect `evaluators list --all` before retrying. Judge dry-runs are refused by Cloud. Jev dry-runs spend model budget. Backfill excludes judge/Jev and accepts at most 90 days. +Human output shows per-session dry-run results before creating/deploying. +Failed previews stop the write unless you explicitly pass `--force` after review. AI drafting requires `fp login` and writes a spec accepted by create/deploy: @@ -96,8 +98,8 @@ fp --json evaluators dry-run --file evaluator.json | --- | --- | | `fp evaluators list` | `--key`, `--all` | | `fp evaluators show KEY[@VERSION]` | Full source/envelope | -| `fp evaluators create` | `--key`, `--name`, `--version`, repeated `--label`, `--condition EXPR\|@file`, one source, repeated `--dry-run-on`, `--disabled` | -| `fp evaluators deploy KEY` | Same source flags; `--version`, code-only `--backfill 7d`; numeric versions auto-increment | +| `fp evaluators create` | `--key`, `--name`, `--version`, repeated `--label`, `--condition EXPR\|@file`, one source, repeated `--dry-run-on`, `--disabled`, explicit `--force` for failed previews | +| `fp evaluators deploy KEY` | Same source flags; `--version`, code-only `--backfill 7d`, `--force`; numeric versions auto-increment; KEY supplies a missing spec-file `eval_key` | | `fp evaluators draft "intent"` | Session; `--kind code\|judge\|jev`, `--out` | | `fp evaluators dry-run` | Source flags or `--file`, repeated `--session` (maximum 10); none = compile only | | `fp evaluators enable\|disable KEY[@VERSION]` | Toggle | @@ -123,6 +125,10 @@ All commands accept global `--json`. Unknown-route 404s exit 6 with “not avail on this FailproofAI Cloud yet”. Missing-record 404s also exit 6 but report that the record was not found. Session-only commands deny key mode before HTTP (exit 2). +Human commands use summaries and tables. For raw JSON exports, pass global +`--json` or save with `--out FILE`; otherwise session and JSON query exports show +a readable view. CSV exports remain plain CSV for piping. + | Commands | Options / notes | | --- | --- | | `fp dashboards list\|show ID\|create\|update ID\|delete ID` | Create `--name`, `--description`, `--file` definition; update preserves unmentioned fields | @@ -139,7 +145,7 @@ record was not found. Session-only commands deny key mode before HTTP (exit 2). | `fp orgs create --name N [--slug S]` | Session; derives omitted slug, does not switch org | | `fp roles list\|show NAME\|create\|update NAME\|delete NAME` | Permission sets; create `--name --permission`; update replace `--permission` or merge `--add/--remove`; `--description`, `--file` | | `fp users delete ID` / `fp users defaults` | Deletion by id | -| `fp usage windows` / `fp plan` / `fp plan claim OFFER` | Offers and trial; Cloud requires actor identity for offers | +| `fp usage windows` / `fp plan` / `fp plan claim OFFER` | Offers and trial; plan/claim require a signed-in session | | `fp settings context-windows list\|set MODEL TOKENS\|delete MODEL` | 0–1000000 tokens | | `fp otel status\|enable\|disable` | Cloud org ingest switch, not local capture | | `fp slack status\|disconnect` | Session | diff --git a/fp-cloud-cli/CHANGELOG.md b/fp-cloud-cli/CHANGELOG.md index 3d4bc74e6..a940b341b 100644 --- a/fp-cloud-cli/CHANGELOG.md +++ b/fp-cloud-cli/CHANGELOG.md @@ -19,6 +19,7 @@ - Remove the unused colour-keyed permission rank, document full-id rendering, emit UTC `Z` timestamps for deploy backfills, and report approval-answer failures without a traceback while restoring the terminal (#885). - Reject natural-language binary judge steps, distinguish unknown routes from missing records on 404, normalize standalone backfill ranges to UTC `Z`, expose all subcommands in help, and render artifact headers before titles with consistent session-only wording (#885). +- New Cloud commands use deliberate human summaries/tables and actionable empty states at 80/140 columns without raw response blobs. Evaluator previews display before writes and require explicit `--force` after failures; plan/claim deny key mode, deploy specs can omit the positional key, and older settings-route 404s degrade clearly. Existing JSON payloads stay unchanged (#885). ## 0.0.2b0 — 2026-09-22 diff --git a/fp-cloud-cli/README.md b/fp-cloud-cli/README.md index 6a5c349ad..227b29a1f 100644 --- a/fp-cloud-cli/README.md +++ b/fp-cloud-cli/README.md @@ -221,6 +221,11 @@ immutable version before disabling older enabled versions. If disabling fails, the CLI names the published id; inspect `list --all` and recover explicitly. `--backfill 7d` is code-only; `evaluators backfill --from T --to T` accepts ≤90 days. +Human output shows each `--dry-run-on` session's outcome before creation or +deployment. Any failed run stops the write; after reviewing the failure, an +explicit `--force` overrides this guard. Existing `--json` response shapes stay +unchanged. + AI drafting is session-only and writes an API spec that create/deploy accept: ```bash @@ -258,8 +263,8 @@ fp roles list|show|create|update|delete fp users delete fp users defaults fp usage windows -fp plan -fp plan claim +fp plan # session +fp plan claim # session fp settings context-windows list|set |delete fp otel status|enable|disable fp slack status|disconnect # session @@ -275,11 +280,17 @@ All new commands accept the global `--json` before the command. Session-only commands fail before HTTP in API-key mode, with exit 2 and a `fp login` hint. Unknown-route 404s exit 6 with “not available on this FailproofAI Cloud yet”. Missing-record 404s also exit 6 but report that the record was not found. -`plan` routes are mapped for keys, but the Cloud also requires an actor email; -an API key alone cannot impersonate a person to claim an offer. +`plan` and `plan claim` require a signed-in person and deny API-key mode before +HTTP with the standard `fp login` message. Human screens use the mint/pink wordmark, grey labels, open tables and full ids; existing JSON shapes are unchanged. +Human commands show summaries, tables and wrapped text rather than response +objects. Use global `--json` for raw API results. For `sessions export` or +`query export --format json`, use `--out FILE` to save the JSON; without either +option, the terminal shows a readable session or query view. CSV exports remain +plain CSV for piping. + ## Configuration | Setting | Flag | Env var | Default | diff --git a/fp-cloud-cli/fp_cli/client.py b/fp-cloud-cli/fp_cli/client.py index 738f46338..d60c855e6 100644 --- a/fp-cloud-cli/fp_cli/client.py +++ b/fp-cloud-cli/fp_cli/client.py @@ -675,6 +675,7 @@ def _cloud_not_found(exc: FpCliError, path: str, *, body=None, params=None) -> N or family == "users" and segments == ["users", "defaults"] or family == "usage" and segments == ["usage", "windows"] or family == "settings" and segments == ["settings", "model-context-windows"] + or family == "settings" and len(segments) == 2 ) # Some collection operations name their target in a query or body. request_body = body if isinstance(body, dict) else {} diff --git a/fp-cloud-cli/fp_cli/cloud_output.py b/fp-cloud-cli/fp_cli/cloud_output.py new file mode 100644 index 000000000..3993dc238 --- /dev/null +++ b/fp-cloud-cli/fp_cli/cloud_output.py @@ -0,0 +1,523 @@ +"""Human views for the additive Cloud CLI surface. + +JSON output never passes through this module. Tables stay tables at narrow widths; +long prose wraps beside grey labels rather than becoming FIELD / VALUE blocks. +""" + +from __future__ import annotations + +import json +from typing import Any, Optional + +from rich.text import Text + +from . import output, theme + + +def begin(title: str) -> None: + output.screen_header() + output._stdout.print(Text(title.upper(), style="bold")) + + +def value(data: Any) -> str: + """Human text, never a Python repr or serialized JSON container.""" + if data is None: + return "—" + if isinstance(data, bool): + return "yes" if data else "no" + if isinstance(data, dict): + return "; ".join(f"{label(k)}: {value(v)}" for k, v in data.items()) + if isinstance(data, (list, tuple)): + return ", ".join(value(v) for v in data) or "none" + return str(data) + + +def label(key: Any) -> str: + return str(key).replace("_", " ").removesuffix(" source") + + +def state(enabled: Any, *, on="enabled", off="disabled") -> Text: + if enabled is None: + return Text("unknown", style=theme.LABEL) + return Text("● " + on, style=theme.ACCENT) if enabled else Text("○ " + off, style=theme.LABEL) + + +def kv(pairs) -> None: + rows = [(label(k), v) for k, v in pairs if v is not None] + if not rows: + return + table = output.Table.grid(padding=(0, 2, 0, 0), pad_edge=False) + table.add_column(width=max(len(k) for k, _ in rows) + 2, style=theme.LABEL) + table.add_column(ratio=1, overflow="fold") + for key, item in rows: + table.add_row(Text(" " + key, style=theme.LABEL), item if isinstance(item, Text) else Text(value(item))) + output._stdout.print(table) + + +def line(text: str) -> None: + output._stdout.print(Text(" " + text)) + + +def next_command(text: str) -> None: + output._stdout.print(Text(" " + text, style=theme.PINK)) + + +def empty(message: str, command: str) -> None: + line(message) + next_command(command) + + +def table(columns, rows, *, empty_message: str, command: str) -> None: + if not rows: + empty(empty_message, command) + return + grid = output.Table(box=None, padding=(0, 2, 0, 0), pad_edge=False, header_style=theme.LABEL) + for column in columns: + grid.add_column(column, overflow="fold") + for row in rows: + grid.add_row(*[v if isinstance(v, Text) else Text(value(v)) for v in row]) + output._stdout.print(grid) + + +def collection(data: Any, *keys: str) -> list: + if isinstance(data, list): + return data + if isinstance(data, dict): + for key in keys: + if isinstance(data.get(key), list): + return data[key] + return [] + + +def detail(data: Any, *, skip=()) -> None: + """Render nested API detail as named sections, not opaque structures.""" + if isinstance(data, dict): + scalar = [(k, v) for k, v in data.items() if k not in skip and not isinstance(v, (dict, list))] + kv(scalar) + for key, item in data.items(): + if key in skip or not isinstance(item, (dict, list)): + continue + if not item: + kv([(key, "none")]) + elif isinstance(item, dict): + line(label(key)) + detail(item) + elif all(not isinstance(v, (dict, list)) for v in item): + kv([(key, item)]) + else: + line(label(key)) + for row in item: + detail(row) + output._stdout.print() + elif isinstance(data, list): + for row in data: + detail(row) + elif data is not None: + line(value(data)) + + +def envelope(row: dict) -> dict: + try: + source = json.loads(row.get("evaluator_source") or "") + except (ValueError, TypeError): + return {} + return source if isinstance(source, dict) else {} + + +def evaluator_kind(row: dict) -> str: + source = envelope(row) + if source.get("kind") == "judge": + return "LLM judge" + if source.get("kind") == "jev": + return "Jev " + str(source.get("type", "")) + return "code" + + +def version(raw: Any) -> str: + text = value(raw) + return text if text.startswith("v") else "v" + text + + +def evaluator(row: dict, title="Evaluator") -> None: + begin(title) + kv([ + ("key", row.get("eval_key")), ("version", row.get("eval_version")), + ("name", row.get("display_name")), ("kind", evaluator_kind(row)), + ("enabled", state(row.get("enabled", True))), ("condition", row.get("condition_source") or "always"), + ("result kind", row.get("result_kind", "score")), ("labels", row.get("labels", [])), + ("id", row.get("id")), + ]) + source = envelope(row) + if source.get("kind") == "judge": + kv([("criteria", source.get("criteria")), + ("threshold", source["threshold"] if source.get("threshold") is not None else "server default")]) + for i, step in enumerate(source.get("steps") or [], 1): + kv([(f"step {i}", step)]) + elif source.get("kind") == "jev": + kv([("instructions", source.get("instructions")), + ("threshold", source["threshold"] if source.get("threshold") is not None else "server default")]) + criteria = source.get("criteria") + if isinstance(criteria, dict): + kv([(str(k), v) for k, v in criteria.items()]) + elif isinstance(criteria, list): + for i, level in enumerate(criteria, 1): + kv([(f"level {i}", level)]) + else: + kv([("source", row.get("evaluator_source")), ("timeout", row.get("timeout_seconds"))]) + + +def evaluators(data: Any) -> None: + begin("Evaluators") + items = collection(data, "definitions") + table(["key", "version", "kind", "enabled", "labels"], [ + [r.get("eval_key"), r.get("eval_version"), evaluator_kind(r), + state(r.get("enabled", True)), r.get("labels", [])] for r in items + ], empty_message="no evaluators yet", command="fp evaluators create --help") + + +def deployed(created: dict, predecessors: list) -> None: + begin("Evaluator deployed") + text = Text(f" deployed {created.get('eval_key', 'evaluator')} {version(created.get('eval_version'))} ") + text.append_text(state(created.get("enabled", True))) + if predecessors: + text.append(" · disabled " + ", ".join(version(r.get("eval_version")) for r in predecessors)) + output._stdout.print(text) + kv([("id", created.get("id"))]) + + +def score_of(result: Any) -> Any: + if not isinstance(result, dict): + return None + score = result.get("score") + if isinstance(score, dict): + return score.get("value") + return score if score is not None else result.get("scores") + + +def dry_run(data: dict) -> None: + begin("Evaluator dry-run") + runs = collection(data, "runs") + if not runs and "ok" in data: + runs = [data] + if not runs: + detail(data) + return + for run in runs: + result = run.get("result") or {} + skipped = run.get("applicable") is False + status = "skipped" if skipped else "passed" if run.get("ok") else "failed" + glyph = "○" if skipped else "✓" if run.get("ok") else "✕" + line(f"{glyph} {run.get('session_id') or 'compile only'} · {status}") + kv([ + ("score", score_of(result)), ("reasoning", result.get("reasoning") if isinstance(result, dict) else None), + ("error", run.get("error_message") or run.get("error_kind")), + ("duration ms", run.get("duration_ms")), + ]) + summary = data.get("summary") + if isinstance(summary, dict): + line(" · ".join(f"{summary.get(k, 0)} {k}" for k in ("succeeded", "failed", "skipped"))) + + +def roles(data: Any, *, listing: bool) -> None: + begin("Roles" if listing else "Role") + if listing: + table(["name", "built-in", "users", "permissions"], [ + [r.get("name"), state(r.get("is_builtin", False), on="yes", off="no"), + r.get("user_count", 0), len(r.get("permissions") or [])] + for r in collection(data, "permission_sets", "roles") + ], empty_message="no custom roles yet", command="fp roles create --name reviewer --permission events:read") + elif isinstance(data, dict): + kv([("name", data.get("name")), ("built-in", data.get("is_builtin")), + ("users", data.get("user_count")), ("description", data.get("description"))]) + kv([("permissions", data.get("permissions", []))]) + + +def dashboards(data: Any, *, listing=False, tile_view=False, dashboard_id=None) -> None: + if tile_view: + begin("Tiles") + items = collection(data, "tiles") if listing else [data] + table(["title", "chart", "query", "id"], [ + [r.get("title") or "untitled", r.get("chart_type"), r.get("query_id"), r.get("id")] + for r in items if isinstance(r, dict) + ], empty_message="no tiles yet", + command=f"fp dashboards tiles {dashboard_id or ''} add --query --chart table") + return + begin("Dashboards" if listing else "Dashboard") + if listing: + table(["name", "tiles", "description", "id"], [ + [r.get("name"), r.get("tile_count", 0), r.get("description") or "—", r.get("id")] + for r in collection(data, "dashboards") + ], empty_message="no dashboards yet", command="fp dashboards create --name Quality") + elif isinstance(data, dict): + row = data.get("dashboard", data) + kv([("name", row.get("name")), ("description", row.get("description")), ("id", row.get("id"))]) + if row.get("definition"): + line("filters and display") + detail(row["definition"]) + if "tiles" in data: + dashboards(data["tiles"], listing=True, tile_view=True, dashboard_id=row.get("id")) + + +def query_result(data: dict, title="Query result") -> None: + begin(title) + columns = [c.get("name", "") if isinstance(c, dict) else str(c) for c in data.get("columns", [])] + table(columns, data.get("rows", []), empty_message="query returned no rows", command="fp query schema") + kv([("elapsed ms", data.get("elapsed_ms")), ("truncated", data.get("truncated"))]) + + +def sentiment(data: Any, mode: str) -> None: + begin("Sentiment" if not mode else "Sentiment " + mode) + mode = "session" if mode.startswith("sessions/") else mode + items = collection(data, mode, "items", "messages", "sessions") + if isinstance(data, list) or items or isinstance(data, dict) and any(k in data for k in (mode, "items")): + columns = ["session", "sentiment", "score", "message"] if mode == "messages" else ["session", "sentiment", "negative", "messages"] + def strongest(row): + scores = row.get("scores") or {} + numeric = [(k, v) for k, v in scores.items() if isinstance(v, (int, float))] + return max(numeric, key=lambda pair: pair[1]) if numeric else ("—", "—") + table(columns, [ + [r.get("session_id"), r.get("sentiment") or r.get("label") or r.get("feeling") or strongest(r)[0], + r.get("score", r.get("negative", r.get("negative_score", strongest(r)[1]))), + r.get("text") or r.get("message") if mode == "messages" else r.get("message_count", r.get("messages_count"))] + for r in items if isinstance(r, dict) + ], empty_message="no sentiment results in this range", command="fp settings set sentiment.enabled --json-value true") + else: + detail(data) + if not data: + empty("no sentiment results yet", "fp settings set sentiment.enabled --json-value true") + + +def plan(data: dict) -> None: + begin("Plan") + offers = data.get("offers", {}) + items = collection(offers, "offers") + table(["offer", "tier", "status", "expires"], [ + [r.get("offer_key", r.get("key")), r.get("tier", r.get("plan")), r.get("status"), + r.get("expires_at")] for r in items if isinstance(r, dict) + ], empty_message="no offers available", command="fp usage") + trial = data.get("team_trial", {}) + line("team trial") + if trial: + detail(trial) + else: + line("no active team trial") + + +def brief(data: dict) -> None: + begin("Home brief") + for key in ("greeting", "summary"): + if data.get(key): + line(str(data[key])) + for block in data.get("blocks", []): + artifact(block) + if not data.get("blocks"): + empty("no brief blocks yet", 'fp agent ask "Summarize my recent sessions"') + + +def artifact(block: dict) -> None: + data = block.get("result") + if isinstance(data, dict): + query_result(data, block.get("title", "Artifact")) + else: + begin(block.get("title", "Home")) + for key in ("text", "note"): + if block.get(key): + line(str(block[key])) + for key in ("items", "sessions"): + for item in block.get(key, []): + detail(item) + + +def resource(path: str, method: str, data: Any, body=None, params=None) -> None: + """Request-specific human views used by shared mutation orchestrators.""" + body = body if isinstance(body, dict) else {} + row = data if isinstance(data, dict) else {} + parts = path.removeprefix("/api/").split("/") + family = parts[0] + if family == "permission-sets": + if method == "DELETE": + begin("Role deleted") + line(f"deleted {parts[-1]} · members keep their permissions") + else: + roles({**body, **row} if method != "GET" else data, listing=len(parts) == 1 and method == "GET") + elif family == "dashboards": + if method == "DELETE": + begin("Tile removed" if "tiles" in parts else "Dashboard deleted") + kv([("id", parts[-1])]) + elif parts[-1] == "layout": + begin("Tile layout updated") + kv([("tiles updated", row.get("updated", len(body.get("tiles", []))))]) + else: + dashboards(data, listing=method == "GET" and (len(parts) == 1 or parts[-1] == "tiles"), + tile_view="tiles" in parts, dashboard_id=parts[1] if len(parts) > 1 else None) + elif family == "otel" or family == "settings" and parts[-1] == "otel.ingest_enabled": + begin("OTEL") + kv([("ingest", state(row.get("enabled", row.get("value", body.get("value"))))), + ("events 24h", row.get("events_24h")), ("last received", row.get("last_received_at")), + ("content missing", row.get("content_missing"))]) + services = collection(row, "services") + table(["service", "events", "last received"], [ + [r.get("name", r.get("service_name")), r.get("event_count", r.get("events")), r.get("last_received_at")] + if isinstance(r, dict) else [r, "—", "—"] for r in services + ], empty_message="no OTEL services yet", command="fp otel enable") + elif family == "integrations": + begin("Slack") + connection = row.get("integration", row.get("connection", row)) + connection = connection if isinstance(connection, dict) else {} + kv([("connection", state(method != "DELETE" and connection.get("connected", connection.get("installed", bool(connection))), + on="connected", off="disconnected")), + ("workspace", connection.get("team_name", connection.get("workspace_name"))), + ("channel", connection.get("channel_name")), ("connected at", connection.get("created_at"))]) + if method == "DELETE": + line("Slack disconnected") + elif not connection: + line("Connect Slack from your Cloud integration settings.") + elif family == "settings": + begin("Context windows") + if method == "GET": + table(["model", "tokens", "updated"], [ + [r.get("model_id"), r.get("context_window"), r.get("updated_at")] + for r in collection(data, "model_context_windows") + ], empty_message="no model overrides yet", command="fp settings context-windows set ") + else: + kv([("model", row.get("model_id", body.get("model_id", (params or {}).get("model")))), + ("tokens", row.get("context_window", body.get("context_window")))]) + if method == "DELETE": + line("override removed") + elif family == "offers": + begin("Offer claimed") + kv([("offer", body.get("offer_key"))]) + detail(data) + elif family == "users": + begin("Member deleted" if method == "DELETE" else "Member defaults") + if method == "DELETE": + kv([("id", parts[-1])]) + else: + detail(data) + elif family == "orgs": + begin("Organization created") + kv([("name", row.get("name", body.get("name"))), ("slug", row.get("slug", body.get("slug"))), + ("id", row.get("id")), ("plan", row.get("plan"))]) + next_command(f"fp orgs switch {row.get('slug', body.get('slug', ''))}") + elif family == "usage": + begin("Usage windows") + table(["from", "to", "events", "evaluations"], [ + [r.get("from", r.get("starts_at")), r.get("to", r.get("ends_at")), + r.get("events_ingested", (r.get("usage") or {}).get("events_ingested")), + r.get("evaluations", (r.get("usage") or {}).get("evaluations"))] + for r in collection(data, "windows") + ], empty_message="no historical usage windows yet", command="fp usage") + elif family == "alerts": + begin("Alert recipients" if parts[-1] == "recipients" else "Alert issues") + items = collection(data, "recipients", "issues") + if method != "GET": + items = [data] + if parts[-1] == "recipients": + table(["recipient", "channel"], [ + [r.get("email", r.get("name")), r.get("channel", "email")] if isinstance(r, dict) else [r, "email"] for r in items + ], empty_message="no recipients available", command="fp users list") + else: + table(["title", "status", "severity", "id"], [ + [r.get("title"), r.get("state", r.get("status")), r.get("severity"), r.get("id")] for r in items + ], empty_message="no issues for this alert", command=f"fp alerts issues {parts[1]} --create --title ") + elif family == "audits": + begin("Agent contracts") + if method == "GET" and len(parts) == 2: + table(["agent", "enabled", "purpose", "id"], [ + [r.get("agent_id"), state(r.get("enabled", True)), r.get("purpose"), r.get("id")] + for r in collection(data, "contracts") + ], empty_message="no agent contracts yet", command="fp audits contracts create --agent-id <agent> --body @agent.md") + else: + kv([("agent", row.get("agent_id", body.get("agent_id"))), ("enabled", state(row.get("enabled", True))), + ("id", row.get("id")), ("body", row.get("body", body.get("body")))]) + else: + begin(label(family)) + detail(data) + + +def render(data: Any, title: str, *, view: Optional[str] = None, body=None, method="GET", params=None) -> None: + """Explicit views for commands without the shared resource orchestrator.""" + if view and view.startswith("/api/"): + resource(view, method, data, body, params) + elif title == "Evaluators": + evaluators(data) + elif title in ("Evaluator", "Evaluator created", "Evaluator labels"): + evaluator(data, title) + elif title == "Evaluator draft": + evaluator(data.get("spec", {}), title) + kv([("file", data.get("out"))]) + next_command(f"fp evaluators create --file {data.get('out')}") + elif title == "Evaluator dry-run": + dry_run(data) + elif title == "Instances": + begin(title) + table(["name", "kind", "enabled", "evaluators", "queued", "id"], [ + [r.get("name"), r.get("kind"), state(r.get("enabled", True)), r.get("eval_count", 0), r.get("queued", 0), r.get("id")] + for r in collection(data, "instances") + ], empty_message="no managed instances available", command="Check your Cloud managed-evaluator deployment.") + elif title == "Schema profile": + begin(title) + kv([(k, data.get(k)) for k in ("window_days", "sampled_events")] if isinstance(data, dict) else []) + for event in collection(data, "event_types"): + line(f"{event.get('event_type')} · {event.get('count', 0)} events") + table(["payload key", "type", "presence"], [ + [r.get("key"), r.get("type"), r.get("presence")] for r in event.get("keys", []) + ], empty_message="no sampled payload keys", command="fp events --since 24h") + if not collection(data, "event_types"): + empty("no sampled event types yet", "fp events --since 24h") + elif title == "Evaluation jobs": + begin(title) + table(["session", "evaluator", "status", "id"], [ + [r.get("session_id"), r.get("eval_key"), r.get("status"), r.get("id")] + for r in collection(data, "jobs", "evaluation_jobs") + ], empty_message="no evaluation jobs", command="fp sessions re-evaluate <session-id>") + elif title == "Evaluation backfill": + begin(title) + detail(data) + next_command("fp evaluators jobs") + elif title == "Session re-evaluation": + begin("Re-evaluate") + count = data.get("v2_assignments_queued", data.get("queued_count")) + session_id = data.get("session_id") or body + if count is not None: + line(f"queued {count} evaluation{'s' if count != 1 else ''} for {session_id}") + else: + line(f"re-evaluation queued for {session_id}" if data.get("queued") or data.get("status") == "queued" else "No evaluations queued.") + elif title == "Session": + begin(title) + row = data.get("overview", data) + kv([(k, row.get(k)) for k in ("session_id", "agent_id", "environment", "status", "event_count", "started_at", "last_event_at")]) + begin("Timeline") + table(["time", "type", "agent", "summary"], [ + [r.get("ts"), r.get("event_type"), r.get("agent_id"), r.get("summary")] + for r in collection(data.get("timeline", data), "events") + ], empty_message="no events in this session", command="fp events --session-id " + str(row.get("session_id", "<id>"))) + elif title == "Evaluation history": + begin(title) + table(["evaluator", "version", "status", "score", "finished"], [ + [r.get("eval_key"), r.get("eval_version"), r.get("status"), score_of(r), r.get("finished_at")] + for r in collection(data, "evaluations", "history", "items") + ], empty_message="no evaluations for this session", command="fp sessions re-evaluate <session-id>") + elif title == "Sentiment": + sentiment(data, view or "") + elif title == "Plan": + plan(data) + elif title == "Composed query": + begin(title) + kv([("SQL", data.get("sql"))]) + if data.get("result"): + query_result(data["result"]) + elif title in ("Query exported", "Session exported"): + begin(title) + kv([("file", data.get("out")), ("format", data.get("format"))]) + elif title in ("Policy draft", "Policy backtest", "Policy candidacy"): + begin(title) + detail(data) + if title == "Policy draft": + line("Draft only — nothing published or deployed.") + elif title == "Event latency": + begin(title) + detail(data) + else: + begin(title) + detail(data) diff --git a/fp-cloud-cli/fp_cli/commands/_cloud.py b/fp-cloud-cli/fp_cli/commands/_cloud.py index 1de691fd6..155bbd57d 100644 --- a/fp-cloud-cli/fp_cli/commands/_cloud.py +++ b/fp-cloud-cli/fp_cli/commands/_cloud.py @@ -2,12 +2,12 @@ from __future__ import annotations -import json from pathlib import Path from typing import Any from .. import _click_compat as click from .. import output +from .. import cloud_output from .._context import deny_in_key_mode from ..errors import KeyModeUnsupportedError @@ -29,21 +29,11 @@ def rows(data: Any, key: str) -> list: raise click.UsageError(f"Cloud returned an invalid {key} collection") -def emit(state, data: Any, title: str) -> None: +def emit(state, data: Any, title: str, **render_options) -> None: if state is not None and state.json: output.emit_json(data) return - if isinstance(data, list) and all(isinstance(r, dict) for r in data): - columns = list(dict.fromkeys(k for row in data for k in row)) - output.print_table(columns, [[cell(r.get(k)) for k in columns] for r in data], title=title) - elif isinstance(data, dict): - output.print_table(["Field", "Value"], [[k, cell(v)] for k, v in data.items()], title=title) - else: - output.print_table(["Value"], [[cell(data)]], title=title) - - -def cell(value: Any) -> str: - return json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) + cloud_output.render(data, title, **render_options) def write_file(path: str, text: str) -> None: diff --git a/fp-cloud-cli/fp_cli/commands/account_cmds.py b/fp-cloud-cli/fp_cli/commands/account_cmds.py index 61e5bbbe7..a6805b615 100644 --- a/fp-cloud-cli/fp_cli/commands/account_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/account_cmds.py @@ -17,7 +17,7 @@ def request(ctx, method, path, body=None, params=None): result = api.cloud_request(require_auth(ctx.obj), method, path, body=body, params=params) if method != "GET": _write.record_action("cloud_resource_changed", resource=path.split("/")[2], success=True, destructive=method == "DELETE") - _cloud.emit(ctx.obj, result, "Cloud") + _cloud.emit(ctx.obj, result, "Cloud", view=path, method=method, body=body, params=params) def roles_list(ctx: typer.Context) -> None: @@ -88,6 +88,7 @@ def plan(ctx: typer.Context) -> None: """Show current offers and team-trial state.""" if ctx.invoked_subcommand: return + _cloud.session_only(ctx.obj, "plan") cctx = require_auth(ctx.obj) data = {"offers": api.cloud_request(cctx, "GET", "/api/offers/me"), "team_trial": api.cloud_request(cctx, "GET", "/api/team-trial/me")} @@ -96,6 +97,7 @@ def plan(ctx: typer.Context) -> None: def claim(ctx: typer.Context, offer: str = typer.Argument(...)) -> None: """Claim a named offer.""" + _cloud.session_only(ctx.obj, "plan claim") request(ctx, "POST", "/api/offers/claim", {"offer_key": offer}) diff --git a/fp-cloud-cli/fp_cli/commands/agent_cmds.py b/fp-cloud-cli/fp_cli/commands/agent_cmds.py index 55dc9609b..8f5d56d40 100644 --- a/fp-cloud-cli/fp_cli/commands/agent_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/agent_cmds.py @@ -21,7 +21,7 @@ from .. import _click_compat as click # the Click Typer is running; see _click_compat from .. import client as api -from .. import output +from .. import cloud_output, output from .._context import GLOBALS_EPILOG, AppState, deny_in_key_mode, require_auth from ..errors import FpCliError, NetworkError, NotFoundError from . import _cloud, _write @@ -463,16 +463,7 @@ def run_artifact(cctx, block): def render_artifact(block): - output.screen_header() - data = block.get("result") - if isinstance(data, dict): - output._stdout.print(str(block.get("title", "Artifact")), markup=False) - output.print_table( - [c.get("name", "") if isinstance(c, dict) else str(c) for c in data.get("columns", [])], - data.get("rows", []), - ) - else: - _cloud.emit(None, block, block.get("title", "Home")) + cloud_output.artifact(block) def agent_brief(ctx: typer.Context, range: str = typer.Option("24h", "--range")) -> None: @@ -486,7 +477,4 @@ def agent_brief(ctx: typer.Context, range: str = typer.Option("24h", "--range")) if ctx.obj.json: output.emit_json({**data, "blocks": blocks}) else: - output.info(str(data.get("greeting", ""))) - output.info(str(data.get("summary", ""))) - for block in blocks: - render_artifact(block) + cloud_output.brief({**data, "blocks": blocks}) diff --git a/fp-cloud-cli/fp_cli/commands/dashboards_cmds.py b/fp-cloud-cli/fp_cli/commands/dashboards_cmds.py index 50c0bd56d..bcc957337 100644 --- a/fp-cloud-cli/fp_cli/commands/dashboards_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/dashboards_cmds.py @@ -20,7 +20,7 @@ def request(ctx, method, path, body=None): data = api.cloud_request(require_auth(ctx.obj), method, path, body=body) if method != "GET": _write.record_action("dashboard_changed", resource="dashboard", success=True, destructive=method == "DELETE") - _cloud.emit(ctx.obj, data, "Dashboards") + _cloud.emit(ctx.obj, data, "Dashboards", view=path, method=method, body=body) def list_dashboards(ctx: typer.Context) -> None: diff --git a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py index fc1cfed1f..0c17354f6 100644 --- a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py @@ -9,7 +9,7 @@ import typer -from .. import client as api, dates, evaluator_spec as spec, output +from .. import client as api, cloud_output, dates, evaluator_spec as spec, output from .._context import GLOBALS_EPILOG, require_auth from ..errors import ApiError, NotFoundError from . import _cloud, _write @@ -47,12 +47,17 @@ def show(ctx: typer.Context, handle: str = typer.Argument(..., metavar="KEY[@VER _cloud.emit(ctx.obj, select(definitions(require_auth(ctx.obj)), handle), "Evaluator") -def _create(state, cctx, body, sessions, disabled=False) -> dict: +def _create(state, cctx, body, sessions, disabled=False, force=False) -> dict: if sessions: if spec.source_kind(body) == "jev": output.warn("Jev dry-runs spend model budget.") preview = api.cloud_request(cctx, "POST", f"{PATH}/dry-run", body=spec.dry_run_body(body, sessions)) - check_dry_run(preview) + if not state.json: + cloud_output.dry_run(preview) + if dry_run_failed(preview) and force: + output.warn("Dry-run failed; --force authorizes creating the evaluator anyway.") + else: + check_dry_run(preview) _write.confirm(state, f"Create evaluator {body['eval_key']}@{body['eval_version']}", assume_yes=False) row = api.cloud_request(cctx, "POST", PATH, body=body) if not isinstance(row, dict) or not row.get("id"): @@ -86,6 +91,7 @@ def create( file: Optional[str] = typer.Option(None, "--file"), dry_run_on: Optional[List[str]] = typer.Option(None, "--dry-run-on"), disabled: bool = typer.Option(False, "--disabled"), + force: bool = typer.Option(False, "--force", help="Create despite failed --dry-run-on results (review the failures first)."), ) -> None: """Create a code, LLM judge or Jev evaluator (enabled by default). @@ -108,8 +114,11 @@ def create( # Validate before any request, including the judge refusal / session bound. if dry_run_on: spec.dry_run_body(body, dry_run_on) - row = _create(ctx.obj, require_auth(ctx.obj), body, dry_run_on, disabled) - _cloud.emit(ctx.obj, row, "Evaluator created") + row = _create(ctx.obj, require_auth(ctx.obj), body, dry_run_on, disabled, force) + if ctx.obj.json: + output.emit_json(row) + else: + cloud_output.evaluator({**body, **row}, "Evaluator created") def deploy( @@ -134,6 +143,7 @@ def deploy( file: Optional[str] = typer.Option(None, "--file"), dry_run_on: Optional[List[str]] = typer.Option(None, "--dry-run-on"), backfill: Optional[str] = typer.Option(None, "--backfill", help="Code only; e.g. 7d (up to 90d)."), + force: bool = typer.Option(False, "--force", help="Deploy despite failed --dry-run-on results (review the failures first)."), ) -> None: """Publish a NEW immutable version, then disable enabled predecessors. @@ -146,7 +156,7 @@ def deploy( judge=judge, criteria=criteria, step=step, threshold_value=threshold, jev=jev, instructions=instructions, true=true, false=false, level=level, file=file, complete=False, ) - if file and body.get("eval_key") != key: + if file and body.get("eval_key") is not None and body["eval_key"] != key: spec.fail("The spec eval_key must match the deploy argument") body["eval_key"] = key if version is not None: @@ -177,7 +187,7 @@ def deploy( spec.fail(f"Version {key}@{body['eval_version']} already exists; deploy needs a NEW version") body.setdefault("display_name", existing[-1].get("display_name", key) if existing else key) spec.validate(body) - created = _create(ctx.obj, cctx, body, dry_run_on) + created = _create(ctx.obj, cctx, body, dry_run_on, force=force) disabled_ids = [] try: for old in existing: @@ -190,7 +200,10 @@ def deploy( output.warn(f"Published {key}@{body['eval_version']} (id {created['id']}); disabled {len(disabled_ids)} predecessors. Follow-up failed; inspect evaluators list --all before retrying.") raise exc _write.record_action("evaluator_deployed", resource="evaluator", success=True) - _cloud.emit(ctx.obj, {"created": created, "disabled": disabled_ids}, "Evaluator deployed") + if ctx.obj.json: + output.emit_json({"created": created, "disabled": disabled_ids}) + else: + cloud_output.deployed({**body, **created}, [r for r in existing if r["id"] in disabled_ids]) def draft( @@ -270,7 +283,10 @@ def toggle(ctx, handle, enabled): _write.confirm(ctx.obj, f"{'Enable' if enabled else 'Disable'} {handle}", assume_yes=False) result = api.cloud_request(cctx, "PATCH", f"{PATH}/{row['id']}", body={"enabled": enabled}) _write.record_action("evaluator_toggled", resource="evaluator", success=True) - _cloud.emit(ctx.obj, result or {**row, "enabled": enabled}, "Evaluator") + if ctx.obj.json: + output.emit_json(result or {**row, "enabled": enabled}) + else: + cloud_output.evaluator({**row, **(result or {}), "enabled": enabled}) def enable(ctx: typer.Context, handle: str = typer.Argument(...)) -> None: @@ -297,7 +313,10 @@ def label_def(ctx: typer.Context, handle: str = typer.Argument(...), spec.validate_labels(labels) _write.confirm(ctx.obj, f"Label {handle}", assume_yes=False) data = api.cloud_request(cctx, "PATCH", f"{PATH}/{row['id']}", body={"labels": labels}) - _cloud.emit(ctx.obj, data or {**row, "labels": labels}, "Evaluator labels") + if ctx.obj.json: + output.emit_json(data or {**row, "labels": labels}) + else: + cloud_output.evaluator({**row, **(data or {}), "labels": labels}, "Evaluator labels") def instances(ctx: typer.Context) -> None: diff --git a/fp-cloud-cli/fp_cli/commands/queries_cmds.py b/fp-cloud-cli/fp_cli/commands/queries_cmds.py index 91941a85e..0a956fda5 100644 --- a/fp-cloud-cli/fp_cli/commands/queries_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/queries_cmds.py @@ -404,7 +404,7 @@ def query_compose(ctx: typer.Context, question: str = typer.Argument(...), elif ctx.obj.json: output.emit_json(draft) else: - print(sql) + _cloud.emit(ctx.obj, {"sql": sql}, "Composed query") def query_export(ctx: typer.Context, id: Optional[str] = typer.Argument(None), @@ -441,5 +441,8 @@ def query_export(ctx: typer.Context, id: Optional[str] = typer.Argument(None), _cloud.emit(ctx.obj, {"out": out, "format": format}, "Query exported") elif ctx.obj.json: output.emit_json(data if format == "json" else {"format": format, "content": content}) + elif format == "json": + from .. import cloud_output + cloud_output.query_result(data, "Query export") else: print(content, end="") diff --git a/fp-cloud-cli/fp_cli/commands/sentiment_cmds.py b/fp-cloud-cli/fp_cli/commands/sentiment_cmds.py index f77c9c7d0..3d14a7113 100644 --- a/fp-cloud-cli/fp_cli/commands/sentiment_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/sentiment_cmds.py @@ -16,7 +16,7 @@ def read(ctx, suffix, since=None, limit=None, session_id=None): frm, to = resolve_dates(since, None, None) data = api.cloud_request(require_auth(ctx.obj), "GET", f"/api/events/sentiment{suffix}", params={"ts_from": frm, "ts_to": to, "limit": limit, "session_id": session_id}) - _cloud.emit(ctx.obj, data, "Sentiment") + _cloud.emit(ctx.obj, data, "Sentiment", view=suffix.strip("/") or "summary") def summary(ctx: typer.Context, since: Optional[str] = typer.Option(None, "--since")) -> None: diff --git a/fp-cloud-cli/fp_cli/commands/sessions_cmds.py b/fp-cloud-cli/fp_cli/commands/sessions_cmds.py index 3796590ad..dfc30992b 100644 --- a/fp-cloud-cli/fp_cli/commands/sessions_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/sessions_cmds.py @@ -154,7 +154,7 @@ def re_evaluate(ctx: typer.Context, session_id: str = typer.Argument(...)) -> No """Re-run evaluators on one session. Needs evaluations:trigger.""" _write.confirm(ctx.obj, f"Re-evaluate session {session_id}", assume_yes=False) _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "POST", - f"/api/sessions/{session_id}/re-evaluate"), "Session re-evaluation") + f"/api/sessions/{session_id}/re-evaluate"), "Session re-evaluation", body=session_id) def show(ctx: typer.Context, id: str = typer.Argument(...)) -> None: @@ -178,7 +178,7 @@ def export(ctx: typer.Context, id: str = typer.Argument(...), _cloud.write_file(out, json.dumps(data, indent=2, ensure_ascii=False) + "\n") _cloud.emit(ctx.obj, {"out": out}, "Session exported") else: - output.emit_json(data) + _cloud.emit(ctx.obj, data, "Session") def history(ctx: typer.Context, id: str = typer.Argument(...)) -> None: diff --git a/fp-cloud-cli/skill/SKILL.md b/fp-cloud-cli/skill/SKILL.md index dfc48db26..146f14964 100644 --- a/fp-cloud-cli/skill/SKILL.md +++ b/fp-cloud-cli/skill/SKILL.md @@ -79,7 +79,7 @@ Throughout this skill, `fp` means "whichever form you resolved." a saved session (Click treats an empty env var as unset, so it falls to rung 4). - **Some commands need a signed-in user.** `login`, `logout`, `orgs *`, the whole - `agent` group, `evaluators draft`, `query compose`, `slack *`, `keys update`, and all of `policies *` / `fleet *` / + `agent` group, `evaluators draft`, `query compose`, `slack *`, `plan`, `plan claim`, `keys update`, and all of `policies *` / `fleet *` / `guardrails *` refuse a key with a usage error (**exit 2**) and make **no network call at all** — there is no user to sign in, no saved active org to switch, no private assistant thread to own, and the enforcement write routes @@ -379,6 +379,10 @@ enabled versions. A partial failure names the new id: inspect `list --all`, then disable any remaining old version; do not retry creation blindly. Backfill excludes judge/Jev and accepts at most 90 days. +Human mode displays dry-run outcomes before creation/deployment. Failed runs +stop the write unless `--force` is explicitly authorized; do not add it merely +to get past a failed check. Plan offers and claims require `fp login`, not a key. + AI drafting needs a signed-in session (`fp login`); evaluator API operations can use a scoped key. Unknown-route 404s exit 6 with “not available on this FailproofAI Cloud yet”; missing-record 404s report that the record was not found diff --git a/fp-cloud-cli/skill/references/commands.md b/fp-cloud-cli/skill/references/commands.md index b2dce7f68..6d4ff47cb 100644 --- a/fp-cloud-cli/skill/references/commands.md +++ b/fp-cloud-cli/skill/references/commands.md @@ -269,8 +269,8 @@ or model-authored source locally merely to inspect it. | --- | --- | | `evaluators list` | `--key K`, `--all` (include disabled versions) | | `evaluators show K[@V]` | full source; specify version when ambiguous | -| `evaluators create` | `--key K --name N`, optional `--version V` (default `1`), repeated `--label`, `--condition EXPR\|@file`, exactly one source below, repeated `--dry-run-on SESSION`, `--disabled` | -| `evaluators deploy K` | same sources/metadata, optional `--version`; numeric versions auto-increment; creates first, then disables predecessors; `--backfill 7d` code only | +| `evaluators create` | `--key K --name N`, optional `--version V` (default `1`), repeated `--label`, `--condition EXPR\|@file`, exactly one source below, repeated `--dry-run-on SESSION`, `--disabled`, explicit `--force` to override failed previews | +| `evaluators deploy K` | same sources/metadata, optional `--version`; numeric versions auto-increment; creates first, then disables predecessors; `--backfill 7d` code only; `--force` to override failed previews | | `evaluators draft "intent"` | session only; `--kind code\|judge\|jev`, `--out spec.json` (refuses to overwrite) | | `evaluators dry-run` | source flags or `--file`; repeated `--session` (max 10), none = compile only; judge refused, Jev spends budget | | `evaluators enable\|disable K[@V]` | toggle definition | @@ -329,8 +329,10 @@ fp --json sessions history run-001 fp --json evaluators list --all ``` -Check draft keys before copying these examples. A failed `--dry-run-on` prevents -creation. A failed predecessor disable leaves the new version published: inspect +Check draft keys before copying these examples. Human output shows each preview's +score/reasoning/error before writing. A failed `--dry-run-on` prevents creation +unless `--force` is explicitly authorized. Deploy's positional key may supply a +missing `eval_key` in a spec file. A failed predecessor disable leaves the new version published: inspect `list --all` and recover with explicit `disable KEY@VERSION`, not another create. API 409 can mean no enabled managed instance or an existing version; 422 can mean the org has 100 enabled keys. Read the error, do not rename/retry silently. @@ -344,6 +346,10 @@ Session-only commands exit 2 before HTTP in key mode. Human output uses mint/pink open screens and full ids; legacy `--full-ids` remains accepted. Exports refuse to overwrite existing files. +Use global `--json` for API objects or `--out` to save a session/query JSON +export. Without either, human mode renders a readable view rather than raw +structures. CSV query export remains pipeable CSV. + | Commands | Options / notes | | --- | --- | | `dashboards list\|show ID\|create\|update ID\|delete ID` | create `--name`, `--description`, optional `--file` definition; update selected fields | @@ -360,7 +366,7 @@ remains accepted. Exports refuse to overwrite existing files. | `orgs create --name N [--slug S]` | session; omitted slug derives from name; never auto-switches | | `roles list\|show NAME\|create\|update NAME\|delete NAME` | permission sets; create `--name --permission`; update `--permission` replaces or `--add/--remove` merges; `--description`, `--file`; builtins immutable | | `users delete ID` / `users defaults` | deletion is by id, not email | -| `usage windows` / `plan` / `plan claim OFFER` | offers + trial; offers still require an actor identity on the Cloud | +| `usage windows` / `plan` / `plan claim OFFER` | offers + trial; plan/claim are session-only (deny key mode before HTTP) | | `settings context-windows list\|set MODEL TOKENS\|delete MODEL` | tokens 0–1000000; model names may contain `/` | | `otel status\|enable\|disable` | Cloud org switch `otel.ingest_enabled`, not local collector configuration | | `slack status\|disconnect` | session only | diff --git a/fp-cloud-cli/tests/fake_cloud_smoke.py b/fp-cloud-cli/tests/fake_cloud_smoke.py index b1bc96848..ff7ee87ac 100644 --- a/fp-cloud-cli/tests/fake_cloud_smoke.py +++ b/fp-cloud-cli/tests/fake_cloud_smoke.py @@ -107,8 +107,15 @@ def dispatch(self, method): "condition_source": "", "evaluator_source": Cloud.source, "reasoning": "Drafted from intent", "kind": "code"}) if route == "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/evaluator-definitions/dry-run": assert len(body["session_ids"]) <= 10 - return self.json_response({"runs": [{"ok": True, "session_id": sid, "mode": "run", "result": {"score": {"value": 1.0}}} for sid in body["session_ids"]], - "summary": {"failed": 0, "succeeded": len(body["session_ids"]), "skipped": 0}}) + runs = [ + {"ok": sid != "failed-session", "session_id": sid, "mode": "run", + "result": {"score": {"value": 1.0}, "reasoning": "No errors found"} if sid != "failed-session" else None, + "error_message": "Missing result score" if sid == "failed-session" else None} + for sid in body["session_ids"] + ] + failed = sum(not run["ok"] for run in runs) + return self.json_response({"runs": runs, + "summary": {"failed": failed, "succeeded": len(runs) - failed, "skipped": 0}}) if route == "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/evaluations/backfill": assert body == { "from": "2026-10-01T00:00:00Z", "to": "2026-10-08T00:00:00Z", @@ -192,7 +199,8 @@ def main(): base = f"http://127.0.0.1:{server.server_port}" transcript = [] with tempfile.TemporaryDirectory(prefix="fp-user-smoke-") as tmp: - env = {**os.environ, "FP_HOME": tmp, "FP_ANALYTICS_DISABLED": "1", "COLUMNS": "140", "NO_COLOR": "1"} + env = {**os.environ, "FP_HOME": tmp, "FP_ANALYTICS_DISABLED": "1", + "COLUMNS": os.environ.get("FP_SMOKE_COLUMNS", "140"), "NO_COLOR": "1"} for var in ("FP_TOKEN", "FP_API_KEY", "FP_ORG", "FP_JSON", "FP_DASHBOARD_URL"): env.pop(var, None) fp = [args.fp, "--base-url", base] @@ -212,6 +220,8 @@ def run(argv, *, input=None, key=False, json_mode=True, expected=0): run(["evaluators", "deploy", "error_free", "--file", spec, "--version", "2"], key=True) assert not Cloud.definitions[0]["enabled"] and Cloud.definitions[1]["enabled"] run(["evaluators", "list", "--all"], key=True) + human_list = run(["evaluators", "list", "--all"], key=True, json_mode=False) + assert "EVALUATORS" in human_list and "error_free" in human_list and "[]" not in human_list run(["evaluators", "disable", "error_free@2"], key=True) run(["evaluators", "backfill", "--from", "2026-10-01T02:00:00+02:00", "--to", "2026-10-07T20:00:00-04:00", "--key", "error_free", "--dry-run"], key=True) @@ -221,6 +231,17 @@ def run(argv, *, input=None, key=False, json_mode=True, expected=0): run(["evaluators", "create", "--key", "binary", "--name", "Binary", "--judge", "--criteria", "Grade relevance", "--step", step], key=True, expected=2) run(["evaluators", "create", "--key", "resolution", "--name", "Resolution", "--jev", "noul", "--instructions", "Was the request resolved?", "--true", "Resolved", "--false", "Unresolved"], key=True) + before = len(Cloud.definitions) + failed = run(["evaluators", "create", "--key", "preview_failure", "--name", "Preview failure", + "--code", "-", "--dry-run-on", "failed-session"], + input=Cloud.source, key=True, json_mode=False, expected=1) + assert "failed-session · failed" in failed and "Missing result score" in failed + assert len(Cloud.definitions) == before + forced = run(["evaluators", "create", "--key", "preview_failure", "--name", "Preview failure", + "--code", "-", "--dry-run-on", "failed-session", "--force"], + input=Cloud.source, key=True, json_mode=False) + assert len(Cloud.definitions) == before + 1 + assert forced.index("Missing result score") < forced.index("EVALUATOR CREATED") run(["dashboards", "create", "--name", "Quality"], key=True) run(["dashboards", "tiles", "board1", "add", "--query", "q1", "--chart", "bar", "--title", "Traffic"], key=True) run(["dashboards", "tiles", "board1", "list"]) @@ -234,6 +255,9 @@ def run(argv, *, input=None, key=False, json_mode=True, expected=0): run(["evaluators", "draft", "Grade quality"], key=True, expected=2) denial = run(["agent", "ask", "Hello"], key=True, expected=2) assert denial["error"] == "`fp agent ask` needs a signed-in session: run `fp login`" + for argv in (["plan"], ["plan", "claim", "offer"]): + denied = run(argv, key=True, expected=2) + assert "needs a signed-in session" in denied["error"] missing = run(["sessions", "re-evaluate", "missing"], key=True, expected=6) assert missing["error"] == "session not found" assert "not available" not in missing["error"] @@ -277,7 +301,7 @@ def run(argv, *, input=None, key=False, json_mode=True, expected=0): # CI may force colour even with NO_COLOR, inserting ANSI spans inside # the wordmark. Compare visible text without weakening the order check. rendered = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", output.decode(errors="replace")) - assert rendered.index("failproof ai") < rendered.index("Events by agent") + assert rendered.index("failproof ai") < rendered.index("EVENTS BY AGENT") server.shutdown() (artifacts / "fp-user-smoke.txt").write_text("\n".join(transcript), encoding="utf-8") (artifacts / "fp-fake-cloud-requests.json").write_text(json.dumps(Cloud.requests, indent=2), encoding="utf-8") diff --git a/fp-cloud-cli/tests/test_cloud_account.py b/fp-cloud-cli/tests/test_cloud_account.py index fc0abdb5b..64273cc2e 100644 --- a/fp-cloud-cli/tests/test_cloud_account.py +++ b/fp-cloud-cli/tests/test_cloud_account.py @@ -42,6 +42,13 @@ @pytest.mark.parametrize("key_mode", [False, True]) @pytest.mark.parametrize("argv,method,path,response", CASES) def test_commands(logged_in, runner, key_mode, argv, method, path, response): + if key_mode and argv[0] == "plan": + with respx.mock as mock: + result = runner.invoke(app, ["--json", "--api-key", "key", *argv]) + assert not mock.calls + assert result.exit_code == 2 + assert "needs a signed-in session" in result.stdout + return prefix = "/v1" if key_mode else "/api" with respx.mock(assert_all_called=False) as mock: mock.get(f"{BASE}{prefix}/permission-sets/reviewer").respond(200, json={"name": "reviewer", "permissions": ["events:read"], "description": "Keep"}) diff --git a/fp-cloud-cli/tests/test_cloud_agent.py b/fp-cloud-cli/tests/test_cloud_agent.py index 6654e2349..464168949 100644 --- a/fp-cloud-cli/tests/test_cloud_agent.py +++ b/fp-cloud-cli/tests/test_cloud_agent.py @@ -63,9 +63,9 @@ class Sys: assert data["answer"] == "Here is the result." assert data["artifacts"][0]["result"] == RESULT else: - assert "Event total" in result.output + assert "EVENT TOTAL" in result.output assert "42" in result.output - assert result.stdout.index("failproof ai v") < result.stdout.index("Event total") + assert result.stdout.index("failproof ai v") < result.stdout.index("EVENT TOTAL") if tty: assert "alert a1" in result.output assert "Approve this change" in result.output @@ -250,4 +250,4 @@ def test_artifact_header_precedes_title_once(capsys): render_artifact({"title": "Second artifact", "result": RESULT}) rendered = capsys.readouterr().out assert rendered.count("failproof ai v") == 1 - assert rendered.index("failproof ai v") < rendered.index("First artifact") < rendered.index("Second artifact") + assert rendered.index("failproof ai v") < rendered.index("FIRST ARTIFACT") < rendered.index("SECOND ARTIFACT") diff --git a/fp-cloud-cli/tests/test_cloud_human_views.py b/fp-cloud-cli/tests/test_cloud_human_views.py new file mode 100644 index 000000000..c47fa5924 --- /dev/null +++ b/fp-cloud-cli/tests/test_cloud_human_views.py @@ -0,0 +1,288 @@ +"""Real command output with server-shaped payloads at both terminal widths.""" + +import json +import os +from pathlib import Path + +import httpx +import pytest +import respx + +from fp_cli.app import app + +BASE = "http://dash.test" +CODE = 'EvalResult(score=Score(0.7), reasoning="Tool usage")' +DEFINITION = { + "eval_key": "tool_usage", "display_name": "Tool usage", "eval_version": "1", + "result_kind": "score", "enabled": True, "labels": ["tools", "reliability"], + "condition_source": 'session.agent_id == "support-bot"', "evaluator_source": CODE, + "id": "b370402a-0ace-4e45-9c1c-d40e7062b5ea", "created_at": "2026-10-14T05:26:14Z", +} +ROLE = {"name": "reviewer", "is_builtin": False, "user_count": 3, + "permissions": ["events:read", "evaluations:read", "queries:run", "users:read"], + "description": "Review agent quality"} +DASHBOARD = {"id": "board1", "name": "Quality", "description": "Review production agents", + "tile_count": 1, "definition": {"filters": {"env": ["prod"]}}} +TILE = {"id": "tile1", "query_id": "query1", "title": "Traffic", "chart_type": "bar", + "chart_config": {"x": "agent", "y": "events"}, "param_bindings": {}, + "pos_x": 0, "pos_y": 0, "pos_w": 6, "pos_h": 4} +DRY_RUN = {"runs": [{"session_id": "restack-check", "ok": True, "mode": "run", + "result": {"score": {"value": 0.7}, "reasoning": "Seven tool calls"}, + "duration_ms": 8}], + "summary": {"succeeded": 1, "failed": 0, "skipped": 0}} +QUERY = {"columns": [{"name": "agent", "type": "String"}, {"name": "events", "type": "UInt64"}], + "rows": [["support-bot", 7]], "elapsed_ms": 9, "truncated": False} +SESSION = {"session_id": "restack-check", "agent_id": "support-bot", "environment": "prod", + "event_count": 7, "status": "done"} +CONTRACT = {"id": "contract1", "agent_id": "support-bot", "body": "# Purpose\nResolve requests", + "enabled": True, "purpose": "Resolve requests"} + + +def response(request): + path = request.url.path + method = request.method + body = json.loads(request.content) if request.content else {} + data = { + "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/api/evaluator-instances": {"instances": [ + {"name": "failproof-managed", "kind": "managed", "enabled": True, "eval_count": 4, + "queued": 2, "id": "instance1"}]}, + "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/api/evaluator-schema-profile": {"window_days": 7, "sampled_events": 100, "event_types": [ + {"event_type": "tool_use", "count": 100, "keys": [{"key": "tool_name", "type": "string", "presence": 1.0}]}]}, + "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/api/evaluation-jobs": {"jobs": [{"session_id": "restack-check", "eval_key": "tool_usage", + "status": "queued", "id": "job1"}]}, + "/api/evaluations/backfill": {"queued": True, "sessions": 12, "eval_key": "tool_usage"}, + "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/api/permission-sets": [ROLE], + "/api/permission-sets/reviewer": ROLE, + "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/api/dashboards": {"dashboards": [DASHBOARD]}, + "/api/dashboards/board1": {"dashboard": DASHBOARD, "tiles": [TILE]}, + "/api/dashboards/board1/tiles": {"tiles": [TILE]}, + "/api/dashboards/board1/tiles/tile1": TILE, + "/api/dashboards/board1/tiles/layout": {"updated": 1}, + "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/api/sessions": {"sessions": [SESSION]}, + "/api/events/summary": {"events": [{"ts": "2026-10-14T05:26:14Z", "event_type": "tool_use", + "agent_id": "support-bot", "summary": "Read customer request"}]}, + "/api/sessions/restack-check/export": {**SESSION, "events": []}, + "/api/sessions/restack-check/evaluation-history": {"evaluations": [ + {"eval_key": "tool_usage", "eval_version": "1", "status": "done", + "score": {"value": 0.7}, "finished_at": "2026-10-14T05:26:14Z"}]}, + "/api/sessions/restack-check/re-evaluate": { + "session_id": "restack-check", "status": "queued", "v2_assignments_queued": 1, + "v2_assignment_ids": ["assignment-uuid"]}, + "/api/events/latency_aggregate": {"p50": 42, "p95": 145, "models": [ + {"model": "model1", "calls": 10, "p95_ms": 145}]}, + "/api/events/sentiment": {"totals": {"messages": 10, "flagged": 2, "sessions": 3}, + "averages": {"frustration": 0.2}, "timeline": [], "by_agent": []}, + "/api/events/sentiment/messages": {"items": [{"session_id": "restack-check", + "text": "Please try again", "scores": {"frustration": 0.8, "satisfaction": 0.1}}]}, + "/api/events/sentiment/sessions": {"sessions": [{"session_id": "restack-check", + "negative_score": 0.8, "message_count": 10, "feeling": "frustration"}]}, + "/api/events/sentiment/sessions/restack-check": {"session_id": "restack-check", + "scores": {"frustration": 0.8}, "messages": []}, + "/api/queries/run": QUERY, + "/api/alerts/recipients": {"recipients": [{"email": "ops@example.test", "channel": "email"}]}, + "/api/alerts/alert1/issues": {"issues": [{"title": "Too many failures", "state": "open", + "severity": "warning", "id": "issue1"}]}, + "/api/audits/contracts": {"contracts": [CONTRACT]}, + "/api/audits/contracts/contract1": CONTRACT, + "/api/agent/policy-backtest": {"matched": 4, "allowed": 6, "denied": 4, + "examples": [{"session_id": "restack-check", "reason": "Force push blocked"}]}, + "/api/agent/policy-candidacy": {"candidate": True, "reason": "Force push can be blocked", "intent": "Prevent force push"}, + "/api/issues/issue1": {"id": "issue1", "title": "Force push incident"}, + "/api/users/defaults": {"permission_set": "reviewer", "permissions": ROLE["permissions"]}, + "/api/usage/windows": {"windows": [{"from": "2026-09-01", "to": "2026-10-01", + "usage": {"events_ingested": 300, "evaluations": 40}}]}, + "/api/offers/me": {"offers": [{"offer_key": "team_offer", "tier": "team", + "status": "available", "expires_at": "2026-10-31"}]}, + "/api/team-trial/me": {"active": True, "ends_at": "2026-10-31"}, + "/api/offers/claim": {"claimed": True, "grant": {"tier": "team", "expires_at": "2026-10-31"}}, + "/api/settings/model-context-windows": {"model_context_windows": [ + {"model_id": "provider/model", "context_window": 64000, "updated_at": "2026-10-14"}]}, + "/api/otel/status": {"enabled": True, "events_24h": 10, "last_received_at": "2026-10-14", + "content_missing": False, "services": [{"service_name": "support-service", "events": 10}]}, + "/api/integrations/slack": {"connected": True, "team_name": "Reliability", "channel_name": "agent-alerts"}, + "/api/agent/home-brief": {"greeting": "Hello", "summary": "One session needs attention", "blocks": [ + {"title": "Traffic", "kind": "table", "sql": "SELECT 1"}, + {"title": "Next steps", "kind": "note", "text": "Inspect the failed session"}]}, + }.get(path) + if path == "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/api/evaluator-definitions": + data = [DEFINITION] if method == "GET" else {**DEFINITION, **body, "id": "new-definition"} + elif path.startswith("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/api/evaluator-definitions/") and path.endswith("dry-run"): + data = DRY_RUN + elif path.startswith("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/api/evaluator-definitions/"): + data = {**DEFINITION, **body} + elif path == "/api/agent/compose-eval": + data = {k: v for k, v in DEFINITION.items() if k not in ("id", "created_at", "enabled")} + elif path in ("/api/agent/compose-sql", "/api/agent/policy-author"): + draft = {"sql": "SELECT agent, events FROM traffic"} if path.endswith("compose-sql") else { + "source": "customPolicies.add({ name: 'no-force-push' });", + "verdict": {"tone": "good", "reason": "Blocks force pushes"}, "tests": {"ran": 4, "passed": 4}} + return httpx.Response(200, text="data: " + json.dumps({"type": "done", "result": draft}) + "\n\n", + headers={"content-type": "text/event-stream"}) + elif path == "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/api/orgs": + data = {"id": "org1", "name": body["name"], "slug": body["slug"], "plan": "free"} + elif path == "/api/users/user1": + data = {"deleted": True} + elif path == "/api/settings/otel.ingest_enabled": + data = {"key": "otel.ingest_enabled", **body} + elif method == "DELETE": + data = {"deleted": True} + elif method in ("POST", "PUT"): + if path == "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/api/dashboards": + data = {**DASHBOARD, **body} + elif path.endswith("/tiles"): + data = {**TILE, **body} + elif path == "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/api/permission-sets": + data = {**ROLE, **body} + elif path == "/api/audits/contracts": + data = {**CONTRACT, **body} + elif path == "/api/settings/model-context-windows": + data = body + elif path == "/api/alerts/alert1/issues": + data = {"id": "issue1", "state": "open", **body} + assert data is not None, f"missing realistic fake payload for {method} {path}" + return httpx.Response(200, json=data) + + +CASES = [ + (["evaluators", "list"], "tool_usage"), + (["evaluators", "show", "tool_usage@1"], "support-bot"), + (["evaluators", "create", "--key", "tool_usage", "--name", "Tool usage", "--code", "{code}"], "code"), + (["evaluators", "create", "--key", "resolution", "--name", "Resolution", "--judge", + "--criteria", "Resolved the request", "--step", "Assess relevance", "--threshold", "0.7"], "LLM judge"), + (["evaluators", "create", "--key", "frustration", "--name", "Frustration", "--jev", "noul", + "--instructions", "Read user messages", "--true", "Frustrated", "--false", "Calm"], "Jev noul"), + (["evaluators", "create", "--key", "quality", "--name", "Quality", "--jev", "score", + "--instructions", "Grade answer", "--level", "Wrong", "--level", "Partial", "--level", "Correct"], "level 3"), + (["evaluators", "deploy", "tool_usage", "--code", "{code}", "--dry-run-on", "restack-check"], "disabled v1"), + (["evaluators", "draft", "Measure tools", "--out", "{draft}"], "create --file"), + (["evaluators", "dry-run", "--code", "{code}", "--session", "restack-check"], "Seven tool calls"), + (["evaluators", "enable", "tool_usage@1"], "● enabled"), + (["evaluators", "disable", "tool_usage@1"], "○ disabled"), + (["evaluators", "label", "tool_usage@1", "--add", "qa"], "qa"), + (["evaluators", "instances"], "failproof-managed"), + (["evaluators", "schema"], "tool_name"), + (["evaluators", "jobs"], "queued"), + (["evaluators", "backfill", "--from", "2026-10-01T00:00:00Z", "--to", "2026-10-08T00:00:00Z", "--dry-run"], "12"), + (["dashboards", "list"], "Quality"), + (["dashboards", "show", "board1"], "Traffic"), + (["dashboards", "create", "--name", "Quality"], "Quality"), + (["dashboards", "update", "board1", "--name", "New quality"], "board1"), + (["dashboards", "delete", "board1"], "board1"), + (["dashboards", "tiles", "board1", "list"], "Traffic"), + (["dashboards", "tiles", "board1", "add", "--query", "query1", "--chart", "bar"], "query1"), + (["dashboards", "tiles", "board1", "update", "tile1", "--title", "New traffic"], "tile1"), + (["dashboards", "tiles", "board1", "remove", "tile1"], "tile1"), + (["dashboards", "tiles", "board1", "layout", "--file", "{layout}"], "tiles updated"), + (["roles", "list"], "reviewer"), + (["roles", "show", "reviewer"], "queries:run"), + (["roles", "create", "--name", "reviewer", "--permission", "events:read"], "events:read"), + (["roles", "update", "reviewer", "--add", "users:read"], "users:read"), + (["roles", "delete", "reviewer"], "members keep"), + (["sessions", "show", "restack-check"], "Read customer"), + (["sessions", "export", "restack-check", "--out", "{export}"], "file"), + (["sessions", "export", "restack-check"], "restack-check"), + (["sessions", "history", "restack-check"], "tool_usage"), + (["sessions", "re-evaluate", "restack-check"], "queued 1 evaluation for restack-check"), + (["events", "latency"], "145"), + (["sentiment", "summary"], "frustration"), + (["sentiment", "messages"], "Please try again"), + (["sentiment", "sessions"], "restack-check"), + (["sentiment", "session", "restack-check"], "frustration"), + (["query", "compose", "Count events"], "SELECT"), + (["query", "compose", "Count events", "--run"], "support-bot"), + (["query", "export", "--sql", "SELECT 1", "--format", "csv"], "support-bot"), + (["query", "export", "--sql", "SELECT 1", "--format", "json"], "support-bot"), + (["query", "export", "--sql", "SELECT 1", "--out", "{export}"], "file"), + (["alerts", "recipients"], "ops@example.test"), + (["alerts", "issues", "alert1"], "Too many failures"), + (["alerts", "issues", "alert1", "--create", "--title", "Failure"], "Failure"), + (["audits", "contracts", "list"], "support-bot"), + (["audits", "contracts", "show", "contract1"], "Resolve requests"), + (["audits", "contracts", "create", "--agent-id", "support-bot", "--body", "Resolve requests"], "Resolve requests"), + (["audits", "contracts", "update", "contract1", "--body", "New purpose"], "support-bot"), + (["policies", "author", "Prevent force pushes"], "nothing published"), + (["policies", "backtest", "{policy}"], "Force push blocked"), + (["policies", "candidacy", "--issue", "issue1"], "can be blocked"), + (["agent", "brief"], "One session needs attention"), + (["orgs", "create", "--name", "Reliability"], "orgs switch"), + (["users", "delete", "user1"], "user1"), + (["users", "defaults"], "queries:run"), + (["usage", "windows"], "300"), + (["plan"], "team_offer"), + (["plan", "claim", "team_offer"], "team_offer"), + (["otel", "status"], "support-service"), + (["otel", "enable"], "● enabled"), + (["otel", "disable"], "○ disabled"), + (["slack", "status"], "Reliability"), + (["slack", "disconnect"], "disconnected"), + (["settings", "context-windows", "list"], "64000"), + (["settings", "context-windows", "set", "provider/model", "64000"], "provider/model"), + (["settings", "context-windows", "delete", "provider/model"], "override removed"), +] + + +@pytest.mark.parametrize("width", [80, 140]) +@pytest.mark.parametrize("argv,expected", CASES, ids=lambda item: " ".join(item[:3]) if isinstance(item, list) else "") +def test_deliberate_human_command_views(logged_in, runner, monkeypatch, tmp_path, width, argv, expected): + monkeypatch.setenv("COLUMNS", str(width)) + code, policy, layout = tmp_path / "source.py", tmp_path / "policy.mjs", tmp_path / "layout.json" + code.write_text(CODE) + policy.write_text("customPolicies.add({})") + layout.write_text(json.dumps({"tiles": [{"id": "tile1", "pos_x": 0, "pos_y": 0, "pos_w": 6, "pos_h": 4}]})) + paths = {"code": code, "policy": policy, "layout": layout, "draft": tmp_path / "draft.json", "export": tmp_path / "export.json"} + args = [v.format(**paths) for v in argv] + with respx.mock as mock: + mock.route().mock(side_effect=response) + result = runner.invoke(app, args) + assert result.exit_code == 0, result.output + assert expected in result.stdout, result.stdout + if argv[:2] != ["query", "export"] or "--out" in argv or "json" in argv: + assert result.stdout.lstrip().startswith("failproof ai"), result.stdout + assert "\nFIELD\n" not in result.stdout and "\nVALUE\n" not in result.stdout + for raw in ('"eval_key":', '"created":', '"permissions":', "dashboards []", "{'"): + assert raw not in result.stdout, result.stdout + if argv[:2] == ["evaluators", "deploy"]: + assert result.stdout.index("restack-check · passed") < result.stdout.index("deployed tool_usage v2") + # Optional transcript capture for the manager, outside the repository. + if os.environ.get("FP_HUMAN_ARTIFACTS"): + directory = Path(os.environ["FP_HUMAN_ARTIFACTS"]) + directory.mkdir(parents=True, exist_ok=True) + name = "-".join(argv[:3]).replace("/", "_").replace("{", "").replace("}", "") + with (directory / f"{width}-{name}.txt").open("a", encoding="utf-8") as dest: + dest.write("$ fp " + " ".join(argv) + "\n" + result.output + "\n") + + +@pytest.mark.parametrize("width", [80, 140]) +@pytest.mark.parametrize("argv,path,next_step", [ + (["dashboards", "list"], "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/api/dashboards", "fp dashboards create --name"), + (["roles", "list"], "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/api/permission-sets", "fp roles create"), + (["evaluators", "list"], "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/api/evaluator-definitions", "fp evaluators create --help"), + (["sentiment", "messages"], "/api/events/sentiment/messages", "sentiment.enabled"), + (["settings", "context-windows", "list"], "/api/settings/model-context-windows", "context-windows set"), +]) +def test_empty_views_give_next_step(logged_in, runner, monkeypatch, width, argv, path, next_step): + monkeypatch.setenv("COLUMNS", str(width)) + data = {"dashboards": []} if argv[0] == "dashboards" else {"items": []} if argv[0] == "sentiment" else [] + with respx.mock as mock: + mock.get(BASE + path).respond(200, json=data) + result = runner.invoke(app, argv) + assert result.exit_code == 0, result.output + assert "no " in result.stdout + assert next_step in result.stdout + assert "[]" not in result.stdout + + +@pytest.mark.parametrize("width", [80, 140]) +def test_roles_permissions_wrap_as_readable_text(logged_in, runner, monkeypatch, width): + monkeypatch.setenv("COLUMNS", str(width)) + from fp_cli.permissions import ASSIGNABLE_PERMISSIONS + with respx.mock as mock: + mock.get(BASE + "/api/permission-sets/admin").respond(200, json={ + "name": "admin", "is_builtin": True, "user_count": 1, + "permissions": ASSIGNABLE_PERMISSIONS, + }) + result = runner.invoke(app, ["roles", "show", "admin"]) + assert result.exit_code == 0 + assert "permissions" in result.stdout + assert "[" not in result.stdout + assert "events:read" in result.stdout and "policies:write" in result.stdout diff --git a/fp-cloud-cli/tests/test_round4_behaviour.py b/fp-cloud-cli/tests/test_round4_behaviour.py new file mode 100644 index 000000000..890316afe --- /dev/null +++ b/fp-cloud-cli/tests/test_round4_behaviour.py @@ -0,0 +1,144 @@ +import json +from io import StringIO + +import pytest +import respx +from rich.console import Console + +from fp_cli import output +from fp_cli.app import app + +BASE = "http://dash.test" +SOURCE = 'EvalResult(score=Score(0.7), reasoning="Tool usage")' + + +@pytest.mark.parametrize("command", ["create", "deploy"]) +@pytest.mark.parametrize("force", [False, True]) +@pytest.mark.parametrize("json_mode", [False, True]) +@pytest.mark.parametrize("width", [80, 140]) +def test_failed_preview_stops_unless_force(logged_in, runner, tmp_path, monkeypatch, command, force, json_mode, width): + monkeypatch.setenv("COLUMNS", str(width)) + file = tmp_path / "source.py" + file.write_text(SOURCE) + old = {"id": "old", "eval_key": "tool_usage", "eval_version": "1", "enabled": True} + preview = { + "runs": [ + {"session_id": "restack-check", "ok": True, + "result": {"score": {"value": 0.7}, "reasoning": "Seven tool calls"}}, + {"session_id": "failed-check", "ok": False, "error_kind": "invalid_result", + "error_message": "Missing result score"}, + ], + "summary": {"succeeded": 1, "failed": 1, "skipped": 0}, + } + capture = StringIO() + original_configure = output.configure + + def configure_capture(**kwargs): + original_configure(**kwargs) + if not json_mode: + output._stdout = Console(file=capture, width=width, no_color=True) + + monkeypatch.setattr(output, "configure", configure_capture) + with respx.mock(assert_all_called=False) as mock: + mock.get(BASE + "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/api/evaluator-definitions").respond(200, json=[old]) + mock.post(BASE + "/api/evaluator-definitions/dry-run").respond(200, json=preview) + created = { + "id": "new", "eval_key": "tool_usage", "eval_version": "2", "enabled": True, + } + create = mock.post(BASE + "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/api/evaluator-definitions") + patch = mock.patch(BASE + "/api/evaluator-definitions/old").respond(200, json={}) + + def preview_happened_first(request): + # Data must already be visible to the user when creation starts. + if not json_mode: + assert "Missing result score" in capture.getvalue() + assert "Seven tool calls" in capture.getvalue() + import httpx + return httpx.Response(201, json=created) + + create.mock(side_effect=preview_happened_first) + flags = ["--key", "tool_usage", "--name", "Tool usage"] if command == "create" else ["tool_usage"] + result = runner.invoke(app, [ + *(["--json"] if json_mode else []), "evaluators", command, *flags, + "--code", str(file), "--dry-run-on", "restack-check", "--dry-run-on", "failed-check", + *(["--force"] if force else []), + ]) + assert result.exit_code == (0 if force else 1), result.output + assert create.called is force + assert patch.called is (force and command == "deploy") + if json_mode: + payload = json.loads(result.stdout) + if force: + assert payload == ( + {"created": {"id": "new", "eval_key": "tool_usage", "eval_version": "2", "enabled": True}, "disabled": ["old"]} + if command == "deploy" else {"id": "new", "eval_key": "tool_usage", "eval_version": "2", "enabled": True} + ) + else: + assert payload["exit_code"] == 1 + assert "Missing result score" in payload["error"] + else: + rendered = capture.getvalue() + assert "failed-check · failed" in rendered + assert "Missing result score" in rendered + if force: + assert "--force" in result.stderr + assert rendered.index("Missing result score") < rendered.index( + "deployed tool_usage" if command == "deploy" else "EVALUATOR CREATED") + + +@pytest.mark.parametrize("width", [80, 140]) +def test_preview_skip_has_deliberate_outcome(logged_in, runner, tmp_path, monkeypatch, width): + monkeypatch.setenv("COLUMNS", str(width)) + file = tmp_path / "source.py" + file.write_text(SOURCE) + with respx.mock as mock: + mock.post(BASE + "/api/evaluator-definitions/dry-run").respond(200, json={ + "runs": [{"session_id": "restack-check", "ok": True, "applicable": False, "result": None}], + "summary": {"succeeded": 0, "failed": 0, "skipped": 1}, + }) + result = runner.invoke(app, ["evaluators", "dry-run", "--code", str(file), "--session", "restack-check"]) + assert result.exit_code == 0, result.output + assert "○ restack-check · skipped" in result.stdout + + +@pytest.mark.parametrize("key_mode", [False, True]) +def test_deploy_file_without_key(logged_in, runner, tmp_path, key_mode): + file = tmp_path / "spec.json" + file.write_text(json.dumps({"display_name": "Tool usage", "evaluator_source": SOURCE, "eval_version": "2"})) + prefix = "/v1" if key_mode else "/api" + with respx.mock as mock: + mock.get(BASE + prefix + "/evaluator-definitions").respond(200, json=[]) + create = mock.post(BASE + prefix + "/evaluator-definitions").respond(201, json={"id": "new"}) + result = runner.invoke(app, [ + "--json", *(["--api-key", "key"] if key_mode else []), + "evaluators", "deploy", "tool_usage", "--file", str(file), + ]) + assert result.exit_code == 0, result.output + assert json.loads(create.calls[0].request.content)["eval_key"] == "tool_usage" + assert json.loads(result.stdout) == {"created": {"id": "new"}, "disabled": []} + + +@pytest.mark.parametrize("command", [["plan"], ["plan", "claim", "team_offer"]]) +@pytest.mark.parametrize("json_mode", [False, True]) +def test_plan_needs_signed_in_person_before_http(logged_in, runner, command, json_mode): + with respx.mock as mock: + result = runner.invoke(app, ["--api-key", "key", *(["--json"] if json_mode else []), *command]) + assert not mock.calls + assert result.exit_code == 2 + expected = f"`fp {' '.join(command[:2])}` needs a signed-in session: run `fp login`" + assert expected in result.output + + +@pytest.mark.parametrize("key_mode", [False, True]) +@pytest.mark.parametrize("body,expected", [ + ({}, "not available on this FailproofAI Cloud yet"), + ({"error": "unknown route"}, "not available on this FailproofAI Cloud yet"), + ({"error": "setting not found"}, "setting not found"), +]) +def test_setting_key_404_on_older_cloud(logged_in, runner, key_mode, body, expected): + prefix = "/v1" if key_mode else "/api" + with respx.mock as mock: + mock.put(BASE + prefix + "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/settings/otel.ingest_enabled").respond(404, json=body) + result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), "otel", "enable"]) + assert result.exit_code == 6, result.output + assert json.loads(result.stdout)["error"] == expected From 82b02ec6b9e5e83402cbad0a6744dcb99dfed045 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:35:36 +0530 Subject: [PATCH 16/16] fix(fp): deploy keeps the previous version's condition and labels A new version deployed with flags dropped the predecessor's condition_source (and labels), silently widening the evaluator to every session. Found in a live run against a real Cloud: v1 scoped to session.agent_id == 'support-bot', v2 came out 'always'. Inherit both from the latest version unless --condition / --label are given; a --file spec still states every field itself. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- fp-cloud-cli/CHANGELOG.md | 1 + .../fp_cli/commands/evaluators_cmds.py | 9 ++++++++ .../tests/test_evaluator_authoring.py | 22 +++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/fp-cloud-cli/CHANGELOG.md b/fp-cloud-cli/CHANGELOG.md index a940b341b..d028972fc 100644 --- a/fp-cloud-cli/CHANGELOG.md +++ b/fp-cloud-cli/CHANGELOG.md @@ -20,6 +20,7 @@ - Remove the unused colour-keyed permission rank, document full-id rendering, emit UTC `Z` timestamps for deploy backfills, and report approval-answer failures without a traceback while restoring the terminal (#885). - Reject natural-language binary judge steps, distinguish unknown routes from missing records on 404, normalize standalone backfill ranges to UTC `Z`, expose all subcommands in help, and render artifact headers before titles with consistent session-only wording (#885). - New Cloud commands use deliberate human summaries/tables and actionable empty states at 80/140 columns without raw response blobs. Evaluator previews display before writes and require explicit `--force` after failures; plan/claim deny key mode, deploy specs can omit the positional key, and older settings-route 404s degrade clearly. Existing JSON payloads stay unchanged (#885). +- `fp evaluators deploy` keeps the previous version's condition and labels unless `--condition`/`--label` change them, so a new version no longer silently widens to every session (#885). ## 0.0.2b0 — 2026-09-22 diff --git a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py index 0c17354f6..8a6c3f005 100644 --- a/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py @@ -186,6 +186,15 @@ def deploy( if any(str(r.get("eval_version")) == body["eval_version"] for r in existing): spec.fail(f"Version {key}@{body['eval_version']} already exists; deploy needs a NEW version") body.setdefault("display_name", existing[-1].get("display_name", key) if existing else key) + if existing and not file: + # A new version keeps the previous version's scope and labels unless a flag + # changes them; silently dropping the condition would widen the evaluator to + # every session. A --file spec states every field itself. + latest = max(existing, key=lambda r: str(r.get("created_at") or "")) + if condition is None: + body["condition_source"] = latest.get("condition_source") + if not label: + body["labels"] = list(latest.get("labels") or []) spec.validate(body) created = _create(ctx.obj, cctx, body, dry_run_on, force=force) disabled_ids = [] diff --git a/fp-cloud-cli/tests/test_evaluator_authoring.py b/fp-cloud-cli/tests/test_evaluator_authoring.py index 03e85c91c..b2b41ff04 100644 --- a/fp-cloud-cli/tests/test_evaluator_authoring.py +++ b/fp-cloud-cli/tests/test_evaluator_authoring.py @@ -95,6 +95,28 @@ def test_deploy_order_and_dry_run(logged_in, runner, code): assert json.loads(result.stdout)["disabled"] == ["d1"] +@pytest.mark.parametrize("flags,expect_condition,expect_labels", [ + ([], "session.agent_id == 'support-bot'", ["prod"]), + (["--condition", "True", "--label", "beta"], "True", ["beta"]), +]) +def test_deploy_inherits_condition_and_labels(logged_in, runner, code, flags, expect_condition, expect_labels): + older = {"id": "d0", "eval_key": "quality", "eval_version": "1", "display_name": "Quality", "enabled": False, + "created_at": "2026-10-01T00:00:00Z", "condition_source": None, "labels": []} + latest = {"id": "d1", "eval_key": "quality", "eval_version": "2", "display_name": "Quality", "enabled": True, + "created_at": "2026-10-02T00:00:00Z", "condition_source": "session.agent_id == 'support-bot'", + "labels": ["prod"]} + with respx.mock as mock: + mock.get(f"{BASE}/v1/evaluator-definitions").respond(200, json=[latest, older]) + create = mock.post(f"{BASE}/v1/evaluator-definitions").respond(201, json={**latest, "id": "d2", "eval_version": "3"}) + mock.patch(f"{BASE}/v1/evaluator-definitions/d1").respond(200, json={}) + result = runner.invoke(app, ["--json", "--api-key", "key", "evaluators", "deploy", "quality", "--code", code, *flags]) + assert result.exit_code == 0, result.output + sent = json.loads(create.calls[0].request.content) + assert sent["condition_source"] == expect_condition + assert sent["labels"] == expect_labels + assert sent["eval_version"] == "3" + + @pytest.mark.parametrize("key_mode", [False, True]) def test_deploy_backfill_uses_utc_z_timestamps(logged_in, runner, code, monkeypatch, key_mode): from fp_cli.commands import evaluators_cmds