From 75626853e1ee7512f0b1d943f9135cdf12e19edd Mon Sep 17 00:00:00 2001 From: syedmustafan Date: Sat, 17 Oct 2026 06:07:17 +0500 Subject: [PATCH 01/10] fix(browser): correct unmade click claims --- sentient/agent/loop.py | 11 +++++++++ sentient/agent/outcomes.py | 19 ++++++++++++++++ tests/test_agent_loop.py | 46 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/sentient/agent/loop.py b/sentient/agent/loop.py index 3bd34a4a1..8a340a551 100644 --- a/sentient/agent/loop.py +++ b/sentient/agent/loop.py @@ -386,6 +386,8 @@ class LoopResult: handed_off: list[dict] = field(default_factory=list) # the outcome checks of the calls that ran (#165, ``sentient.agent.outcomes``), in order; written by code only checks: list[dict] = field(default_factory=list) + # browser clicks attempted in this run, recorded by code so a final reply cannot invent one (#402) + browser_clicks: list[dict] = field(default_factory=list) def _hosts(raw: Any) -> set[str]: @@ -900,6 +902,9 @@ async def run_loop( result.from_findings = True log.info("%s: the model gave no answer after its tool calls; answering from the results", source) yield TextDelta(text=text_acc, **ev) + if note := outcomes.unmade_browser_click_note(text_acc, result.browser_clicks): + text_acc = f"{text_acc.rstrip()}\n\nCorrection: {note}" + yield TextDelta(text=f"\n\nCorrection: {note}", **ev) result.text = text_acc result.messages = messages return @@ -1434,6 +1439,12 @@ async def _record( if ran and p.host: ctx.visited.add(p.host) err = _error_text(res) + if tc.name == "browser_click": + result.browser_clicks.append({ + "name": tc.name, + "declined": declined, + "ok": ran and not is_error and not err and outcomes.result_problem(res) is None, + }) if ran and p.tool is not None and not ctx.untrusted: if brings_untrusted(p.tool): ctx.untrusted = untrusted_source(p.tool, self.registry) diff --git a/sentient/agent/outcomes.py b/sentient/agent/outcomes.py index 3c359a183..cc55b43d2 100644 --- a/sentient/agent/outcomes.py +++ b/sentient/agent/outcomes.py @@ -50,6 +50,7 @@ "home": re.compile(r"\b(turned|switched|set|done|ran)\b", re.I), } _CLAIM_ANY = re.compile(r"\b(done|completed|finished|success(?:fully)?|sent|saved|created|updated)\b", re.I) +_BROWSER_CLICK_CLAIM = re.compile(r"\b(?:clicked|was clicked)\b", re.I) # an MCP tool's kind from its name (servers declare nothing like it) _NAME_KINDS: list[tuple[str, re.Pattern, re.Pattern]] = [ @@ -293,6 +294,24 @@ def claimed(text: str, kind: str | None) -> bool: return bool((_CLAIMS.get(kind or "") or _CLAIM_ANY).search(text or "")) +def unmade_browser_click_note(text: str, calls: list[dict]) -> str: + """A short correction when a reply says it clicked although this turn did not. + + Small local models occasionally state that a browser click happened without making + the call at all, or after the user declined its approval. This is deliberately + narrow: it only corrects the completed form of ``click`` and only looks at calls + recorded by the loop, never at text supplied by a page or the user. + """ + if not _BROWSER_CLICK_CLAIM.search(text or ""): + return "" + clicks = [call for call in calls if call.get("name") == "browser_click"] + if clicks and clicks[-1].get("ok"): + return "" + if clicks and clicks[-1].get("declined"): + return "I didn't click it: you declined." + return "I couldn't click it." if clicks else "I haven't clicked anything yet." + + def _settled(failed: dict, later: list[dict]) -> bool: """A failed check counts as settled when a later successful call of the same tool covered what it missed. A failure that names nothing it acted on (a call with arguments the tool couldn't read) is settled only by the very diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index 1490d6d78..cfda5c51b 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -1,4 +1,5 @@ +from sentient.agent import outcomes from sentient.app import SentientApp from sentient.llm.events import ApprovalRequest, Done, TextDelta, ToolResultEvent from tests.conftest import FakeProvider, tool_call @@ -101,6 +102,51 @@ async def allow(ev): await s.stop() +async def test_reply_corrects_a_browser_click_it_never_made(config, isolated_home): + llm = FakeProvider(replies=["The subscription button was clicked."]) + s = await SentientApp(config, llm=llm, db_path=isolated_home / "never-clicked.db").start() + try: + sid = await s.store.create_session(channel="cli") + events = await _collect(s.agent, sid, "click the subscription button") + text = "".join(e.text for e in events if isinstance(e, TextDelta)) + assert text.endswith("Correction: I haven't clicked anything yet.") + assert (await s.store.recent_messages(sid, 1))[-1]["content"] == text + finally: + await s.stop() + + +async def test_reply_corrects_a_browser_click_the_user_declined(config, isolated_home): + config.tools.approvals.mode = "ask" + llm = FakeProvider(replies=[[tool_call("browser_click", ref="e1")], "It was clicked, but the action was declined."]) + s = await SentientApp(config, llm=llm, db_path=isolated_home / "declined-click.db").start() + try: + sid = await s.store.create_session(channel="cli") + + async def deny(ev): + if isinstance(ev, ApprovalRequest): + s.approvals.resolve(ev.approval_id, "deny") + + events = await _collect(s.agent, sid, "click the subscription button", deny) + text = "".join(e.text for e in events if isinstance(e, TextDelta)) + assert text.endswith("Correction: I didn't click it: you declined.") + result = next(e for e in events if isinstance(e, ToolResultEvent)) + assert result.result["declined"] is True + finally: + await s.stop() + + +def test_browser_click_correction_uses_the_latest_call(): + assert outcomes.unmade_browser_click_note( + "The button was clicked.", [{"name": "browser_click", "ok": True, "declined": False}] + ) == "" + assert outcomes.unmade_browser_click_note( + "The button was clicked.", [ + {"name": "browser_click", "ok": True, "declined": False}, + {"name": "browser_click", "ok": False, "declined": True}, + ] + ) == "I didn't click it: you declined." + + async def test_internal_tools_do_not_prompt_in_ask_mode(config, isolated_home): config.tools.approvals.mode = "ask" llm = FakeProvider(replies=[[tool_call("file_write", name="note.txt", content="hi")], "saved"]) From c7626662f3919d70b857d11ee0ee03fb9affb5a6 Mon Sep 17 00:00:00 2001 From: syedmustafan Date: Sat, 17 Oct 2026 06:20:18 +0500 Subject: [PATCH 02/10] fix(browser): cover click claims at step limit --- sentient/agent/loop.py | 5 +++++ sentient/agent/outcomes.py | 2 ++ tests/test_agent_loop.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/sentient/agent/loop.py b/sentient/agent/loop.py index 8a340a551..0254c0041 100644 --- a/sentient/agent/loop.py +++ b/sentient/agent/loop.py @@ -1033,6 +1033,10 @@ async def run_loop( result.hit_step_limit = True result.text = text_acc or "I reached the step limit before finishing. Tell me how to continue." + if note := outcomes.unmade_browser_click_note(result.text, result.browser_clicks): + correction = f"\n\nCorrection: {note}" + result.text = f"{result.text.rstrip()}{correction}" + yield TextDelta(text=correction, **ev) result.messages = messages async def _model_stream( @@ -1444,6 +1448,7 @@ async def _record( "name": tc.name, "declined": declined, "ok": ran and not is_error and not err and outcomes.result_problem(res) is None, + "uncertain": ran and (is_error or bool(err)), }) if ran and p.tool is not None and not ctx.untrusted: if brings_untrusted(p.tool): diff --git a/sentient/agent/outcomes.py b/sentient/agent/outcomes.py index cc55b43d2..599f3ccea 100644 --- a/sentient/agent/outcomes.py +++ b/sentient/agent/outcomes.py @@ -309,6 +309,8 @@ def unmade_browser_click_note(text: str, calls: list[dict]) -> str: return "" if clicks and clicks[-1].get("declined"): return "I didn't click it: you declined." + if clicks and clicks[-1].get("uncertain"): + return "I couldn't confirm whether it was clicked." return "I couldn't click it." if clicks else "I haven't clicked anything yet." diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index cfda5c51b..3a2040213 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -2,6 +2,7 @@ from sentient.agent import outcomes from sentient.app import SentientApp from sentient.llm.events import ApprovalRequest, Done, TextDelta, ToolResultEvent +from sentient.llm.provider import StreamChunk, ToolCall from tests.conftest import FakeProvider, tool_call @@ -103,6 +104,7 @@ async def allow(ev): async def test_reply_corrects_a_browser_click_it_never_made(config, isolated_home): + """A natural-language click claim needs evidence from a browser click call.""" llm = FakeProvider(replies=["The subscription button was clicked."]) s = await SentientApp(config, llm=llm, db_path=isolated_home / "never-clicked.db").start() try: @@ -116,6 +118,7 @@ async def test_reply_corrects_a_browser_click_it_never_made(config, isolated_hom async def test_reply_corrects_a_browser_click_the_user_declined(config, isolated_home): + """A denied browser-click approval must remain visible in the final reply.""" config.tools.approvals.mode = "ask" llm = FakeProvider(replies=[[tool_call("browser_click", ref="e1")], "It was clicked, but the action was declined."]) s = await SentientApp(config, llm=llm, db_path=isolated_home / "declined-click.db").start() @@ -135,7 +138,35 @@ async def deny(ev): await s.stop() +async def test_step_limit_corrects_a_declined_browser_click_claim(config, isolated_home): + """A streamed claim is corrected even when its tool call exhausts the final round.""" + class ClaimAndClick(FakeProvider): + async def stream(self, role, messages, tools=None, *, model=None): + """Stream a textual click claim and its tool call in one final round.""" + self.calls.append({"role": role, "messages": messages, "tools": tools, "model": model}) + yield StreamChunk(text="The subscription button was clicked.", model="fake") + yield StreamChunk(done=True, tool_calls=[ToolCall(**tool_call("browser_click", ref="e1"))], model="fake") + + config.models.max_tool_rounds = 1 + config.tools.approvals.mode = "ask" + s = await SentientApp(config, llm=ClaimAndClick(), db_path=isolated_home / "limit-click.db").start() + try: + sid = await s.store.create_session(channel="cli") + + async def deny(ev): + if isinstance(ev, ApprovalRequest): + s.approvals.resolve(ev.approval_id, "deny") + + events = await _collect(s.agent, sid, "click the subscription button", deny) + text = "".join(e.text for e in events if isinstance(e, TextDelta)) + assert text.endswith("Correction: I didn't click it: you declined.") + assert isinstance(events[-1], Done) and events[-1].content == text + finally: + await s.stop() + + def test_browser_click_correction_uses_the_latest_call(): + """The latest known click outcome controls its correction wording.""" assert outcomes.unmade_browser_click_note( "The button was clicked.", [{"name": "browser_click", "ok": True, "declined": False}] ) == "" @@ -145,6 +176,9 @@ def test_browser_click_correction_uses_the_latest_call(): {"name": "browser_click", "ok": False, "declined": True}, ] ) == "I didn't click it: you declined." + assert outcomes.unmade_browser_click_note( + "The button was clicked.", [{"name": "browser_click", "ok": False, "uncertain": True}] + ) == "I couldn't confirm whether it was clicked." async def test_internal_tools_do_not_prompt_in_ask_mode(config, isolated_home): From a14b47dfe60cc25ef9eedc5378ad2ff5ec23f40c Mon Sep 17 00:00:00 2001 From: syedmustafan Date: Sat, 17 Oct 2026 11:05:01 +0500 Subject: [PATCH 03/10] fix(browser): ground click corrections in context --- CHANGELOG.md | 6 ++++++ sentient/agent/loop.py | 40 ++++++++++++++++++++++++++++++++++---- sentient/agent/outcomes.py | 15 ++++++++++++-- tests/test_agent_loop.py | 34 +++++++++++++++++++++++++++----- 4 files changed, 84 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c6307197..bcc887b9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ and versions follow [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Fixed +- **Browser click claims now stay grounded across a chat.** Sentient does not correct a user or another person's + click, or a sentence that already says no click happened. When it does need to correct its own unsupported browser + click claim, it says plainly that nothing changed on the page; it also remembers the latest browser-click outcome + from the chat, so a later claim after you declined a click says that instead of saying no click was attempted. + ### Added - **Home Assistant devices are checked after Sentient changes them.** A moment after it turns on a light, locks a door, opens the blinds, arms the alarm or sets the thermostat, Sentient asks Home Assistant for the device's state diff --git a/sentient/agent/loop.py b/sentient/agent/loop.py index 0254c0041..6cb2cfe6a 100644 --- a/sentient/agent/loop.py +++ b/sentient/agent/loop.py @@ -293,6 +293,36 @@ def history_to_openai(rows: list[dict]) -> list[dict]: return fixed +def browser_click_history(rows: list[dict]) -> list[dict]: + """Completed browser-click outcomes in stored chat history, oldest first (#402).""" + calls: dict[str, dict] = {} + out: list[dict] = [] + for row in rows: + if row.get("role") == "assistant": + for call in _normalize_tool_calls(row.get("tool_calls")): + if call["function"]["name"] == "browser_click": + calls[call["id"]] = {"name": "browser_click"} + continue + if row.get("role") != "tool": + continue + call = calls.pop(str(row.get("tool_call_id") or ""), None) + if call is None: + continue + try: + res = json.loads(str(row.get("content") or "")) + except (TypeError, ValueError): + res = {} + declined = isinstance(res, dict) and bool(res.get("declined")) + err = _error_text(res) + call.update( + declined=declined, + ok=isinstance(res, dict) and res.get("ok") is True and not err and outcomes.result_problem(res) is None, + uncertain=not declined and bool(err), + ) + out.append(call) + return out + + def is_tool_format_error(exc: Exception) -> bool: """True when the provider rejected the request because the model cannot do tool calling (e.g. an old Ollama model template emitting malformed tool calls).""" @@ -903,8 +933,9 @@ async def run_loop( log.info("%s: the model gave no answer after its tool calls; answering from the results", source) yield TextDelta(text=text_acc, **ev) if note := outcomes.unmade_browser_click_note(text_acc, result.browser_clicks): - text_acc = f"{text_acc.rstrip()}\n\nCorrection: {note}" - yield TextDelta(text=f"\n\nCorrection: {note}", **ev) + correction = f"\n\nCorrection: {note}" + text_acc += correction + yield TextDelta(text=correction, **ev) result.text = text_acc result.messages = messages return @@ -1035,7 +1066,7 @@ async def run_loop( result.text = text_acc or "I reached the step limit before finishing. Tell me how to continue." if note := outcomes.unmade_browser_click_note(result.text, result.browser_clicks): correction = f"\n\nCorrection: {note}" - result.text = f"{result.text.rstrip()}{correction}" + result.text += correction yield TextDelta(text=correction, **ev) result.messages = messages @@ -1448,7 +1479,7 @@ async def _record( "name": tc.name, "declined": declined, "ok": ran and not is_error and not err and outcomes.result_problem(res) is None, - "uncertain": ran and (is_error or bool(err)), + "uncertain": ran and is_error, }) if ran and p.tool is not None and not ctx.untrusted: if brings_untrusted(p.tool): @@ -1710,6 +1741,7 @@ def release() -> list[str]: system, sources, notes = await self._system_prompt_with_sources(user_text, channel, session) history = await self.store.recent_messages(session_id, self.config.chat.history_window) convo = self.mark_outside_messages(history_to_openai(history)) + result.browser_clicks.extend(browser_click_history(history)) # Replace the just-stored user message with the rich version (attachments inlined). # Keep the stored user message unchanged. Add the current time to the diff --git a/sentient/agent/outcomes.py b/sentient/agent/outcomes.py index 599f3ccea..aab8b7594 100644 --- a/sentient/agent/outcomes.py +++ b/sentient/agent/outcomes.py @@ -51,6 +51,13 @@ } _CLAIM_ANY = re.compile(r"\b(done|completed|finished|success(?:fully)?|sent|saved|created|updated)\b", re.I) _BROWSER_CLICK_CLAIM = re.compile(r"\b(?:clicked|was clicked)\b", re.I) +_NEGATED_BROWSER_CLICK = re.compile( + r"\b(?:not|didn't|hasn't|wasn't|haven't|never)\b[^.!?]{0,50}\bclicked\b", re.I +) +_OTHER_PERSON_BROWSER_CLICK = re.compile( + r"\b(?:you|he|she|they|someone|anyone|a friend|my friend)\b[^.!?]{0,50}\bclicked\b", re.I +) +_ASSISTANT_BROWSER_CLICK = re.compile(r"\b(?:i|we|sentient)\s+(?:have\s+)?clicked\b|\bclicked\s+(?:it|the\b)", re.I) # an MCP tool's kind from its name (servers declare nothing like it) _NAME_KINDS: list[tuple[str, re.Pattern, re.Pattern]] = [ @@ -304,14 +311,18 @@ def unmade_browser_click_note(text: str, calls: list[dict]) -> str: """ if not _BROWSER_CLICK_CLAIM.search(text or ""): return "" + if _NEGATED_BROWSER_CLICK.search(text) or _OTHER_PERSON_BROWSER_CLICK.search(text): + return "" + if not calls and not _ASSISTANT_BROWSER_CLICK.search(text): + return "" clicks = [call for call in calls if call.get("name") == "browser_click"] if clicks and clicks[-1].get("ok"): return "" if clicks and clicks[-1].get("declined"): - return "I didn't click it: you declined." + return "I didn't click it: you declined, so nothing changed on the page." if clicks and clicks[-1].get("uncertain"): return "I couldn't confirm whether it was clicked." - return "I couldn't click it." if clicks else "I haven't clicked anything yet." + return "I couldn't click it, so nothing changed on the page." if clicks else "I didn't click it, so nothing changed on the page." def _settled(failed: dict, later: list[dict]) -> bool: diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index 3a2040213..a1a5faeb1 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -105,13 +105,13 @@ async def allow(ev): async def test_reply_corrects_a_browser_click_it_never_made(config, isolated_home): """A natural-language click claim needs evidence from a browser click call.""" - llm = FakeProvider(replies=["The subscription button was clicked."]) + llm = FakeProvider(replies=["I clicked the subscription button. "]) s = await SentientApp(config, llm=llm, db_path=isolated_home / "never-clicked.db").start() try: sid = await s.store.create_session(channel="cli") events = await _collect(s.agent, sid, "click the subscription button") text = "".join(e.text for e in events if isinstance(e, TextDelta)) - assert text.endswith("Correction: I haven't clicked anything yet.") + assert text == "I clicked the subscription button. \n\nCorrection: I didn't click it, so nothing changed on the page." assert (await s.store.recent_messages(sid, 1))[-1]["content"] == text finally: await s.stop() @@ -131,7 +131,7 @@ async def deny(ev): events = await _collect(s.agent, sid, "click the subscription button", deny) text = "".join(e.text for e in events if isinstance(e, TextDelta)) - assert text.endswith("Correction: I didn't click it: you declined.") + assert text.endswith("Correction: I didn't click it: you declined, so nothing changed on the page.") result = next(e for e in events if isinstance(e, ToolResultEvent)) assert result.result["declined"] is True finally: @@ -159,7 +159,7 @@ async def deny(ev): events = await _collect(s.agent, sid, "click the subscription button", deny) text = "".join(e.text for e in events if isinstance(e, TextDelta)) - assert text.endswith("Correction: I didn't click it: you declined.") + assert text.endswith("Correction: I didn't click it: you declined, so nothing changed on the page.") assert isinstance(events[-1], Done) and events[-1].content == text finally: await s.stop() @@ -175,10 +175,34 @@ def test_browser_click_correction_uses_the_latest_call(): {"name": "browser_click", "ok": True, "declined": False}, {"name": "browser_click", "ok": False, "declined": True}, ] - ) == "I didn't click it: you declined." + ) == "I didn't click it: you declined, so nothing changed on the page." assert outcomes.unmade_browser_click_note( "The button was clicked.", [{"name": "browser_click", "ok": False, "uncertain": True}] ) == "I couldn't confirm whether it was clicked." + assert outcomes.unmade_browser_click_note("The button hasn't been clicked yet.", []) == "" + assert outcomes.unmade_browser_click_note("My friend clicked a link in a phishing email.", []) == "" + assert outcomes.unmade_browser_click_note("The subscription button was clicked.", []) == "" + + +async def test_reply_uses_the_chat_history_for_a_declined_browser_click(config, isolated_home): + """A later unsupported click claim retains the preceding declined-click context.""" + config.tools.approvals.mode = "ask" + llm = FakeProvider(replies=[[tool_call("browser_click", ref="e1")], "I didn't click it: you declined.", "I clicked it."]) + s = await SentientApp(config, llm=llm, db_path=isolated_home / "prior-declined-click.db").start() + try: + sid = await s.store.create_session(channel="cli") + + async def deny(ev): + if isinstance(ev, ApprovalRequest): + s.approvals.resolve(ev.approval_id, "deny") + + first = await _collect(s.agent, sid, "click the subscription button", deny) + assert "Correction:" not in "".join(e.text for e in first if isinstance(e, TextDelta)) + second = await _collect(s.agent, sid, "did it work?") + text = "".join(e.text for e in second if isinstance(e, TextDelta)) + assert text.endswith("Correction: I didn't click it: you declined, so nothing changed on the page.") + finally: + await s.stop() async def test_internal_tools_do_not_prompt_in_ask_mode(config, isolated_home): From 6aada6f726c076dad7fb9c485979a44567bec9b0 Mon Sep 17 00:00:00 2001 From: syedmustafan Date: Sat, 17 Oct 2026 15:49:10 +0500 Subject: [PATCH 04/10] fix(browser): scope click evidence to each turn --- sentient/agent/loop.py | 12 +++++++++--- sentient/agent/outcomes.py | 27 +++++++++++++++++++-------- tests/test_agent_loop.py | 15 +++++++++++++++ 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/sentient/agent/loop.py b/sentient/agent/loop.py index 6cb2cfe6a..0e7cfe38e 100644 --- a/sentient/agent/loop.py +++ b/sentient/agent/loop.py @@ -418,6 +418,8 @@ class LoopResult: checks: list[dict] = field(default_factory=list) # browser clicks attempted in this run, recorded by code so a final reply cannot invent one (#402) browser_clicks: list[dict] = field(default_factory=list) + # completed browser clicks from earlier chat turns, consulted only for an explicit reference to one (#402) + browser_click_history: list[dict] = field(default_factory=list) def _hosts(raw: Any) -> set[str]: @@ -932,7 +934,9 @@ async def run_loop( result.from_findings = True log.info("%s: the model gave no answer after its tool calls; answering from the results", source) yield TextDelta(text=text_acc, **ev) - if note := outcomes.unmade_browser_click_note(text_acc, result.browser_clicks): + if note := outcomes.unmade_browser_click_note( + text_acc, result.browser_clicks, result.browser_click_history + ): correction = f"\n\nCorrection: {note}" text_acc += correction yield TextDelta(text=correction, **ev) @@ -1064,7 +1068,9 @@ async def run_loop( result.hit_step_limit = True result.text = text_acc or "I reached the step limit before finishing. Tell me how to continue." - if note := outcomes.unmade_browser_click_note(result.text, result.browser_clicks): + if note := outcomes.unmade_browser_click_note( + result.text, result.browser_clicks, result.browser_click_history + ): correction = f"\n\nCorrection: {note}" result.text += correction yield TextDelta(text=correction, **ev) @@ -1741,7 +1747,7 @@ def release() -> list[str]: system, sources, notes = await self._system_prompt_with_sources(user_text, channel, session) history = await self.store.recent_messages(session_id, self.config.chat.history_window) convo = self.mark_outside_messages(history_to_openai(history)) - result.browser_clicks.extend(browser_click_history(history)) + result.browser_click_history.extend(browser_click_history(history)) # Replace the just-stored user message with the rich version (attachments inlined). # Keep the stored user message unchanged. Add the current time to the diff --git a/sentient/agent/outcomes.py b/sentient/agent/outcomes.py index aab8b7594..d59488b17 100644 --- a/sentient/agent/outcomes.py +++ b/sentient/agent/outcomes.py @@ -57,7 +57,8 @@ _OTHER_PERSON_BROWSER_CLICK = re.compile( r"\b(?:you|he|she|they|someone|anyone|a friend|my friend)\b[^.!?]{0,50}\bclicked\b", re.I ) -_ASSISTANT_BROWSER_CLICK = re.compile(r"\b(?:i|we|sentient)\s+(?:have\s+)?clicked\b|\bclicked\s+(?:it|the\b)", re.I) +_ASSISTANT_BROWSER_CLICK = re.compile(r"\b(?:i|we|sentient)\s+(?:have\s+)?clicked\b", re.I) +_PRIOR_BROWSER_CLICK_REFERENCE = re.compile(r"\b(?:it|that|previous(?:ly)?|earlier|last)\b", re.I) # an MCP tool's kind from its name (servers declare nothing like it) _NAME_KINDS: list[tuple[str, re.Pattern, re.Pattern]] = [ @@ -301,7 +302,7 @@ def claimed(text: str, kind: str | None) -> bool: return bool((_CLAIMS.get(kind or "") or _CLAIM_ANY).search(text or "")) -def unmade_browser_click_note(text: str, calls: list[dict]) -> str: +def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] | None = None) -> str: """A short correction when a reply says it clicked although this turn did not. Small local models occasionally state that a browser click happened without making @@ -309,13 +310,23 @@ def unmade_browser_click_note(text: str, calls: list[dict]) -> str: narrow: it only corrects the completed form of ``click`` and only looks at calls recorded by the loop, never at text supplied by a page or the user. """ - if not _BROWSER_CLICK_CLAIM.search(text or ""): + current_clicks = [call for call in calls if call.get("name") == "browser_click"] + claims = re.split(r"(?<=[.!?])\s+", text or "") + has_claim = any( + _ASSISTANT_BROWSER_CLICK.search(claim) + or ( + current_clicks + and _BROWSER_CLICK_CLAIM.search(claim) + and not _NEGATED_BROWSER_CLICK.search(claim) + and not _OTHER_PERSON_BROWSER_CLICK.search(claim) + ) + for claim in claims + ) + if not has_claim: return "" - if _NEGATED_BROWSER_CLICK.search(text) or _OTHER_PERSON_BROWSER_CLICK.search(text): - return "" - if not calls and not _ASSISTANT_BROWSER_CLICK.search(text): - return "" - clicks = [call for call in calls if call.get("name") == "browser_click"] + clicks = current_clicks + if not clicks and _PRIOR_BROWSER_CLICK_REFERENCE.search(text): + clicks = [call for call in history or [] if call.get("name") == "browser_click"] if clicks and clicks[-1].get("ok"): return "" if clicks and clicks[-1].get("declined"): diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index a1a5faeb1..a168bed87 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -81,6 +81,7 @@ class Postcards(ToolPlugin): sid = await s.store.create_session(channel="cli") async def deny(ev): + """Decline the browser-click approval shown by this test.""" if isinstance(ev, ApprovalRequest): assert ev.risk == "write" s.approvals.resolve(ev.approval_id, "deny") @@ -91,6 +92,7 @@ async def deny(ev): assert sent == [] async def allow(ev): + """Allow the postcard approval shown by this test.""" if isinstance(ev, ApprovalRequest): s.approvals.resolve(ev.approval_id, "allow_session") @@ -126,6 +128,7 @@ async def test_reply_corrects_a_browser_click_the_user_declined(config, isolated sid = await s.store.create_session(channel="cli") async def deny(ev): + """Decline the browser-click approval shown by this test.""" if isinstance(ev, ApprovalRequest): s.approvals.resolve(ev.approval_id, "deny") @@ -182,6 +185,17 @@ def test_browser_click_correction_uses_the_latest_call(): assert outcomes.unmade_browser_click_note("The button hasn't been clicked yet.", []) == "" assert outcomes.unmade_browser_click_note("My friend clicked a link in a phishing email.", []) == "" assert outcomes.unmade_browser_click_note("The subscription button was clicked.", []) == "" + assert outcomes.unmade_browser_click_note( + "I did not click A, but I clicked B.", [] + ) == "I didn't click it, so nothing changed on the page." + assert outcomes.unmade_browser_click_note( + "You clicked A, and I clicked B.", [] + ) == "I didn't click it, so nothing changed on the page." + prior_success = [{"name": "browser_click", "ok": True}] + assert outcomes.unmade_browser_click_note( + "I clicked the upgrade button.", [], prior_success + ) == "I didn't click it, so nothing changed on the page." + assert outcomes.unmade_browser_click_note("I clicked it.", [], prior_success) == "" async def test_reply_uses_the_chat_history_for_a_declined_browser_click(config, isolated_home): @@ -193,6 +207,7 @@ async def test_reply_uses_the_chat_history_for_a_declined_browser_click(config, sid = await s.store.create_session(channel="cli") async def deny(ev): + """Decline the browser-click approval shown by this test.""" if isinstance(ev, ApprovalRequest): s.approvals.resolve(ev.approval_id, "deny") From c99d066c500d05e47f3681a190e03495d036bd9d Mon Sep 17 00:00:00 2001 From: syedmustafan Date: Sat, 17 Oct 2026 16:59:22 +0500 Subject: [PATCH 05/10] chore: use changelog fragment for #402 --- CHANGELOG.md | 6 ------ changes/402.fixed.md | 3 +++ 2 files changed, 3 insertions(+), 6 deletions(-) create mode 100644 changes/402.fixed.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bcc887b9b..0c6307197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,6 @@ and versions follow [Semantic Versioning](https://semver.org/). ## [Unreleased] -### Fixed -- **Browser click claims now stay grounded across a chat.** Sentient does not correct a user or another person's - click, or a sentence that already says no click happened. When it does need to correct its own unsupported browser - click claim, it says plainly that nothing changed on the page; it also remembers the latest browser-click outcome - from the chat, so a later claim after you declined a click says that instead of saying no click was attempted. - ### Added - **Home Assistant devices are checked after Sentient changes them.** A moment after it turns on a light, locks a door, opens the blinds, arms the alarm or sets the thermostat, Sentient asks Home Assistant for the device's state diff --git a/changes/402.fixed.md b/changes/402.fixed.md new file mode 100644 index 000000000..745ce2bc2 --- /dev/null +++ b/changes/402.fixed.md @@ -0,0 +1,3 @@ +**Browser click claims now stay grounded across a chat.** Sentient does not correct a user or another person's +click, or a sentence that already says no click happened. When it needs to correct its own unsupported browser-click +claim, it says plainly that nothing changed on the page and keeps the latest click outcome from the chat in context. From c8529c6ab0bb11ad65d6b48fc17e423f9c13c60a Mon Sep 17 00:00:00 2001 From: syedmustafan Date: Sat, 17 Oct 2026 22:00:23 +0500 Subject: [PATCH 06/10] fix(browser): match historical click claims to targets --- sentient/agent/loop.py | 8 ++++++- sentient/agent/outcomes.py | 45 ++++++++++++++++++++++++++++++-------- tests/test_agent_loop.py | 10 +++++++++ 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/sentient/agent/loop.py b/sentient/agent/loop.py index 0e7cfe38e..c4a1feea1 100644 --- a/sentient/agent/loop.py +++ b/sentient/agent/loop.py @@ -301,7 +301,10 @@ def browser_click_history(rows: list[dict]) -> list[dict]: if row.get("role") == "assistant": for call in _normalize_tool_calls(row.get("tool_calls")): if call["function"]["name"] == "browser_click": - calls[call["id"]] = {"name": "browser_click"} + calls[call["id"]] = { + "name": "browser_click", + "target": _arguments(call["function"].get("arguments")).get("ref", ""), + } continue if row.get("role") != "tool": continue @@ -319,6 +322,8 @@ def browser_click_history(rows: list[dict]) -> list[dict]: ok=isinstance(res, dict) and res.get("ok") is True and not err and outcomes.result_problem(res) is None, uncertain=not declined and bool(err), ) + if isinstance(res, dict) and res.get("clicked"): + call["target"] = str(res["clicked"]) out.append(call) return out @@ -1486,6 +1491,7 @@ async def _record( "declined": declined, "ok": ran and not is_error and not err and outcomes.result_problem(res) is None, "uncertain": ran and is_error, + "target": str(res.get("clicked") or tc.arguments.get("ref", "")) if isinstance(res, dict) else tc.arguments.get("ref", ""), }) if ran and p.tool is not None and not ctx.untrusted: if brings_untrusted(p.tool): diff --git a/sentient/agent/outcomes.py b/sentient/agent/outcomes.py index d59488b17..6e2fb0d7e 100644 --- a/sentient/agent/outcomes.py +++ b/sentient/agent/outcomes.py @@ -55,10 +55,13 @@ r"\b(?:not|didn't|hasn't|wasn't|haven't|never)\b[^.!?]{0,50}\bclicked\b", re.I ) _OTHER_PERSON_BROWSER_CLICK = re.compile( - r"\b(?:you|he|she|they|someone|anyone|a friend|my friend)\b[^.!?]{0,50}\bclicked\b", re.I + r"\b(?:you|he|she|they|someone|anyone|(?:my|your|his|her|their)\s+\w+)\s+" + r"(?:have\s+|had\s+|just\s+)?clicked\b", re.I ) -_ASSISTANT_BROWSER_CLICK = re.compile(r"\b(?:i|we|sentient)\s+(?:have\s+)?clicked\b", re.I) -_PRIOR_BROWSER_CLICK_REFERENCE = re.compile(r"\b(?:it|that|previous(?:ly)?|earlier|last)\b", re.I) +_ASSISTANT_BROWSER_CLICK = re.compile( + r"\b(?:i(?:'ve)?|we|sentient)\s+(?:(?:have|had|just|only)\s+)?clicked\b", re.I +) +_HYPOTHETICAL_BROWSER_CLICK = re.compile(r"\b(?:if|would|could|might|may)\s+(?:i\s+)?(?:have\s+)?clicked\b", re.I) # an MCP tool's kind from its name (servers declare nothing like it) _NAME_KINDS: list[tuple[str, re.Pattern, re.Pattern]] = [ @@ -302,6 +305,31 @@ def claimed(text: str, kind: str | None) -> bool: return bool((_CLAIMS.get(kind or "") or _CLAIM_ANY).search(text or "")) +def _claimed_click_target(text: str) -> str: + """The named target in an explicit assistant claim, or empty for ``clicked it``.""" + match = _ASSISTANT_BROWSER_CLICK.search(text) + if match is None: + return "" + tail = re.split(r"\b(?:on|and|but)\b|[,;.!?]", text[match.end():], maxsplit=1, flags=re.I)[0] + target = re.sub(r"^\s*(?:only|just|the|a|an)\s+", "", tail, flags=re.I).strip() + return "" if target.lower() in {"", "it", "that", "this"} else target + + +def _matches_click_target(claim: str, click: dict) -> bool: + """Whether a named claim plausibly refers to this click's recorded browser element.""" + if not claim: + return True + + def words(value: str) -> set[str]: + """Meaningful target words, without generic element labels.""" + return { + word for word in re.findall(r"[a-z0-9]+", value.lower()) + if word not in {"a", "an", "the", "button", "link", "only", "just"} + } + + return bool(words(claim) & words(str(click.get("target") or ""))) + + def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] | None = None) -> str: """A short correction when a reply says it clicked although this turn did not. @@ -311,11 +339,12 @@ def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] recorded by the loop, never at text supplied by a page or the user. """ current_clicks = [call for call in calls if call.get("name") == "browser_click"] + previous_clicks = [call for call in history or [] if call.get("name") == "browser_click"] claims = re.split(r"(?<=[.!?])\s+", text or "") has_claim = any( - _ASSISTANT_BROWSER_CLICK.search(claim) + (_ASSISTANT_BROWSER_CLICK.search(claim) and not _HYPOTHETICAL_BROWSER_CLICK.search(claim)) or ( - current_clicks + (current_clicks or previous_clicks) and _BROWSER_CLICK_CLAIM.search(claim) and not _NEGATED_BROWSER_CLICK.search(claim) and not _OTHER_PERSON_BROWSER_CLICK.search(claim) @@ -324,11 +353,9 @@ def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] ) if not has_claim: return "" - clicks = current_clicks - if not clicks and _PRIOR_BROWSER_CLICK_REFERENCE.search(text): - clicks = [call for call in history or [] if call.get("name") == "browser_click"] + clicks = current_clicks or previous_clicks if clicks and clicks[-1].get("ok"): - return "" + return "" if current_clicks or _matches_click_target(_claimed_click_target(text), clicks[-1]) else "I didn't click it, so nothing changed on the page." if clicks and clicks[-1].get("declined"): return "I didn't click it: you declined, so nothing changed on the page." if clicks and clicks[-1].get("uncertain"): diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index a168bed87..4bbf3671f 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -196,6 +196,16 @@ def test_browser_click_correction_uses_the_latest_call(): "I clicked the upgrade button.", [], prior_success ) == "I didn't click it, so nothing changed on the page." assert outcomes.unmade_browser_click_note("I clicked it.", [], prior_success) == "" + prior_subscribe = [{"name": "browser_click", "ok": True, "target": 'button "Subscribe"'}] + assert outcomes.unmade_browser_click_note("I clicked the Subscribe button.", [], prior_subscribe) == "" + assert outcomes.unmade_browser_click_note( + "I clicked the Delete account button.", [], prior_subscribe + ) == "I didn't click it, so nothing changed on the page." + assert outcomes.unmade_browser_click_note( + "The Subscribe button was clicked, but the action failed.", [], [{"name": "browser_click", "declined": True}] + ) == "I didn't click it: you declined, so nothing changed on the page." + assert outcomes.unmade_browser_click_note("If I clicked it, you would be charged $19.", []) == "" + assert outcomes.unmade_browser_click_note("I've clicked the button.", []) == "I didn't click it, so nothing changed on the page." async def test_reply_uses_the_chat_history_for_a_declined_browser_click(config, isolated_home): From 2136b4934c7105519d91a07c38806c0fa111bb78 Mon Sep 17 00:00:00 2001 From: syedmustafan Date: Sun, 18 Oct 2026 15:34:39 +0500 Subject: [PATCH 07/10] fix(browser): select click evidence by target --- sentient/agent/outcomes.py | 85 ++++++++++++++++++++++++-------------- tests/test_agent_loop.py | 22 ++++++++++ 2 files changed, 77 insertions(+), 30 deletions(-) diff --git a/sentient/agent/outcomes.py b/sentient/agent/outcomes.py index 6e2fb0d7e..83be29c91 100644 --- a/sentient/agent/outcomes.py +++ b/sentient/agent/outcomes.py @@ -61,7 +61,9 @@ _ASSISTANT_BROWSER_CLICK = re.compile( r"\b(?:i(?:'ve)?|we|sentient)\s+(?:(?:have|had|just|only)\s+)?clicked\b", re.I ) -_HYPOTHETICAL_BROWSER_CLICK = re.compile(r"\b(?:if|would|could|might|may)\s+(?:i\s+)?(?:have\s+)?clicked\b", re.I) +_HYPOTHETICAL_BROWSER_CLICK = re.compile( + r"\b(?:if|would|could|might|may)\b[^.!?]{0,50}\b(?:clicked|was clicked|were clicked)\b", re.I +) # an MCP tool's kind from its name (servers declare nothing like it) _NAME_KINDS: list[tuple[str, re.Pattern, re.Pattern]] = [ @@ -306,13 +308,25 @@ def claimed(text: str, kind: str | None) -> bool: def _claimed_click_target(text: str) -> str: - """The named target in an explicit assistant claim, or empty for ``clicked it``.""" + """The named target in an assistant or passive click claim, or empty for ``clicked it``.""" match = _ASSISTANT_BROWSER_CLICK.search(text) - if match is None: - return "" - tail = re.split(r"\b(?:on|and|but)\b|[,;.!?]", text[match.end():], maxsplit=1, flags=re.I)[0] + if match is not None: + tail = re.split(r"\b(?:on|and|but)\b|[,;.!?]", text[match.end():], maxsplit=1, flags=re.I)[0] + else: + passive = re.search(r"(?P.+?)\s+was clicked\b", text, re.I) + if passive is None: + return "" + tail = passive.group("target") target = re.sub(r"^\s*(?:only|just|the|a|an)\s+", "", tail, flags=re.I).strip() - return "" if target.lower() in {"", "it", "that", "this"} else target + return "" if target.lower() in {"", "it", "that", "this", "button", "link"} else target + + +def _target_words(value: str) -> set[str]: + """Meaningful target words, without generic element labels.""" + return { + word for word in re.findall(r"[a-z0-9]+", value.lower()) + if word not in {"a", "an", "the", "button", "link", "only", "just"} and not re.fullmatch(r"e\d+", word) + } def _matches_click_target(claim: str, click: dict) -> bool: @@ -320,14 +334,31 @@ def _matches_click_target(claim: str, click: dict) -> bool: if not claim: return True - def words(value: str) -> set[str]: - """Meaningful target words, without generic element labels.""" - return { - word for word in re.findall(r"[a-z0-9]+", value.lower()) - if word not in {"a", "an", "the", "button", "link", "only", "just"} - } - - return bool(words(claim) & words(str(click.get("target") or ""))) + claim_words = _target_words(claim) + return bool(claim_words) and claim_words <= _target_words(str(click.get("target") or "")) + + +def _click_note(clicks: list[dict], target: str) -> str: + """The correction for a click claim after selecting evidence for its named target.""" + matches = [click for click in clicks if not target or _matches_click_target(target, click)] + if target and not matches: + # A declined call may retain only its transient ref; it is still evidence that the named action was not made. + matches = [ + click for click in clicks + if not click.get("ok") and not _target_words(str(click.get("target") or "")) + ] + if target and not matches: + return "I didn't click it, so nothing changed on the page." + if not matches: + return "I didn't click it, so nothing changed on the page." + click = matches[-1] + if click.get("ok"): + return "" + if click.get("declined"): + return "I didn't click it: you declined, so nothing changed on the page." + if click.get("uncertain"): + return "I couldn't confirm whether it was clicked." + return "I couldn't click it, so nothing changed on the page." def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] | None = None) -> str: @@ -341,26 +372,20 @@ def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] current_clicks = [call for call in calls if call.get("name") == "browser_click"] previous_clicks = [call for call in history or [] if call.get("name") == "browser_click"] claims = re.split(r"(?<=[.!?])\s+", text or "") - has_claim = any( - (_ASSISTANT_BROWSER_CLICK.search(claim) and not _HYPOTHETICAL_BROWSER_CLICK.search(claim)) - or ( - (current_clicks or previous_clicks) + clicks = current_clicks or previous_clicks + for claim in claims: + if _HYPOTHETICAL_BROWSER_CLICK.search(claim): + continue + explicit = _ASSISTANT_BROWSER_CLICK.search(claim) + passive = ( + clicks and _BROWSER_CLICK_CLAIM.search(claim) and not _NEGATED_BROWSER_CLICK.search(claim) and not _OTHER_PERSON_BROWSER_CLICK.search(claim) ) - for claim in claims - ) - if not has_claim: - return "" - clicks = current_clicks or previous_clicks - if clicks and clicks[-1].get("ok"): - return "" if current_clicks or _matches_click_target(_claimed_click_target(text), clicks[-1]) else "I didn't click it, so nothing changed on the page." - if clicks and clicks[-1].get("declined"): - return "I didn't click it: you declined, so nothing changed on the page." - if clicks and clicks[-1].get("uncertain"): - return "I couldn't confirm whether it was clicked." - return "I couldn't click it, so nothing changed on the page." if clicks else "I didn't click it, so nothing changed on the page." + if (explicit or passive) and (note := _click_note(clicks, _claimed_click_target(claim))): + return note + return "" def _settled(failed: dict, later: list[dict]) -> bool: diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index 4bbf3671f..c89872d52 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -206,6 +206,28 @@ def test_browser_click_correction_uses_the_latest_call(): ) == "I didn't click it: you declined, so nothing changed on the page." assert outcomes.unmade_browser_click_note("If I clicked it, you would be charged $19.", []) == "" assert outcomes.unmade_browser_click_note("I've clicked the button.", []) == "I didn't click it, so nothing changed on the page." + subscribe_then_declined_delete = [ + {"name": "browser_click", "ok": True, "target": 'button "Subscribe"'}, + {"name": "browser_click", "declined": True, "target": 'button "Delete account"'}, + ] + assert outcomes.unmade_browser_click_note( + "I clicked the Subscribe button.", subscribe_then_declined_delete + ) == "" + assert outcomes.unmade_browser_click_note( + "I clicked the Delete account button.", subscribe_then_declined_delete + ) == "I didn't click it: you declined, so nothing changed on the page." + assert outcomes.unmade_browser_click_note( + "I clicked the Subscribe button.", [], subscribe_then_declined_delete + ) == "" + assert outcomes.unmade_browser_click_note( + "I clicked the Delete account button.", [], subscribe_then_declined_delete + ) == "I didn't click it: you declined, so nothing changed on the page." + assert outcomes.unmade_browser_click_note( + "I clicked the account settings button.", [{"name": "browser_click", "ok": True, "target": 'button "Delete account"'}] + ) == "I didn't click it, so nothing changed on the page." + assert outcomes.unmade_browser_click_note( + "If it were clicked, you would be charged $19.", [{"name": "browser_click", "declined": True}] + ) == "" async def test_reply_uses_the_chat_history_for_a_declined_browser_click(config, isolated_home): From 95430e976b30607f4dd0c07a68e71ce678c0ba29 Mon Sep 17 00:00:00 2001 From: syedmustafan Date: Mon, 19 Oct 2026 14:38:00 +0500 Subject: [PATCH 08/10] Fix click correction after successful browser actions --- sentient/agent/outcomes.py | 29 +++++++++++++++++++++++++---- tests/test_agent_loop.py | 32 +++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/sentient/agent/outcomes.py b/sentient/agent/outcomes.py index 83be29c91..d290fc759 100644 --- a/sentient/agent/outcomes.py +++ b/sentient/agent/outcomes.py @@ -52,7 +52,7 @@ _CLAIM_ANY = re.compile(r"\b(done|completed|finished|success(?:fully)?|sent|saved|created|updated)\b", re.I) _BROWSER_CLICK_CLAIM = re.compile(r"\b(?:clicked|was clicked)\b", re.I) _NEGATED_BROWSER_CLICK = re.compile( - r"\b(?:not|didn't|hasn't|wasn't|haven't|never)\b[^.!?]{0,50}\bclicked\b", re.I + r"\b(?:not|didn't|hasn't|wasn't|haven't|never|nothing)\b[^.!?]{0,50}\bclicked\b", re.I ) _OTHER_PERSON_BROWSER_CLICK = re.compile( r"\b(?:you|he|she|they|someone|anyone|(?:my|your|his|her|their)\s+\w+)\s+" @@ -335,19 +335,38 @@ def _matches_click_target(claim: str, click: dict) -> bool: return True claim_words = _target_words(claim) - return bool(claim_words) and claim_words <= _target_words(str(click.get("target") or "")) + target_words = _target_words(str(click.get("target") or "")) + + def same_word(left: str, right: str) -> bool: + """Accept the small morphological variations common in a natural-language reply.""" + return left == right or ( + len(left) >= 6 and len(right) >= 6 and left[:6] == right[:6] + ) + + # The recorded element name is the dependable evidence. It need only appear + # in the claim: a reply naturally adds context such as "for the newsletter" + # or an element ref after naming the clicked control. + return bool(target_words) and all( + any(same_word(target_word, claim_word) for claim_word in claim_words) + for target_word in target_words + ) def _click_note(clicks: list[dict], target: str) -> str: """The correction for a click claim after selecting evidence for its named target.""" matches = [click for click in clicks if not target or _matches_click_target(target, click)] - if target and not matches: + has_success = any(click.get("ok") for click in clicks) + if target and not matches and not has_success: # A declined call may retain only its transient ref; it is still evidence that the named action was not made. matches = [ click for click in clicks if not click.get("ok") and not _target_words(str(click.get("target") or "")) ] if target and not matches: + if has_success: + # A successful click of another named control makes an unmatched + # claim ambiguous. Do not turn an honest report into a correction. + return "" return "I didn't click it, so nothing changed on the page." if not matches: return "I didn't click it, so nothing changed on the page." @@ -372,7 +391,9 @@ def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] current_clicks = [call for call in calls if call.get("name") == "browser_click"] previous_clicks = [call for call in history or [] if call.get("name") == "browser_click"] claims = re.split(r"(?<=[.!?])\s+", text or "") - clicks = current_clicks or previous_clicks + # A reply can summarize a successful click from an earlier turn alongside a + # declined click from this one, so consider the complete chronological record. + clicks = previous_clicks + current_clicks for claim in claims: if _HYPOTHETICAL_BROWSER_CLICK.search(claim): continue diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index c89872d52..51a473295 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -194,13 +194,13 @@ def test_browser_click_correction_uses_the_latest_call(): prior_success = [{"name": "browser_click", "ok": True}] assert outcomes.unmade_browser_click_note( "I clicked the upgrade button.", [], prior_success - ) == "I didn't click it, so nothing changed on the page." + ) == "" assert outcomes.unmade_browser_click_note("I clicked it.", [], prior_success) == "" prior_subscribe = [{"name": "browser_click", "ok": True, "target": 'button "Subscribe"'}] assert outcomes.unmade_browser_click_note("I clicked the Subscribe button.", [], prior_subscribe) == "" assert outcomes.unmade_browser_click_note( "I clicked the Delete account button.", [], prior_subscribe - ) == "I didn't click it, so nothing changed on the page." + ) == "" assert outcomes.unmade_browser_click_note( "The Subscribe button was clicked, but the action failed.", [], [{"name": "browser_click", "declined": True}] ) == "I didn't click it: you declined, so nothing changed on the page." @@ -224,10 +224,36 @@ def test_browser_click_correction_uses_the_latest_call(): ) == "I didn't click it: you declined, so nothing changed on the page." assert outcomes.unmade_browser_click_note( "I clicked the account settings button.", [{"name": "browser_click", "ok": True, "target": 'button "Delete account"'}] - ) == "I didn't click it, so nothing changed on the page." + ) == "" assert outcomes.unmade_browser_click_note( "If it were clicked, you would be charged $19.", [{"name": "browser_click", "declined": True}] ) == "" + successful_subscribe_then_declined_delete = [ + {"name": "browser_click", "ok": True, "target": 'button "Subscribe"'}, + {"name": "browser_click", "declined": True, "target": 'button "Delete account"'}, + ] + for reply in ( + 'I clicked the "Subscribe" button (e1) for the newsletter and attempted to click the "Delete account" button (e2).', + "The subscription button was clicked successfully.", + "Yes, I clicked the Subscribe button for the free weekly newsletter.", + 'Today I clicked just one thing: the "Subscribe" button on the Garden Notes page.', + ): + assert outcomes.unmade_browser_click_note(reply, successful_subscribe_then_declined_delete) == "" + assert outcomes.unmade_browser_click_note( + "Nothing was clicked.", successful_subscribe_then_declined_delete + ) == "" + assert outcomes.unmade_browser_click_note( + "Yes, I clicked the Subscribe button for the free weekly newsletter.", + [{"name": "browser_click", "declined": True, "target": 'button "Delete account"'}], + [{"name": "browser_click", "ok": True, "target": 'button "Subscribe"'}], + ) == "" + assert outcomes.unmade_browser_click_note( + "I clicked the Delete account button.", + [ + {"name": "browser_click", "ok": True, "target": 'button "Subscribe"'}, + {"name": "browser_click", "declined": True}, + ], + ) == "" async def test_reply_uses_the_chat_history_for_a_declined_browser_click(config, isolated_home): From 9bba3db1c3acf282497fcd18ef3bfaffbf1686dd Mon Sep 17 00:00:00 2001 From: syedmustafan Date: Mon, 19 Oct 2026 17:26:17 +0500 Subject: [PATCH 09/10] Harden browser click correction coverage --- sentient/agent/loop.py | 10 +++++++++- sentient/agent/outcomes.py | 17 ++++++++++++++--- tests/test_agent_loop.py | 16 ++++++++++++++++ tests/test_gateway.py | 21 +++++++++++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/sentient/agent/loop.py b/sentient/agent/loop.py index c9a6ab273..d58e6dd79 100644 --- a/sentient/agent/loop.py +++ b/sentient/agent/loop.py @@ -318,7 +318,15 @@ def browser_click_history(rows: list[dict]) -> list[dict]: try: res = json.loads(str(row.get("content") or "")) except (TypeError, ValueError): - res = {} + # A short result may end after the stable leading fields. Preserve + # those fields rather than treating a recorded success as a failed + # click solely because its trailing URL was cut for model context. + content = str(row.get("content") or "") + res = {"ok": True} if re.search(r'"ok"\s*:\s*true\b', content) else {} + clicked = re.search(r'"clicked"\s*:\s*"((?:[^"\\]|\\.)*)"', content) + if clicked is not None: + with contextlib.suppress(ValueError): + res["clicked"] = json.loads(f'"{clicked.group(1)}"') declined = isinstance(res, dict) and bool(res.get("declined")) err = _error_text(res) call.update( diff --git a/sentient/agent/outcomes.py b/sentient/agent/outcomes.py index d290fc759..4f973685a 100644 --- a/sentient/agent/outcomes.py +++ b/sentient/agent/outcomes.py @@ -397,14 +397,25 @@ def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] for claim in claims: if _HYPOTHETICAL_BROWSER_CLICK.search(claim): continue - explicit = _ASSISTANT_BROWSER_CLICK.search(claim) + explicit = list(_ASSISTANT_BROWSER_CLICK.finditer(claim)) + # Each self-attributed claim needs its own evidence. In particular, a + # successful "I clicked Subscribe" must not hide a later false + # "I clicked Delete account" in the same sentence. + explicit_parts = [ + claim[match.start(): explicit[i + 1].start() if i + 1 < len(explicit) else None] + for i, match in enumerate(explicit) + ] + for part in explicit_parts: + if note := _click_note(clicks, _claimed_click_target(part)): + return note passive = ( - clicks + not explicit + and clicks and _BROWSER_CLICK_CLAIM.search(claim) and not _NEGATED_BROWSER_CLICK.search(claim) and not _OTHER_PERSON_BROWSER_CLICK.search(claim) ) - if (explicit or passive) and (note := _click_note(clicks, _claimed_click_target(claim))): + if passive and (note := _click_note(clicks, _claimed_click_target(claim))): return note return "" diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index 51a473295..c5fb26799 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -1,5 +1,6 @@ from sentient.agent import outcomes +from sentient.agent.loop import browser_click_history from sentient.app import SentientApp from sentient.llm.events import ApprovalRequest, Done, TextDelta, ToolResultEvent from sentient.llm.provider import StreamChunk, ToolCall @@ -254,6 +255,21 @@ def test_browser_click_correction_uses_the_latest_call(): {"name": "browser_click", "declined": True}, ], ) == "" + assert outcomes.unmade_browser_click_note( + "I clicked the Subscribe button and I clicked the Delete account button.", + successful_subscribe_then_declined_delete, + ) == "I didn't click it: you declined, so nothing changed on the page." + + +def test_browser_click_history_retains_a_truncated_successful_result(): + """A context-truncated result must not turn a recorded click into a false failure.""" + history = browser_click_history([ + {"role": "assistant", "tool_calls": [tool_call("browser_click", ref="e1")]}, + {"role": "tool", "tool_call_id": "call_browser_click", "content": + '{"ok": true, "clicked": "[e1] button \\"Subscribe\\"", "url": "https://example.test/'}, + ]) + assert history == [{"name": "browser_click", "target": '[e1] button "Subscribe"', "declined": False, + "ok": True, "uncertain": False}] async def test_reply_uses_the_chat_history_for_a_declined_browser_click(config, isolated_home): diff --git a/tests/test_gateway.py b/tests/test_gateway.py index a5a4bfc98..fed8747de 100644 --- a/tests/test_gateway.py +++ b/tests/test_gateway.py @@ -50,6 +50,27 @@ def turn(ws, text): assert "tool_call" in second and "tool_result" in second +def test_websocket_delivers_browser_click_correction(config, isolated_home): + """The final WebSocket message includes corrections appended after model text streamed.""" + app = create_app(SentientApp( + config, llm=FakeProvider(replies=["I clicked the subscription button."]), db_path=isolated_home / "ws.db" + )) + with TestClient(app) as client: + token = client.app.state.token + with client.websocket_connect(f"/ws?token={token}") as ws: + assert ws.receive_json()["type"] == "hello" + ws.send_json({"type": "chat.send", "text": "click the subscription button"}) + messages = [] + while True: + message = ws.receive_json() + messages.append(message) + if message["type"] == "done": + break + correction = "Correction: I didn't click it, so nothing changed on the page." + assert messages[-1]["content"].endswith(correction) + assert any(message["type"] == "text_delta" and correction in message.get("text", "") for message in messages) + + def test_websocket_rejects_bad_token(client): with pytest.raises(WebSocketDisconnect), client.websocket_connect("/ws?token=nope") as ws: ws.receive_json() From eb3e50a0d72926c5ae09b18930d3e54444a8d672 Mon Sep 17 00:00:00 2001 From: syedmustafan Date: Tue, 20 Oct 2026 17:06:37 +0500 Subject: [PATCH 10/10] Refine browser click claim matching --- sentient/agent/loop.py | 8 +++++++- sentient/agent/outcomes.py | 11 ++++++++++- tests/test_agent_loop.py | 19 ++++++++++++++++++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/sentient/agent/loop.py b/sentient/agent/loop.py index d58e6dd79..df299f07e 100644 --- a/sentient/agent/loop.py +++ b/sentient/agent/loop.py @@ -315,6 +315,9 @@ def browser_click_history(rows: list[dict]) -> list[dict]: call = calls.pop(str(row.get("tool_call_id") or ""), None) if call is None: continue + approval = row.get("approval") + if isinstance(approval, dict) and approval.get("target"): + call["target"] = str(approval["target"]) try: res = json.loads(str(row.get("content") or "")) except (TypeError, ValueError): @@ -1526,12 +1529,15 @@ async def _record( ctx.visited.add(p.host) err = _error_text(res) if tc.name == "browser_click": + approval = p.approval or {} result.browser_clicks.append({ "name": tc.name, "declined": declined, "ok": ran and not is_error and not err and outcomes.result_problem(res) is None, "uncertain": ran and is_error, - "target": str(res.get("clicked") or tc.arguments.get("ref", "")) if isinstance(res, dict) else tc.arguments.get("ref", ""), + "target": str( + res.get("clicked") or approval.get("target") or tc.arguments.get("ref", "") + ) if isinstance(res, dict) else tc.arguments.get("ref", ""), }) if ran and p.tool is not None and not ctx.untrusted: if brings_untrusted(p.tool): diff --git a/sentient/agent/outcomes.py b/sentient/agent/outcomes.py index 4f973685a..262ae8767 100644 --- a/sentient/agent/outcomes.py +++ b/sentient/agent/outcomes.py @@ -59,7 +59,7 @@ r"(?:have\s+|had\s+|just\s+)?clicked\b", re.I ) _ASSISTANT_BROWSER_CLICK = re.compile( - r"\b(?:i(?:'ve)?|we|sentient)\s+(?:(?:have|had|just|only)\s+)?clicked\b", re.I + r"\b(?:i(?:'ve)?|we|sentient)\s+(?:(?:have|had|just|only|also|then|already|recently|finally|even|really|\w+ly)\s+){0,2}clicked\b", re.I ) _HYPOTHETICAL_BROWSER_CLICK = re.compile( r"\b(?:if|would|could|might|may)\b[^.!?]{0,50}\b(?:clicked|was clicked|were clicked)\b", re.I @@ -356,6 +356,10 @@ def _click_note(clicks: list[dict], target: str) -> str: """The correction for a click claim after selecting evidence for its named target.""" matches = [click for click in clicks if not target or _matches_click_target(target, click)] has_success = any(click.get("ok") for click in clicks) + if not target and has_success: + # A targetless claim could refer to any completed action. A later + # declined click is not enough evidence to contradict it. + return "" if target and not matches and not has_success: # A declined call may retain only its transient ref; it is still evidence that the named action was not made. matches = [ @@ -406,6 +410,11 @@ def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] for i, match in enumerate(explicit) ] for part in explicit_parts: + # The reply already tells the user that this attempted action was + # declined, so a second correction would add no information and + # can contradict an earlier successful click in the same reply. + if re.search(r"\b(?:declined|denied)\b", part, re.I): + continue if note := _click_note(clicks, _claimed_click_target(part)): return note passive = ( diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index c5fb26799..1cfe7cb83 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -179,7 +179,7 @@ def test_browser_click_correction_uses_the_latest_call(): {"name": "browser_click", "ok": True, "declined": False}, {"name": "browser_click", "ok": False, "declined": True}, ] - ) == "I didn't click it: you declined, so nothing changed on the page." + ) == "" assert outcomes.unmade_browser_click_note( "The button was clicked.", [{"name": "browser_click", "ok": False, "uncertain": True}] ) == "I couldn't confirm whether it was clicked." @@ -259,6 +259,16 @@ def test_browser_click_correction_uses_the_latest_call(): "I clicked the Subscribe button and I clicked the Delete account button.", successful_subscribe_then_declined_delete, ) == "I didn't click it: you declined, so nothing changed on the page." + assert outcomes.unmade_browser_click_note( + "I successfully clicked the Subscribe button.", successful_subscribe_then_declined_delete + ) == "" + assert outcomes.unmade_browser_click_note( + "Today I clicked Subscribe, which succeeded. I also clicked Delete account, but that one was declined.", + successful_subscribe_then_declined_delete, + ) == "" + assert outcomes.unmade_browser_click_note( + "I also clicked the Delete account button.", successful_subscribe_then_declined_delete + ) == "I didn't click it: you declined, so nothing changed on the page." def test_browser_click_history_retains_a_truncated_successful_result(): @@ -270,6 +280,13 @@ def test_browser_click_history_retains_a_truncated_successful_result(): ]) assert history == [{"name": "browser_click", "target": '[e1] button "Subscribe"', "declined": False, "ok": True, "uncertain": False}] + declined = browser_click_history([ + {"role": "assistant", "tool_calls": [tool_call("browser_click", ref="e2")]}, + {"role": "tool", "tool_call_id": "call_browser_click", "content": '{"declined": true}', + "approval": {"target": "Delete account"}}, + ]) + assert declined == [{"name": "browser_click", "target": "Delete account", "declined": True, + "ok": False, "uncertain": False}] async def test_reply_uses_the_chat_history_for_a_declined_browser_click(config, isolated_home):