fix(browser): correct unmade click claims

changes/402.fixed.md

@@ -0,0 +1,3 @@
+**Browser click claims now stay grounded across a chat.** Sentient does not correct a user or another person's
+click, or a sentence that already says no click happened. When it needs to correct its own unsupported browser-click
+claim, it says plainly that nothing changed on the page and keeps the latest click outcome from the chat in context.

sentient/agent/loop.py

@@ -297,6 +297,52 @@ def history_to_openai(rows: list[dict]) -> list[dict]:
     return fixed
 
 
+def browser_click_history(rows: list[dict]) -> list[dict]:
+    """Completed browser-click outcomes in stored chat history, oldest first (#402)."""
+    calls: dict[str, dict] = {}
+    out: list[dict] = []
+    for row in rows:
+        if row.get("role") == "assistant":
+            for call in _normalize_tool_calls(row.get("tool_calls")):
+                if call["function"]["name"] == "browser_click":
+                    calls[call["id"]] = {
+                        "name": "browser_click",
+                        "target": _arguments(call["function"].get("arguments")).get("ref", ""),
+                    }
+            continue
+        if row.get("role") != "tool":
+            continue
+        call = calls.pop(str(row.get("tool_call_id") or ""), None)
+        if call is None:
+            continue
+        approval = row.get("approval")
+        if isinstance(approval, dict) and approval.get("target"):
+            call["target"] = str(approval["target"])
+        try:
+            res = json.loads(str(row.get("content") or ""))
+        except (TypeError, ValueError):
+            # A short result may end after the stable leading fields.  Preserve
+            # those fields rather than treating a recorded success as a failed
+            # click solely because its trailing URL was cut for model context.
+            content = str(row.get("content") or "")
+            res = {"ok": True} if re.search(r'"ok"\s*:\s*true\b', content) else {}
+            clicked = re.search(r'"clicked"\s*:\s*"((?:[^"\\]|\\.)*)"', content)
+            if clicked is not None:
+                with contextlib.suppress(ValueError):
+                    res["clicked"] = json.loads(f'"{clicked.group(1)}"')
+        declined = isinstance(res, dict) and bool(res.get("declined"))
+        err = _error_text(res)
+        call.update(
+            declined=declined,
+            ok=isinstance(res, dict) and res.get("ok") is True and not err and outcomes.result_problem(res) is None,
+            uncertain=not declined and bool(err),
+        )
+        if isinstance(res, dict) and res.get("clicked"):
+            call["target"] = str(res["clicked"])
+        out.append(call)
+    return out
+
+
 def is_tool_format_error(exc: Exception) -> bool:
     """True when the provider rejected the request because the model cannot do tool calling
     (e.g. an old Ollama model template emitting malformed tool calls)."""
@@ -392,6 +438,10 @@ class LoopResult:
     handed_off: list[dict] = field(default_factory=list)
     # the outcome checks of the calls that ran (#165, ``sentient.agent.outcomes``), in order; written by code only
     checks: list[dict] = field(default_factory=list)
+    # browser clicks attempted in this run, recorded by code so a final reply cannot invent one (#402)
+    browser_clicks: list[dict] = field(default_factory=list)
+    # completed browser clicks from earlier chat turns, consulted only for an explicit reference to one (#402)
+    browser_click_history: list[dict] = field(default_factory=list)
 
 
 def _hosts(raw: Any) -> set[str]:
@@ -922,6 +972,12 @@ def interrupted() -> bool:
                     result.from_findings = True
                     log.info("%s: the model gave no answer after its tool calls; answering from the results", source)
                     yield TextDelta(text=text_acc, **ev)
+                if note := outcomes.unmade_browser_click_note(
+                    text_acc, result.browser_clicks, result.browser_click_history
+                ):
+                    correction = f"\n\nCorrection: {note}"
+                    text_acc += correction
+                    yield TextDelta(text=correction, **ev)
                 result.text = text_acc
                 result.messages = messages
                 return
@@ -1054,6 +1110,12 @@ def interrupted() -> bool:
 
         result.hit_step_limit = True
         result.text = text_acc or "I reached the step limit before finishing. Tell me how to continue."
+        if note := outcomes.unmade_browser_click_note(
+            result.text, result.browser_clicks, result.browser_click_history
+        ):
+            correction = f"\n\nCorrection: {note}"
+            result.text += correction
+            yield TextDelta(text=correction, **ev)
         result.messages = messages
 
     async def _model_stream(
@@ -1466,6 +1528,17 @@ async def _record(
         if ran and p.host:
             ctx.visited.add(p.host)
         err = _error_text(res)
+        if tc.name == "browser_click":
+            approval = p.approval or {}
+            result.browser_clicks.append({
+                "name": tc.name,
+                "declined": declined,
+                "ok": ran and not is_error and not err and outcomes.result_problem(res) is None,
+                "uncertain": ran and is_error,
+                "target": str(
+                    res.get("clicked") or approval.get("target") or tc.arguments.get("ref", "")
+                ) if isinstance(res, dict) else tc.arguments.get("ref", ""),
+            })
         if ran and p.tool is not None and not ctx.untrusted:
             if brings_untrusted(p.tool):
                 ctx.untrusted = untrusted_source(p.tool, self.registry)
@@ -1726,6 +1799,7 @@ def release() -> list[str]:
             system, sources, notes = await self._system_prompt_with_sources(user_text, channel, session)
             history = await self.store.recent_messages(session_id, self.config.chat.history_window)
             convo = self.mark_outside_messages(history_to_openai(history))
+            result.browser_click_history.extend(browser_click_history(history))
 
             # Replace the just-stored user message with the rich version (attachments inlined).
             # Keep the stored user message unchanged. Add the current time to the

sentient/agent/outcomes.py

@@ -50,6 +50,20 @@
     "home": re.compile(r"\b(turned|switched|set|done|ran)\b", re.I),
 }
 _CLAIM_ANY = re.compile(r"\b(done|completed|finished|success(?:fully)?|sent|saved|created|updated)\b", re.I)
+_BROWSER_CLICK_CLAIM = re.compile(r"\b(?:clicked|was clicked)\b", re.I)
+_NEGATED_BROWSER_CLICK = re.compile(
+    r"\b(?:not|didn't|hasn't|wasn't|haven't|never|nothing)\b[^.!?]{0,50}\bclicked\b", re.I
+)
+_OTHER_PERSON_BROWSER_CLICK = re.compile(
+    r"\b(?:you|he|she|they|someone|anyone|(?:my|your|his|her|their)\s+\w+)\s+"
+    r"(?:have\s+|had\s+|just\s+)?clicked\b", re.I
+)
+_ASSISTANT_BROWSER_CLICK = re.compile(
+    r"\b(?:i(?:'ve)?|we|sentient)\s+(?:(?:have|had|just|only|also|then|already|recently|finally|even|really|\w+ly)\s+){0,2}clicked\b", re.I
+)
+_HYPOTHETICAL_BROWSER_CLICK = re.compile(
+    r"\b(?:if|would|could|might|may)\b[^.!?]{0,50}\b(?:clicked|was clicked|were clicked)\b", re.I
+)
 
 # an MCP tool's kind from its name (servers declare nothing like it)
 _NAME_KINDS: list[tuple[str, re.Pattern, re.Pattern]] = [
@@ -293,6 +307,128 @@ def claimed(text: str, kind: str | None) -> bool:
     return bool((_CLAIMS.get(kind or "") or _CLAIM_ANY).search(text or ""))
 
 
+def _claimed_click_target(text: str) -> str:
+    """The named target in an assistant or passive click claim, or empty for ``clicked it``."""
+    match = _ASSISTANT_BROWSER_CLICK.search(text)
+    if match is not None:
+        tail = re.split(r"\b(?:on|and|but)\b|[,;.!?]", text[match.end():], maxsplit=1, flags=re.I)[0]
+    else:
+        passive = re.search(r"(?P<target>.+?)\s+was clicked\b", text, re.I)
+        if passive is None:
+            return ""
+        tail = passive.group("target")
+    target = re.sub(r"^\s*(?:only|just|the|a|an)\s+", "", tail, flags=re.I).strip()
+    return "" if target.lower() in {"", "it", "that", "this", "button", "link"} else target
+
+
+def _target_words(value: str) -> set[str]:
+    """Meaningful target words, without generic element labels."""
+    return {
+        word for word in re.findall(r"[a-z0-9]+", value.lower())
+        if word not in {"a", "an", "the", "button", "link", "only", "just"} and not re.fullmatch(r"e\d+", word)
+    }
+
+
+def _matches_click_target(claim: str, click: dict) -> bool:
+    """Whether a named claim plausibly refers to this click's recorded browser element."""
+    if not claim:
+        return True
+
+    claim_words = _target_words(claim)
+    target_words = _target_words(str(click.get("target") or ""))
+
+    def same_word(left: str, right: str) -> bool:
+        """Accept the small morphological variations common in a natural-language reply."""
+        return left == right or (
+            len(left) >= 6 and len(right) >= 6 and left[:6] == right[:6]
+        )
+
+    # The recorded element name is the dependable evidence.  It need only appear
+    # in the claim: a reply naturally adds context such as "for the newsletter"
+    # or an element ref after naming the clicked control.
+    return bool(target_words) and all(
+        any(same_word(target_word, claim_word) for claim_word in claim_words)
+        for target_word in target_words
+    )
+
+
+def _click_note(clicks: list[dict], target: str) -> str:
+    """The correction for a click claim after selecting evidence for its named target."""
+    matches = [click for click in clicks if not target or _matches_click_target(target, click)]
+    has_success = any(click.get("ok") for click in clicks)
+    if not target and has_success:
+        # A targetless claim could refer to any completed action. A later
+        # declined click is not enough evidence to contradict it.
+        return ""
+    if target and not matches and not has_success:
+        # A declined call may retain only its transient ref; it is still evidence that the named action was not made.
+        matches = [
+            click for click in clicks
+            if not click.get("ok") and not _target_words(str(click.get("target") or ""))
+        ]
+    if target and not matches:
+        if has_success:
+            # A successful click of another named control makes an unmatched
+            # claim ambiguous.  Do not turn an honest report into a correction.
+            return ""
+        return "I didn't click it, so nothing changed on the page."
+    if not matches:
+        return "I didn't click it, so nothing changed on the page."
+    click = matches[-1]
+    if click.get("ok"):
+        return ""
+    if click.get("declined"):
+        return "I didn't click it: you declined, so nothing changed on the page."
+    if click.get("uncertain"):
+        return "I couldn't confirm whether it was clicked."
+    return "I couldn't click it, so nothing changed on the page."
+
+
+def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] | None = None) -> str:
+    """A short correction when a reply says it clicked although this turn did not.
+
+    Small local models occasionally state that a browser click happened without making
+    the call at all, or after the user declined its approval.  This is deliberately
+    narrow: it only corrects the completed form of ``click`` and only looks at calls
+    recorded by the loop, never at text supplied by a page or the user.
+    """
+    current_clicks = [call for call in calls if call.get("name") == "browser_click"]
+    previous_clicks = [call for call in history or [] if call.get("name") == "browser_click"]
+    claims = re.split(r"(?<=[.!?])\s+", text or "")
+    # A reply can summarize a successful click from an earlier turn alongside a
+    # declined click from this one, so consider the complete chronological record.
+    clicks = previous_clicks + current_clicks
+    for claim in claims:
+        if _HYPOTHETICAL_BROWSER_CLICK.search(claim):
+            continue
+        explicit = list(_ASSISTANT_BROWSER_CLICK.finditer(claim))
+        # Each self-attributed claim needs its own evidence.  In particular, a
+        # successful "I clicked Subscribe" must not hide a later false
+        # "I clicked Delete account" in the same sentence.
+        explicit_parts = [
+            claim[match.start(): explicit[i + 1].start() if i + 1 < len(explicit) else None]
+            for i, match in enumerate(explicit)
+        ]
+        for part in explicit_parts:
+            # The reply already tells the user that this attempted action was
+            # declined, so a second correction would add no information and
+            # can contradict an earlier successful click in the same reply.
+            if re.search(r"\b(?:declined|denied)\b", part, re.I):
+                continue
+            if note := _click_note(clicks, _claimed_click_target(part)):
+                return note
+        passive = (
+            not explicit
+            and clicks
+            and _BROWSER_CLICK_CLAIM.search(claim)
+            and not _NEGATED_BROWSER_CLICK.search(claim)
+            and not _OTHER_PERSON_BROWSER_CLICK.search(claim)
+        )
+        if passive and (note := _click_note(clicks, _claimed_click_target(claim))):
+            return note
+    return ""
+
+
 def _settled(failed: dict, later: list[dict]) -> bool:
     """A failed check counts as settled when a later successful call of the same tool covered what it missed. A
     failure that names nothing it acted on (a call with arguments the tool couldn't read) is settled only by the very