fix(browser): correct unmade click claims

CHANGELOG.md

@@ -5,6 +5,12 @@ and versions follow [Semantic Versioning](https://semver.org/).
 
 ## [Unreleased]
 
+### Fixed
+- **Browser click claims now stay grounded across a chat.** Sentient does not correct a user or another person's
+  click, or a sentence that already says no click happened. When it does need to correct its own unsupported browser
+  click claim, it says plainly that nothing changed on the page; it also remembers the latest browser-click outcome
+  from the chat, so a later claim after you declined a click says that instead of saying no click was attempted.
+
 ### Added
 - **Home Assistant devices are checked after Sentient changes them.** A moment after it turns on a light, locks a
   door, opens the blinds, arms the alarm or sets the thermostat, Sentient asks Home Assistant for the device's state

sentient/agent/loop.py

@@ -293,6 +293,36 @@ def history_to_openai(rows: list[dict]) -> list[dict]:
     return fixed
 
 
+def browser_click_history(rows: list[dict]) -> list[dict]:
+    """Completed browser-click outcomes in stored chat history, oldest first (#402)."""
+    calls: dict[str, dict] = {}
+    out: list[dict] = []
+    for row in rows:
+        if row.get("role") == "assistant":
+            for call in _normalize_tool_calls(row.get("tool_calls")):
+                if call["function"]["name"] == "browser_click":
+                    calls[call["id"]] = {"name": "browser_click"}
+            continue
+        if row.get("role") != "tool":
+            continue
+        call = calls.pop(str(row.get("tool_call_id") or ""), None)
+        if call is None:
+            continue
+        try:
+            res = json.loads(str(row.get("content") or ""))
+        except (TypeError, ValueError):
+            res = {}
+        declined = isinstance(res, dict) and bool(res.get("declined"))
+        err = _error_text(res)
+        call.update(
+            declined=declined,
+            ok=isinstance(res, dict) and res.get("ok") is True and not err and outcomes.result_problem(res) is None,
+            uncertain=not declined and bool(err),
+        )
+        out.append(call)
+    return out
+
+
 def is_tool_format_error(exc: Exception) -> bool:
     """True when the provider rejected the request because the model cannot do tool calling
     (e.g. an old Ollama model template emitting malformed tool calls)."""
@@ -386,6 +416,10 @@ class LoopResult:
     handed_off: list[dict] = field(default_factory=list)
     # the outcome checks of the calls that ran (#165, ``sentient.agent.outcomes``), in order; written by code only
     checks: list[dict] = field(default_factory=list)
+    # browser clicks attempted in this run, recorded by code so a final reply cannot invent one (#402)
+    browser_clicks: list[dict] = field(default_factory=list)
+    # completed browser clicks from earlier chat turns, consulted only for an explicit reference to one (#402)
+    browser_click_history: list[dict] = field(default_factory=list)
 
 
 def _hosts(raw: Any) -> set[str]:
@@ -900,6 +934,12 @@ async def run_loop(
                     result.from_findings = True
                     log.info("%s: the model gave no answer after its tool calls; answering from the results", source)
                     yield TextDelta(text=text_acc, **ev)
+                if note := outcomes.unmade_browser_click_note(
+                    text_acc, result.browser_clicks, result.browser_click_history
+                ):
+                    correction = f"\n\nCorrection: {note}"
+                    text_acc += correction
+                    yield TextDelta(text=correction, **ev)
                 result.text = text_acc
                 result.messages = messages
                 return
@@ -1028,6 +1068,12 @@ async def run_loop(
 
         result.hit_step_limit = True
         result.text = text_acc or "I reached the step limit before finishing. Tell me how to continue."
+        if note := outcomes.unmade_browser_click_note(
+            result.text, result.browser_clicks, result.browser_click_history
+        ):
+            correction = f"\n\nCorrection: {note}"
+            result.text += correction
+            yield TextDelta(text=correction, **ev)
         result.messages = messages
 
     async def _model_stream(
@@ -1434,6 +1480,13 @@ async def _record(
         if ran and p.host:
             ctx.visited.add(p.host)
         err = _error_text(res)
+        if tc.name == "browser_click":
+            result.browser_clicks.append({
+                "name": tc.name,
+                "declined": declined,
+                "ok": ran and not is_error and not err and outcomes.result_problem(res) is None,
+                "uncertain": ran and is_error,
+            })
         if ran and p.tool is not None and not ctx.untrusted:
             if brings_untrusted(p.tool):
                 ctx.untrusted = untrusted_source(p.tool, self.registry)
@@ -1694,6 +1747,7 @@ def release() -> list[str]:
             system, sources, notes = await self._system_prompt_with_sources(user_text, channel, session)
             history = await self.store.recent_messages(session_id, self.config.chat.history_window)
             convo = self.mark_outside_messages(history_to_openai(history))
+            result.browser_click_history.extend(browser_click_history(history))
 
             # Replace the just-stored user message with the rich version (attachments inlined).
             # Keep the stored user message unchanged. Add the current time to the

sentient/agent/outcomes.py

@@ -50,6 +50,15 @@
     "home": re.compile(r"\b(turned|switched|set|done|ran)\b", re.I),
 }
 _CLAIM_ANY = re.compile(r"\b(done|completed|finished|success(?:fully)?|sent|saved|created|updated)\b", re.I)
+_BROWSER_CLICK_CLAIM = re.compile(r"\b(?:clicked|was clicked)\b", re.I)
+_NEGATED_BROWSER_CLICK = re.compile(
+    r"\b(?:not|didn't|hasn't|wasn't|haven't|never)\b[^.!?]{0,50}\bclicked\b", re.I
+)
+_OTHER_PERSON_BROWSER_CLICK = re.compile(
+    r"\b(?:you|he|she|they|someone|anyone|a friend|my friend)\b[^.!?]{0,50}\bclicked\b", re.I
+)
+_ASSISTANT_BROWSER_CLICK = re.compile(r"\b(?:i|we|sentient)\s+(?:have\s+)?clicked\b", re.I)
+_PRIOR_BROWSER_CLICK_REFERENCE = re.compile(r"\b(?:it|that|previous(?:ly)?|earlier|last)\b", re.I)
 
 # an MCP tool's kind from its name (servers declare nothing like it)
 _NAME_KINDS: list[tuple[str, re.Pattern, re.Pattern]] = [
@@ -293,6 +302,40 @@ def claimed(text: str, kind: str | None) -> bool:
     return bool((_CLAIMS.get(kind or "") or _CLAIM_ANY).search(text or ""))
 
 
+def unmade_browser_click_note(text: str, calls: list[dict], history: list[dict] | None = None) -> str:
+    """A short correction when a reply says it clicked although this turn did not.
+
+    Small local models occasionally state that a browser click happened without making
+    the call at all, or after the user declined its approval.  This is deliberately
+    narrow: it only corrects the completed form of ``click`` and only looks at calls
+    recorded by the loop, never at text supplied by a page or the user.
+    """
+    current_clicks = [call for call in calls if call.get("name") == "browser_click"]
+    claims = re.split(r"(?<=[.!?])\s+", text or "")
+    has_claim = any(
+        _ASSISTANT_BROWSER_CLICK.search(claim)
+        or (
+            current_clicks
+            and _BROWSER_CLICK_CLAIM.search(claim)
+            and not _NEGATED_BROWSER_CLICK.search(claim)
+            and not _OTHER_PERSON_BROWSER_CLICK.search(claim)
+        )
+        for claim in claims
+    )
+    if not has_claim:
+        return ""
+    clicks = current_clicks
+    if not clicks and _PRIOR_BROWSER_CLICK_REFERENCE.search(text):
+        clicks = [call for call in history or [] if call.get("name") == "browser_click"]
+    if clicks and clicks[-1].get("ok"):
+        return ""
+    if clicks and clicks[-1].get("declined"):
+        return "I didn't click it: you declined, so nothing changed on the page."
+    if clicks and clicks[-1].get("uncertain"):
+        return "I couldn't confirm whether it was clicked."
+    return "I couldn't click it, so nothing changed on the page." if clicks else "I didn't click it, so nothing changed on the page."
+
+
 def _settled(failed: dict, later: list[dict]) -> bool:
     """A failed check counts as settled when a later successful call of the same tool covered what it missed. A
     failure that names nothing it acted on (a call with arguments the tool couldn't read) is settled only by the very

tests/test_agent_loop.py

@@ -1,6 +1,8 @@
 
+from sentient.agent import outcomes
 from sentient.app import SentientApp
 from sentient.llm.events import ApprovalRequest, Done, TextDelta, ToolResultEvent
+from sentient.llm.provider import StreamChunk, ToolCall
 from tests.conftest import FakeProvider, tool_call
 
 
@@ -79,6 +81,7 @@ class Postcards(ToolPlugin):
         sid = await s.store.create_session(channel="cli")
 
         async def deny(ev):
+            """Decline the browser-click approval shown by this test."""
             if isinstance(ev, ApprovalRequest):
                 assert ev.risk == "write"
                 s.approvals.resolve(ev.approval_id, "deny")
@@ -89,6 +92,7 @@ async def deny(ev):
         assert sent == []
 
         async def allow(ev):
+            """Allow the postcard approval shown by this test."""
             if isinstance(ev, ApprovalRequest):
                 s.approvals.resolve(ev.approval_id, "allow_session")
 
@@ -101,6 +105,121 @@ async def allow(ev):
         await s.stop()
 
 
+async def test_reply_corrects_a_browser_click_it_never_made(config, isolated_home):
+    """A natural-language click claim needs evidence from a browser click call."""
+    llm = FakeProvider(replies=["I clicked the subscription button. "])
+    s = await SentientApp(config, llm=llm, db_path=isolated_home / "never-clicked.db").start()
+    try:
+        sid = await s.store.create_session(channel="cli")
+        events = await _collect(s.agent, sid, "click the subscription button")
+        text = "".join(e.text for e in events if isinstance(e, TextDelta))
+        assert text == "I clicked the subscription button. \n\nCorrection: I didn't click it, so nothing changed on the page."
+        assert (await s.store.recent_messages(sid, 1))[-1]["content"] == text
+    finally:
+        await s.stop()
+
+
+async def test_reply_corrects_a_browser_click_the_user_declined(config, isolated_home):
+    """A denied browser-click approval must remain visible in the final reply."""
+    config.tools.approvals.mode = "ask"
+    llm = FakeProvider(replies=[[tool_call("browser_click", ref="e1")], "It was clicked, but the action was declined."])
+    s = await SentientApp(config, llm=llm, db_path=isolated_home / "declined-click.db").start()
+    try:
+        sid = await s.store.create_session(channel="cli")
+
+        async def deny(ev):
+            """Decline the browser-click approval shown by this test."""
+            if isinstance(ev, ApprovalRequest):
+                s.approvals.resolve(ev.approval_id, "deny")
+
+        events = await _collect(s.agent, sid, "click the subscription button", deny)
+        text = "".join(e.text for e in events if isinstance(e, TextDelta))
+        assert text.endswith("Correction: I didn't click it: you declined, so nothing changed on the page.")
+        result = next(e for e in events if isinstance(e, ToolResultEvent))
+        assert result.result["declined"] is True
+    finally:
+        await s.stop()
+
+
+async def test_step_limit_corrects_a_declined_browser_click_claim(config, isolated_home):
+    """A streamed claim is corrected even when its tool call exhausts the final round."""
+    class ClaimAndClick(FakeProvider):
+        async def stream(self, role, messages, tools=None, *, model=None):
+            """Stream a textual click claim and its tool call in one final round."""
+            self.calls.append({"role": role, "messages": messages, "tools": tools, "model": model})
+            yield StreamChunk(text="The subscription button was clicked.", model="fake")
+            yield StreamChunk(done=True, tool_calls=[ToolCall(**tool_call("browser_click", ref="e1"))], model="fake")
+
+    config.models.max_tool_rounds = 1
+    config.tools.approvals.mode = "ask"
+    s = await SentientApp(config, llm=ClaimAndClick(), db_path=isolated_home / "limit-click.db").start()
+    try:
+        sid = await s.store.create_session(channel="cli")
+
+        async def deny(ev):
+            if isinstance(ev, ApprovalRequest):
+                s.approvals.resolve(ev.approval_id, "deny")
+
+        events = await _collect(s.agent, sid, "click the subscription button", deny)
+        text = "".join(e.text for e in events if isinstance(e, TextDelta))
+        assert text.endswith("Correction: I didn't click it: you declined, so nothing changed on the page.")
+        assert isinstance(events[-1], Done) and events[-1].content == text
+    finally:
+        await s.stop()
+
+
+def test_browser_click_correction_uses_the_latest_call():
+    """The latest known click outcome controls its correction wording."""
+    assert outcomes.unmade_browser_click_note(
+        "The button was clicked.", [{"name": "browser_click", "ok": True, "declined": False}]
+    ) == ""
+    assert outcomes.unmade_browser_click_note(
+        "The button was clicked.", [
+            {"name": "browser_click", "ok": True, "declined": False},
+            {"name": "browser_click", "ok": False, "declined": True},
+        ]
+    ) == "I didn't click it: you declined, so nothing changed on the page."
+    assert outcomes.unmade_browser_click_note(
+        "The button was clicked.", [{"name": "browser_click", "ok": False, "uncertain": True}]
+    ) == "I couldn't confirm whether it was clicked."
+    assert outcomes.unmade_browser_click_note("The button hasn't been clicked yet.", []) == ""
+    assert outcomes.unmade_browser_click_note("My friend clicked a link in a phishing email.", []) == ""
+    assert outcomes.unmade_browser_click_note("The subscription button was clicked.", []) == ""
+    assert outcomes.unmade_browser_click_note(
+        "I did not click A, but I clicked B.", []
+    ) == "I didn't click it, so nothing changed on the page."
+    assert outcomes.unmade_browser_click_note(
+        "You clicked A, and I clicked B.", []
+    ) == "I didn't click it, so nothing changed on the page."
+    prior_success = [{"name": "browser_click", "ok": True}]
+    assert outcomes.unmade_browser_click_note(
+        "I clicked the upgrade button.", [], prior_success
+    ) == "I didn't click it, so nothing changed on the page."
+    assert outcomes.unmade_browser_click_note("I clicked it.", [], prior_success) == ""
+
+
+async def test_reply_uses_the_chat_history_for_a_declined_browser_click(config, isolated_home):
+    """A later unsupported click claim retains the preceding declined-click context."""
+    config.tools.approvals.mode = "ask"
+    llm = FakeProvider(replies=[[tool_call("browser_click", ref="e1")], "I didn't click it: you declined.", "I clicked it."])
+    s = await SentientApp(config, llm=llm, db_path=isolated_home / "prior-declined-click.db").start()
+    try:
+        sid = await s.store.create_session(channel="cli")
+
+        async def deny(ev):
+            """Decline the browser-click approval shown by this test."""
+            if isinstance(ev, ApprovalRequest):
+                s.approvals.resolve(ev.approval_id, "deny")
+
+        first = await _collect(s.agent, sid, "click the subscription button", deny)
+        assert "Correction:" not in "".join(e.text for e in first if isinstance(e, TextDelta))
+        second = await _collect(s.agent, sid, "did it work?")
+        text = "".join(e.text for e in second if isinstance(e, TextDelta))
+        assert text.endswith("Correction: I didn't click it: you declined, so nothing changed on the page.")
+    finally:
+        await s.stop()
+
+
 async def test_internal_tools_do_not_prompt_in_ask_mode(config, isolated_home):
     config.tools.approvals.mode = "ask"
     llm = FakeProvider(replies=[[tool_call("file_write", name="note.txt", content="hi")], "saved"])