Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions crates/agent/src/acp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1303,6 +1303,7 @@ async fn finish_turn(
let (status, message, usage) = match outcome.result {
Ok(response) => {
let usage = response.usage.as_ref().map(|usage| TokenUsage {
freshness: crate::ContextFreshness::Current,
total_processed_tokens: Some(usage.total_tokens),
input_tokens: Some(usage.input_tokens),
cached_input_tokens: usage.cached_read_tokens,
Expand Down Expand Up @@ -1817,6 +1818,7 @@ impl State {
}
acp::SessionUpdate::UsageUpdate(usage) => {
let usage = TokenUsage {
freshness: crate::ContextFreshness::Current,
used_tokens: Some(usage.used),
context_window: Some(usage.size),
cost_usd: usage
Expand Down
352 changes: 329 additions & 23 deletions crates/agent/src/claude.rs

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion crates/agent/src/codex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1562,7 +1562,9 @@ impl Actor {
}
Some("contextCompaction") => {
if method == "item/completed" {
self.events.emit(AgentEvent::ContextCompacted).await;
self.events
.emit(AgentEvent::ContextCompacted(Default::default()))
.await;
}
return;
}
Expand Down Expand Up @@ -2466,6 +2468,7 @@ fn map_usage(value: &Value) -> Option<TokenUsage> {
// The session-cumulative running total lives in a sibling `total` object.
let total_processed_tokens = value.pointer("/total/totalTokens").and_then(Value::as_u64);
Some(TokenUsage {
freshness: crate::ContextFreshness::Current,
context_window: value.get("modelContextWindow").and_then(Value::as_u64),
total_processed_tokens,
input_tokens: last.get("inputTokens").and_then(Value::as_u64),
Expand Down
67 changes: 65 additions & 2 deletions crates/agent/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1085,7 +1085,7 @@ pub enum AgentEvent {
},
/// The provider compacted its context window (Claude `system/compact_boundary`;
/// Codex `contextCompaction` item). Rendered as a "Context compacted" work-log row.
ContextCompacted,
ContextCompacted(Compaction),
/// A tcode-level context-window change. This is never emitted by an adapter;
/// the runtime persists it after the user message that selected the window.
ContextWindowChanged {
Expand Down Expand Up @@ -1451,17 +1451,44 @@ pub enum ApprovalDecision {
Option(String),
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ContextFreshness {
/// Older logs lack provenance; their counts may be turn aggregates.
#[default]
Unknown,
Current,
LastKnown,
Compacting,
AwaitingObservation,
}

#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct Compaction {
pub in_progress: bool,
pub trigger: Option<String>,
pub pre_tokens: Option<u64>,
/// Provider-reported compaction output size; not a new request observation.
pub post_tokens: Option<u64>,
pub dropped_tokens: Option<u64>,
pub duration_ms: Option<u64>,
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct TokenUsage {
pub freshness: ContextFreshness,
/// Main-loop traffic for the completed turn, separate from occupancy.
pub turn_processed_tokens: Option<u64>,
pub input_tokens: Option<u64>,
pub cached_input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
/// Total context currently in use, if the provider reports it.
pub used_tokens: Option<u64>,
pub context_window: Option<u64>,
/// Cumulative tokens processed over the session's lifetime, if known (Codex
/// `thread/tokenUsage` running total; Claude accumulated per-turn usage).
/// `thread/tokenUsage` running total; timeline accumulation of Claude turns).
/// Shown as "Total processed" in the context-meter popover.
pub total_processed_tokens: Option<u64>,
/// Provider-reported cost for this turn/message, in US dollars.
Expand All @@ -1473,6 +1500,7 @@ pub struct TokenUsage {
impl TokenUsage {
#[cfg(feature = "process")]
fn merge(&mut self, usage: Self) {
self.freshness = usage.freshness;
self.input_tokens = add_token_counts(self.input_tokens, usage.input_tokens);
self.cached_input_tokens =
add_token_counts(self.cached_input_tokens, usage.cached_input_tokens);
Expand Down Expand Up @@ -1868,3 +1896,38 @@ mod thread_item_serde_tests {
);
}
}

#[cfg(test)]
mod usage_compatibility_tests {
use super::*;

#[test]
fn compaction_and_usage_read_old_literal_records_without_inventing_provenance() {
let event: AgentEvent = serde_json::from_str(r#"{"type":"context_compacted"}"#).unwrap();
assert!(
matches!(event, AgentEvent::ContextCompacted(c) if !c.in_progress && c.pre_tokens.is_none())
);
let usage: TokenUsage = serde_json::from_str(
r#"{"used_tokens":4100000,"context_window":1000000,"total_processed_tokens":4100000}"#,
)
.unwrap();
assert_eq!(usage.freshness, ContextFreshness::Unknown);
assert_eq!(usage.total_processed_tokens, Some(4100000));
// Existing readers use a unit variant with this tag. Extra fields are
// additive and do not introduce an unknown protocol event tag.
#[derive(Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
enum LegacyEvent {
ContextCompacted,
}
let _: LegacyEvent = serde_json::from_str(r#"{"type":"context_compacted","in_progress":false,"trigger":"manual","pre_tokens":500}"#).unwrap();
let event = AgentEvent::ContextCompacted(Compaction {
trigger: Some("manual".into()),
pre_tokens: Some(500),
..Default::default()
});
let wire = serde_json::to_value(event).unwrap();
assert_eq!(wire["type"], "context_compacted");
assert_eq!(wire["pre_tokens"], 500);
}
}
6 changes: 5 additions & 1 deletion crates/agent/src/opencode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,9 @@ impl OpenCodeMapper {
.push((request_id.to_owned(), None));
}
}
"session.compacted" => mapped.events.push(AgentEvent::ContextCompacted),
"session.compacted" => mapped
.events
.push(AgentEvent::ContextCompacted(Default::default())),
_ => {}
}
mapped
Expand Down Expand Up @@ -903,6 +905,7 @@ impl OpenCodeMapper {
aggregate.merge(usage);
}
let aggregate = TokenUsage {
freshness: crate::ContextFreshness::Current,
total_processed_tokens: Some(self.cumulative_processed),
..aggregate
};
Expand Down Expand Up @@ -1123,6 +1126,7 @@ fn usage_from_tokens(tokens: Option<&Value>) -> Option<TokenUsage> {
let cache_read = crate::json_u64(tokens.pointer("/cache/read"));
let cache_write = crate::json_u64(tokens.pointer("/cache/write")).unwrap_or(0);
(input.is_some() || output.is_some() || cache_read.is_some()).then_some(TokenUsage {
freshness: crate::ContextFreshness::Current,
input_tokens: input,
cached_input_tokens: cache_read,
output_tokens: output,
Expand Down
4 changes: 3 additions & 1 deletion crates/agent/src/pi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -866,7 +866,7 @@ impl PiMapper {
}],
"compaction_end" => {
if message.get("result").is_some_and(|value| !value.is_null()) {
vec![AgentEvent::ContextCompacted]
vec![AgentEvent::ContextCompacted(Default::default())]
} else {
vec![AgentEvent::Warning {
message: format!(
Expand Down Expand Up @@ -1086,6 +1086,7 @@ impl PiMapper {
let processed = crate::processed_tokens(usage);
self.cumulative_processed = self.cumulative_processed.saturating_add(processed);
let usage = TokenUsage {
freshness: crate::ContextFreshness::Current,
total_processed_tokens: Some(self.cumulative_processed),
..usage
};
Expand Down Expand Up @@ -1409,6 +1410,7 @@ fn map_usage(usage: Option<&Value>) -> Option<TokenUsage> {
let output = crate::json_u64(usage.get("output"));
let cache_read = crate::json_u64(usage.get("cacheRead"));
(input.is_some() || output.is_some() || cache_read.is_some()).then_some(TokenUsage {
freshness: crate::ContextFreshness::Current,
input_tokens: input,
cached_input_tokens: cache_read,
output_tokens: output,
Expand Down
44 changes: 44 additions & 0 deletions crates/agent/tests/fixtures/claude/USAGE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Usage fixtures

`usage_recorded.jsonl` and `compaction_recorded.jsonl` were captured on
2026-09-08 from new Claude Code 2.1.263 sessions in `/tmp/tcode-usage-provider-evidence`,
with `TCODE_DATA_DIR` pointing to a throwaway profile. No existing session was
opened or resumed. The probe used `--no-session-persistence`, `--verbose`,
`--output-format stream-json`, and `--include-partial-messages`.

The first prompt asked for two separate Bash `printf` calls and a DONE response.
Only message IDs (replaced with stable fixture IDs), model names, usage and result
accounting fields were retained. Prompts, content, initialization payloads,
environment, stderr, account identifiers and credentials were not saved.
The independently calculated latest input/cache contexts are 20,250, 20,635 and
20,764. Turn traffic is 6 + 10,840 + 50,803 + 435 = 62,084. Repeated assistant
blocks share usage; their output values are placeholders, while stream deltas
are cumulative within each request.

The second probe used `--input-format stream-json`, sent a new harmless printf
turn, then `/compact` in that same newly created process. The recorded lifecycle
is compacting → compact boundary, with manual trigger, 19,555 pre-tokens, 2,948
post-tokens, 16,607 cumulative dropped tokens and 24,455 ms duration. Transcript
UUIDs in preserved-segment metadata were omitted. Post-tokens describe the
compaction result; they are not a fresh request context observation.

`usage_scope.jsonl` and `usage_resume.jsonl` are constructed protocol fixtures,
not live captures. They protect >capacity turn traffic, output-only deltas,
repeated message/result IDs, main/subagent routing, a fresh process on an existing
timeline, compaction and interrupted completion. Their expected counts are
literal assertions in the production adapter → timeline → composer replay.

Contract references:
- https://code.claude.com/docs/en/agent-sdk/cost-tracking
- https://platform.claude.com/docs/en/build-with-claude/streaming
- https://code.claude.com/docs/en/statusline#context-window-fields

The exact reporter's 2.5M/4.1M run and automatic compaction were not captured.
The fixtures deliberately distinguish observed evidence from constructed cases.

A separate read-only native account probe ran the existing ignored
`provider_usage::tests::live_provider_usage` test explicitly. Both Codex and
Claude returned non-empty windows with no error; only normalized usage output
was captured outside the repository. The unsupported endpoint and temporary
failure cases remain deterministic local fixtures, not claims about a live
custom account.
2 changes: 2 additions & 0 deletions crates/agent/tests/fixtures/claude/compaction_recorded.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{"type":"system","subtype":"status","status":"compacting"}
{"type":"system","subtype":"compact_boundary","compact_metadata":{"trigger":"manual","pre_tokens":19555,"post_tokens":2948,"cumulative_dropped_tokens":16607,"duration_ms":24455}}
11 changes: 11 additions & 0 deletions crates/agent/tests/fixtures/claude/usage_recorded.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{"type":"stream_event","event":{"type":"message_start","message":{"id":"recorded-request-1","usage":{"input_tokens":2,"cache_creation_input_tokens":10326,"cache_read_input_tokens":9922,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":10326},"output_tokens":2,"service_tier":"standard","inference_geo":"not_available"}}}}
{"type":"assistant","message":{"id":"recorded-request-1","usage":{"input_tokens":2,"cache_creation_input_tokens":10326,"cache_read_input_tokens":9922,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":10326},"output_tokens":2,"service_tier":"standard","inference_geo":"not_available"},"model":"claude-opus-5"}}
{"type":"assistant","message":{"id":"recorded-request-1","usage":{"input_tokens":2,"cache_creation_input_tokens":10326,"cache_read_input_tokens":9922,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":10326},"output_tokens":2,"service_tier":"standard","inference_geo":"not_available"},"model":"claude-opus-5"}}
{"type":"stream_event","event":{"type":"message_delta","usage":{"input_tokens":2,"cache_creation_input_tokens":10326,"cache_read_input_tokens":9922,"output_tokens":343,"output_tokens_details":{"thinking_tokens":255},"iterations":[{"input_tokens":2,"output_tokens":343,"cache_read_input_tokens":9922,"cache_creation_input_tokens":10326,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":10326},"type":"message"}]}}}
{"type":"stream_event","event":{"type":"message_start","message":{"id":"recorded-request-2","usage":{"input_tokens":2,"cache_creation_input_tokens":385,"cache_read_input_tokens":20248,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":385},"output_tokens":17,"service_tier":"standard","inference_geo":"not_available"}}}}
{"type":"assistant","message":{"id":"recorded-request-2","usage":{"input_tokens":2,"cache_creation_input_tokens":385,"cache_read_input_tokens":20248,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":385},"output_tokens":17,"service_tier":"standard","inference_geo":"not_available"},"model":"claude-opus-5"}}
{"type":"stream_event","event":{"type":"message_delta","usage":{"input_tokens":2,"cache_creation_input_tokens":385,"cache_read_input_tokens":20248,"output_tokens":87,"output_tokens_details":{"thinking_tokens":0},"iterations":[{"input_tokens":2,"output_tokens":87,"cache_read_input_tokens":20248,"cache_creation_input_tokens":385,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":385},"type":"message"}]}}}
{"type":"stream_event","event":{"type":"message_start","message":{"id":"recorded-request-3","usage":{"input_tokens":2,"cache_creation_input_tokens":129,"cache_read_input_tokens":20633,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":129},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"}}}}
{"type":"assistant","message":{"id":"recorded-request-3","usage":{"input_tokens":2,"cache_creation_input_tokens":129,"cache_read_input_tokens":20633,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":129},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"model":"claude-opus-5"}}
{"type":"stream_event","event":{"type":"message_delta","usage":{"input_tokens":2,"cache_creation_input_tokens":129,"cache_read_input_tokens":20633,"output_tokens":5,"output_tokens_details":{"thinking_tokens":0},"iterations":[{"input_tokens":2,"output_tokens":5,"cache_read_input_tokens":20633,"cache_creation_input_tokens":129,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":129},"type":"message"}]}}}
{"type":"result","subtype":"success","usage":{"input_tokens":6,"cache_creation_input_tokens":10840,"cache_read_input_tokens":50803,"output_tokens":435,"output_tokens_details":{"thinking_tokens":255},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":10840,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":5,"cache_read_input_tokens":20633,"cache_creation_input_tokens":129,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":129},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-opus-5[1m]":{"inputTokens":6,"outputTokens":435,"cacheReadInputTokens":50803,"cacheCreationInputTokens":10840,"webSearchRequests":0,"costUSD":0.1447065,"contextWindow":1000000,"maxOutputTokens":64000,"thinkingTokens":255,"canonicalModel":"claude-opus-5","provider":"firstParty","costBasis":"list"}},"duration_ms":10822}
6 changes: 6 additions & 0 deletions crates/agent/tests/fixtures/claude/usage_resume.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{"type":"system","subtype":"init","session_id":"fixture-session","model":"claude-opus-5"}
{"type":"system","subtype":"status","status":"compacting"}
{"type":"system","subtype":"compact_boundary","compact_metadata":{"trigger":"manual","pre_tokens":500260}}
{"type":"stream_event","event":{"type":"message_start","message":{"id":"request-three","usage":{"input_tokens":20,"cache_read_input_tokens":30,"cache_creation_input_tokens":10,"output_tokens":0}}}}
{"type":"stream_event","event":{"type":"message_delta","usage":{"output_tokens":5}}}
{"type":"result","uuid":"result-two","subtype":"error_during_execution","is_error":true,"result":"Interrupted test turn","usage":{"input_tokens":20,"cache_read_input_tokens":30,"cache_creation_input_tokens":10,"output_tokens":5}}
10 changes: 10 additions & 0 deletions crates/agent/tests/fixtures/claude/usage_scope.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{"type":"system","subtype":"init","session_id":"fixture-session","model":"claude-opus-5"}
{"type":"stream_event","event":{"type":"message_start","message":{"id":"request-one","usage":{"input_tokens":100,"cache_read_input_tokens":400000,"cache_creation_input_tokens":50,"output_tokens":0}}}}
{"type":"stream_event","event":{"type":"message_delta","usage":{"output_tokens":20}}}
{"type":"stream_event","parent_tool_use_id":"child-task","event":{"type":"message_start","message":{"id":"child-request","usage":{"input_tokens":900000,"output_tokens":500}}}}
{"type":"assistant","message":{"id":"request-one","content":[],"usage":{"input_tokens":100,"cache_read_input_tokens":400000,"cache_creation_input_tokens":50,"output_tokens":1}}}
{"type":"stream_event","event":{"type":"message_start","message":{"id":"request-two","usage":{"input_tokens":200,"cache_read_input_tokens":500000,"cache_creation_input_tokens":60,"output_tokens":0}}}}
{"type":"stream_event","event":{"type":"message_delta","usage":{"output_tokens":30}}}
{"type":"assistant","message":{"id":"request-one","content":[],"usage":{"input_tokens":100,"cache_read_input_tokens":400000,"cache_creation_input_tokens":50,"output_tokens":1}}}
{"type":"result","uuid":"result-one","subtype":"success","usage":{"input_tokens":1000,"cache_read_input_tokens":4000000,"cache_creation_input_tokens":100,"output_tokens":200},"modelUsage":{"claude-opus-5":{"contextWindow":1000000}}}
{"type":"result","uuid":"result-one","subtype":"success","usage":{"input_tokens":1000,"cache_read_input_tokens":4000000,"cache_creation_input_tokens":100,"output_tokens":200},"modelUsage":{"claude-opus-5":{"contextWindow":1000000}}}
18 changes: 15 additions & 3 deletions crates/core/src/relay.rs
Original file line number Diff line number Diff line change
Expand Up @@ -233,9 +233,21 @@ fn render_turn(number: usize, entries: &[&TimelineEntry], timeline: &Timeline) -
to_provider.display_name()
));
}
EntryContent::ContextCompacted => {
activity(&mut body, "context", "provider", "compacted")
}
EntryContent::ContextCompacted(c) => activity(
&mut body,
"context",
"provider",
&format!(
"{}; trigger={:?}; pre_tokens={:?}",
if c.in_progress {
"compacting"
} else {
"compacted"
},
c.trigger,
c.pre_tokens
),
),
EntryContent::ContextWindowChanged { window } => activity(
&mut body,
"context",
Expand Down
Loading
Loading