diff --git a/.github/workflows/fork-ci.yml b/.github/workflows/fork-ci.yml index f21fe8921a0b..6973eb072d04 100644 --- a/.github/workflows/fork-ci.yml +++ b/.github/workflows/fork-ci.yml @@ -93,6 +93,22 @@ jobs: working-directory: ${{ github.workspace }} run: sudo apt-get update && sudo apt-get install -y libcap-dev pkg-config libasound2-dev bubblewrap + # The `rust-heavy` label is also served by container runners (for example + # nodef-gha-public-docker) that lack the baked packages, so codex-bwrap's + # build.rs fails on "libcap not available via pkg-config". Those runners + # may run as root with no sudo. Skip when libcap is already present (the + # baked rust-heavy image, whose jobs have no sudo). + - name: Install build libraries on self-hosted runners + if: runner.environment == 'self-hosted' + working-directory: ${{ github.workspace }} + run: | + if command -v pkg-config >/dev/null && pkg-config --exists libcap; then + echo "libcap already available via pkg-config" + exit 0 + fi + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO=sudo + $SUDO apt-get update && $SUDO apt-get install -y libcap-dev pkg-config + # Ubuntu 24.04's AppArmor user-namespace restriction blocks bwrap, so # every sandbox test fails without this. Copied from # .github/actions/setup-ci/action.yml, which upstream's Linux nextest diff --git a/FORK.md b/FORK.md index e3b61347b4b8..11a91af3a2fc 100644 --- a/FORK.md +++ b/FORK.md @@ -67,6 +67,7 @@ passing on its own is enough: a cherry-pick does not move the base. | — | `agent-state` terminal title item (`codex: ready\|working\|awaiting approval`), written only on change, with the user's title pushed (`CSI 22;0t`) at the first title write and popped (`CSI 23;0t`) on exit. Idle with `animations = false` schedules no refresh. Landed on `fork/wi-1127-agent-state` at `b5f9b014`. | fork-only; Klaudia contract from vogt `internal/tui/idle.go` | | — | `wire_api = "chat"`: Chat Completions client restored for providers whose Responses path drops tool calls (grok via theclawbay/LiteLLM, WI-1139). New files only, plus one dispatch arm: `core/src/chat_request.rs` (request conversion; namespaced tools flattened to `ns__name`, freeform tools such as apply_patch exposed as a function taking one `input` string, calls restored by `ChatToolMap`), `codex-api/src/{endpoint,sse}/chat.rs` (client and SSE parser, fixture tests in `sse/chat_tests.rs`). Reasoning, web search and tool search are not sent over chat. Branch `fork/wi-1286-chat-wire`. | fork-only; upstream removed the chat client | | — | Headless MCP startup (issue #26, WI-1514): `codex exec` defaults `mcp_optional_startup_grace_ms` to 0 unless set, so every enabled server's startup is awaited (each bounded by its `startup_timeout_sec`) before the first request; a failed or timed-out server is printed as a warning and the run continues. Tool dispatch resolves un-namespaced `mcp____` names and bare names unique across MCP servers, and answers a call to a server still handshaking with "MCP server `x` is still starting" instead of "unsupported call". `exec/src/lib.rs`, `core/src/tools/registry.rs`, `codex-mcp` `pending_startup_servers`. Branch `fix/issue-26-mcp-startup`. | fork-only | +| — | Auto-compaction loop guard (WI-2832): after a context-limit compaction (pre-turn or mid-turn), if the first request's prompt alone is still at or over `model_auto_compact_token_limit` (body after prefix for that scope) or the usable window, the turn ends with a warning and a `ContextWindowExceeded` error instead of compacting again. Before this, a fixed prefix (tool specs) over the limit looped compaction -> one tool call -> compaction (235 compactions in one turn). `core/src/session/turn.rs`; tests `turn_tests.rs` `prompt_over_limit_*`, `suite::compact::auto_compact_stops_when_compaction_cannot_get_below_limit`. Branch `wi-2832-compaction-thrash-guard`. | fork-only; upstream has the same loop | ## External adapters diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 64e8786a15b4..fd533a9ec182 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -180,14 +180,17 @@ pub(crate) async fn run_turn( // new user message are recorded. Estimate pending incoming items (context // diffs/full reinjection + user input) and trigger compaction preemptively // when they would push the thread over the compaction threshold. - if let Err(err) = run_pre_sampling_compact( + let pre_sampling_compact = run_pre_sampling_compact( &sess, &turn_context, &mut client_session, &cancellation_token, ) - .await - { + .await; + // Set when a context-limit compaction ran and the next sampling request has not been + // checked against the limit yet (see `compaction_left_prompt_over_limit`). + let mut context_limit_compaction_pending_check = matches!(pre_sampling_compact, Ok(true)); + if let Err(err) = pre_sampling_compact { // Compaction runs before the new input is recorded, so preserve it on every failure. run_hooks_and_record_inputs( &sess, @@ -616,7 +619,56 @@ pub(crate) async fn run_turn( ) .await; - // as long as compaction works well in getting us way below the token limit, we shouldn't worry about being in an infinite loop. + // Compacting again cannot help when the first request after a compaction was + // already over the limit: the part compaction cannot shrink (tool specs, + // instructions, retained context) fills the window by itself. End the turn with + // an error instead of looping compaction -> one step -> compaction. + let compaction_pending_check = + std::mem::take(&mut context_limit_compaction_pending_check); + if should_roll_over + && token_limit_reached + && compaction_pending_check + && let Some(prompt_tokens) = compaction_left_prompt_over_limit( + sess.as_ref(), + turn_context.config.model_auto_compact_token_limit_scope, + &token_status, + ) + .await + { + let limit = token_status + .auto_compact_scope_limit + .or(token_status.full_context_window_limit) + .unwrap_or_default(); + warn!( + turn_id = %turn_context.sub_id, + prompt_tokens, + limit, + "auto-compaction left the prompt over the limit; not compacting again" + ); + sess.send_event( + &turn_context, + EventMsg::Warning(WarningEvent { + message: format!( + "Auto-compaction could not bring the context below the limit: the first request after compaction used {prompt_tokens} prompt tokens against a limit of {limit}. What compaction cannot shrink (tool definitions, instructions, retained context) is too large for this limit, so Codex stopped instead of compacting again. Reduce the enabled MCP servers or tools, or raise model_auto_compact_token_limit." + ), + }), + ) + .await; + let err = CodexErr::ContextWindowExceeded; + sess.emit_turn_error_lifecycle( + turn_context.as_ref(), + err.to_codex_protocol_error(), + err.details(), + ) + .await; + sess.send_event( + turn_context.as_ref(), + EventMsg::Error(err.to_error_event(/*message_prefix*/ None)), + ) + .await; + return Ok(None); + } + if should_roll_over { if let Err(err) = run_auto_compact( &sess, @@ -640,6 +692,7 @@ pub(crate) async fn run_turn( .await; return Ok(None); } + context_limit_compaction_pending_check = token_limit_reached; if run_pending_session_start_hooks(&sess, &turn_context).await { return Ok(None); } @@ -1305,7 +1358,7 @@ async fn run_pre_sampling_compact( turn_context: &Arc, client_session: &mut ModelClientSession, cancellation_token: &CancellationToken, -) -> CodexResult<()> { +) -> CodexResult { maybe_run_previous_model_inline_compact(sess, turn_context, client_session, cancellation_token) .await?; let token_status = @@ -1326,8 +1379,44 @@ async fn run_pre_sampling_compact( CompactionPhase::PreTurn, ) .await?; + return Ok(true); } - Ok(()) + Ok(false) +} + +/// Returns the prompt size of the last sampling request when that prompt alone already +/// reached the auto-compaction limit (or the usable context window). +/// +/// Called for the first request after a context-limit compaction: if its prompt is still +/// over the limit, another compaction would produce the same prompt again. +async fn compaction_left_prompt_over_limit( + sess: &Session, + scope: AutoCompactTokenLimitScope, + token_status: &super::context_window::ContextWindowTokenStatus, +) -> Option { + let prompt_tokens = sess.token_usage_info().await?.last_token_usage.input_tokens; + prompt_over_limit(scope, prompt_tokens, token_status).then_some(prompt_tokens) +} + +fn prompt_over_limit( + scope: AutoCompactTokenLimitScope, + prompt_tokens: i64, + token_status: &super::context_window::ContextWindowTokenStatus, +) -> bool { + let scope_prompt_tokens = match scope { + AutoCompactTokenLimitScope::Total => Some(prompt_tokens), + // The prefix does not count against this scope; compare only the body after it. + AutoCompactTokenLimitScope::BodyAfterPrefix => token_status + .auto_compact_window_prefill_tokens + .map(|prefill| prompt_tokens.saturating_sub(prefill)), + }; + let over_scope_limit = scope_prompt_tokens + .zip(token_status.auto_compact_scope_limit) + .is_some_and(|(tokens, limit)| tokens >= limit); + let over_context_window = token_status + .full_context_window_limit + .is_some_and(|limit| prompt_tokens >= limit); + over_scope_limit || over_context_window } /// Returns true only when both turns declare compaction compatibility hashes and they differ. diff --git a/codex-rs/core/src/session/turn_tests.rs b/codex-rs/core/src/session/turn_tests.rs index 85d8fac51138..9f832afa6bd7 100644 --- a/codex-rs/core/src/session/turn_tests.rs +++ b/codex-rs/core/src/session/turn_tests.rs @@ -111,3 +111,65 @@ fn realtime_user_verification_notice_excludes_request_payload() { )), ); } + +fn token_status_for_prompt_check( + scope_limit: Option, + prefill: Option, + context_window: Option, +) -> super::super::context_window::ContextWindowTokenStatus { + super::super::context_window::ContextWindowTokenStatus { + active_context_tokens: 0, + auto_compact_scope_tokens: 0, + auto_compact_scope_limit: scope_limit, + full_context_window_limit: context_window, + base_window_tokens_remaining: None, + auto_compact_window_prefill_tokens: prefill, + full_context_window_limit_reached: false, + token_limit_reached: true, + turn_end_compaction_threshold_reached: false, + } +} + +#[test] +fn prompt_over_limit_detects_a_prefix_that_compaction_cannot_shrink() { + // WI-2832: a 130.5k-token prompt after compaction against a 128k limit. + let status = token_status_for_prompt_check(Some(128_000), None, Some(258_400)); + assert!(prompt_over_limit( + AutoCompactTokenLimitScope::Total, + 130_516, + &status + )); + // A normal post-compaction prompt leaves room for the turn to make progress. + assert!(!prompt_over_limit( + AutoCompactTokenLimitScope::Total, + 43_000, + &status + )); +} + +#[test] +fn prompt_over_limit_ignores_the_prefix_for_body_after_prefix_scope() { + let status = token_status_for_prompt_check(Some(20_000), Some(110_000), Some(258_400)); + assert!(!prompt_over_limit( + AutoCompactTokenLimitScope::BodyAfterPrefix, + 125_000, + &status + )); + assert!(prompt_over_limit( + AutoCompactTokenLimitScope::BodyAfterPrefix, + 130_000, + &status + )); + // Without a recorded prefix the body cannot be measured; only the hard window applies. + let status = token_status_for_prompt_check(Some(20_000), None, Some(100_000)); + assert!(!prompt_over_limit( + AutoCompactTokenLimitScope::BodyAfterPrefix, + 90_000, + &status + )); + assert!(prompt_over_limit( + AutoCompactTokenLimitScope::BodyAfterPrefix, + 100_000, + &status + )); +} diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index b46bc107e979..b52fa8004103 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -4560,6 +4560,91 @@ async fn auto_compact_allows_multiple_attempts_when_interleaved_with_other_turn_ ); } +/// WI-2832: when the part of the prompt that compaction cannot shrink (tool specs, +/// instructions) is already over `model_auto_compact_token_limit`, every request after a +/// compaction is over the limit again. Codex used to loop compaction -> one tool call -> +/// compaction until the session was stopped (235 compactions in one turn). The turn must +/// stop after the first ineffective compaction instead. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auto_compact_stops_when_compaction_cannot_get_below_limit() { + skip_if_no_network!(); + + let server = start_mock_server().await; + + let over_limit_step = || { + sse(vec![ + ev_function_call(DUMMY_CALL_ID, DUMMY_FUNCTION_NAME, "{}"), + ev_completed_with_tokens("over-limit", /*total_tokens*/ 400), + ]) + }; + let summary_step = || { + sse(vec![ + ev_assistant_message("summary", &auto_summary(AUTO_SUMMARY_TEXT)), + ev_completed_with_tokens("summary", /*total_tokens*/ 50), + ]) + }; + // One step, one compaction, one step. The old loop compacted again here; the sequence + // has no fourth response, so a further request fails the turn with a different error. + let request_log = mount_sse_sequence( + &server, + vec![over_limit_step(), summary_step(), over_limit_step()], + ) + .await; + + let model_provider = non_openai_model_provider(&server); + let codex = test_codex() + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + config.model_auto_compact_token_limit = Some(300); + }) + .build(&server) + .await + .unwrap() + .codex; + + codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: FUNCTION_CALL_LIMIT_MSG.into(), + text_elements: Vec::new(), + }])) + .await + .unwrap(); + + let mut warnings = Vec::new(); + let mut errors = Vec::new(); + let turn_error = loop { + match wait_for_event(&codex, |_| true).await { + EventMsg::Warning(WarningEvent { message }) => warnings.push(message), + EventMsg::Error(error) => errors.push(error.codex_error_info), + EventMsg::TurnComplete(event) => { + break event.error.and_then(|error| error.codex_error_info); + } + _ => {} + } + }; + + assert_eq!(turn_error, Some(CodexErrorInfo::ContextWindowExceeded)); + assert_eq!(errors, vec![Some(CodexErrorInfo::ContextWindowExceeded)]); + assert!( + warnings.iter().any(|message| message + .contains("Auto-compaction could not bring the context below the limit") + && message.contains("400 prompt tokens against a limit of 300")), + "expected an ineffective-compaction warning, got {warnings:?}" + ); + + let requests = request_log.requests(); + assert_eq!( + requests.len(), + 3, + "expected one step, one compaction and one step, then a stop" + ); + assert!(body_contains_text( + &requests[1].body_json().to_string(), + SUMMARIZATION_PROMPT + )); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn paginated_compaction_cold_resume_from_bounded_suffix() -> Result<()> { skip_if_no_network!(Ok(()));