Ensure problemStatement sent to coding agent is always within the character limit (#7874)

* truncate everything

* remove integration id

* test:plus2

* user agent to both jobApi routes

Author: joshspicer

src/github/copilotApi.ts

@@ -68,6 +68,12 @@ export class CopilotApi {
 		return this.makeApiCallFullUrl(`${this.baseUrl}${api}`, init);
 	}
 
+	private get userAgent(): string {
+		const extensionVersion = vscode.extensions.getExtension('GitHub.vscode-pull-request-github')?.packageJSON.version ?? 'unknown';
+		return `vscode-pull-request-github/${extensionVersion}`;
+	}
+
+
 	async postRemoteAgentJob(
 		owner: string,
 		name: string,
@@ -86,10 +92,10 @@ export class CopilotApi {
 			const response = await this.makeApiCall(apiUrl, {
 				method: 'POST',
 				headers: {
-					'Copilot-Integration-Id': 'copilot-developer-dev',
 					'Authorization': `Bearer ${this.token}`,
 					'Content-Type': 'application/json',
-					'Accept': 'application/json'
+					'Accept': 'application/json',
+					'User-Agent': this.userAgent,
 				},
 				body: payloadJson
 			});
@@ -269,10 +275,10 @@ export class CopilotApi {
 			const response = await this.makeApiCall(`/agents/swe/v0/jobs/${owner}/${repo}/session/${sessionId}`, {
 				method: 'GET',
 				headers: {
-					'Copilot-Integration-Id': 'copilot-developer-dev',
 					'Authorization': `Bearer ${this.token}`,
 					'Content-Type': 'application/json',
-					'Accept': 'application/json'
+					'Accept': 'application/json',
+					'User-Agent': this.userAgent,
 				}
 			});
 			if (!response.ok) {

src/github/copilotRemoteAgent.ts

@@ -19,10 +19,11 @@ import { CODING_AGENT, CODING_AGENT_AUTO_COMMIT_AND_PUSH } from '../common/setti
 import { ITelemetry } from '../common/telemetry';
 import { toOpenPullRequestWebviewUri } from '../common/uri';
 import { copilotPRStatusToSessionStatus, IAPISessionLogs, ICopilotRemoteAgentCommandArgs, ICopilotRemoteAgentCommandResponse, OctokitCommon, RemoteAgentResult, RepoInfo } from './common';
-import { ChatSessionWithPR, CopilotApi, getCopilotApi, MAX_PROBLEM_STATEMENT_LENGTH, RemoteAgentJobPayload, SessionInfo, SessionSetupStep } from './copilotApi';
+import { ChatSessionWithPR, CopilotApi, getCopilotApi, RemoteAgentJobPayload, SessionInfo, SessionSetupStep } from './copilotApi';
 import { CodingAgentPRAndStatus, CopilotPRWatcher, CopilotStateModel } from './copilotPrWatcher';
 import { ChatSessionContentBuilder } from './copilotRemoteAgent/chatSessionContentBuilder';
 import { GitOperationsManager } from './copilotRemoteAgent/gitOperationsManager';
+import { extractTitle, formatBodyPlaceholder, truncatePrompt } from './copilotRemoteAgentUtils';
 import { CredentialStore } from './credentials';
 import { FolderRepositoryManager, ReposManagerState } from './folderRepositoryManager';
 import { GitHubRepository } from './githubRepository';
@@ -40,6 +41,8 @@ const PUSH_CHANGES = vscode.l10n.t('Include changes');
 const CONTINUE_WITHOUT_PUSHING = vscode.l10n.t('Ignore changes');
 const CONTINUE_AND_DO_NOT_ASK_AGAIN = vscode.l10n.t('Continue and don\'t ask again');
 
+const CONTINUE_TRUNCATION = vscode.l10n.t('Continue with truncation');
+
 const COPILOT = '@copilot';
 
 const body_suffix = vscode.l10n.t('Created from VS Code via the [GitHub Pull Request](https://marketplace.visualstudio.com/items?itemName=GitHub.vscode-pull-request-github) extension.');
@@ -709,33 +712,32 @@ export class CopilotRemoteAgentManager extends Disposable {
 			return { error: vscode.l10n.t('Failed to configure base branch \'{0}\' does not exist on the remote repository \'{1}/{2}\'. Please create the remote branch first.', base_ref, owner, repo), state: 'error' };
 		}
 
-		let title = prompt;
-		const titleMatch = problemContext?.match(/TITLE: \s*(.*)/i);
-		if (titleMatch && titleMatch[1]) {
-			title = titleMatch[1].trim();
-		}
+		const title = extractTitle(problemContext);
+		const { problemStatement, isTruncated } = truncatePrompt(prompt, problemContext);
 
-		const formatBodyPlaceholder = (problemContext: string): string => {
-			const header = vscode.l10n.t('Coding agent has begun work on **{0}** and will replace this description as work progresses.', title);
-			const collapsedContext = `<details><summary>${vscode.l10n.t('See problem context')}</summary>\n\n${problemContext}\n\n</details>`;
-			return `${header}\n\n${collapsedContext}`;
-		};
-
-		let isTruncated = false;
-		if (problemContext && (problemContext.length + prompt.length >= MAX_PROBLEM_STATEMENT_LENGTH)) {
-			isTruncated = true;
-			Logger.warn(`Truncating problemContext as it will cause us to exceed maximum problem_statement length (${MAX_PROBLEM_STATEMENT_LENGTH})`, CopilotRemoteAgentManager.ID);
-			const availableLength = MAX_PROBLEM_STATEMENT_LENGTH - prompt.length;
-			problemContext = problemContext.slice(-availableLength);
+		if (isTruncated) {
+			const truncationResult = await vscode.window.showWarningMessage(
+				vscode.l10n.t('Prompt size exceeded'), { modal: true, detail: vscode.l10n.t('Your prompt will be truncated to fit within coding agent\'s context window. This may affect the quality of the response.') }, CONTINUE_TRUNCATION);
+			const userCancelled = token?.isCancellationRequested || !truncationResult || truncationResult !== CONTINUE_TRUNCATION;
+			/* __GDPR__
+				"remoteAgent.truncation" : {
+					"isCancelled" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
+				}
+			*/
+			this.telemetry.sendTelemetryEvent('remoteAgent.truncation', {
+				isCancelled: String(userCancelled),
+			});
+			if (userCancelled) {
+				return { error: vscode.l10n.t('User cancelled due to truncation'), state: 'error' };
+			}
 		}
 
-		const problemStatement: string = `${prompt}\n${problemContext ?? ''}`;
 		const payload: RemoteAgentJobPayload = {
 			problem_statement: problemStatement,
 			event_type: 'visual_studio_code_remote_agent_tool_invoked',
 			pull_request: {
 				title,
-				body_placeholder: formatBodyPlaceholder(problemContext || prompt),
+				body_placeholder: formatBodyPlaceholder(title),
 				base_ref,
 				body_suffix,
 				...(head_ref && { head_ref })

src/github/copilotRemoteAgentUtils.ts

@@ -0,0 +1,56 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import Logger from '../common/logger';
+import { MAX_PROBLEM_STATEMENT_LENGTH } from './copilotApi';
+
+/**
+ * Truncation utility to ensure the problem statement sent to Copilot API is under the maximum length.
+ * Truncation is not ideal. The caller providing the prompt/context should be summarizing so this is a no-op whenever possible.
+ *
+ * @param prompt The final message submitted by the user
+ * @param context Any additional context collected by the caller (chat history, open files, etc...)
+ * @returns A complete 'problem statement' string that is under the maximum length, and a flag indicating if truncation occurred
+ */
+export function truncatePrompt(prompt: string, context?: string): { problemStatement: string; isTruncated: boolean } {
+	// Prioritize the userPrompt
+	// Take the last n characters that fit within the limit
+	if (prompt.length >= MAX_PROBLEM_STATEMENT_LENGTH) {
+		Logger.warn(`Truncation: Prompt length ${prompt.length} exceeds max of ${MAX_PROBLEM_STATEMENT_LENGTH}`);
+		prompt = prompt.slice(-MAX_PROBLEM_STATEMENT_LENGTH);
+		return { problemStatement: prompt, isTruncated: true };
+	}
+
+	if (context && (prompt.length + context.length >= MAX_PROBLEM_STATEMENT_LENGTH)) {
+		const availableLength = MAX_PROBLEM_STATEMENT_LENGTH - prompt.length - 2 /* new lines */;
+		Logger.warn(`Truncation: Combined prompt and context length ${prompt.length + context.length} exceeds max of ${MAX_PROBLEM_STATEMENT_LENGTH}`);
+		context = context.slice(-availableLength);
+		return {
+			problemStatement: prompt + (context ? `\n\n${context}` : ''),
+			isTruncated: true
+		};
+	}
+
+	// No truncation occurred
+	return {
+		problemStatement: prompt + (context ? `\n\n${context}` : ''),
+		isTruncated: false
+	};
+}
+
+export function extractTitle(context: string | undefined): string | undefined {
+	if (!context) {
+		return;
+	}
+	const titleMatch = context.match(/TITLE: \s*(.*)/i);
+	if (titleMatch && titleMatch[1]) {
+		return titleMatch[1].trim();
+	}
+}
+
+export function formatBodyPlaceholder(title: string | undefined): string {
+	return vscode.l10n.t('Coding agent has begun work on **{0}** and will update this pull request as work progresses.', title || vscode.l10n.t('your request'));
+}

src/test/github/copilotRemoteAgentUtils.test.ts

@@ -0,0 +1,106 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { default as assert } from 'assert';
+import { truncatePrompt, extractTitle, formatBodyPlaceholder } from '../../github/copilotRemoteAgentUtils';
+import { MAX_PROBLEM_STATEMENT_LENGTH } from '../../github/copilotApi';
+
+describe('copilotRemoteAgentUtils', () => {
+
+	describe('truncatePrompt', () => {
+		it('should return prompt and context unchanged when under limit', () => {
+			const prompt = 'This is a short prompt';
+			const context = 'This is some additional context';
+			const result = truncatePrompt(prompt, context);
+			assert.strictEqual(result.problemStatement, `${prompt}\n\n${context}`);
+			assert.strictEqual(result.isTruncated, false);
+		});
+
+		it('should return only prompt when no context provided and under limit', () => {
+			const prompt = 'This is a short prompt';
+			const result = truncatePrompt(prompt);
+			assert.strictEqual(result.problemStatement, 'This is a short prompt');
+			assert.strictEqual(result.isTruncated, false);
+		});
+
+		it('should truncate prompt when it exceeds the maximum length', () => {
+			const longPrompt = 'a'.repeat(MAX_PROBLEM_STATEMENT_LENGTH + 100);
+			const result = truncatePrompt(longPrompt);
+			assert.strictEqual(result.problemStatement.length, MAX_PROBLEM_STATEMENT_LENGTH);
+			assert.strictEqual(result.isTruncated, true);
+		});
+
+		it('should truncate context when combined length exceeds limit', () => {
+			const prompt = 'Short prompt';
+			const longContext = 'b'.repeat(MAX_PROBLEM_STATEMENT_LENGTH);
+
+			const result = truncatePrompt(prompt, longContext);
+
+			assert.strictEqual(result.isTruncated, true);
+			assert(result.problemStatement.startsWith(prompt));
+			assert(result.problemStatement.includes('\n\n'));
+			const expectedAvailableLength = MAX_PROBLEM_STATEMENT_LENGTH - prompt.length;
+			const expectedContext = longContext.slice(-expectedAvailableLength + 2);
+			assert.strictEqual(result.problemStatement, `${prompt}\n\n${expectedContext}`);
+		});
+
+		it('long prompts are prioritized when truncating', () => {
+			const longPrompt = 'a'.repeat(MAX_PROBLEM_STATEMENT_LENGTH + 100);
+			const context = 'B';
+
+			const result = truncatePrompt(longPrompt, context);
+
+			assert.strictEqual(result.isTruncated, true);
+			assert.strictEqual(result.problemStatement.length, MAX_PROBLEM_STATEMENT_LENGTH);
+			assert(!result.problemStatement.includes(context));
+		});
+	});
+
+	describe('extractTitle', () => {
+		it('should extract title from context with TITLE prefix', () => {
+			const context = 'Some initial text\nTITLE: Fix authentication bug\nSome other content';
+
+			const result = extractTitle(context);
+
+			assert.strictEqual(result, 'Fix authentication bug');
+		});
+
+		it('should extract title with case insensitive matching', () => {
+			const context = 'Some text\ntitle: Add new feature\nMore text';
+
+			const result = extractTitle(context);
+
+			assert.strictEqual(result, 'Add new feature');
+		});
+
+		it('should extract title with extra whitespace', () => {
+			const context = 'TITLE:   Refactor code structure   \n';
+
+			const result = extractTitle(context);
+
+			assert.strictEqual(result, 'Refactor code structure');
+		});
+
+		it('should return undefined when no title is found', () => {
+			const context = 'Some text without any title marker\nJust regular content';
+
+			const result = extractTitle(context);
+
+			assert.strictEqual(result, undefined);
+		});
+
+		it('should return undefined when context is undefined', () => {
+			const result = extractTitle(undefined);
+
+			assert.strictEqual(result, undefined);
+		});
+
+		it('should return undefined when context is empty string', () => {
+			const result = extractTitle('');
+
+			assert.strictEqual(result, undefined);
+		});
+	});
+});