Skip to content

Commit 0f5c833

Browse files
rajbosCopilot
rajbos
and
Copilot
committed
feat: improve fluency spiderweb for Claude-only users
- workspaceHelpers: fix isMcpTool() to detect mcp__ (Claude Code) prefix - workspaceHelpers: fix extractMcpServerName() to parse mcp__server__tool format - claudeCodeAdapter: add extractClaudeSlashCommand() + CLAUDE_SLASH_ALLOWLIST Slash commands (/review, /bug, /think, /compact, /pr_comments) are stored as __slash__<cmd> in toolCalls.byTool without incrementing toolCalls.total - claudeCodeAdapter: route mcp__ tools to mcpTools instead of toolCalls - claudeDesktopAdapter: detect slash commands using shared extractClaudeSlashCommand - maturityScoring: recognize __slash__* entries as Prompt Engineering evidence (combined with VS Code slash commands in both personal + team scorers) - maturityScoring: exclude __slash__* from nonAutoToolCount to avoid inflating Tool Usage / Agentic scores for Claude users - maturityScoring: update customization Stage 1 tips + evidence to mention CLAUDE.md - cli/helpers: export buildCustomizationMatrix (extracted from fluency.ts + all.ts) - detects Claude Code workspaces via cwd field in JSONL events - checks for CLAUDE.md in addition to copilot-instructions.md and agents.md - cli/helpers: fix ClaudeDesktopAdapter constructor (was missing 3 required args) - cli/commands/fluency.ts + all.ts: use shared buildCustomizationMatrix from helpers - tests: add mcp__ prefix tests in workspaceHelpers.test.ts - tests: add extractClaudeSlashCommand tests in ecosystemAdapters.test.ts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 3a8a3fc commit 0f5c833

9 files changed

Lines changed: 239 additions & 131 deletions

File tree

‎cli/src/commands/all.ts‎

Lines changed: 1 addition & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -3,72 +3,16 @@
33
* Used by the Visual Studio extension to load every view in one CLI call
44
* instead of spawning four separate processes.
55
*/
6-
import * as fs from 'fs';
7-
import * as path from 'path';
86
import { Command } from 'commander';
97
import {
108
discoverSessionFiles,
119
calculateDetailedStats,
1210
calculateDailyStats,
1311
buildChartPayload,
1412
calculateUsageAnalysisStats,
13+
buildCustomizationMatrix,
1514
} from '../helpers';
1615
import { calculateMaturityScores } from '../../../vscode-extension/src/maturityScoring';
17-
import type { WorkspaceCustomizationMatrix } from '../../../vscode-extension/src/types';
18-
19-
/**
20-
* Builds a WorkspaceCustomizationMatrix by deriving workspace folder paths from
21-
* VS Code-style session file paths (workspaceStorage/<hash>/chatSessions/<file>),
22-
* then checking each workspace for .github/copilot-instructions.md or agents.md.
23-
*
24-
* Non-VS Code session files (Crush, OpenCode, Copilot CLI, Visual Studio) are skipped.
25-
*/
26-
async function buildCustomizationMatrix(sessionFiles: string[]): Promise<WorkspaceCustomizationMatrix | undefined> {
27-
const workspacePaths = new Set<string>();
28-
29-
for (const sessionFile of sessionFiles) {
30-
// Expected structure: .../workspaceStorage/<hash>/chatSessions/<file>
31-
const chatSessionsDir = path.dirname(sessionFile);
32-
if (path.basename(chatSessionsDir) !== 'chatSessions') { continue; }
33-
const hashDir = path.dirname(chatSessionsDir);
34-
const workspaceJsonPath = path.join(hashDir, 'workspace.json');
35-
36-
try {
37-
if (!fs.existsSync(workspaceJsonPath)) { continue; }
38-
const content = JSON.parse(await fs.promises.readFile(workspaceJsonPath, 'utf-8'));
39-
const folderUri: string | undefined = content.folder;
40-
if (!folderUri || !folderUri.startsWith('file://')) { continue; }
41-
42-
// Convert file URI to a local path, handling Windows drive letters
43-
let folderPath = decodeURIComponent(folderUri.replace(/^file:\/\//, ''));
44-
// On Windows, file:///C:/... becomes /C:/... — strip the leading slash
45-
if (/^\/[A-Za-z]:/.test(folderPath)) { folderPath = folderPath.slice(1); }
46-
workspacePaths.add(folderPath);
47-
} catch {
48-
// Skip unreadable workspace.json files
49-
}
50-
}
51-
52-
if (workspacePaths.size === 0) { return undefined; }
53-
54-
let workspacesWithIssues = 0;
55-
for (const wsPath of workspacePaths) {
56-
try {
57-
const hasInstructions = fs.existsSync(path.join(wsPath, '.github', 'copilot-instructions.md'));
58-
const hasAgentsMd = fs.existsSync(path.join(wsPath, 'agents.md'));
59-
if (!hasInstructions && !hasAgentsMd) { workspacesWithIssues++; }
60-
} catch {
61-
workspacesWithIssues++; // Count inaccessible workspaces as lacking customization
62-
}
63-
}
64-
65-
return {
66-
customizationTypes: [],
67-
workspaces: [],
68-
totalWorkspaces: workspacePaths.size,
69-
workspacesWithIssues,
70-
};
71-
}
7216

7317
export const allCommand = new Command('all')
7418
.description('Output all view data in a single JSON response (for Visual Studio extension)')

‎cli/src/commands/fluency.ts‎

Lines changed: 1 addition & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,10 @@
11
/**
22
* `fluency` command - Show Copilot Fluency Score based on usage patterns.
33
*/
4-
import * as fs from 'fs';
5-
import * as path from 'path';
64
import { Command } from 'commander';
75
import chalk from 'chalk';
8-
import { discoverSessionFiles, calculateUsageAnalysisStats, fmt } from '../helpers';
6+
import { discoverSessionFiles, calculateUsageAnalysisStats, fmt, buildCustomizationMatrix } from '../helpers';
97
import { calculateMaturityScores } from '../../../vscode-extension/src/maturityScoring';
10-
import type { WorkspaceCustomizationMatrix } from '../../../vscode-extension/src/types';
11-
12-
/**
13-
* Builds a WorkspaceCustomizationMatrix by deriving workspace folder paths from
14-
* VS Code-style session file paths (workspaceStorage/<hash>/chatSessions/<file>),
15-
* then checking each workspace for .github/copilot-instructions.md or agents.md.
16-
*
17-
* Non-VS Code session files (Crush, OpenCode, Copilot CLI, Visual Studio) are skipped.
18-
*/
19-
async function buildCustomizationMatrix(sessionFiles: string[]): Promise<WorkspaceCustomizationMatrix | undefined> {
20-
const workspacePaths = new Set<string>();
21-
22-
for (const sessionFile of sessionFiles) {
23-
// Expected structure: .../workspaceStorage/<hash>/chatSessions/<file>
24-
// Go up 2 levels to reach the hash directory, then read workspace.json
25-
const chatSessionsDir = path.dirname(sessionFile);
26-
if (path.basename(chatSessionsDir) !== 'chatSessions') { continue; }
27-
const hashDir = path.dirname(chatSessionsDir);
28-
const workspaceJsonPath = path.join(hashDir, 'workspace.json');
29-
30-
try {
31-
if (!fs.existsSync(workspaceJsonPath)) { continue; }
32-
const content = JSON.parse(await fs.promises.readFile(workspaceJsonPath, 'utf-8'));
33-
const folderUri: string | undefined = content.folder;
34-
if (!folderUri || !folderUri.startsWith('file://')) { continue; }
35-
36-
// Convert file URI to a local path, handling Windows drive letters
37-
let folderPath = decodeURIComponent(folderUri.replace(/^file:\/\//, ''));
38-
// On Windows, file:///C:/... becomes /C:/... — strip the leading slash
39-
if (/^\/[A-Za-z]:/.test(folderPath)) { folderPath = folderPath.slice(1); }
40-
workspacePaths.add(folderPath);
41-
} catch {
42-
// Skip unreadable workspace.json files
43-
}
44-
}
45-
46-
if (workspacePaths.size === 0) { return undefined; }
47-
48-
let workspacesWithIssues = 0;
49-
for (const wsPath of workspacePaths) {
50-
try {
51-
const hasInstructions = fs.existsSync(path.join(wsPath, '.github', 'copilot-instructions.md'));
52-
const hasAgentsMd = fs.existsSync(path.join(wsPath, 'agents.md'));
53-
if (!hasInstructions && !hasAgentsMd) { workspacesWithIssues++; }
54-
} catch {
55-
workspacesWithIssues++; // Count inaccessible workspaces as lacking customization
56-
}
57-
}
58-
59-
return {
60-
customizationTypes: [],
61-
workspaces: [],
62-
totalWorkspaces: workspacePaths.size,
63-
workspacesWithIssues,
64-
};
65-
}
668

679
export const fluencyCommand = new Command('fluency')
6810
.description('Show your Copilot Fluency Score and improvement tips')

‎cli/src/helpers.ts‎

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,10 @@ import { ClaudeDesktopCoworkDataAccess } from '../../vscode-extension/src/claude
1616
import { MistralVibeDataAccess } from '../../vscode-extension/src/mistralvibe';
1717
import type { IEcosystemAdapter } from '../../vscode-extension/src/ecosystemAdapter';
1818
import { OpenCodeAdapter, CrushAdapter, ContinueAdapter, ClaudeDesktopAdapter, ClaudeCodeAdapter, VisualStudioAdapter, MistralVibeAdapter, CopilotChatAdapter, CopilotCliAdapter } from '../../vscode-extension/src/adapters';
19+
import { isMcpTool, extractMcpServerName } from '../../vscode-extension/src/workspaceHelpers';
1920
import { parseSessionFileContent } from '../../vscode-extension/src/sessionParser';
2021
import { estimateTokensFromText, getModelFromRequest, isJsonlContent, estimateTokensFromJsonlSession, calculateEstimatedCost, getModelTier } from '../../vscode-extension/src/tokenEstimation';
21-
import type { DetailedStats, PeriodStats, ModelUsage, EditorUsage, SessionFileCache, UsageAnalysisStats, UsageAnalysisPeriod } from '../../vscode-extension/src/types';
22+
import type { DetailedStats, PeriodStats, ModelUsage, EditorUsage, SessionFileCache, UsageAnalysisStats, UsageAnalysisPeriod, WorkspaceCustomizationMatrix } from '../../vscode-extension/src/types';
2223
import { analyzeSessionUsage, mergeUsageAnalysis, calculateModelSwitching, trackEnhancedMetrics } from '../../vscode-extension/src/usageAnalysis';
2324
import { createEmptyContextRefs } from '../../vscode-extension/src/tokenEstimation';
2425
import * as vscodeStub from './vscode-stub';
@@ -95,7 +96,12 @@ const _ecosystems: IEcosystemAdapter[] = [
9596
new CrushAdapter(_crushInstance),
9697
new VisualStudioAdapter(_visualStudioInstance, (t, m) => estimateTokensFromText(t, m ?? 'gpt-4', tokenEstimators)),
9798
new ContinueAdapter(_continueInstance),
98-
new ClaudeDesktopAdapter(_claudeDesktopCoworkInstance),
99+
new ClaudeDesktopAdapter(
100+
_claudeDesktopCoworkInstance,
101+
isMcpTool,
102+
extractMcpServerName,
103+
(t, m) => estimateTokensFromText(t, m ?? 'gpt-4', tokenEstimators)
104+
),
99105
new ClaudeCodeAdapter(_claudeCodeInstance),
100106
new MistralVibeAdapter(_mistralVibeInstance),
101107
// Copilot Chat / CLI adapters: discovery-only. Their handles() returns
@@ -116,6 +122,79 @@ export async function discoverSessionFiles(): Promise<string[]> {
116122
return discovery.getCopilotSessionFiles();
117123
}
118124

125+
/**
126+
* Builds a WorkspaceCustomizationMatrix from session file paths.
127+
*
128+
* - For VS Code sessions: derives workspace folder from workspaceStorage/<hash>/workspace.json,
129+
* then checks for .github/copilot-instructions.md, agents.md, or CLAUDE.md.
130+
* - For Claude Code sessions (~/.claude/projects/<hash>/): reads the JSONL to extract the
131+
* `cwd` workspace path, then checks for CLAUDE.md there.
132+
*/
133+
export async function buildCustomizationMatrix(sessionFiles: string[]): Promise<WorkspaceCustomizationMatrix | undefined> {
134+
const workspacePaths = new Set<string>();
135+
const claudeBasePath = path.join(os.homedir(), '.claude', 'projects');
136+
137+
for (const sessionFile of sessionFiles) {
138+
// Claude Code session: ~/.claude/projects/<hash>/<uuid>.jsonl
139+
if (sessionFile.startsWith(claudeBasePath + path.sep) || sessionFile.startsWith(claudeBasePath + '/')) {
140+
try {
141+
const content = await fs.promises.readFile(sessionFile, 'utf-8');
142+
const lines = content.split('\n').slice(0, 30);
143+
for (const line of lines) {
144+
if (!line.trim()) { continue; }
145+
try {
146+
const event = JSON.parse(line);
147+
if (event.cwd && typeof event.cwd === 'string') {
148+
workspacePaths.add(event.cwd);
149+
break;
150+
}
151+
} catch { /* skip malformed lines */ }
152+
}
153+
} catch { /* skip unreadable files */ }
154+
continue;
155+
}
156+
157+
// VS Code session: .../workspaceStorage/<hash>/chatSessions/<file>
158+
const chatSessionsDir = path.dirname(sessionFile);
159+
if (path.basename(chatSessionsDir) !== 'chatSessions') { continue; }
160+
const hashDir = path.dirname(chatSessionsDir);
161+
const workspaceJsonPath = path.join(hashDir, 'workspace.json');
162+
163+
try {
164+
if (!fs.existsSync(workspaceJsonPath)) { continue; }
165+
const content = JSON.parse(await fs.promises.readFile(workspaceJsonPath, 'utf-8'));
166+
const folderUri: string | undefined = content.folder;
167+
if (!folderUri || !folderUri.startsWith('file://')) { continue; }
168+
169+
let folderPath = decodeURIComponent(folderUri.replace(/^file:\/\//, ''));
170+
// On Windows, file:///C:/... becomes /C:/... — strip the leading slash
171+
if (/^\/[A-Za-z]:/.test(folderPath)) { folderPath = folderPath.slice(1); }
172+
workspacePaths.add(folderPath);
173+
} catch { /* skip unreadable workspace.json files */ }
174+
}
175+
176+
if (workspacePaths.size === 0) { return undefined; }
177+
178+
let workspacesWithIssues = 0;
179+
for (const wsPath of workspacePaths) {
180+
try {
181+
const hasInstructions = fs.existsSync(path.join(wsPath, '.github', 'copilot-instructions.md'));
182+
const hasAgentsMd = fs.existsSync(path.join(wsPath, 'agents.md'));
183+
const hasClaudeMd = fs.existsSync(path.join(wsPath, 'CLAUDE.md'));
184+
if (!hasInstructions && !hasAgentsMd && !hasClaudeMd) { workspacesWithIssues++; }
185+
} catch {
186+
workspacesWithIssues++;
187+
}
188+
}
189+
190+
return {
191+
customizationTypes: [],
192+
workspaces: [],
193+
totalWorkspaces: workspacePaths.size,
194+
workspacesWithIssues,
195+
};
196+
}
197+
119198
/** Get diagnostic candidate paths info */
120199
export function getDiagnosticPaths(): { path: string; exists: boolean; source: string }[] {
121200
const discovery = createSessionDiscovery();

‎vscode-extension/src/adapters/claudeCodeAdapter.ts‎

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,39 @@ import type { ModelUsage } from '../types';
33
import type { IEcosystemAdapter, IDiscoverableEcosystem, IAnalyzableEcosystem, DiscoveryResult, CandidatePath, UsageAnalysisAdapterContext } from '../ecosystemAdapter';
44
import { ClaudeCodeDataAccess, normalizeClaudeModelId } from '../claudecode';
55
import { readClaudeCodeEventsForAnalysis, createEmptySessionUsageAnalysis, applyModelTierClassification } from '../usageAnalysis';
6+
import { isMcpTool, extractMcpServerName } from '../workspaceHelpers';
7+
8+
/**
9+
* Claude Code slash commands that map to Prompt Engineering fluency.
10+
* These are stored in toolCalls.byTool with a __slash__ prefix so they
11+
* are tracked without inflating tool call counts or agentic metrics.
12+
*/
13+
const CLAUDE_SLASH_ALLOWLIST = new Set(['review', 'bug', 'think', 'compact', 'pr_comments']);
14+
15+
/**
16+
* Extract a Claude slash command from the first non-empty text line of a user message.
17+
* Returns the command name (without leading '/') if found and in the allowlist, else null.
18+
* Only matches at the very start of the message to avoid false-positives from pasted code.
19+
* Exported for reuse by other Claude-family adapters.
20+
*/
21+
export function extractClaudeSlashCommand(content: unknown): string | null {
22+
let text = '';
23+
if (typeof content === 'string') {
24+
text = content;
25+
} else if (Array.isArray(content)) {
26+
for (const block of content) {
27+
if (block?.type === 'text' && typeof block.text === 'string') {
28+
text = block.text;
29+
break;
30+
}
31+
}
32+
}
33+
const firstLine = text.trimStart().split('\n')[0].trim();
34+
const m = firstLine.match(/^\/([a-z_]+)(?:\s|$)/i);
35+
if (!m) { return null; }
36+
const cmd = m[1].toLowerCase();
37+
return CLAUDE_SLASH_ALLOWLIST.has(cmd) ? cmd : null;
38+
}
639

740
export class ClaudeCodeAdapter implements IEcosystemAdapter, IDiscoverableEcosystem, IAnalyzableEcosystem {
841
readonly id = 'claudecode';
@@ -75,15 +108,30 @@ export class ClaudeCodeAdapter implements IEcosystemAdapter, IDiscoverableEcosys
75108
for (const event of events) {
76109
if (event.type === 'user' && event.message?.role === 'user' && !event.isSidechain) {
77110
analysis.modeUsage.cli++;
111+
// Detect Claude Code slash commands from the first line of user messages
112+
const cmd = extractClaudeSlashCommand(event.message?.content);
113+
if (cmd) {
114+
const key = `__slash__${cmd}`;
115+
analysis.toolCalls.byTool[key] = (analysis.toolCalls.byTool[key] || 0) + 1;
116+
// Note: do NOT increment analysis.toolCalls.total — slash commands are not tool calls
117+
}
78118
} else if (event.type === 'assistant') {
79119
const model = normalizeClaudeModelId(event.message?.model || 'unknown');
80120
models.push(model);
81121
const content: any[] = Array.isArray(event.message?.content) ? event.message.content : [];
82122
for (const c of content) {
83123
if (c?.type === 'tool_use') {
84-
analysis.toolCalls.total++;
85124
const toolName = String(c.name || 'tool');
86-
analysis.toolCalls.byTool[toolName] = (analysis.toolCalls.byTool[toolName] || 0) + 1;
125+
if (isMcpTool(toolName)) {
126+
// Route MCP tools to mcpTools tracking
127+
const server = extractMcpServerName(toolName, ctx.toolNameMap);
128+
analysis.mcpTools.total++;
129+
analysis.mcpTools.byServer[server] = (analysis.mcpTools.byServer[server] || 0) + 1;
130+
analysis.mcpTools.byTool[toolName] = (analysis.mcpTools.byTool[toolName] || 0) + 1;
131+
} else {
132+
analysis.toolCalls.total++;
133+
analysis.toolCalls.byTool[toolName] = (analysis.toolCalls.byTool[toolName] || 0) + 1;
134+
}
87135
}
88136
}
89137
}

‎vscode-extension/src/adapters/claudeDesktopAdapter.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { ClaudeDesktopCoworkDataAccess } from '../claudedesktop';
55
import { createEmptyContextRefs } from '../tokenEstimation';
66
import { readClaudeCodeEventsForAnalysis, createEmptySessionUsageAnalysis, applyModelTierClassification } from '../usageAnalysis';
77
import { normalizeClaudeModelId } from '../claudecode';
8+
import { extractClaudeSlashCommand } from './claudeCodeAdapter';
89

910
export class ClaudeDesktopAdapter implements IEcosystemAdapter, IDiscoverableEcosystem, IAnalyzableEcosystem {
1011
readonly id = 'claudedesktop';
@@ -170,6 +171,13 @@ export class ClaudeDesktopAdapter implements IEcosystemAdapter, IDiscoverableEco
170171
for (const event of events) {
171172
if (event.type === 'user' && event.message?.role === 'user' && !event.isSidechain) {
172173
analysis.modeUsage.ask++;
174+
// Detect Claude slash commands from the first line of user messages
175+
const cmd = extractClaudeSlashCommand(event.message?.content);
176+
if (cmd) {
177+
const key = `__slash__${cmd}`;
178+
analysis.toolCalls.byTool[key] = (analysis.toolCalls.byTool[key] || 0) + 1;
179+
// Note: do NOT increment analysis.toolCalls.total — slash commands are not tool calls
180+
}
173181
} else if (event.type === 'assistant') {
174182
const model = normalizeClaudeModelId(event.message?.model || 'unknown');
175183
models.push(model);

0 commit comments

Comments
 (0)