From fbb092522c460f95c60abfffb6e8f174f3a05e79 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 05:21:16 +0700 Subject: [PATCH] feat: add opt-in PromptSyntax study analytics --- Cargo.lock | 14 +- Cargo.toml | 2 +- apps/gui/frontend/src/api/client.ts | 8 + apps/gui/frontend/src/api/fixtures.ts | 1 + apps/gui/frontend/src/api/index.ts | 3 + apps/gui/frontend/src/api/mock.test.ts | 19 + apps/gui/frontend/src/api/mock.ts | 31 + apps/gui/frontend/src/api/tauri.ts | 3 + .../frontend/src/features/draft/DraftTab.tsx | 2 +- .../frontend/src/features/home/HomeTab.tsx | 13 +- .../src/features/project/Composer.test.tsx | 34 +- .../src/features/project/Composer.tsx | 14 +- .../src/features/project/ProjectTab.tsx | 2 +- .../features/settings/SettingsTab.test.tsx | 69 ++ .../src/features/settings/SettingsTab.tsx | 156 +++++ apps/gui/frontend/src/lib/promptEditor.ts | 5 +- apps/gui/frontend/src/stores/workspace.tsx | 52 +- apps/gui/frontend/src/types/index.ts | 27 + apps/gui/src/db/fingerprint.rs | 1 + apps/gui/src/db/schema/mod.rs | 1 + apps/gui/src/db/schema/study_event.rs | 51 ++ apps/gui/src/db/tables.rs | 73 ++ apps/gui/src/directives.rs | 23 + apps/gui/src/main.rs | 32 +- apps/gui/src/projects.rs | 477 +++++++++++++- apps/gui/src/prs.rs | 31 +- apps/gui/src/settings.rs | 22 + apps/gui/src/study.rs | 623 ++++++++++++++++++ 28 files changed, 1726 insertions(+), 63 deletions(-) create mode 100644 apps/gui/frontend/src/features/settings/SettingsTab.test.tsx create mode 100644 apps/gui/src/db/schema/study_event.rs create mode 100644 apps/gui/src/study.rs diff --git a/Cargo.lock b/Cargo.lock index 2540633..8185ef1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,25 +156,25 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "az-agent" -version = "0.1.59" +version = "0.1.60" dependencies = [ "az-core", ] [[package]] name = "az-agent-proxy" -version = "0.1.59" +version = "0.1.60" dependencies = [ "az-core", ] [[package]] name = "az-core" -version = "0.1.59" +version = "0.1.60" [[package]] name = "az-gui" -version = "0.1.59" +version = "0.1.60" dependencies = [ "agent-abstraction", "agent-experimental", @@ -199,7 +199,7 @@ dependencies = [ [[package]] name = "az-mcp-proxy" -version = "0.1.59" +version = "0.1.60" dependencies = [ "az-core", ] @@ -5587,7 +5587,7 @@ dependencies = [ [[package]] name = "wt-migrate" -version = "0.1.59" +version = "0.1.60" dependencies = [ "derive_more 2.1.1", "eyre", @@ -5601,7 +5601,7 @@ dependencies = [ [[package]] name = "wt-tools" -version = "0.1.59" +version = "0.1.60" dependencies = [ "derive_more 2.1.1", "dirs", diff --git a/Cargo.toml b/Cargo.toml index 9914ac5..2f43700 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ members = [ ] [workspace.package] -version = "0.1.59" +version = "0.1.60" edition = "2024" publish = false diff --git a/apps/gui/frontend/src/api/client.ts b/apps/gui/frontend/src/api/client.ts index 305e5a7..60ee35e 100644 --- a/apps/gui/frontend/src/api/client.ts +++ b/apps/gui/frontend/src/api/client.ts @@ -20,6 +20,8 @@ import type { QuotaReport, RateLimit, RunningTask, + StudySummary, + StudyTurnMetadata, TableSize, TaskLogEntry, TaskManagerState, @@ -58,6 +60,7 @@ export interface AgencyZeroApi { permission?: Permission; /** Reasoning effort, as `Request::effort`. Absent means the CLI's default. */ effort?: string; + study?: StudyTurnMetadata; }): Promise; deleteProject(id: string): Promise; /** Stage 3 of the naming design: a manual rename outranks both derived stages. */ @@ -103,6 +106,7 @@ export interface AgencyZeroApi { permission?: Permission; /** Reasoning effort, as `Request::effort`. Absent means the CLI's default. */ effort?: string; + study?: StudyTurnMetadata; }): Promise; /** Approve once / Deny on a moderator hold. */ resolveModeration(messageId: string, approve: boolean): Promise; @@ -110,6 +114,10 @@ export interface AgencyZeroApi { // — Settings ———————————————————————————————————————————————— getSettings(): Promise; setSettings(patch: DeepPartial): Promise; + getStudySummary(): Promise; + /** Native save picker; `null` means it was cancelled. */ + exportStudyEvents(): Promise; + clearStudyEvents(): Promise; /** Experimental profile only. Fetches usage through Claude Code's managed login. */ claudeUsage(): Promise; /** Probes the installed CLIs. `recheck` forces a fresh probe. */ diff --git a/apps/gui/frontend/src/api/fixtures.ts b/apps/gui/frontend/src/api/fixtures.ts index d28315e..554db45 100644 --- a/apps/gui/frontend/src/api/fixtures.ts +++ b/apps/gui/frontend/src/api/fixtures.ts @@ -586,6 +586,7 @@ export const SETTINGS: GlobalSettings = { forwardProxyVars: false, completedItems: "resolve", theme: { accent: "", softness: 0, wash: 10, textBrightness: 0 }, + studyAnalytics: { enabled: false, sessionId: "", enabledAt: "" }, notifications: { onHold: true, onRunFinished: true, diff --git a/apps/gui/frontend/src/api/index.ts b/apps/gui/frontend/src/api/index.ts index 44aecf6..8c71896 100644 --- a/apps/gui/frontend/src/api/index.ts +++ b/apps/gui/frontend/src/api/index.ts @@ -41,6 +41,9 @@ const COMMAND_FOR: Partial> = { resolveModeration: "resolve_moderation", getSettings: "get_settings", setSettings: "set_settings", + getStudySummary: "get_study_summary", + exportStudyEvents: "export_study_events", + clearStudyEvents: "clear_study_events", claudeUsage: "claude_usage", listAgentStatus: "list_agent_status", listModels: "list_models", diff --git a/apps/gui/frontend/src/api/mock.test.ts b/apps/gui/frontend/src/api/mock.test.ts index 308fc0e..127a66d 100644 --- a/apps/gui/frontend/src/api/mock.test.ts +++ b/apps/gui/frontend/src/api/mock.test.ts @@ -199,6 +199,25 @@ describe("settings", () => { const after = await api.listAgentStatus(true); expect(Date.parse(after[0].checkedAt)).toBeGreaterThanOrEqual(Date.parse(before[0].checkedAt)); }); + + it("keeps study collection off until an explicit transition", async () => { + const before = await api.getStudySummary(); + expect(before).toMatchObject({ enabled: false, eventCount: 0, studyId: null }); + + const settings = await api.setSettings({ studyAnalytics: { enabled: true } }); + const started = await api.getStudySummary(); + + expect(settings.studyAnalytics.sessionId).toMatch(/^study-/); + expect(started).toMatchObject({ enabled: true, eventCount: 1 }); + }); + + it("deletes study rows without silently changing consent", async () => { + await api.setSettings({ studyAnalytics: { enabled: true } }); + await api.setSettings({ studyAnalytics: { enabled: false } }); + await api.clearStudyEvents(); + + expect(await api.getStudySummary()).toMatchObject({ enabled: false, eventCount: 0 }); + }); }); describe("rate limits", () => { diff --git a/apps/gui/frontend/src/api/mock.ts b/apps/gui/frontend/src/api/mock.ts index 56d9f8c..9066413 100644 --- a/apps/gui/frontend/src/api/mock.ts +++ b/apps/gui/frontend/src/api/mock.ts @@ -11,6 +11,7 @@ import type { ProjectStatus, QuotaReport, RunningTask, + StudySummary, TaskLogEntry, } from "~/types"; import { @@ -83,6 +84,7 @@ export function createMockApi(): AgencyZeroApi { const agentStatus = clone(fixtures.AGENT_STATUS); const models = clone(fixtures.MODEL_CATALOGUE); let settings = clone(fixtures.SETTINGS); + let studyEventCount = 0; const pullRequests = clone(fixtures.PULL_REQUESTS); /* * What a project's agent kept across a compaction, per project. @@ -394,10 +396,39 @@ export function createMockApi(): AgencyZeroApi { getSettings: () => settle(settings), async setSettings(patch): Promise { + const wasEnabled = settings.studyAnalytics.enabled; settings = deepMerge(settings, patch); + if (!wasEnabled && settings.studyAnalytics.enabled) { + settings.studyAnalytics.sessionId = nextId("study"); + settings.studyAnalytics.enabledAt = new Date().toISOString(); + studyEventCount += 1; + } else if (wasEnabled && !settings.studyAnalytics.enabled) { + studyEventCount += 1; + } return settle(settings); }, + getStudySummary: () => + settle({ + enabled: settings.studyAnalytics.enabled, + studyId: settings.studyAnalytics.sessionId || null, + enabledAt: settings.studyAnalytics.enabledAt || null, + eventCount: studyEventCount, + firstAt: settings.studyAnalytics.enabledAt || null, + lastAt: settings.studyAnalytics.enabledAt || null, + } satisfies StudySummary), + + // The fixture has no filesystem or durable study table. Cancelling is the + // honest save result rather than inventing a file that does not exist. + exportStudyEvents: () => settle(null), + clearStudyEvents: () => { + if (settings.studyAnalytics.enabled) { + return Promise.reject(new Error("stop study collection before deleting its stored events")); + } + studyEventCount = 0; + return settle(undefined); + }, + claudeUsage: () => settle({ fiveHour: { diff --git a/apps/gui/frontend/src/api/tauri.ts b/apps/gui/frontend/src/api/tauri.ts index 6741c5f..cdfd94b 100644 --- a/apps/gui/frontend/src/api/tauri.ts +++ b/apps/gui/frontend/src/api/tauri.ts @@ -70,6 +70,9 @@ export function createTauriApi(): AgencyZeroApi { getSettings: () => call("get_settings"), setSettings: (patch) => call("set_settings", { patch }), + getStudySummary: () => call("get_study_summary"), + exportStudyEvents: () => call("export_study_events"), + clearStudyEvents: () => call("clear_study_events"), claudeUsage: () => call("claude_usage"), listAgentStatus: (recheck) => call("list_agent_status", { recheck }), listModels: (discover) => call("list_models", { discover }), diff --git a/apps/gui/frontend/src/features/draft/DraftTab.tsx b/apps/gui/frontend/src/features/draft/DraftTab.tsx index 9a48a91..21b05e4 100644 --- a/apps/gui/frontend/src/features/draft/DraftTab.tsx +++ b/apps/gui/frontend/src/features/draft/DraftTab.tsx @@ -50,7 +50,7 @@ export function DraftTab(props: { tab: Tab }): JSX.Element { effort, ) } - onSend={(body) => actions.createProject(body, props.tab.key)} + onSend={(body, study) => actions.createProject(body, props.tab.key, study)} /> diff --git a/apps/gui/frontend/src/features/home/HomeTab.tsx b/apps/gui/frontend/src/features/home/HomeTab.tsx index 0471ce3..433e097 100644 --- a/apps/gui/frontend/src/features/home/HomeTab.tsx +++ b/apps/gui/frontend/src/features/home/HomeTab.tsx @@ -9,6 +9,7 @@ import { AgentIoList } from "~/features/project/ProjectPanel"; import { relativeTime } from "~/lib/format"; import { AGENT_LABELS, nextStatus, statusSuffix } from "~/lib/labels"; import { describeError, log } from "~/lib/log"; +import { compileAdvancedPrompt } from "~/lib/promptEditor"; import { prefs, setPrefs, togglePanelSection } from "~/stores/prefs"; import { TASK_MANAGER_ID, useWorkspace } from "~/stores/workspace"; import type { Project, ProjectItem } from "~/types"; @@ -265,9 +266,10 @@ function TaskManagerComposer(): JSX.Element { const waitsForRun = () => isRunning() && !canFollowUp(); const submit = async (): Promise => { + const authored = draft(); // The pills become prose on the way out; a file alone is a sendable // prompt ("eat this"). - const body = [draft().trim(), attachments().join("\n")] + const body = [authored.trim(), attachments().join("\n")] .filter((part) => part.length > 0) .join("\n\n"); if (!body || isSending() || waitsForRun()) return; @@ -275,7 +277,14 @@ function TaskManagerComposer(): JSX.Element { setError(null); setIsSending(true); try { - await actions.sendTaskPrompt(body); + const parsedAuthored = compileAdvancedPrompt(authored, []); + await actions.sendTaskPrompt(body, { + authoredCharacterCount: [...authored].length, + authoredLineCount: + authored.length === 0 ? 0 : authored.replaceAll("\r\n", "\n").split("\n").length, + attachmentCount: attachments().length, + userAuthoredPs: parsedAuthored.segments.some((segment) => segment.type === "directive"), + }); setDraft(""); setAttachments([]); } catch (cause) { diff --git a/apps/gui/frontend/src/features/project/Composer.test.tsx b/apps/gui/frontend/src/features/project/Composer.test.tsx index fcdc56f..5491723 100644 --- a/apps/gui/frontend/src/features/project/Composer.test.tsx +++ b/apps/gui/frontend/src/features/project/Composer.test.tsx @@ -76,10 +76,33 @@ describe("Composer", () => { field.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true })); - await waitFor(() => expect(onSend).toHaveBeenCalledWith("Review the upgrade")); + await waitFor(() => + expect(onSend).toHaveBeenCalledWith("Review the upgrade", { + authoredCharacterCount: 18, + authoredLineCount: 1, + attachmentCount: 0, + userAuthoredPs: false, + }), + ); await waitFor(() => expect(field.value).toBe("")); }); + it("detects authored PromptSyntax even when Advanced leaves the message unchanged", async () => { + const { field, onSend } = mount(); + type(field, "@model:sonnet Review this"); + + field.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true })); + + await waitFor(() => + expect(onSend).toHaveBeenCalledWith("@model:sonnet Review this", { + authoredCharacterCount: 25, + authoredLineCount: 1, + attachmentCount: 0, + userAuthoredPs: true, + }), + ); + }); + /* * A prompt is often long and carefully written. Clearing on dispatch and * discovering the failure afterwards means it is already gone. @@ -353,7 +376,14 @@ describe("the alert slot means failure", () => { expect(send.disabled).toBe(false); fireEvent.click(send); - await waitFor(() => expect(onSend).toHaveBeenCalledWith("and this should wait its turn")); + await waitFor(() => + expect(onSend).toHaveBeenCalledWith("and this should wait its turn", { + authoredCharacterCount: 29, + authoredLineCount: 1, + attachmentCount: 0, + userAuthoredPs: false, + }), + ); }); it("says nothing when a compaction succeeds", async () => { diff --git a/apps/gui/frontend/src/features/project/Composer.tsx b/apps/gui/frontend/src/features/project/Composer.tsx index de7791f..35a957d 100644 --- a/apps/gui/frontend/src/features/project/Composer.tsx +++ b/apps/gui/frontend/src/features/project/Composer.tsx @@ -7,7 +7,7 @@ import { compileAdvancedPrompt, type PromptModelOption } from "~/lib/promptEdito import { parseSlash } from "~/lib/slash"; import { prefs, setPrefs } from "~/stores/prefs"; import { useWorkspace } from "~/stores/workspace"; -import type { Agent, Permission } from "~/types"; +import type { Agent, Permission, StudyTurnMetadata } from "~/types"; const PERMISSION_HINTS: Record = { read_only: "Reads only. The crate default.", @@ -75,7 +75,7 @@ export type ComposerProps = { * Resolves on success. The draft is held until then, so an IPC, database or * backend failure cannot swallow a prompt someone spent minutes writing. */ - onSend: (body: string) => Promise; + onSend: (body: string, study: StudyTurnMetadata) => Promise; /** Shown on the right of the control row, e.g. "31.4k / 200k ctx · 16%". */ usage?: string; /** A run is in flight: the send button becomes Stop. */ @@ -359,7 +359,15 @@ export function Composer(props: ComposerProps): JSX.Element { setErrorFor(key, "The prompt contains controls but no message to send."); return; } - await props.onSend(body); + const authored = draft(); + const parsedAuthored = advancedPrompt ?? compileAdvancedPrompt(authored, props.modelOptions); + await props.onSend(body, { + authoredCharacterCount: [...authored].length, + authoredLineCount: + authored.length === 0 ? 0 : authored.replaceAll("\r\n", "\n").split("\n").length, + attachmentCount: attachments().length, + userAuthoredPs: parsedAuthored.segments.some((segment) => segment.type === "directive"), + }); remember(""); setAttachments([]); resize(); diff --git a/apps/gui/frontend/src/features/project/ProjectTab.tsx b/apps/gui/frontend/src/features/project/ProjectTab.tsx index d661959..e265ee6 100644 --- a/apps/gui/frontend/src/features/project/ProjectTab.tsx +++ b/apps/gui/frontend/src/features/project/ProjectTab.tsx @@ -309,7 +309,7 @@ export function ProjectTab(props: { tab: Tab; project: Project }): JSX.Element { onPermissionChange={(permission) => actions.setTabModel(props.tab.key, props.tab.agent, props.tab.model, permission) } - onSend={(body) => actions.send(props.project.id, body)} + onSend={(body, study) => actions.send(props.project.id, body, study)} /> diff --git a/apps/gui/frontend/src/features/settings/SettingsTab.test.tsx b/apps/gui/frontend/src/features/settings/SettingsTab.test.tsx new file mode 100644 index 0000000..3e3c9d5 --- /dev/null +++ b/apps/gui/frontend/src/features/settings/SettingsTab.test.tsx @@ -0,0 +1,69 @@ +import { fireEvent, render, waitFor } from "@solidjs/testing-library"; +import { describe, expect, it } from "vitest"; +import { SettingsTab } from "~/features/settings/SettingsTab"; +import { useWorkspace, type Workspace, WorkspaceProvider } from "~/stores/workspace"; + +async function mountSettings() { + let workspace!: Workspace; + + function Probe() { + workspace = useWorkspace(); + return null; + } + + const screen = render(() => ( + + + + + )); + + await waitFor(() => expect(workspace.state.boot.status).toBe("ready"), { timeout: 5_000 }); + await waitFor(() => expect(screen.getByText("no study interval has been started")).toBeTruthy()); + return { ...screen, workspace }; +} + +describe("PS deployment study settings", () => { + it("starts off and explains the content boundary", async () => { + const screen = await mountSettings(); + const toggle = screen.getByLabelText("PS deployment study") as HTMLInputElement; + + expect(toggle.checked).toBe(false); + expect(screen.getByText(/does not copy prompt text/)).toBeTruthy(); + expect(screen.getByText(/Nothing is uploaded/)).toBeTruthy(); + }); + + it("creates a study interval only after explicit opt-in", async () => { + const screen = await mountSettings(); + const toggle = screen.getByLabelText("PS deployment study") as HTMLInputElement; + + fireEvent.click(toggle); + + await waitFor(() => expect(toggle.checked).toBe(true)); + await waitFor(() => expect(screen.getByText("Collection started locally.")).toBeTruthy()); + expect(screen.workspace.state.settings?.studyAnalytics.sessionId).toMatch(/^study-/); + expect(screen.workspace.state.settings?.studyAnalytics.enabledAt).not.toBe(""); + }); + + it("requires a second action before deleting local study rows", async () => { + const screen = await mountSettings(); + const toggle = screen.getByLabelText("PS deployment study") as HTMLInputElement; + fireEvent.click(toggle); + await waitFor(() => expect(screen.getByText("Collection started locally.")).toBeTruthy()); + expect(screen.getByRole("button", { name: "Delete data" })).toBeDisabled(); + + fireEvent.click(toggle); + await waitFor(() => expect(screen.getByText("Collection stopped.")).toBeTruthy()); + + fireEvent.click(screen.getByRole("button", { name: "Delete data" })); + expect(screen.getByRole("button", { name: "Confirm delete" })).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { name: "Confirm delete" })); + await waitFor(() => + expect( + screen.getByText("Stored study events deleted. The collection setting was not changed."), + ).toBeTruthy(), + ); + expect(toggle.checked).toBe(false); + }); +}); diff --git a/apps/gui/frontend/src/features/settings/SettingsTab.tsx b/apps/gui/frontend/src/features/settings/SettingsTab.tsx index abd348c..bf784ff 100644 --- a/apps/gui/frontend/src/features/settings/SettingsTab.tsx +++ b/apps/gui/frontend/src/features/settings/SettingsTab.tsx @@ -37,6 +37,7 @@ import type { ModelSelection, ModelSource, Permission, + StudySummary, TableSize, TaskManagerSettings, } from "~/types"; @@ -600,6 +601,8 @@ export function SettingsTab(): JSX.Element { + +
(null); + const [busy, setBusy] = createSignal(false); + const [confirmingDelete, setConfirmingDelete] = createSignal(false); + const [note, setNote] = createSignal(null); + const available = () => state.backend === "mock" || isLive("getStudySummary"); + + const refresh = async (): Promise => { + if (!available()) return; + setSummary(await actions.getStudySummary()); + }; + + onMount(() => { + void refresh().catch((cause) => { + setNote(`Study status unavailable: ${describeError(cause)}`); + }); + }); + + const toggle = async (enabled: boolean): Promise => { + setBusy(true); + setNote(null); + try { + await actions.saveSettings({ studyAnalytics: { enabled } }); + await refresh(); + setNote(enabled ? "Collection started locally." : "Collection stopped."); + } catch (cause) { + setNote(`Could not change collection: ${describeError(cause)}`); + } finally { + setBusy(false); + } + }; + + const exportEvents = async (): Promise => { + setBusy(true); + setNote(null); + try { + const path = await actions.exportStudyEvents(); + setNote(path ? "De-identified JSONL exported." : "Export canceled."); + } catch (cause) { + setNote(`Could not export: ${describeError(cause)}`); + } finally { + setBusy(false); + } + }; + + const clearEvents = async (): Promise => { + if (!confirmingDelete()) { + setConfirmingDelete(true); + setNote("Choose Confirm delete to remove every stored study event."); + return; + } + setBusy(true); + try { + await actions.clearStudyEvents(); + setConfirmingDelete(false); + await refresh(); + setNote("Stored study events deleted. The collection setting was not changed."); + } catch (cause) { + setNote(`Could not delete study data: ${describeError(cause)}`); + } finally { + setBusy(false); + } + }; + + const enabled = () => state.settings?.studyAnalytics.enabled ?? false; + + return ( +
+
+ Records timestamps, prompt character and line counts, attachment counts, whether you used + PromptSyntax, operation types, providers, opaque links, timing and explicit outcomes. It + does not copy prompt text, agent prose, task titles, paths, URLs, tool calls, tool output or + attachment contents. Nothing is uploaded. +
+ + void toggle(checked)} + /> + + + + {summary()?.eventCount ?? 0} + + + +
+ + +
+ +
+ ); +} + /** Claude usage controls compiled and advertised only by the experimental profile. */ function ExperimentalSettings(): JSX.Element { const { state, actions } = useWorkspace(); @@ -1343,12 +1497,14 @@ function Row(props: { function SettingToggle(props: { label: string; checked: boolean; + disabled?: boolean; onChange: (checked: boolean) => void; }): JSX.Element { return ( props.onChange(event.currentTarget.checked)} diff --git a/apps/gui/frontend/src/lib/promptEditor.ts b/apps/gui/frontend/src/lib/promptEditor.ts index f2baf93..b44f1ed 100644 --- a/apps/gui/frontend/src/lib/promptEditor.ts +++ b/apps/gui/frontend/src/lib/promptEditor.ts @@ -54,7 +54,10 @@ export function compileAdvancedPrompt( options: PromptModelOption[], ): CompiledAdvancedPrompt { const entityNames = options.flatMap((option) => [option.model, option.label]); - const parsed = new PromptSyntaxParser({ entities: entityNames }).parse(source); + const parsed = new PromptSyntaxParser({ + entities: entityNames, + authoringNamespaces: ["agency"], + }).parse(source); const errors = parsed.diagnostics.map((diagnostic) => diagnostic.message); const selected: PromptModelOption[] = []; diff --git a/apps/gui/frontend/src/stores/workspace.tsx b/apps/gui/frontend/src/stores/workspace.tsx index 4ecd0cc..0854bb2 100644 --- a/apps/gui/frontend/src/stores/workspace.tsx +++ b/apps/gui/frontend/src/stores/workspace.tsx @@ -35,6 +35,7 @@ import type { QuotaReport, RateLimit, RunningTask, + StudyTurnMetadata, Tab, TabStatus, TaskLogEntry, @@ -207,7 +208,11 @@ export type RunStatus = { export type QueueReason = "busy" | "compacting"; /** A prompt held back, and what it is held back for. */ -export type QueuedPrompt = { body: string; reason: QueueReason }; +export type QueuedPrompt = { + body: string; + reason: QueueReason; + study?: StudyTurnMetadata; +}; /** What the chip above the composer says while a prompt waits. */ export const QUEUE_REASONS: Record = { @@ -1393,7 +1398,11 @@ function createWorkspace() { // Each of these fires a command and lets the resulting event update the // store, so a change made by the agent and one made here land the same way. - async function createProject(firstMessage: string, tabKey: string): Promise { + async function createProject( + firstMessage: string, + tabKey: string, + study?: StudyTurnMetadata, + ): Promise { const tab = state.tabs.find((candidate) => candidate.key === tabKey); const created = await client().createProject({ firstMessage, @@ -1401,6 +1410,7 @@ function createWorkspace() { model: tab?.model, permission: tab?.permission, effort: tab?.effort, + study, }); batch(() => { /* @@ -1496,7 +1506,11 @@ function createWorkspace() { * No optimistic `runStatus` here: the backend's `run:accepted` starts the * status line, so a backend that fakes no run (the mock) shows no run. */ - const dispatch = async (projectId: string, body: string): Promise => { + const dispatch = async ( + projectId: string, + body: string, + study?: StudyTurnMetadata, + ): Promise => { const tab = state.tabs.find((candidate) => candidate.projectId === projectId); await client().sendMessage({ projectId, @@ -1507,15 +1521,25 @@ function createWorkspace() { // The tab's effort, which was being dropped here: every run reached the // agent with `effort=` while the composer showed a level selected. effort: tab?.effort, + study, }); }; /** Hold a prompt, and say what it is waiting for. */ - function enqueue(projectId: string, body: string, reason: QueueReason): void { - setState("queued", projectId, (waiting = []) => [...waiting, { body, reason }]); + function enqueue( + projectId: string, + body: string, + reason: QueueReason, + study?: StudyTurnMetadata, + ): void { + setState("queued", projectId, (waiting = []) => [...waiting, { body, reason, study }]); } - const send = async (projectId: string, body: string): Promise => { + const send = async ( + projectId: string, + body: string, + study?: StudyTurnMetadata, + ): Promise => { /* * A compaction is the one busy state worth checking *before* dispatching. * It is not a turn that can be interrupted — the words would go into a run @@ -1523,7 +1547,7 @@ function createWorkspace() { * that a message vanishing into it looks like the app dropping it. */ if (state.compacting[projectId]) { - enqueue(projectId, body, "compacting"); + enqueue(projectId, body, "compacting", study); return; } @@ -1533,7 +1557,7 @@ function createWorkspace() { runningAgent !== undefined && !capabilitiesFor(runningAgent)?.liveFollowUp ) { - enqueue(projectId, body, "busy"); + enqueue(projectId, body, "busy", study); return; } @@ -1546,11 +1570,11 @@ function createWorkspace() { * back. */ try { - await dispatch(projectId, body); + await dispatch(projectId, body, study); } catch (cause) { const reason = queueReason(cause); if (reason) { - enqueue(projectId, body, reason); + enqueue(projectId, body, reason, study); return; } throw cause; @@ -1571,7 +1595,7 @@ function createWorkspace() { if (isBusy(projectId)) return; // a newer run took the slot; its stop will re-cue setState("queued", projectId, waiting.slice(1)); try { - await dispatch(projectId, next.body); + await dispatch(projectId, next.body, next.study); } catch (cause) { setState("queued", projectId, (rest = []) => [next, ...rest]); if (attempt < 4) { @@ -1593,7 +1617,7 @@ function createWorkspace() { * Its conversation is separate from project tabs, including one native * session per provider. */ - const sendTaskPrompt = async (body: string): Promise => { + const sendTaskPrompt = async (body: string, study?: StudyTurnMetadata): Promise => { const taskManager = state.settings?.taskManager; await client().sendMessage({ projectId: TASK_MANAGER_ID, @@ -1602,6 +1626,7 @@ function createWorkspace() { model: taskManager?.model, permission: taskManager?.permission, effort: taskManager?.effort, + study, }); }; @@ -1713,6 +1738,9 @@ function createWorkspace() { // rejected or clamped theme shows as the value that was actually stored. applyTheme(next.theme); }, + getStudySummary: () => client().getStudySummary(), + exportStudyEvents: () => client().exportStudyEvents(), + clearStudyEvents: () => client().clearStudyEvents(), async recheckAgents() { setState("agents", reconcile(await client().listAgentStatus(true))); }, diff --git a/apps/gui/frontend/src/types/index.ts b/apps/gui/frontend/src/types/index.ts index 62dfcaa..68e921c 100644 --- a/apps/gui/frontend/src/types/index.ts +++ b/apps/gui/frontend/src/types/index.ts @@ -446,6 +446,31 @@ export interface NotificationSettings { sound: boolean; } +/** Explicit local consent and backend-owned interval for the PS study. */ +export interface StudyAnalyticsSettings { + enabled: boolean; + sessionId: string; + enabledAt: string; +} + +/** Content-free study status shown in Settings. */ +export interface StudySummary { + enabled: boolean; + studyId: string | null; + enabledAt: string | null; + eventCount: number; + firstAt: string | null; + lastAt: string | null; +} + +/** Content-free composer facts captured before controls and paths are compiled. */ +export interface StudyTurnMetadata { + authoredCharacterCount: number; + authoredLineCount: number; + attachmentCount: number; + userAuthoredPs: boolean; +} + /** One record, persisted. Every new tab starts from it. */ export interface GlobalSettings { defaultAgent: Agent; @@ -483,6 +508,8 @@ export interface GlobalSettings { completedItems: "resolve" | "delete"; /** How the workspace is coloured. See {@link ThemeSettings}. */ theme: ThemeSettings; + /** Off by default; events stay local until an explicit export. */ + studyAnalytics: StudyAnalyticsSettings; } /** diff --git a/apps/gui/src/db/fingerprint.rs b/apps/gui/src/db/fingerprint.rs index 91db4aa..ac3cc49 100644 --- a/apps/gui/src/db/fingerprint.rs +++ b/apps/gui/src/db/fingerprint.rs @@ -39,6 +39,7 @@ pub const SCHEMA_FINGERPRINT: &str = concat!( "usage_ledger(id,at,day,project_id,model,cost_micro,input_tokens,output_tokens);", "approval_rule(id,project_id,signature,created_at);", "pull_request(id,project_id,url,repo,number,branch,state,additions,deletions,ci,dismissed,updated_at);", + "study_event(id,study_id,at,project_id,turn_id,interaction_id,agent,pathway,operation,stage,outcome,code,target_kind,target_id,latency_ms,detail,app_version,parser_version,protocol_version);", ); /// What opening the tables found, so the caller can say something useful. diff --git a/apps/gui/src/db/schema/mod.rs b/apps/gui/src/db/schema/mod.rs index 1542889..9fc771c 100644 --- a/apps/gui/src/db/schema/mod.rs +++ b/apps/gui/src/db/schema/mod.rs @@ -11,5 +11,6 @@ pub mod message; pub mod project; pub mod project_item; pub mod pull_request; +pub mod study_event; pub mod task_log; pub mod usage_ledger; diff --git a/apps/gui/src/db/schema/study_event.rs b/apps/gui/src/db/schema/study_event.rs new file mode 100644 index 0000000..47560f5 --- /dev/null +++ b/apps/gui/src/db/schema/study_event.rs @@ -0,0 +1,51 @@ +//! Content-free events for the opt-in PromptSyntax deployment study. +//! +//! The table records how a declared operation travelled through AgencyZero, +//! not what the user or agent said. Prompt bodies, task titles, project names, +//! paths, URLs, tool output and agent prose do not belong here. Local ids are +//! kept only so an export can link a later manual correction to the operation +//! it followed; the export command replaces every one with a per-export +//! pseudonym. +//! +//! Collection is disabled by default. Callers go through `crate::study`, which +//! checks the persisted setting before any row is written. + +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: StudyEvent, + persist: true, + columns: { + id: String primary_key, + study_id: String, + at: String, + project_id: String, + turn_id: String, + // Links one parsed PS segment to its one terminal result. + interaction_id: String, + agent: String, + pathway: String, + operation: String, + stage: String, + outcome: String, + code: String, + target_kind: String, + target_id: String, + latency_ms: i64, + // Allow-listed JSON counters and booleans, never source text. + detail: String, + app_version: String, + parser_version: String, + protocol_version: String, + }, + indexes: { + study_idx: study_id, + project_idx: project_id, + }, + queries: { + delete: { + ByStudy() by study_id, + } + } +); diff --git a/apps/gui/src/db/tables.rs b/apps/gui/src/db/tables.rs index f3e41f0..8648913 100644 --- a/apps/gui/src/db/tables.rs +++ b/apps/gui/src/db/tables.rs @@ -20,6 +20,7 @@ use crate::db::schema::message::{MessagePersistenceEngine, MessageWorkTable}; use crate::db::schema::project::{ProjectPersistenceEngine, ProjectWorkTable}; use crate::db::schema::project_item::{ProjectItemPersistenceEngine, ProjectItemWorkTable}; use crate::db::schema::pull_request::{PullRequestPersistenceEngine, PullRequestWorkTable}; +use crate::db::schema::study_event::{StudyEventPersistenceEngine, StudyEventWorkTable}; use crate::db::schema::task_log::{TaskLogPersistenceEngine, TaskLogWorkTable}; use crate::db::schema::usage_ledger::{UsageLedgerPersistenceEngine, UsageLedgerWorkTable}; @@ -47,6 +48,8 @@ pub struct Tables { pub approval_rule: Arc, /// One row per PR cut during a run. See `schema/pull_request.rs`. pub pull_request: Arc, + /// Content-free records from an explicitly enabled deployment study. + pub study_event: Arc, } impl Tables { @@ -87,6 +90,7 @@ impl Tables { usage_ledger: open!(UsageLedgerPersistenceEngine, UsageLedgerWorkTable), approval_rule: open!(ApprovalRulePersistenceEngine, ApprovalRuleWorkTable), pull_request: open!(PullRequestPersistenceEngine, PullRequestWorkTable), + study_event: open!(StudyEventPersistenceEngine, StudyEventWorkTable), }) } } @@ -233,6 +237,74 @@ mod tests { "project_item changed shape without the fingerprint changing, so every \ row on disk would be read through the wrong layout" ); + + let row = crate::db::schema::study_event::StudyEventRow { + id: String::new(), + study_id: String::new(), + at: String::new(), + project_id: String::new(), + turn_id: String::new(), + interaction_id: String::new(), + agent: String::new(), + pathway: String::new(), + operation: String::new(), + stage: String::new(), + outcome: String::new(), + code: String::new(), + target_kind: String::new(), + target_id: String::new(), + latency_ms: -1, + detail: String::new(), + app_version: String::new(), + parser_version: String::new(), + protocol_version: String::new(), + }; + let crate::db::schema::study_event::StudyEventRow { + id: _, + study_id: _, + at: _, + project_id: _, + turn_id: _, + interaction_id: _, + agent: _, + pathway: _, + operation: _, + stage: _, + outcome: _, + code: _, + target_kind: _, + target_id: _, + latency_ms: _, + detail: _, + app_version: _, + parser_version: _, + protocol_version: _, + } = row; + assert_eq!( + columns_in_fingerprint("study_event"), + vec![ + "id", + "study_id", + "at", + "project_id", + "turn_id", + "interaction_id", + "agent", + "pathway", + "operation", + "stage", + "outcome", + "code", + "target_kind", + "target_id", + "latency_ms", + "detail", + "app_version", + "parser_version", + "protocol_version", + ], + "study_event changed shape without the fingerprint changing" + ); } /// Opening a fresh directory must produce every table, and a blob must @@ -500,5 +572,6 @@ impl Tables { self.usage_ledger.wait_for_ops().await; self.approval_rule.wait_for_ops().await; self.pull_request.wait_for_ops().await; + self.study_event.wait_for_ops().await; } } diff --git a/apps/gui/src/directives.rs b/apps/gui/src/directives.rs index 7a288b6..cce60a3 100644 --- a/apps/gui/src/directives.rs +++ b/apps/gui/src/directives.rs @@ -130,6 +130,20 @@ pub enum Directive { ItemRetire { id: String }, } +impl Directive { + /// Stable, content-free labels for deployment-study records. + #[must_use] + pub fn operation(&self) -> &'static str { + match self { + Self::ItemState { .. } => "items.state", + Self::ItemAdd { .. } => "items.add", + Self::ItemRetire { .. } => "items.retire", + Self::PrLink { .. } => "pr.link", + Self::IssueLink { .. } => "issue.link", + } + } +} + /// What became of one directive, in the agent's own words back to it. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Outcome { @@ -161,6 +175,15 @@ impl Outcome { Self::Refused { what, code } => format!("rejected: {what} [{code}]"), } } + + /// Terminal state and typed code for a content-free study row. + #[must_use] + pub fn study_result(&self) -> (&'static str, String) { + match self { + Self::Done(_) => ("applied", String::new()), + Self::Refused { code, .. } => ("refused", code.clone()), + } + } } fn scalar_text(value: &Scalar) -> Option { diff --git a/apps/gui/src/main.rs b/apps/gui/src/main.rs index 1b82a1b..ef334c2 100644 --- a/apps/gui/src/main.rs +++ b/apps/gui/src/main.rs @@ -11,6 +11,7 @@ mod projects; mod prs; mod quota; mod settings; +mod study; mod tasks; mod update; @@ -84,6 +85,9 @@ const IMPLEMENTED: &[&str] = &[ "send_message", "get_settings", "set_settings", + "get_study_summary", + "export_study_events", + "clear_study_events", "list_agent_status", "list_models", "log_frontend", @@ -555,6 +559,9 @@ async fn set_settings( .and_then(|raw| serde_json::from_str(&raw).ok()) .unwrap_or_else(|| serde_json::to_value(GlobalSettings::default()).unwrap_or_default()); + let previous: GlobalSettings = + serde_json::from_value(current.clone()).unwrap_or_else(|_| GlobalSettings::default()); + let mut merged = current; settings::merge(&mut merged, &patch); @@ -563,13 +570,25 @@ async fn set_settings( let mut parsed: GlobalSettings = serde_json::from_value(merged.clone()).map_err(|error| error.to_string())?; settings::normalize_task_manager(&mut parsed); + let boundary = study::normalize_setting(&previous.study_analytics, &mut parsed.study_analytics); let merged = serde_json::to_value(&parsed).map_err(|error| error.to_string())?; - state - .tables - .kv_put(settings::KEY, merged.to_string()) - .await - .map_err(|error| error.to_string())?; + let boundary_id = boundary + .map(|boundary| study::record_boundary(&state.tables, &parsed.study_analytics, boundary)) + .transpose()?; + + if let Err(error) = state.tables.kv_put(settings::KEY, merged.to_string()).await { + if let Some(id) = boundary_id + && let Err(cleanup) = state.tables.study_event.delete(id).await + { + crate::log!( + log::Level::Error, + "study", + "settings failed and its boundary event could not be removed: {cleanup}" + ); + } + return Err(error.to_string()); + } Ok(parsed) } @@ -1067,6 +1086,9 @@ fn main() { projects::create_project, projects::send_message, get_settings, + study::get_study_summary, + study::export_study_events, + study::clear_study_events, relaunch_app, set_settings, list_agent_status, diff --git a/apps/gui/src/projects.rs b/apps/gui/src/projects.rs index 33d5525..f896f5f 100644 --- a/apps/gui/src/projects.rs +++ b/apps/gui/src/projects.rs @@ -250,6 +250,18 @@ pub struct SendMessageInput { pub model: Option, pub permission: Option, pub effort: Option, + /// Content-free facts computed before the composer compiles controls or + /// appends attachment paths. Absent callers fall back to the sent body. + pub study: Option, +} + +#[derive(Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StudyTurnMetadata { + pub authored_character_count: usize, + pub authored_line_count: usize, + pub attachment_count: usize, + pub user_authored_ps: bool, } #[derive(Deserialize)] @@ -261,6 +273,7 @@ pub struct CreateProjectInput { pub permission: Option, /// Reasoning effort, as `Request::effort`. `None` means the CLI's default. pub effort: Option, + pub study: Option, } #[derive(Serialize)] @@ -549,6 +562,7 @@ pub fn create_item( title: String, state: State<'_, AppState>, ) -> Result { + let started = std::time::Instant::now(); let title = title.trim().to_string(); if title.is_empty() { return Err("an item needs a title".into()); @@ -576,6 +590,10 @@ pub fn create_item( .map_err(|error| error.to_string())?; let dto = ProjectItemDto::from(row); let _ = app.emit("item:created", dto.clone()); + let mut study = + crate::study::Record::manual(dto.project_id.clone(), "items.add", "item", dto.id.clone()); + study.latency = Some(started.elapsed()); + crate::study::record(&state.tables, study); Ok(dto) } @@ -591,6 +609,7 @@ pub async fn set_item_status( status: String, state: State<'_, AppState>, ) -> Result { + let started = std::time::Instant::now(); /* * Every status the ladder can reach, including `questions` and `canceled`. * @@ -645,6 +664,10 @@ pub async fn set_item_status( "item:deleted", serde_json::json!({ "id": id, "projectId": row.project_id }), ); + let mut study = + crate::study::Record::manual(row.project_id.clone(), "items.state", "item", id.clone()); + study.latency = Some(started.elapsed()); + crate::study::record(&state.tables, study); // Preserve the command's return shape for callers that await it even // though the event removes the row from the live store. row.status = status; @@ -663,6 +686,14 @@ pub async fn set_item_status( .ok_or_else(|| format!("no item {id}"))?; let dto = ProjectItemDto::from(row); let _ = app.emit("item:updated", dto.clone()); + let mut study = crate::study::Record::manual( + dto.project_id.clone(), + "items.state", + "item", + dto.id.clone(), + ); + study.latency = Some(started.elapsed()); + crate::study::record(&state.tables, study); Ok(dto) } @@ -677,6 +708,7 @@ pub async fn update_item( title: String, state: State<'_, AppState>, ) -> Result { + let started = std::time::Instant::now(); let title = title.trim().to_string(); if title.is_empty() { return Err("an item needs a title".into()); @@ -694,6 +726,14 @@ pub async fn update_item( .ok_or_else(|| format!("no item {id}"))?; let dto = ProjectItemDto::from(row); let _ = app.emit("item:updated", dto.clone()); + let mut study = crate::study::Record::manual( + dto.project_id.clone(), + "items.update", + "item", + dto.id.clone(), + ); + study.latency = Some(started.elapsed()); + crate::study::record(&state.tables, study); Ok(dto) } @@ -754,7 +794,12 @@ pub async fn set_item_issue( url: String, state: State<'_, AppState>, ) -> Result { - link_item_issue_inner(&app, &state.tables, &id, &url).await + let started = std::time::Instant::now(); + let dto = link_item_issue_inner(&app, &state.tables, &id, &url).await?; + let mut study = crate::study::Record::manual(dto.project_id.clone(), "issue.link", "item", id); + study.latency = Some(started.elapsed()); + crate::study::record(&state.tables, study); + Ok(dto) } /// Remove one item. @@ -767,6 +812,7 @@ pub async fn delete_item( id: String, state: State<'_, AppState>, ) -> Result<(), String> { + let started = std::time::Instant::now(); let row = state .tables .project_item @@ -782,6 +828,9 @@ pub async fn delete_item( "item:deleted", serde_json::json!({ "id": id, "projectId": row.project_id }), ); + let mut study = crate::study::Record::manual(row.project_id, "items.retire", "item", id); + study.latency = Some(started.elapsed()); + crate::study::record(&state.tables, study); Ok(()) } @@ -802,6 +851,8 @@ pub async fn reorder_items( ids: Vec, state: State<'_, AppState>, ) -> Result, String> { + let started = std::time::Instant::now(); + let moved = ids.len(); for (index, item_id) in ids.iter().enumerate() { state .tables @@ -815,10 +866,15 @@ pub async fn reorder_items( .await .map_err(|error| error.to_string())?; } - let items = list_items(project_id, state); + let items = list_items(project_id.clone(), state.clone()); for item in &items { let _ = app.emit("item:updated", item.clone()); } + let mut study = + crate::study::Record::manual(project_id.clone(), "items.reorder", "project", project_id); + study.latency = Some(started.elapsed()); + study.detail = serde_json::json!({ "itemCount": moved }); + crate::study::record(&state.tables, study); Ok(items) } @@ -1475,11 +1531,94 @@ fn authored_directive_line<'a>(line: &'a str, fenced: &mut FenceState) -> Option Some(trimmed) } +struct StudyTarget { + kind: &'static str, + id: String, + before_add: std::collections::HashSet, +} + +/// Resolve an authored prefix to the application id it actually changed. +/// +/// The study table must not retain an arbitrary `id:` argument, and a prefix +/// would not join to a later manual correction anyway. Adds have no id until +/// execution, so their prior id set is kept and the one new row is resolved +/// afterwards. +fn study_target_before( + tables: &crate::db::tables::Tables, + directive: &crate::directives::Directive, +) -> StudyTarget { + use crate::directives::Directive; + + let rows: Vec = tables + .project_item + .select_all() + .execute() + .unwrap_or_default(); + let known: Vec<&str> = rows.iter().map(|row| row.id.as_str()).collect(); + let resolve = |named: &str| { + crate::directives::resolve(&known, named) + .map(str::to_string) + .unwrap_or_default() + }; + + match directive { + Directive::ItemState { id, .. } | Directive::ItemRetire { id } => StudyTarget { + kind: "item", + id: resolve(id), + before_add: std::collections::HashSet::new(), + }, + Directive::ItemAdd { .. } => StudyTarget { + kind: "item", + id: String::new(), + before_add: rows.iter().map(|row| row.id.clone()).collect(), + }, + Directive::PrLink { item, .. } => StudyTarget { + kind: if item.is_some() { + "item" + } else { + "pull_request" + }, + id: item.as_deref().map(resolve).unwrap_or_default(), + before_add: std::collections::HashSet::new(), + }, + Directive::IssueLink { item, .. } => StudyTarget { + kind: "item", + id: resolve(item), + before_add: std::collections::HashSet::new(), + }, + } +} + +fn study_target_after_add( + tables: &crate::db::tables::Tables, + target: &mut StudyTarget, + operation: &str, + applied: bool, +) { + if operation != "items.add" || !applied { + return; + } + let added: Vec = tables + .project_item + .select_all() + .execute() + .unwrap_or_default() + .into_iter() + .map(|row| row.id) + .filter(|id| !target.before_add.contains(id)) + .collect(); + if let [id] = added.as_slice() { + target.id.clone_from(id); + } +} + /// Read every directive out of a stretch of reply text and carry it out. async fn apply_directives_with_state( app: &AppHandle, tables: &crate::db::tables::Tables, project_id: &str, + turn_id: &str, + agent: &str, text: &str, fenced: &mut FenceState, ) -> Vec { @@ -1487,9 +1626,92 @@ async fn apply_directives_with_state( for line in text.lines() { match authored_directive_line(line, fenced).and_then(crate::directives::parse_authored) { Some(crate::directives::Authored::Directive(directive)) => { - done.push(apply_directive(app, tables, project_id, directive).await); + let operation = directive.operation(); + let mut target = study_target_before(tables, &directive); + let interaction_id = id("interaction"); + crate::study::record( + tables, + crate::study::Record { + project_id: project_id.into(), + turn_id: turn_id.into(), + interaction_id: interaction_id.clone(), + agent: agent.into(), + pathway: "ps", + operation, + stage: "parsed", + outcome: "observed", + code: String::new(), + target_kind: target.kind, + target_id: target.id.clone(), + latency: None, + detail: serde_json::json!({}), + }, + ); + let started = std::time::Instant::now(); + let outcome = apply_directive(app, tables, project_id, directive).await; + let (result, code) = outcome.study_result(); + study_target_after_add(tables, &mut target, operation, result == "applied"); + crate::study::record( + tables, + crate::study::Record { + project_id: project_id.into(), + turn_id: turn_id.into(), + interaction_id, + agent: agent.into(), + pathway: "ps", + operation, + stage: "completed", + outcome: result, + code, + target_kind: target.kind, + target_id: target.id, + latency: Some(started.elapsed()), + detail: serde_json::json!({}), + }, + ); + done.push(outcome); + } + Some(crate::directives::Authored::Refused(outcome)) => { + let interaction_id = id("interaction"); + crate::study::record( + tables, + crate::study::Record { + project_id: project_id.into(), + turn_id: turn_id.into(), + interaction_id: interaction_id.clone(), + agent: agent.into(), + pathway: "ps", + operation: "authoring.segment", + stage: "parsed", + outcome: "observed", + code: String::new(), + target_kind: "", + target_id: String::new(), + latency: None, + detail: serde_json::json!({}), + }, + ); + let (result, code) = outcome.study_result(); + crate::study::record( + tables, + crate::study::Record { + project_id: project_id.into(), + turn_id: turn_id.into(), + interaction_id, + agent: agent.into(), + pathway: "ps", + operation: "authoring.segment", + stage: "completed", + outcome: result, + code, + target_kind: "", + target_id: String::new(), + latency: Some(std::time::Duration::ZERO), + detail: serde_json::json!({}), + }, + ); + done.push(outcome); } - Some(crate::directives::Authored::Refused(outcome)) => done.push(outcome), None => {} } } @@ -1500,9 +1722,76 @@ async fn apply_directives( app: &AppHandle, tables: &crate::db::tables::Tables, project_id: &str, + turn_id: &str, + agent: &str, text: &str, ) -> Vec { - apply_directives_with_state(app, tables, project_id, text, &mut FenceState::default()).await + apply_directives_with_state( + app, + tables, + project_id, + turn_id, + agent, + text, + &mut FenceState::default(), + ) + .await +} + +/// Whether a submitted turn itself contains a live AgencyZero authoring line. +/// +/// This records one boolean, never the line. The same quote, indentation and +/// fence rules as the reverse-channel parser keep an example in prose from +/// being counted as direct syntax use. +fn user_authored_ps(text: &str) -> bool { + let mut fenced = FenceState::default(); + text.lines().any(|line| { + authored_directive_line(line, &mut fenced) + .and_then(crate::directives::parse_authored) + .is_some() + }) +} + +fn record_study_turn( + tables: &crate::db::tables::Tables, + project_id: &str, + turn_id: &str, + agent: &str, + body: &str, + authored: Option<&StudyTurnMetadata>, + followup: bool, +) { + let fallback = StudyTurnMetadata { + authored_character_count: body.chars().count(), + authored_line_count: body.lines().count(), + attachment_count: 0, + user_authored_ps: user_authored_ps(body), + }; + let authored = authored.unwrap_or(&fallback); + crate::study::record( + tables, + crate::study::Record { + project_id: project_id.into(), + turn_id: turn_id.into(), + interaction_id: String::new(), + agent: agent.into(), + pathway: "study", + operation: "turn.submit", + stage: "submitted", + outcome: "observed", + code: String::new(), + target_kind: "", + target_id: String::new(), + latency: None, + detail: serde_json::json!({ + "characterCount": authored.authored_character_count, + "lineCount": authored.authored_line_count, + "attachmentCount": authored.attachment_count, + "followup": followup, + "userAuthoredPs": authored.user_authored_ps, + }), + }, + ); } /// Find the project a line named, by id or by name, case-insensitively. @@ -2001,6 +2290,43 @@ impl RateLimitReport { /// the turn that just happened and is worthless a day later. pub type Receipts = std::sync::Mutex>>; +fn queue_directive_receipts( + receipts: &Receipts, + tables: &crate::db::tables::Tables, + project_id: &str, + turn_id: &str, + agent: &str, + outcomes: &[crate::directives::Outcome], +) { + let (outcome, code) = match receipts.lock() { + Ok(mut kept) => { + kept.entry(project_id.to_string()) + .or_default() + .extend(outcomes.iter().map(crate::directives::Outcome::line)); + ("applied", String::new()) + } + Err(_) => ("failed", "RECEIPT_LOCK_FAILED".into()), + }; + crate::study::record( + tables, + crate::study::Record { + project_id: project_id.into(), + turn_id: turn_id.into(), + interaction_id: String::new(), + agent: agent.into(), + pathway: "ps", + operation: "receipt.queue", + stage: "queued", + outcome, + code, + target_kind: "", + target_id: String::new(), + latency: None, + detail: serde_json::json!({ "outcomeCount": outcomes.len() }), + }, + ); +} + pub type RunningTasks = std::sync::Mutex>>; /// One question a run is blocked on: the way to answer it, and what @@ -2154,7 +2480,13 @@ pub struct ActiveRun { /// delivered into that turn would be read by nobody and would disappear /// with it, so the send is refused and the frontend holds them for the /// session that comes out the other side. - pub inject: Option>, + pub inject: Option>, +} + +/// A live follow-up and the persisted user-message row that owns it. +pub struct InjectedMessage { + body: String, + turn_id: String, } pub type ActiveRuns = std::sync::Mutex>; @@ -3571,6 +3903,7 @@ pub async fn create_project( model: input.model, permission: input.permission, effort: input.effort, + study: input.study, }, state, ) @@ -3668,6 +4001,15 @@ pub async fn send_message( .insert(user_row.clone()) .map_err(|error| error.to_string())?; let user_message = MessageDto::from(user_row); + record_study_turn( + &state.tables, + &input.project_id, + &user_message.id, + agent_name, + &input.body, + input.study.as_ref(), + true, + ); note_gui( &app, &state, @@ -3681,7 +4023,13 @@ pub async fn send_message( // echo — the crate deliberately requests none. let _ = app.emit("message:appended", &user_message); - if inject.send(input.body.clone()).is_err() { + if inject + .send(InjectedMessage { + body: input.body.clone(), + turn_id: user_message.id.clone(), + }) + .is_err() + { // The run tore down in the race window. The row stands (the // words were said); the refusal tells the frontend to queue // the body for a fresh turn so the agent actually hears it. @@ -3733,6 +4081,15 @@ pub async fn send_message( .map_err(|error| error.to_string())?; let user_message = MessageDto::from(user_row); + record_study_turn( + &state.tables, + &input.project_id, + &user_message.id, + agent_name, + &input.body, + input.study.as_ref(), + false, + ); note_gui( &app, &state, @@ -3868,6 +4225,7 @@ pub async fn send_message( } } let project_id = input.project_id.clone(); + let turn_id = user_message.id.clone(); let effort = input.effort.clone(); tauri::async_runtime::spawn(async move { @@ -3884,6 +4242,7 @@ pub async fn send_message( cancel, inject_rx, project_id, + turn_id, input.body, agent, model, @@ -3926,8 +4285,11 @@ async fn drive_run( _reservation: RunReservation, mut cancel: tokio::sync::watch::Receiver, // Messages typed while this run is live, to deliver into the open turn. - mut inject_rx: tokio::sync::mpsc::UnboundedReceiver, + mut inject_rx: tokio::sync::mpsc::UnboundedReceiver, project_id: String, + // The user message that opened this run. PS outcomes emitted by the agent + // link back to it without retaining the message body in the study table. + turn_id: String, prompt: String, agent: Agent, model: String, @@ -3943,6 +4305,7 @@ async fn drive_run( // or working directory. Told to the agent every turn; see below. memory_dir: std::path::PathBuf, ) { + let mut directive_turn_id = turn_id; /* * Home's conversation is the task manager, and its replies have to become * rows. The user's own words go out unchanged with the output contract @@ -4301,7 +4664,7 @@ async fn drive_run( /// `run.send` cannot be called until that future is dropped. enum Wake { Event(Event), - Inject(String), + Inject(InjectedMessage), } loop { @@ -4327,11 +4690,12 @@ async fn drive_run( }; let event = match wake { Wake::Event(event) => event, - Wake::Inject(body) => { + Wake::Inject(injected) => { // A correction typed mid-turn. The user row was persisted and // broadcast by `send_message`; delivery and its failure modes // live in the helper, shared with the approval-wait arm. - deliver_injection(&app, &io, &run, &project_id, body).await; + directive_turn_id = injected.turn_id; + deliver_injection(&app, &io, &run, &project_id, injected.body).await; // A user message is a block boundary: the next streamed text // starts a new paragraph rather than gluing to the old one. last_was_text = false; @@ -4445,8 +4809,16 @@ async fn drive_run( break None; } injected = inject_rx.recv() => { - if let Some(body) = injected { - deliver_injection(&app, &io, &run, &project_id, body).await; + if let Some(injected) = injected { + directive_turn_id = injected.turn_id; + deliver_injection( + &app, + &io, + &run, + &project_id, + injected.body, + ) + .await; } } } @@ -4535,6 +4907,8 @@ async fn drive_run( &app, &tables, &project_id, + &directive_turn_id, + agent_wire_name(agent), &line, &mut directives_fenced, ) @@ -4551,11 +4925,14 @@ async fn drive_run( .collect::>() .join("; "), ); - if let Ok(mut kept) = receipts.lock() { - kept.entry(project_id.clone()) - .or_default() - .extend(done.iter().map(crate::directives::Outcome::line)); - } + queue_directive_receipts( + &receipts, + &tables, + &project_id, + &directive_turn_id, + agent_wire_name(agent), + &done, + ); } } @@ -5175,12 +5552,22 @@ async fn drive_run( &app, &tables, &project_id, + &directive_turn_id, + agent_wire_name(agent), &body[tail_at..], &mut directives_fenced, ) .await } else { - apply_directives(&app, &tables, &project_id, &body).await + apply_directives( + &app, + &tables, + &project_id, + &directive_turn_id, + agent_wire_name(agent), + &body, + ) + .await }; if !done.is_empty() { note_io( @@ -5194,11 +5581,14 @@ async fn drive_run( .collect::>() .join("; "), ); - if let Ok(mut kept) = receipts.lock() { - kept.entry(project_id.clone()) - .or_default() - .extend(done.iter().map(crate::directives::Outcome::line)); - } + queue_directive_receipts( + &receipts, + &tables, + &project_id, + &directive_turn_id, + agent_wire_name(agent), + &done, + ); } emit_run_stopped( &app, @@ -6160,6 +6550,45 @@ mod tests { ); } + #[test] + fn direct_use_flag_obeys_the_same_inert_content_rules() { + let directive = r#""#; + assert!(user_authored_ps(directive)); + assert!(!user_authored_ps(&format!("```text\n{directive}\n```"))); + assert!(!user_authored_ps(&format!("> {directive}"))); + } + + #[tokio::test] + async fn study_targets_use_resolved_ids_instead_of_authored_prefixes() { + let dir = std::env::temp_dir().join(format!("az-study-target-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let tables = crate::db::tables::Tables::open(&dir) + .await + .expect("study target store opens"); + tables + .project_item + .insert(ProjectItemRow { + id: "item-a3f9-canonical".into(), + project_id: "project-private".into(), + title: "not collected".into(), + status: "active".into(), + position: 0, + reference: String::new(), + }) + .expect("item inserts"); + + let target = study_target_before( + &tables, + &crate::directives::Directive::ItemState { + id: "item-a3f9".into(), + status: "shipped".into(), + pr: None, + }, + ); + assert_eq!(target.id, "item-a3f9-canonical"); + assert_eq!(target.kind, "item"); + } + #[test] fn a_different_or_shorter_fence_cannot_promote_quoted_ps() { let directive = r#""#; diff --git a/apps/gui/src/prs.rs b/apps/gui/src/prs.rs index 8b5e884..63c0f73 100644 --- a/apps/gui/src/prs.rs +++ b/apps/gui/src/prs.rs @@ -507,14 +507,19 @@ pub async fn dismiss_pull_request( id: String, state: State<'_, AppState>, ) -> Result<(), String> { + let started = std::time::Instant::now(); state .tables .pull_request .update_pr_dismissed_by_id(PrDismissedByIdQuery { dismissed: true }, id.clone()) .await .map_err(|error| error.to_string())?; - if let Some(row) = state.tables.pull_request.select(id) { + if let Some(row) = state.tables.pull_request.select(id.clone()) { + let project_id = row.project_id.clone(); let _ = app.emit("pr:updated", PullRequestDto::from(row)); + let mut study = crate::study::Record::manual(project_id, "pr.dismiss", "pull_request", id); + study.latency = Some(started.elapsed()); + crate::study::record(&state.tables, study); } Ok(()) } @@ -524,13 +529,31 @@ pub async fn dismiss_pull_request( pub fn refresh_pull_request(app: AppHandle, id: String) { // Asked about one, answered for its whole project: the query costs the // same either way, and a chip nobody clicked is no less stale. - let project = app - .state::() + let state = app.state::(); + let project = state .tables .pull_request - .select(id) + .select(id.clone()) .map(|row| row.project_id); if let Some(project) = project { + crate::study::record( + &state.tables, + crate::study::Record { + project_id: project.clone(), + turn_id: String::new(), + interaction_id: String::new(), + agent: String::new(), + pathway: "manual", + operation: "pr.refresh", + stage: "submitted", + outcome: "observed", + code: String::new(), + target_kind: "pull_request", + target_id: id, + latency: None, + detail: serde_json::json!({}), + }, + ); refresh_project(app, project); } } diff --git a/apps/gui/src/settings.rs b/apps/gui/src/settings.rs index d057189..84123fb 100644 --- a/apps/gui/src/settings.rs +++ b/apps/gui/src/settings.rs @@ -43,6 +43,21 @@ pub struct GlobalSettings { pub completed_items: String, /// How the workspace is coloured. See [`Theme`]. pub theme: Theme, + /// Explicit local consent for the content-free PS deployment study. + pub study_analytics: StudyAnalytics, +} + +/// The opt-in boundary for the PromptSyntax deployment study. +/// +/// The session fields are assigned by the backend on an off-to-on transition. +/// They are not user-editable labels: every recorded row carries the id so a +/// stopped and later restarted study cannot be mistaken for one interval. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", default)] +pub struct StudyAnalytics { + pub enabled: bool, + pub session_id: String, + pub enabled_at: String, } /// The theme picker's two axes, as the webview applies them. @@ -252,6 +267,7 @@ impl Default for GlobalSettings { notifications: Notifications::default(), completed_items: "resolve".into(), theme: Theme::default(), + study_analytics: StudyAnalytics::default(), } } } @@ -333,6 +349,11 @@ mod tests { assert_eq!(back.task_manager.model, "gpt-5.6-luna"); assert_eq!(back.task_manager.effort, "low"); assert_eq!(back.task_manager.permission, "ask"); + assert!( + !back.study_analytics.enabled, + "research collection is opt-in" + ); + assert!(back.study_analytics.session_id.is_empty()); assert!(json.contains("defaultAgent"), "must be camelCase: {json}"); } @@ -346,6 +367,7 @@ mod tests { assert_eq!(loaded.default_agent, "codex"); assert_eq!(loaded.task_manager.agent, "codex"); assert_eq!(loaded.task_manager.permission, "ask"); + assert!(!loaded.study_analytics.enabled); assert_eq!( loaded.moderator.model, "haiku", "absent blocks use defaults" diff --git a/apps/gui/src/study.rs b/apps/gui/src/study.rs new file mode 100644 index 0000000..f736d00 --- /dev/null +++ b/apps/gui/src/study.rs @@ -0,0 +1,623 @@ +//! Opt-in, content-free instrumentation for the PromptSyntax deployment study. +//! +//! This is not general telemetry. Nothing is uploaded, collection is off by +//! default, and the event shape has no field for prompt text, agent prose, +//! titles, paths, URLs or tool output. It measures the declared control path: +//! a turn, a PS or manual operation, and the operation's explicit outcome. + +use std::collections::BTreeMap; +use std::time::Duration; + +use serde::Serialize; +use serde_json::{Value, json}; +use tauri::{AppHandle, State}; +use tauri_plugin_dialog::DialogExt; +use worktable::prelude::*; + +use crate::AppState; +use crate::db::schema::study_event::StudyEventRow; +use crate::db::tables::Tables; +use crate::settings::{GlobalSettings, StudyAnalytics}; + +pub const PROTOCOL_VERSION: &str = "agencyzero-ps-deployment-study/0.1"; +pub const PARSER_VERSION: &str = "promptsyntax-rs/0.1.0"; +const MAX_DETAIL_BYTES: usize = 2_000; +const DETAIL_KEYS: &[&str] = &[ + "attachmentCount", + "characterCount", + "followup", + "itemCount", + "lineCount", + "outcomeCount", + "userAuthoredPs", +]; + +/// One operation observed while the study setting is enabled. +/// +/// Every field is application-owned metadata. Callers cannot put arbitrary +/// prose into the row: `detail` accepts only objects made from counters, +/// booleans and nulls, and is checked again before persistence. +pub struct Record { + pub project_id: String, + pub turn_id: String, + pub interaction_id: String, + pub agent: String, + pub pathway: &'static str, + pub operation: &'static str, + pub stage: &'static str, + pub outcome: &'static str, + pub code: String, + pub target_kind: &'static str, + pub target_id: String, + pub latency: Option, + pub detail: Value, +} + +impl Record { + #[must_use] + pub fn manual( + project_id: impl Into, + operation: &'static str, + target_kind: &'static str, + target_id: impl Into, + ) -> Self { + Self { + project_id: project_id.into(), + turn_id: String::new(), + interaction_id: String::new(), + agent: String::new(), + pathway: "manual", + operation, + stage: "completed", + outcome: "applied", + code: String::new(), + target_kind, + target_id: target_id.into(), + latency: None, + detail: json!({}), + } + } +} + +/// The only setting transition that creates a boundary event. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Boundary { + Enabled, + Disabled, +} + +/// Keep backend-owned session fields stable and mint a new interval on enable. +/// +/// A generic settings patch can carry `studyAnalytics.enabled`, but it cannot +/// choose or rewrite the session id. This is what makes two opt-in intervals +/// distinguishable even if a webview sends a stale whole settings object. +pub fn normalize_setting(previous: &StudyAnalytics, next: &mut StudyAnalytics) -> Option { + let changed = match (previous.enabled, next.enabled) { + (false, true) => { + next.session_id = crate::projects::id("study"); + next.enabled_at = crate::projects::now(); + Some(Boundary::Enabled) + } + (true, false) => { + next.session_id.clone_from(&previous.session_id); + next.enabled_at.clone_from(&previous.enabled_at); + Some(Boundary::Disabled) + } + _ => None, + }; + if changed.is_none() { + next.session_id.clone_from(&previous.session_id); + next.enabled_at.clone_from(&previous.enabled_at); + } + changed +} + +fn current_setting(tables: &Tables) -> StudyAnalytics { + tables + .kv_get(crate::settings::KEY) + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .unwrap_or_default() + .study_analytics +} + +fn detail_is_content_free(value: &Value) -> bool { + match value { + Value::Null | Value::Bool(_) | Value::Number(_) => true, + Value::Object(fields) => fields.iter().all(|(key, value)| { + DETAIL_KEYS.contains(&key.as_str()) + && matches!(value, Value::Null | Value::Bool(_) | Value::Number(_)) + }), + Value::String(_) | Value::Array(_) => false, + } +} + +/// Reduce runtime errors to a stable category before they reach the table. +/// +/// A WorkTable failure can contain paths or engine details after a colon. The +/// full error still goes to the normal log; the study row keeps only the +/// machine-readable category needed for the failure taxonomy. +fn failure_code(value: String) -> String { + if value.is_empty() { + return value; + } + let category = value.split(':').next().unwrap_or_default().trim(); + if category.len() <= 64 + && !category.is_empty() + && category + .chars() + .all(|character| character.is_ascii_uppercase() || character == '_') + { + category.to_string() + } else { + "UNCLASSIFIED_FAILURE".into() + } +} + +fn detail_json(detail: Value) -> String { + if !detail_is_content_free(&detail) { + crate::log!( + crate::log::Level::Warn, + "study", + "refused a study detail outside the fixed metadata allowlist" + ); + return "{}".into(); + } + let encoded = serde_json::to_string(&detail).unwrap_or_else(|_| "{}".into()); + if encoded.len() > MAX_DETAIL_BYTES { + crate::log!( + crate::log::Level::Warn, + "study", + "refused an oversized study detail" + ); + "{}".into() + } else { + encoded + } +} + +/// Keep only application-style opaque ids. A malformed directive may put any +/// prose in an `id:` argument; recording that value would create a content +/// channel in a table whose contract explicitly has none. +fn opaque_id(value: String) -> String { + if value.len() <= 160 + && value + .chars() + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_')) + { + value + } else { + String::new() + } +} + +fn row(setting: &StudyAnalytics, record: Record) -> StudyEventRow { + StudyEventRow { + id: crate::projects::id("study-event"), + study_id: setting.session_id.clone(), + at: crate::projects::now(), + project_id: record.project_id, + turn_id: record.turn_id, + interaction_id: opaque_id(record.interaction_id), + agent: record.agent, + pathway: record.pathway.into(), + operation: record.operation.into(), + stage: record.stage.into(), + outcome: record.outcome.into(), + code: failure_code(record.code), + target_kind: record.target_kind.into(), + target_id: opaque_id(record.target_id), + latency_ms: record + .latency + .and_then(|elapsed| i64::try_from(elapsed.as_millis()).ok()) + .unwrap_or(-1), + detail: detail_json(record.detail), + app_version: az_core::VERSION.into(), + parser_version: PARSER_VERSION.into(), + protocol_version: PROTOCOL_VERSION.into(), + } +} + +/// Record an event if, and only if, the persisted opt-in setting is active. +/// +/// Instrumentation never changes the product operation's result. A failed +/// study write is logged loudly and the requested task or PR mutation still +/// stands, because research collection must not become application authority. +pub fn record(tables: &Tables, record: Record) { + let setting = current_setting(tables); + if !setting.enabled || setting.session_id.is_empty() { + return; + } + if let Err(error) = tables.study_event.insert(row(&setting, record)) { + crate::log!( + crate::log::Level::Error, + "study", + "could not record an enabled study event: {error}" + ); + } +} + +/// Insert the enable or disable marker before the matching settings write. +/// +/// The caller removes this row if the settings write fails, giving the two +/// WorkTable writes transaction-like cleanup without claiming cross-table +/// transactions the engine does not provide. +pub fn record_boundary( + tables: &Tables, + setting: &StudyAnalytics, + boundary: Boundary, +) -> Result { + let record = Record { + project_id: String::new(), + turn_id: String::new(), + interaction_id: String::new(), + agent: String::new(), + pathway: "study", + operation: "study.session", + stage: "boundary", + outcome: match boundary { + Boundary::Enabled => "enabled", + Boundary::Disabled => "disabled", + }, + code: String::new(), + target_kind: "", + target_id: String::new(), + latency: None, + detail: json!({}), + }; + let row = row(setting, record); + let id = row.id.clone(); + tables + .study_event + .insert(row) + .map_err(|error| error.to_string())?; + Ok(id) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StudySummary { + enabled: bool, + study_id: Option, + enabled_at: Option, + event_count: usize, + first_at: Option, + last_at: Option, +} + +fn rows(tables: &Tables) -> Vec { + let mut rows = tables + .study_event + .select_all() + .execute() + .unwrap_or_default(); + rows.sort_by(|left, right| left.at.cmp(&right.at).then(left.id.cmp(&right.id))); + rows +} + +#[tauri::command] +pub fn get_study_summary(state: State<'_, AppState>) -> StudySummary { + let setting = current_setting(&state.tables); + let rows = rows(&state.tables); + StudySummary { + enabled: setting.enabled, + study_id: (!setting.session_id.is_empty()).then_some(setting.session_id), + enabled_at: (!setting.enabled_at.is_empty()).then_some(setting.enabled_at), + event_count: rows.len(), + first_at: rows.first().map(|row| row.at.clone()), + last_at: rows.last().map(|row| row.at.clone()), + } +} + +#[derive(Default)] +struct Pseudonyms { + events: BTreeMap, + studies: BTreeMap, + projects: BTreeMap, + turns: BTreeMap, + interactions: BTreeMap, + targets: BTreeMap, +} + +fn pseudonym(map: &mut BTreeMap, prefix: &str, value: &str) -> String { + if value.is_empty() { + return String::new(); + } + let next = map.len() + 1; + map.entry(value.to_string()) + .or_insert_with(|| format!("{prefix}-{next:03}")) + .clone() +} + +fn render_export(rows: &[StudyEventRow], exported_at: &str) -> Result { + let mut out = String::new(); + let meta = json!({ + "record": "metadata", + "protocolVersion": PROTOCOL_VERSION, + "exportedAt": exported_at, + "appVersion": az_core::VERSION, + "parserVersion": PARSER_VERSION, + "eventCount": rows.len(), + "deidentified": true, + "containsPromptOrToolContent": false, + }); + out.push_str(&serde_json::to_string(&meta).map_err(|error| error.to_string())?); + out.push('\n'); + + let mut names = Pseudonyms::default(); + for row in rows { + let detail: Value = serde_json::from_str(&row.detail).unwrap_or_else(|_| json!({})); + let event = json!({ + "record": "event", + "id": pseudonym(&mut names.events, "event", &row.id), + "studyId": pseudonym(&mut names.studies, "study", &row.study_id), + "at": row.at, + "projectId": pseudonym(&mut names.projects, "project", &row.project_id), + "turnId": pseudonym(&mut names.turns, "turn", &row.turn_id), + "interactionId": pseudonym(&mut names.interactions, "interaction", &row.interaction_id), + "agent": row.agent, + "pathway": row.pathway, + "operation": row.operation, + "stage": row.stage, + "outcome": row.outcome, + "code": row.code, + "targetKind": row.target_kind, + "targetId": pseudonym(&mut names.targets, "target", &row.target_id), + "latencyMs": row.latency_ms, + "detail": detail, + "appVersion": row.app_version, + "parserVersion": row.parser_version, + "protocolVersion": row.protocol_version, + }); + out.push_str(&serde_json::to_string(&event).map_err(|error| error.to_string())?); + out.push('\n'); + } + Ok(out) +} + +/// Save a de-identified JSONL copy through the native picker. +/// +/// `None` is a cancelled picker. No upload or background destination exists. +#[tauri::command] +pub async fn export_study_events( + app: AppHandle, + state: State<'_, AppState>, +) -> Result, String> { + let export = render_export(&rows(&state.tables), &crate::projects::now())?; + let (tx, rx) = tokio::sync::oneshot::channel(); + app.dialog() + .file() + .set_title("Export de-identified PS deployment study data") + .set_file_name("agencyzero-ps-study.jsonl") + .add_filter("JSON Lines", &["jsonl"]) + .save_file(move |picked| { + let _ = tx.send(picked); + }); + let Some(picked) = rx + .await + .map_err(|_| "the save dialog closed without answering".to_string())? + else { + return Ok(None); + }; + let path = picked.into_path().map_err(|error| error.to_string())?; + std::fs::write(&path, export).map_err(|error| error.to_string())?; + Ok(Some(path.to_string_lossy().into_owned())) +} + +/// Delete every locally stored study event. This does not change the toggle. +async fn clear_rows(tables: &Tables) -> Result<(), String> { + for row in rows(tables) { + tables + .study_event + .delete(row.id) + .await + .map_err(|error| error.to_string())?; + } + Ok(()) +} + +#[tauri::command] +pub async fn clear_study_events(state: State<'_, AppState>) -> Result<(), String> { + if current_setting(&state.tables).enabled { + return Err("stop study collection before deleting its stored events".into()); + } + clear_rows(&state.tables).await +} + +#[cfg(test)] +mod tests { + use super::*; + + async fn tables(label: &str) -> Tables { + let dir = std::env::temp_dir().join(format!("az-study-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + Tables::open(&dir).await.expect("study test store opens") + } + + fn sample() -> Record { + Record { + project_id: "private-project".into(), + turn_id: "private-turn".into(), + interaction_id: "private-interaction".into(), + agent: "codex".into(), + pathway: "ps", + operation: "items.state", + stage: "completed", + outcome: "applied", + code: String::new(), + target_kind: "item", + target_id: "private-item".into(), + latency: Some(Duration::from_millis(12)), + detail: json!({"userAuthoredPs": false, "lineCount": 2}), + } + } + + #[test] + fn enabling_mints_a_new_backend_owned_session() { + let previous = StudyAnalytics::default(); + let mut next = StudyAnalytics { + enabled: true, + session_id: "caller-chosen".into(), + enabled_at: "caller-chosen".into(), + }; + assert_eq!( + normalize_setting(&previous, &mut next), + Some(Boundary::Enabled) + ); + assert!(next.session_id.starts_with("study-")); + assert_ne!(next.enabled_at, "caller-chosen"); + } + + #[test] + fn an_enabled_client_cannot_rewrite_backend_session_fields() { + let previous = StudyAnalytics { + enabled: true, + session_id: "study-kept".into(), + enabled_at: "2026-08-03T00:00:00Z".into(), + }; + let mut next = StudyAnalytics { + enabled: true, + session_id: "caller-rewrite".into(), + enabled_at: "caller-rewrite".into(), + }; + + assert_eq!(normalize_setting(&previous, &mut next), None); + assert_eq!(next.session_id, "study-kept"); + assert_eq!(next.enabled_at, "2026-08-03T00:00:00Z"); + } + + #[tokio::test] + async fn disabled_collection_writes_nothing() { + let tables = tables("off").await; + record(&tables, sample()); + assert!(rows(&tables).is_empty()); + } + + #[tokio::test] + async fn enabled_collection_keeps_metadata_without_content() { + let tables = tables("on").await; + let settings = GlobalSettings { + study_analytics: StudyAnalytics { + enabled: true, + session_id: "study-local".into(), + enabled_at: "2026-08-03T00:00:00Z".into(), + }, + ..GlobalSettings::default() + }; + tables + .kv_put( + crate::settings::KEY, + serde_json::to_string(&settings).expect("settings serialize"), + ) + .await + .expect("settings persist"); + + record(&tables, sample()); + let kept = rows(&tables); + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].operation, "items.state"); + assert!(!kept[0].detail.contains("private")); + } + + #[tokio::test] + async fn boundaries_survive_the_disabled_side_of_a_transition() { + let tables = tables("boundaries").await; + let enabled = StudyAnalytics { + enabled: true, + session_id: "study-boundary".into(), + enabled_at: "2026-08-03T00:00:00Z".into(), + }; + let disabled = StudyAnalytics { + enabled: false, + ..enabled.clone() + }; + + record_boundary(&tables, &enabled, Boundary::Enabled).expect("enable boundary inserts"); + record_boundary(&tables, &disabled, Boundary::Disabled).expect("disable boundary inserts"); + + let kept = rows(&tables); + assert_eq!(kept.len(), 2); + assert_eq!(kept[0].outcome, "enabled"); + assert_eq!(kept[1].outcome, "disabled"); + assert!(kept.iter().all(|row| row.study_id == "study-boundary")); + } + + #[tokio::test] + async fn clearing_removes_every_stored_row() { + let tables = tables("clear").await; + let setting = StudyAnalytics { + enabled: false, + session_id: "study-clear".into(), + enabled_at: "2026-08-03T00:00:00Z".into(), + }; + tables + .study_event + .insert(row(&setting, sample())) + .expect("sample inserts"); + + clear_rows(&tables).await.expect("rows clear"); + assert!(rows(&tables).is_empty()); + } + + #[test] + fn detail_refuses_arbitrary_text() { + assert_eq!(detail_json(json!({"prompt": "do the private thing"})), "{}"); + assert_eq!(detail_json(json!({"private words as a key": 1})), "{}"); + assert_eq!(detail_json(json!({"lineCount": {"covert": 2}})), "{}"); + assert_eq!(detail_json(json!({"chars": 42, "followup": true})), "{}"); + assert_eq!( + detail_json(json!({"characterCount": 42, "followup": true})), + r#"{"characterCount":42,"followup":true}"# + ); + } + + #[test] + fn failure_codes_cannot_carry_runtime_content() { + assert_eq!( + failure_code("WRITE_FAILED: /private/path in engine".into()), + "WRITE_FAILED" + ); + assert_eq!(failure_code("ENTITY_NOT_FOUND".into()), "ENTITY_NOT_FOUND"); + assert_eq!( + failure_code("customer-specific failure".into()), + "UNCLASSIFIED_FAILURE" + ); + } + + #[test] + fn target_ids_cannot_become_a_content_channel() { + assert_eq!(opaque_id("item-a3f9".into()), "item-a3f9"); + assert_eq!(opaque_id("the customer's private title".into()), ""); + assert_eq!(opaque_id("x".repeat(161)), ""); + } + + #[test] + fn export_replaces_every_linkable_local_id() { + let setting = StudyAnalytics { + enabled: true, + session_id: "private-study".into(), + enabled_at: "2026-08-03T00:00:00Z".into(), + }; + let mut kept = row(&setting, sample()); + kept.id = "private-event".into(); + let export = render_export(&[kept], "2026-08-10T00:00:00Z").expect("export renders"); + assert!(export.contains("study-001")); + assert!(export.contains("project-001")); + assert!(export.contains("turn-001")); + assert!(export.contains("interaction-001")); + assert!(export.contains("target-001")); + assert!(export.contains("event-001")); + assert!(!export.contains("private-event")); + assert!(!export.contains("private-study")); + assert!(!export.contains("private-project")); + assert!(!export.contains("private-turn")); + assert!(!export.contains("private-interaction")); + assert!(!export.contains("private-item")); + } + + #[test] + fn parser_version_is_tied_to_the_declared_dependency() { + let manifest = include_str!("../Cargo.toml"); + assert!(manifest.contains("promptsyntax = \"0.1.0\"")); + assert_eq!(PARSER_VERSION, "promptsyntax-rs/0.1.0"); + } +}