Merge pull request #671 from rajbos/rajbos/fluency-shared-server

feat: add AI fluency score to sharing server dashboard

Author: rajbos

sharing-server/.env.example

@@ -12,6 +12,25 @@ SESSION_SECRET=change_me_to_a_random_secret
 # Public base URL of this server (used in OAuth redirect URI and links)
 BASE_URL=http://localhost:3000
 
-# Optional: restrict to members of a specific GitHub organization
-# Leave empty to allow any GitHub user
+# Optional: restrict dashboard access and data uploads to members of this GitHub org.
+# Leave empty to allow any authenticated GitHub user.
+# Example: ALLOWED_GITHUB_ORG=my-company
 ALLOWED_GITHUB_ORG=
+
+# Optional: server-side GitHub PAT used to verify org membership (see ALLOWED_GITHUB_ORG).
+#
+# When ALLOWED_GITHUB_ORG is set, the server calls the GitHub API to check whether
+# the authenticated user is a member of that org. By default it uses the user's own
+# OAuth token, which fails for orgs that enforce SAML SSO (GitHub returns 403 because
+# the user's token is not SSO-authorized for the org).
+#
+# Setting this to a PAT that belongs to the server operator (who IS a member and has
+# authorized the token for SSO) bypasses that limitation: the server checks membership
+# on the user's behalf without requiring anything extra from the end user.
+#
+# How to create:
+#   1. Go to https://github.com/settings/tokens → Generate new token (classic)
+#   2. Grant the "read:org" scope
+#   3. Click "Configure SSO" next to the token and authorize it for ALLOWED_GITHUB_ORG
+#   4. Paste the generated token below
+GITHUB_ORG_CHECK_TOKEN=

sharing-server/src/auth.ts

@@ -30,6 +30,14 @@ const ipRateMap = new Map<string, { count: number; resetAt: number }>();
 const IP_RATE_MAX = 200;
 const IP_RATE_WINDOW_MS = 60 * 1000; // 1 minute per IP
 
+/**
+ * Validates a GitHub Bearer token supplied by the client (e.g. the VS Code extension).
+ * Resolves the token to a local user row, or returns null if the token is invalid,
+ * the GitHub API is unreachable, or the user is not a member of ALLOWED_GITHUB_ORG.
+ *
+ * Results are cached for 10 minutes (positive) or 1 minute (negative) to reduce
+ * outbound GitHub API calls.
+ */
 export async function validateGitHubToken(token: string): Promise<UserRow | null> {
 	const cacheKey = createHash('sha256').update(token).digest('hex');
 
@@ -83,11 +91,25 @@ export async function validateGitHubToken(token: string): Promise<UserRow | null
 	return user;
 }
 
-async function checkOrgMembership(token: string, username: string, org: string): Promise<boolean> {
+/**
+ * Checks whether `username` is an active public member of `org`.
+ *
+ * Uses GITHUB_ORG_CHECK_TOKEN (a server-configured PAT) when set, so that the
+ * check works even when the org enforces SAML SSO — the server operator's PAT
+ * is pre-authorized for the org, meaning the end user's token never needs
+ * read:org scope or SSO authorization.
+ *
+ * Falls back to the user's own token if no server PAT is configured (works for
+ * orgs with public membership and no SAML enforcement).
+ */
+async function checkOrgMembership(userToken: string, username: string, org: string): Promise<boolean> {
+	// Prefer a server-side PAT (already SSO-authorized) so the user's OAuth token
+	// doesn't need read:org or SAML SSO authorization.
+	const checkToken = process.env.GITHUB_ORG_CHECK_TOKEN || userToken;
 	try {
 		const res = await fetch(`https://api.github.com/orgs/${org}/members/${username}`, {
 			headers: {
-				Authorization: `Bearer ${token}`,
+				Authorization: `Bearer ${checkToken}`,
 				'User-Agent': 'copilot-sharing-server/1.0',
 				Accept: 'application/vnd.github+json',
 			},
@@ -99,6 +121,7 @@ async function checkOrgMembership(token: string, username: string, org: string):
 	}
 }
 
+/** Returns true if the IP address is within the pre-auth rate limit window, false if it should be blocked. */
 export function checkIpRateLimit(ip: string): boolean {
 	const now = Date.now();
 	const entry = ipRateMap.get(ip);
@@ -111,6 +134,7 @@ export function checkIpRateLimit(ip: string): boolean {
 	return true;
 }
 
+/** Returns true if the user is within the upload rate limit window, false if they should be blocked. */
 export function checkUploadRateLimit(userId: number): boolean {
 	const now = Date.now();
 	const entry = uploadRateMap.get(userId);
@@ -125,6 +149,11 @@ export function checkUploadRateLimit(userId: number): boolean {
 
 export type AuthVariables = { user: UserRow };
 
+/**
+ * Hono middleware that enforces Bearer token authentication on API routes.
+ * Applies IP-level rate limiting before token validation, then resolves the
+ * token to a user and stores it in the Hono context for downstream handlers.
+ */
 export async function requireBearerAuth(c: Context, next: Next): Promise<Response | void> {
 	const ip = c.req.header('x-forwarded-for') ?? c.req.header('x-real-ip') ?? 'unknown';
 

sharing-server/src/db.ts

@@ -11,6 +11,7 @@ export interface UserRow {
 	created_at: string;
 	last_seen_at: string | null;
 	is_admin: number;
+	fluency_score_json: string | null;
 }
 
 export interface UploadRow {
@@ -29,6 +30,7 @@ export interface UploadRow {
 	interactions: number;
 	schema_version: number;
 	uploaded_at: string;
+	fluency_json: string | null;
 }
 
 export interface UploadEntry {
@@ -43,6 +45,7 @@ export interface UploadEntry {
 	inputTokens: number;
 	outputTokens: number;
 	interactions: number;
+	fluencyMetrics?: Record<string, unknown>;
 }
 
 let _db: DatabaseSync | undefined;
@@ -79,6 +82,7 @@ const UPLOADS_TABLE_DDL = `
 		interactions   INTEGER NOT NULL DEFAULT 0,
 		schema_version INTEGER NOT NULL DEFAULT 3,
 		uploaded_at    TEXT DEFAULT (datetime('now')),
+		fluency_json   TEXT,
 		UNIQUE(user_id, dataset_id, day, model, workspace_id, machine_id, editor)
 	)`;
 
@@ -147,6 +151,22 @@ function initSchema(db: DatabaseSync): void {
 		CREATE INDEX IF NOT EXISTS idx_uploads_user_day ON usage_uploads(user_id, day);
 		CREATE INDEX IF NOT EXISTS idx_uploads_dataset   ON usage_uploads(dataset_id, day);
 	`);
+
+	// Add fluency_json column if it doesn't exist (migration for existing DBs)
+	const cols = db
+		.prepare("PRAGMA table_info(usage_uploads)")
+		.all() as unknown as Array<{ name: string }>;
+	if (!cols.some(c => c.name === 'fluency_json')) {
+		db.exec('ALTER TABLE usage_uploads ADD COLUMN fluency_json TEXT');
+	}
+
+	// Add fluency_score_json column to users if it doesn't exist (migration for existing DBs)
+	const userCols = db
+		.prepare("PRAGMA table_info(users)")
+		.all() as unknown as Array<{ name: string }>;
+	if (!userCols.some(c => c.name === 'fluency_score_json')) {
+		db.exec('ALTER TABLE users ADD COLUMN fluency_score_json TEXT');
+	}
 }
 
 export function upsertUser(
@@ -178,17 +198,19 @@ export function getUserByGithubId(githubId: number): UserRow | undefined {
 
 export function upsertUpload(userId: number, entry: UploadEntry): void {
 	const editor = ((entry.editor ?? '').trim() || 'VS Code').slice(0, 100);
+	const fluencyJson = entry.fluencyMetrics ? JSON.stringify(entry.fluencyMetrics) : null;
 	getDb().prepare(`
 		INSERT INTO usage_uploads
 			(user_id, dataset_id, day, model, workspace_id, workspace_name, machine_id, machine_name,
-			 editor, input_tokens, output_tokens, interactions)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+			 editor, input_tokens, output_tokens, interactions, fluency_json)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 		ON CONFLICT(user_id, dataset_id, day, model, workspace_id, machine_id, editor) DO UPDATE SET
 			workspace_name = excluded.workspace_name,
 			machine_name   = excluded.machine_name,
 			input_tokens   = excluded.input_tokens,
 			output_tokens  = excluded.output_tokens,
 			interactions   = excluded.interactions,
+			fluency_json   = excluded.fluency_json,
 			uploaded_at    = datetime('now')
 	`).run(
 		userId,
@@ -203,6 +225,7 @@ export function upsertUpload(userId: number, entry: UploadEntry): void {
 		entry.inputTokens,
 		entry.outputTokens,
 		entry.interactions,
+		fluencyJson,
 	);
 }
 
@@ -233,3 +256,14 @@ export function getUploadsForUser(userId: number, days = 30): UploadRow[] {
 export function getAllUsers(): UserRow[] {
 	return getDb().prepare('SELECT * FROM users ORDER BY created_at DESC').all() as unknown as UserRow[];
 }
+
+export function upsertUserFluencyScore(userId: number, scoreJson: string): void {
+	getDb().prepare(`
+		UPDATE users SET fluency_score_json = ? WHERE id = ?
+	`).run(scoreJson, userId);
+}
+
+export function getUserFluencyScore(userId: number): string | null {
+	const row = getDb().prepare('SELECT fluency_score_json FROM users WHERE id = ?').get(userId) as unknown as { fluency_score_json: string | null } | undefined;
+	return row?.fluency_score_json ?? null;
+}

sharing-server/src/routes/api.ts

@@ -1,6 +1,6 @@
 import { Hono } from 'hono';
 import { requireBearerAuth, checkUploadRateLimit, type AuthVariables } from '../auth.js';
-import { upsertUpload, deleteUploadsForDays, getUploadsForUser, getDb, type UploadEntry } from '../db.js';
+import { upsertUpload, deleteUploadsForDays, getUploadsForUser, getDb, upsertUserFluencyScore, type UploadEntry } from '../db.js';
 
 const MAX_STRING_LENGTHS = {
 	model: 128,
@@ -111,6 +111,35 @@ api.get('/data', requireBearerAuth, (c) => {
 	return c.json(data);
 });
 
+/**
+ * POST /api/fluency-score — Store the extension's locally-computed fluency score.
+ * Body: { overallStage, overallLabel, categories, computedAt }
+ * This is the authoritative score: the server dashboard uses it directly instead of
+ * re-computing from aggregated upload blobs.
+ */
+api.post('/fluency-score', requireBearerAuth, async (c) => {
+	const user = c.get('user');
+
+	let body: unknown;
+	try {
+		body = await c.req.json();
+	} catch {
+		return c.json({ error: 'Invalid JSON body.' }, 400);
+	}
+
+	if (typeof body !== 'object' || body === null || Array.isArray(body)) {
+		return c.json({ error: 'Body must be a JSON object.' }, 400);
+	}
+
+	const b = body as Record<string, unknown>;
+	if (typeof b.overallStage !== 'number' || !Array.isArray(b.categories)) {
+		return c.json({ error: '"overallStage" (number) and "categories" (array) are required.' }, 400);
+	}
+
+	upsertUserFluencyScore(user.id, JSON.stringify(body));
+	return c.json({ ok: true });
+});
+
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
 function clampDays(raw: string | undefined): number {
@@ -175,6 +204,11 @@ function validateEntry(entry: unknown): string | null {
 			return `"editor" too long (max ${MAX_STRING_LENGTHS.editor})`;
 		}
 	}
+	if (e.fluencyMetrics !== undefined && e.fluencyMetrics !== null) {
+		if (typeof e.fluencyMetrics !== 'object' || Array.isArray(e.fluencyMetrics)) {
+			return '"fluencyMetrics" must be an object';
+		}
+	}
 
 	return null;
 }