Devtools screencast feature

vishnuv688 · vishnuv688 · commit 0566c12ef309 · 2026-04-13T18:05:10.000+05:30
diff --git a/packages/app/src/components/browser/snapshot.ts b/packages/app/src/components/browser/snapshot.ts
@@ -1,5 +1,5 @@
 import { Element } from '@core/element'
-import { html, css } from 'lit'
+import { html, css, nothing } from 'lit'
 import { consume } from '@lit/context'
 
 import { type ComponentChildren, h, render, type VNode } from 'preact'
@@ -19,6 +19,12 @@ import '../placeholder.js'
 
 const MUTATION_SELECTOR = '__mutation-highlight__'
 
+declare global {
+  interface WindowEventMap {
+    'screencast-ready': CustomEvent<{ sessionId: string }>
+  }
+}
+
 function transform(node: any): VNode<{}> {
   if (typeof node !== 'object' || node === null) {
     // Plain string/number text node — return as-is for Preact to render as text.
@@ -47,6 +53,20 @@ export class DevtoolsBrowser extends Element {
   #activeUrl?: string
   /** Base64 PNG of the screenshot for the currently selected command, or null. */
   #screenshotData: string | null = null
+  /**
+   * All recorded videos received from the backend, in arrival order.
+   * Each entry is { sessionId, url } — a new entry is pushed for every
+   * browser session (initial + after every reloadSession() call).
+   */
+  #videos: Array<{ sessionId: string; url: string }> = []
+  /** Index into #videos of the currently displayed video. */
+  #activeVideoIdx = 0
+  /**
+   * Which view is active in the browser panel.
+   * 'video'    — always show the screencast player (default when a recording exists)
+   * 'snapshot' — show DOM mutations replay and per-command screenshots
+   */
+  #viewMode: 'snapshot' | 'video' = 'snapshot'
 
   @consume({ context: metadataContext, subscribe: true })
   metadata: Metadata | undefined = undefined
@@ -136,13 +156,63 @@ export class DevtoolsBrowser extends Element {
         display: block;
       }
 
+      .screencast-player {
+        width: 100%;
+        height: 100%;
+        object-fit: contain;
+        background: #111;
+        border-radius: 0 0 0.5rem 0.5rem;
+        display: block;
+      }
+
       .iframe-wrapper {
         position: relative;
         flex: 1;
         min-height: 0;
         display: flex;
         flex-direction: column;
       }
+
+      .view-toggle {
+        display: flex;
+        gap: 2px;
+        margin-left: 0.5rem;
+        flex-shrink: 0;
+      }
+
+      .view-toggle button {
+        padding: 2px 10px;
+        font-size: 11px;
+        font-family: inherit;
+        border: 1px solid var(--vscode-editorSuggestWidget-border, #454545);
+        background: transparent;
+        color: var(--vscode-input-foreground, #ccc);
+        cursor: pointer;
+        border-radius: 3px;
+        line-height: 20px;
+        transition:
+          background 0.1s,
+          color 0.1s;
+      }
+
+      .view-toggle button.active {
+        background: var(--vscode-button-background, #0e639c);
+        color: var(--vscode-button-foreground, #fff);
+        border-color: transparent;
+      }
+
+      .video-select {
+        font-size: 11px;
+        font-family: inherit;
+        padding: 2px 4px;
+        border: 1px solid var(--vscode-dropdown-border, #454545);
+        border-radius: 3px;
+        background: var(--vscode-dropdown-background, #3c3c3c);
+        color: var(--vscode-dropdown-foreground, #ccc);
+        cursor: pointer;
+        line-height: 20px;
+        margin-left: 4px;
+      }
     `
   ]
 
@@ -170,6 +240,10 @@ export class DevtoolsBrowser extends Element {
       'show-command',
       this.#handleShowCommand as EventListener
     )
+    window.addEventListener(
+      'screencast-ready',
+      this.#handleScreencastReady as EventListener
+    )
     await this.updateComplete
   }
 
@@ -215,8 +289,34 @@ export class DevtoolsBrowser extends Element {
       (event as CustomEvent<{ command?: CommandLog }>).detail?.command
     )
 
+  #handleScreencastReady = (event: Event) => {
+    const { sessionId } = (event as CustomEvent<{ sessionId: string }>).detail
+    this.#videos.push({ sessionId, url: `/api/video/${sessionId}` })
+    // Always show the latest video and switch to video mode automatically
+    this.#activeVideoIdx = this.#videos.length - 1
+    this.#viewMode = 'video'
+    this.requestUpdate()
+  }
+
+  #setViewMode(mode: 'snapshot' | 'video') {
+    this.#viewMode = mode
+    this.requestUpdate()
+  }
+
+  #setActiveVideo(idx: number) {
+    this.#activeVideoIdx = idx
+    this.requestUpdate()
+  }
+
+  /** URL of the currently selected video, or null when no videos exist. */
+  get #activeVideoUrl(): string | null {
+    return this.#videos[this.#activeVideoIdx]?.url ?? null
+  }
+
   async #renderCommandScreenshot(command?: CommandLog) {
     this.#screenshotData = command?.screenshot ?? null
+    // Switch to snapshot mode so the command screenshot is visible instead of the video.
+    this.#viewMode = 'snapshot'
     this.requestUpdate()
   }
 
@@ -461,32 +561,79 @@ export class DevtoolsBrowser extends Element {
             ></icon-mdi-world>
             <span class="truncate">${this.#activeUrl}</span>
           </div>
+          ${this.#videos.length > 0
+            ? html`
+                <div class="view-toggle">
+                  <button
+                    class=${this.#viewMode === 'snapshot' ? 'active' : ''}
+                    @click=${() => this.#setViewMode('snapshot')}
+                  >
+                    Snapshot
+                  </button>
+                  <button
+                    class=${this.#viewMode === 'video' ? 'active' : ''}
+                    @click=${() => this.#setViewMode('video')}
+                  >
+                    Video
+                  </button>
+                  ${this.#videos.length > 1
+                    ? html`<select
+                        class="video-select"
+                        @change=${(e: Event) => {
+                          this.#setActiveVideo(
+                            Number((e.target as HTMLSelectElement).value)
+                          )
+                          this.#setViewMode('video')
+                        }}
+                      >
+                        ${this.#videos.map(
+                          (_v, i) =>
+                            html`<option
+                              value=${i}
+                              ?selected=${this.#activeVideoIdx === i}
+                            >
+                              Recording ${i + 1}
+                            </option>`
+                        )}
+                      </select>`
+                    : nothing}
+                </div>
+              `
+            : nothing}
         </header>
-        ${this.#screenshotData
-          ? html` <div class="iframe-wrapper">
-              <div
-                class="screenshot-overlay"
-                style="position:relative;flex:1;min-height:0;"
-              >
-                <img src="data:image/png;base64,${this.#screenshotData}" />
-              </div>
+        ${this.#viewMode === 'video' && this.#activeVideoUrl
+          ? html`<div class="iframe-wrapper">
+              <video
+                class="screencast-player"
+                src="${this.#activeVideoUrl}"
+                controls
+              ></video>
             </div>`
-          : hasMutations
-            ? html` <div class="iframe-wrapper">
-                <iframe class="origin-top-left"></iframe>
+          : this.#screenshotData
+            ? html`<div class="iframe-wrapper">
+                <div
+                  class="screenshot-overlay"
+                  style="position:relative;flex:1;min-height:0;"
+                >
+                  <img src="data:image/png;base64,${this.#screenshotData}" />
+                </div>
               </div>`
-            : displayScreenshot
-              ? html` <div class="iframe-wrapper">
-                  <div
-                    class="screenshot-overlay"
-                    style="position:relative;flex:1;min-height:0;"
-                  >
-                    <img src="data:image/png;base64,${displayScreenshot}" />
-                  </div>
+            : hasMutations
+              ? html`<div class="iframe-wrapper">
+                  <iframe class="origin-top-left"></iframe>
                 </div>`
-              : html`<wdio-devtools-placeholder
-                  style="height: 100%"
-                ></wdio-devtools-placeholder>`}
+              : displayScreenshot
+                ? html`<div class="iframe-wrapper">
+                    <div
+                      class="screenshot-overlay"
+                      style="position:relative;flex:1;min-height:0;"
+                    >
+                      <img src="data:image/png;base64,${displayScreenshot}" />
+                    </div>
+                  </div>`
+                : html`<wdio-devtools-placeholder
+                    style="height: 100%"
+                  ></wdio-devtools-placeholder>`}
       </section>
     `
   }
diff --git a/packages/app/src/controller/DataManager.ts b/packages/app/src/controller/DataManager.ts
@@ -308,6 +308,14 @@ export class DataManagerController implements ReactiveController {
         return
       }
 
+      if (scope === 'screencast') {
+        const { sessionId } = data as { sessionId: string }
+        window.dispatchEvent(
+          new CustomEvent('screencast-ready', { detail: { sessionId } })
+        )
+        return
+      }
+
       if (scope === 'clearExecutionData') {
         const { uid, entryType } =
           data as SocketMessage<'clearExecutionData'>['data']
diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts
@@ -1,3 +1,4 @@
+import fs from 'node:fs'
 import url from 'node:url'
 
 import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify'
@@ -22,6 +23,13 @@ interface DevtoolsBackendOptions {
 const log = logger('@wdio/devtools-backend')
 const clients = new Set<WebSocket>()
 
+/**
+ * Registry mapping sessionId → absolute path of the encoded .webm file.
+ * Populated when the service sends { scope: 'screencast', data: { sessionId, videoPath } }.
+ * Queried by GET /api/video/:sessionId.
+ */
+const videoRegistry = new Map<string, string>()
+
 export function broadcastToClients(message: string) {
   clients.forEach((client) => {
     if (client.readyState === WebSocket.OPEN) {
@@ -102,28 +110,49 @@ export async function start(
           `received ${message.length} byte message from worker to ${clients.size} client${clients.size > 1 ? 's' : ''}`
         )
 
-        // Parse message to check if it's a clearCommands message
+        // Parse message to check if it needs special handling
         try {
           const parsed = JSON.parse(message.toString())
 
-          // If this is a clearCommands message, transform it to clear-execution-data format
+          // Transform clearCommands → clearExecutionData for the UI
           if (parsed.scope === 'clearCommands') {
             const testUid = parsed.data?.testUid
             log.info(`Clearing commands for test: ${testUid || 'all'}`)
-
-            // Create a synthetic message that DataManager will understand
             const clearMessage = JSON.stringify({
               scope: 'clearExecutionData',
               data: { uid: testUid }
             })
-
             clients.forEach((client) => {
               if (client.readyState === WebSocket.OPEN) {
                 client.send(clearMessage)
               }
             })
             return
           }
+
+          // Intercept screencast messages: store the absolute videoPath in the
+          // registry (backend-only), then forward only the sessionId to the UI
+          // so the UI can request the video via GET /api/video/:sessionId.
+          if (parsed.scope === 'screencast' && parsed.data?.sessionId) {
+            const { sessionId, videoPath } = parsed.data
+            if (videoPath) {
+              videoRegistry.set(sessionId, videoPath)
+              log.info(
+                `Screencast registered for session ${sessionId}: ${videoPath}`
+              )
+            }
+            // Forward trimmed message (no videoPath) to UI clients
+            const uiMessage = JSON.stringify({
+              scope: 'screencast',
+              data: { sessionId }
+            })
+            clients.forEach((client) => {
+              if (client.readyState === WebSocket.OPEN) {
+                client.send(uiMessage)
+              }
+            })
+            return
+          }
         } catch {
           // Not JSON or parsing failed, forward as-is
         }
@@ -138,6 +167,29 @@ export async function start(
     }
   )
 
+  // Serve recorded screencast videos. The service sends an absolute videoPath
+  // which is stored in videoRegistry; the UI only knows the sessionId and
+  // requests the file through this endpoint.
+  server.get(
+    '/api/video/:sessionId',
+    async (
+      request: FastifyRequest<{ Params: { sessionId: string } }>,
+      reply
+    ) => {
+      const { sessionId } = request.params
+      const videoPath = videoRegistry.get(sessionId)
+      if (!videoPath) {
+        return reply.code(404).send({ error: 'Video not found' })
+      }
+      if (!fs.existsSync(videoPath)) {
+        return reply.code(404).send({ error: 'Video file missing from disk' })
+      }
+      return reply
+        .header('Content-Type', 'video/webm')
+        .send(fs.createReadStream(videoPath))
+    }
+  )
+
   log.info(`Starting WebdriverIO Devtools application on port ${port}`)
   await server.listen({ port, host })
   return { server, port }
diff --git a/packages/service/package.json b/packages/service/package.json
@@ -42,6 +42,7 @@
     "@wdio/logger": "9.18.0",
     "@wdio/reporter": "9.27.0",
     "@wdio/types": "9.27.0",
+    "fluent-ffmpeg": "^2.1.3",
     "import-meta-resolve": "^4.1.0",
     "stack-trace": "1.0.0-pre2",
     "ws": "^8.18.3"
@@ -50,6 +51,7 @@
   "devDependencies": {
     "@types/babel__core": "^7.20.5",
     "@types/babel__traverse": "^7.28.0",
+    "@types/fluent-ffmpeg": "^2.1.27",
     "@types/stack-trace": "^0.0.33",
     "@types/ws": "^8.18.1",
     "@wdio/globals": "9.27.0",
diff --git a/packages/service/src/screencast.ts b/packages/service/src/screencast.ts
diff --git a/packages/service/src/video-encoder.ts b/packages/service/src/video-encoder.ts
