fixup! feat: add ncu-ci resume command

Author: panva

lib/ci/failure_file_scanner.js

@@ -2,9 +2,8 @@ import { StringDecoder } from 'node:string_decoder';
 
 import { FAILURE_PATTERNS } from './ci_failure_parser.js';
 
-// Drop stateful flags: repeated streaming checks must not share lastIndex state
-// with the whole-log parser or with another scanner. Preserve matching flags.
 function createMatcher(patterns) {
+  // Preserve matching flags without sharing RegExp.lastIndex between scans.
   const matchers = patterns.map(pattern =>
     new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, '')));
   return text => matchers.some(pattern => pattern.test(text));
@@ -13,123 +12,142 @@ function createMatcher(patterns) {
 const diagnostic = createMatcher(FAILURE_PATTERNS.diagnostic);
 const infrastructure = createMatcher(FAILURE_PATTERNS.infrastructure);
 
-export class FailureFileScanner {
-  constructor(filenames) {
-    this.windowSize = 256;
-    this.matchers = [...filenames].map(filename => {
-      const paths = new Set([filename]);
-      if (filename.startsWith('test/')) {
-        paths.add(filename.slice(5));
-        paths.add(filename.slice(5).replace(/\.(?:js|mjs|cjs|out)$/, ''));
-      } else if (filename.startsWith('lib/')) {
-        paths.add(filename.slice(4));
-      }
-      const patterns = [...paths].map(path => {
-        this.windowSize = Math.max(this.windowSize, path.length + 2);
-        const escaped = path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-        // Require a real delimiter, rather than treating a chunk end as EOF.
-        return new RegExp(`(?:^|[^\\w.-])${escaped}(?=[^\\w./-])`);
-      });
-      return { filename, patterns };
-    });
-    this.line = {};
-    this.tail = '';
-    this.history = [];
-    this.context = 0;
-    this.tap = null;
-    this.git = null;
+function fileAliases(filename) {
+  const paths = new Set([filename]);
+  if (filename.startsWith('test/')) {
+    paths.add(filename.slice(5));
+    paths.add(filename.slice(5).replace(/\.(?:js|mjs|cjs|out)$/, ''));
+  } else if (filename.startsWith('lib/')) {
+    paths.add(filename.slice(4));
   }
+  return [...paths];
+}
+
+// Emit overlapping, normalized windows rather than buffering whole log lines.
+// Explicit line boundaries keep network chunk boundaries out of the matching rules.
+async function * logWindows(source, overlap) {
+  const decoder = new StringDecoder('utf8');
+  let tail = '';
+  let lineStart = true;
+  let afterBackslash = false;
 
-  fragment(raw, end) {
-    // Normalize literal and escaped Windows separators, including runs split
-    // across network chunks. Preserve UTF-8 with StringDecoder in scan().
+  function * consume(raw) {
+    raw = raw.replace(/\r/g, '');
     let text = raw.replace(/\\+/g, '/');
-    if (this.backslash && raw.startsWith('\\')) text = text.slice(1);
-    if (raw) this.backslash = raw.endsWith('\\');
-    const line = this.line;
-    line.prefix = ((line.prefix ?? '') + text).slice(0, 128);
-    const window = (this.continuation ? 'x' : '') + this.tail + text + (end ? '\n' : '');
-    line.match ??= this.matchers.find(({ patterns }) =>
-      patterns.some(pattern => pattern.test(window)))?.filename;
-    line.diagnostic ||= diagnostic(window);
-    line.infrastructure ||= infrastructure(window);
-    line.cpp ||= /\[ {2}FAILED {2}\]/.test(window);
-    line.todo ||= window.includes('# TODO :');
-    line.tapEnd ||= / {2}\.\.\.\n$/.test(window);
-    line.gitStart ||= window.includes('Changes not staged for commit:') ||
-      window.includes('error: Your local changes to the following files');
-    line.gitEnd ||= window.includes('no changes added to commit') ||
-      window.includes('Failed to merge in the changes.');
-    this.continuation = window.length > this.windowSize;
-    this.tail = window.slice(-this.windowSize);
-    if (!end) return;
+    if (afterBackslash && raw.startsWith('\\')) text = text.slice(1);
+    if (raw) afterBackslash = raw.endsWith('\\');
+    for (let offset = 0; offset < text.length;) {
+      const newline = text.indexOf('\n', offset);
+      const end = Math.min(offset + 8192, newline < 0 ? text.length : newline + 1);
+      const window = tail + text.slice(offset, end);
+      const lineEnd = end === newline + 1;
+      yield { text: window, lineStart, lineEnd };
+      if (lineEnd) {
+        tail = '';
+        lineStart = true;
+      } else {
+        lineStart &&= window.length <= overlap;
+        tail = window.slice(-overlap);
+      }
+      offset = end;
+    }
+  }
+
+  for await (const chunk of source) yield * consume(decoder.write(chunk));
+  yield * consume(decoder.end());
+  if (tail) yield { text: `${tail}\n`, lineStart, lineEnd: true };
+}
 
-    this.tail = '';
-    this.continuation = false;
-    this.backslash = false;
-    this.line = {};
-    return this.finishLine(line);
+// Each line is reduced to a filename and failure markers. Neither a giant line
+// nor a giant TAP block needs to survive in memory.
+async function * failureLines(windows, matchFile) {
+  let line = {};
+  for await (const { text, lineStart, lineEnd } of windows) {
+    line.file ??= matchFile(text, lineStart);
+    line.diagnostic ||= diagnostic(text);
+    line.infrastructure ||= infrastructure(text);
+    line.cpp ||= /\[ {2}FAILED {2}\]/.test(text);
+    line.tapStart ||= lineStart && /^not ok \d+/.test(text);
+    line.tapEnd ||= lineEnd && / {2}\.\.\.\n$/.test(text);
+    line.todo ||= text.includes('# TODO :');
+    line.gitStart ||= text.includes('Changes not staged for commit:') ||
+      text.includes('error: Your local changes to the following files');
+    line.gitEnd ||= text.includes('no changes added to commit') ||
+      text.includes('Failed to merge in the changes.');
+    if (lineEnd) {
+      yield line;
+      line = {};
+    }
   }
+}
+
+async function findFailure(lines) {
+  let history = [];
+  let followingLines = 0;
+  let tap = null;
+  let git = null;
 
-  finishLine(line) {
-    if (/^not ok \d+/.test(line.prefix)) {
-      this.tap = {};
-      this.context = 0;
+  for await (const line of lines) {
+    const precedingLines = history;
+    history = [...history, line.file].slice(-5);
+    if (line.tapStart) {
+      tap = {};
+      followingLines = 0;
     }
-    if (this.tap) {
-      this.tap.match ??= line.match;
-      this.tap.todo ||= line.todo;
-      // Wait for the block to finish: a later TODO marks an expected failure.
+    if (tap) {
+      tap.file ??= line.file;
+      tap.todo ||= line.todo;
+      // A later TODO can mark this as an expected failure; wait for the ending.
       if (line.tapEnd) {
-        const match = !this.tap.todo && this.tap.match;
-        this.tap = null;
-        if (match) return match;
+        if (!tap.todo && tap.file) return tap.file;
+        tap = null;
       }
-    } else {
-      if (line.gitStart) this.git = {};
-      if (this.git) {
-        this.git.match ??= line.match;
-        if (line.gitEnd) {
-          const { match } = this.git;
-          this.git = null;
-          if (match) return match;
-        }
-      }
-      if (line.infrastructure || line.cpp) {
-        const before = line.cpp ? 5 : 1;
-        const match = line.match || this.history.slice(-before).find(Boolean);
-        if (match) return match;
-      }
-      if (line.diagnostic) this.context = 6;
-      if (this.context > 0) {
-        this.context--;
-        if (line.match) return line.match;
+      continue;
+    }
+
+    if (line.gitStart) git = {};
+    if (git) {
+      git.file ??= line.file;
+      if (line.gitEnd) {
+        if (git.file) return git.file;
+        git = null;
       }
     }
-    this.history.push(line.match);
-    if (this.history.length > 5) this.history.shift();
+    if (line.infrastructure || line.cpp) {
+      const before = line.cpp ? 5 : 1;
+      const file = line.file || precedingLines.slice(-before).find(Boolean);
+      if (file) return file;
+    }
+    if (line.diagnostic) followingLines = 6;
+    if (followingLines > 0) {
+      followingLines--;
+      if (line.file) return line.file;
+    }
   }
+}
 
-  async scan(source) {
-    const decoder = new StringDecoder('utf8');
-    const consume = (text) => {
-      let offset = 0;
-      while (offset < text.length) {
-        const newline = text.indexOf('\n', offset);
-        // Split even a giant single line into bounded pieces. Only the small
-        // overlap window and per-line flags survive between pieces.
-        const end = Math.min(offset + 8192, newline < 0 ? text.length : newline);
-        const complete = end === newline;
-        const match = this.fragment(text.slice(offset, end).replace(/\r/g, ''), complete);
-        if (match) return match;
-        offset = end + (complete ? 1 : 0);
-      }
+export class FailureFileScanner {
+  constructor(filenames) {
+    const aliases = [...filenames].map(filename => ({ filename, paths: fileAliases(filename) }));
+    this.overlap = Math.max(256,
+      ...aliases.flatMap(({ paths }) => paths.map(path => path.length + 2)));
+    const matchers = aliases.map(({ filename, paths }) => ({
+      filename,
+      patterns: paths.map(path => {
+        const escaped = path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+        // Both delimiters must be real: a window boundary is not a path boundary.
+        return new RegExp(`[^\\w.-]${escaped}(?=[^\\w./-])`);
+      })
+    }));
+    this.matchFile = (text, lineStart) => {
+      // The start of a real log line is also a valid left delimiter.
+      if (lineStart) text = `\n${text}`;
+      return matchers.find(({ patterns }) =>
+        patterns.some(pattern => pattern.test(text)))?.filename;
     };
-    for await (const chunk of source) {
-      const match = consume(decoder.write(chunk));
-      if (match) return match;
-    }
-    const match = consume(decoder.end());
-    return match || this.fragment('', true);
+  }
+
+  scan(source) {
+    return findFailure(failureLines(logWindows(source, this.overlap), this.matchFile));
   }
 }

test/unit/ci_failure_file_scanner.test.js

@@ -91,6 +91,14 @@ describe('Streaming failure file scanner', () => {
     assert.equal(await scan('not ok 1 parallel/test-example\n  partial output'), undefined);
   });
 
+  it('does not carry an unfinished failure into another log when reused', async() => {
+    const scanner = new FailureFileScanner([filename]);
+    assert.equal(await scanner.scan([Buffer.from('not ok 1 parallel/test-example\n')]),
+      undefined);
+    assert.equal(await scanner.scan([Buffer.from('unrelated output\n  ...\n')]), undefined);
+    assert.equal(await scanner.scan([Buffer.from(tap('  actual failure'))]), filename);
+  });
+
   it('matches a compiler diagnostic without a final newline', async() => {
     assert.equal(await scan('../src/node.cc:42: error: failure', ['src/node.cc'], 1),
       'src/node.cc');