Skip to content

Commit 5a8d4d8

Browse files
pedjaradenkovicpedjaradenkovic
committed
teach golang and cypress conformance runners to run a folder of suites
Degenerate case (root conformance_tests.go / cypress.config.*) preserves today's single-suite behavior byte-for-byte. Otherwise the runner iterates non-hidden subfolders that look like suites, runs each with an '=== conformance suite: <name> ===' header, keeps going past failures so the full failure set is reported, and exits with the first failure's code (exit 1 when no suites are discovered). Preparation work for whole-module conformance execution; the renderer still passes per-FRID folders, so behavior is unchanged.
1 parent b074d00 commit 5a8d4d8

4 files changed

Lines changed: 351 additions & 131 deletions

File tree

‎test_scripts/run_conformance_tests_cypress.ps1‎

Lines changed: 115 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -256,80 +256,143 @@ try {
256256
# Move back to the original directory
257257
Set-Location $current_dir
258258

259+
if (-not (Test-Path $ConformanceTestsFolder)) {
260+
Write-Host "Error: Conformance tests folder '$ConformanceTestsFolder' does not exist."
261+
exit $UNRECOVERABLE_ERROR_EXIT_CODE
262+
}
263+
259264
# Define the path to the conformance tests subfolder
260265
$script:NODE_CONFORMANCE_TESTS_SUBFOLDER = Join-Path ([System.IO.Path]::GetTempPath()) "node_$(Split-Path $ConformanceTestsFolder -Leaf)"
261266

262-
if ($env:VERBOSE -eq "1") {
263-
Write-Host "Preparing conformance tests Node subfolder: $script:NODE_CONFORMANCE_TESTS_SUBFOLDER"
264-
}
265-
266-
# Check if the conformance tests node subfolder exists
267-
if (Test-Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER) {
268-
# Delete all files and folders except "node_modules", "plain_modules", and "package-lock.json"
269-
Get-ChildItem -Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER -Force |
270-
Where-Object {
271-
$_.Name -ne "node_modules" -and
272-
$_.Name -ne "plain_modules" -and
273-
$_.Name -ne "package-lock.json"
274-
} | Remove-Item -Recurse -Force
267+
# Stage a single conformance test suite into the scratch subfolder and run
268+
# it. Returns the cypress run exit code.
269+
function Invoke-CypressSuite {
270+
param([string]$SuiteFolder)
275271

276272
if ($env:VERBOSE -eq "1") {
277-
Write-Host "Cleanup completed, keeping 'node_modules' and 'package-lock.json'."
273+
Write-Host "Preparing conformance tests Node subfolder: $script:NODE_CONFORMANCE_TESTS_SUBFOLDER"
278274
}
279-
} else {
280-
if ($env:VERBOSE -eq "1") {
281-
Write-Host "Subfolder does not exist. Creating it..."
275+
276+
# Check if the conformance tests node subfolder exists
277+
if (Test-Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER) {
278+
# Delete all files and folders except "node_modules", "plain_modules", and "package-lock.json"
279+
Get-ChildItem -Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER -Force |
280+
Where-Object {
281+
$_.Name -ne "node_modules" -and
282+
$_.Name -ne "plain_modules" -and
283+
$_.Name -ne "package-lock.json"
284+
} | Remove-Item -Recurse -Force
285+
286+
if ($env:VERBOSE -eq "1") {
287+
Write-Host "Cleanup completed, keeping 'node_modules' and 'package-lock.json'."
288+
}
289+
} else {
290+
if ($env:VERBOSE -eq "1") {
291+
Write-Host "Subfolder does not exist. Creating it..."
292+
}
293+
294+
New-Item -ItemType Directory -Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER -Force | Out-Null
282295
}
283296

284-
New-Item -ItemType Directory -Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER -Force | Out-Null
285-
}
297+
Copy-Item -Path "$SuiteFolder/*" -Destination $script:NODE_CONFORMANCE_TESTS_SUBFOLDER -Recurse -Force
286298

287-
Copy-Item -Path "$ConformanceTestsFolder/*" -Destination $script:NODE_CONFORMANCE_TESTS_SUBFOLDER -Recurse -Force
299+
# Move to the subfolder with Cypress tests
300+
if (-not (Test-Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER)) {
301+
Write-Host "Error: conformance tests Node folder '$script:NODE_CONFORMANCE_TESTS_SUBFOLDER' does not exist."
302+
exit $UNRECOVERABLE_ERROR_EXIT_CODE
303+
}
288304

289-
# Move to the subfolder with Cypress tests
290-
if (-not (Test-Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER)) {
291-
Write-Host "Error: conformance tests Node folder '$script:NODE_CONFORMANCE_TESTS_SUBFOLDER' does not exist."
292-
exit $UNRECOVERABLE_ERROR_EXIT_CODE
293-
}
305+
Push-Location $script:NODE_CONFORMANCE_TESTS_SUBFOLDER
294306

295-
Push-Location $script:NODE_CONFORMANCE_TESTS_SUBFOLDER
307+
try {
308+
# Temporarily allow stderr output without throwing (npm may write warnings to stderr)
309+
# ForEach-Object converts ErrorRecord objects (from stderr) to plain strings to avoid verbose error formatting
310+
$script:ErrorActionPreference = 'Continue'
311+
$npmInstallOutput = npm install cypress --save-dev --prefer-offline --no-audit --no-fund --loglevel error 2>&1 | ForEach-Object { if ($_ -is [System.Management.Automation.ErrorRecord]) { $_.Exception.Message } else { $_ } } | Out-String
312+
$script:ErrorActionPreference = 'Stop'
313+
$npmInstallOutput -split "`n" | Where-Object { $_ -notmatch $NPM_INSTALL_OUTPUT_FILTER } | ForEach-Object {
314+
if ($_.Trim()) { Write-Host $_ }
315+
}
296316

297-
# Temporarily allow stderr output without throwing (npm may write warnings to stderr)
298-
# ForEach-Object converts ErrorRecord objects (from stderr) to plain strings to avoid verbose error formatting
299-
$ErrorActionPreference = 'Continue'
300-
$npmInstallOutput = npm install cypress --save-dev --prefer-offline --no-audit --no-fund --loglevel error 2>&1 | ForEach-Object { if ($_ -is [System.Management.Automation.ErrorRecord]) { $_.Exception.Message } else { $_ } } | Out-String
301-
$ErrorActionPreference = 'Stop'
302-
$npmInstallOutput -split "`n" | Where-Object { $_ -notmatch $NPM_INSTALL_OUTPUT_FILTER } | ForEach-Object {
303-
if ($_.Trim()) { Write-Host $_ }
304-
}
317+
if ($env:VERBOSE -eq "1") {
318+
Write-Host "Running Cypress conformance tests..."
319+
}
305320

306-
if ($env:VERBOSE -eq "1") {
307-
Write-Host "Running Cypress conformance tests..."
321+
$script:ErrorActionPreference = 'Continue'
322+
$cypress_info_output = npx cypress info 2>&1 | ForEach-Object { if ($_ -is [System.Management.Automation.ErrorRecord]) { $_.Exception.Message } else { $_ } } | Out-String
323+
$script:ErrorActionPreference = 'Stop'
324+
$CYPRESS_BROWSER_FLAG = ""
325+
if ($cypress_info_output -match "(?i)chrome") {
326+
$CYPRESS_BROWSER_FLAG = "--browser=chrome"
327+
}
328+
Write-Host "CYPRESS_BROWSER_FLAG: $(if ($CYPRESS_BROWSER_FLAG) { $CYPRESS_BROWSER_FLAG } else { 'none' })"
329+
330+
$env:BROWSERSLIST_IGNORE_OLD_DATA = "1"
331+
if ($CYPRESS_BROWSER_FLAG) {
332+
npx cypress run $CYPRESS_BROWSER_FLAG --config video=false 2>$null
333+
} else {
334+
npx cypress run --config video=false 2>$null
335+
}
336+
return $LASTEXITCODE
337+
} finally {
338+
Pop-Location
339+
}
308340
}
309341

310-
$ErrorActionPreference = 'Continue'
311-
$cypress_info_output = npx cypress info 2>&1 | ForEach-Object { if ($_ -is [System.Management.Automation.ErrorRecord]) { $_.Exception.Message } else { $_ } } | Out-String
312-
$ErrorActionPreference = 'Stop'
313-
$CYPRESS_BROWSER_FLAG = ""
314-
if ($cypress_info_output -match "(?i)chrome") {
315-
$CYPRESS_BROWSER_FLAG = "--browser=chrome"
342+
function Test-HasCypressConfig {
343+
param([string]$Folder)
344+
return [bool](Get-ChildItem -Path $Folder -Filter "cypress.config.*" -File -ErrorAction SilentlyContinue)
316345
}
317-
Write-Host "CYPRESS_BROWSER_FLAG: $(if ($CYPRESS_BROWSER_FLAG) { $CYPRESS_BROWSER_FLAG } else { 'none' })"
318346

319-
$env:BROWSERSLIST_IGNORE_OLD_DATA = "1"
320-
if ($CYPRESS_BROWSER_FLAG) {
321-
npx cypress run $CYPRESS_BROWSER_FLAG --config video=false 2>$null
322-
} else {
323-
npx cypress run --config video=false 2>$null
347+
if (Test-HasCypressConfig $ConformanceTestsFolder) {
348+
# Single conformance test suite ($ConformanceTestsFolder is the suite folder itself).
349+
$cypress_run_result = Invoke-CypressSuite $ConformanceTestsFolder
350+
351+
if ($cypress_run_result -ne 0) {
352+
if ($env:VERBOSE -eq "1") {
353+
Write-Host "Error: Cypress conformance tests have failed."
354+
}
355+
exit 1
356+
}
357+
358+
exit 0
324359
}
325-
$cypress_run_result = $LASTEXITCODE
326360

327-
if ($cypress_run_result -ne 0) {
328-
if ($env:VERBOSE -eq "1") {
329-
Write-Host "Error: Cypress conformance tests have failed."
361+
# $ConformanceTestsFolder is a folder of conformance test suites: run every
362+
# non-hidden subfolder that contains a cypress config file.
363+
$suites_run = 0
364+
$aggregated_exit_code = 0
365+
366+
$suite_folders = Get-ChildItem -Path $ConformanceTestsFolder -Directory |
367+
Where-Object { -not $_.Name.StartsWith(".") } |
368+
Sort-Object Name
369+
370+
foreach ($suite in $suite_folders) {
371+
if (-not (Test-HasCypressConfig $suite.FullName)) {
372+
continue
373+
}
374+
375+
Write-Host "=== conformance suite: $($suite.Name) ==="
376+
377+
$cypress_run_result = Invoke-CypressSuite $suite.FullName
378+
$suites_run += 1
379+
380+
# Keep running the remaining suites so the full set of failures is
381+
# reported, but exit non-zero if any suite failed.
382+
if ($cypress_run_result -ne 0) {
383+
if ($env:VERBOSE -eq "1") {
384+
Write-Host "Error: Cypress conformance tests have failed."
385+
}
386+
$aggregated_exit_code = 1
330387
}
388+
}
389+
390+
if ($suites_run -eq 0) {
391+
Write-Host "`nError: No conformance test suites discovered."
331392
exit 1
332393
}
394+
395+
exit $aggregated_exit_code
333396
} finally {
334397
Cleanup
335398
}

‎test_scripts/run_conformance_tests_cypress.sh‎

Lines changed: 100 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -209,51 +209,122 @@ printf "### Step 2: Running Cypress conformance tests $2...\n"
209209
# Move back to the original directory
210210
cd $current_dir
211211

212-
# Define the path to the conformance tests subfolder
213-
NODE_CONFORMANCE_TESTS_SUBFOLDER="/tmp/node_$(basename "$2")"
212+
# Resolve the conformance tests folder to an absolute path
213+
CONFORMANCE_TESTS_FOLDER=$(cd "$2" 2>/dev/null && pwd)
214214

215-
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
216-
printf "Preparing conformance tests Node subfolder: $NODE_CONFORMANCE_TESTS_SUBFOLDER\n"
215+
if [ -z "$CONFORMANCE_TESTS_FOLDER" ]; then
216+
printf "Error: Conformance tests folder '$2' does not exist.\n"
217+
exit $UNRECOVERABLE_ERROR_EXIT_CODE
217218
fi
218219

219-
# Check if the conformance tests node subfolder exists
220-
if [ -d "$NODE_CONFORMANCE_TESTS_SUBFOLDER" ]; then
221-
# Find and delete all files and folders except "node_modules", "plain_modules", and "package-lock.json"
222-
find "$NODE_CONFORMANCE_TESTS_SUBFOLDER" -mindepth 1 ! -path "$NODE_CONFORMANCE_TESTS_SUBFOLDER/node_modules*" ! -path "$NODE_CONFORMANCE_TESTS_SUBFOLDER/plain_modules*" ! -name "package-lock.json" -exec rm -rf {} +
220+
# Define the path to the conformance tests subfolder
221+
NODE_CONFORMANCE_TESTS_SUBFOLDER="/tmp/node_$(basename "$2")"
222+
223+
# Stage a single conformance test suite into the scratch subfolder and run it.
224+
# Returns the cypress run exit code (or exits the script on environment errors).
225+
stage_and_run_suite() {
226+
suite_folder="$1"
223227

224228
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
225-
printf "Cleanup completed, keeping 'node_modules' and 'package-lock.json'.\n"
229+
printf "Preparing conformance tests Node subfolder: $NODE_CONFORMANCE_TESTS_SUBFOLDER\n"
226230
fi
227-
else
231+
232+
# Check if the conformance tests node subfolder exists
233+
if [ -d "$NODE_CONFORMANCE_TESTS_SUBFOLDER" ]; then
234+
# Find and delete all files and folders except "node_modules", "plain_modules", and "package-lock.json"
235+
find "$NODE_CONFORMANCE_TESTS_SUBFOLDER" -mindepth 1 ! -path "$NODE_CONFORMANCE_TESTS_SUBFOLDER/node_modules*" ! -path "$NODE_CONFORMANCE_TESTS_SUBFOLDER/plain_modules*" ! -name "package-lock.json" -exec rm -rf {} +
236+
237+
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
238+
printf "Cleanup completed, keeping 'node_modules' and 'package-lock.json'.\n"
239+
fi
240+
else
241+
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
242+
printf "Subfolder does not exist. Creating it...\n"
243+
fi
244+
245+
mkdir -p $NODE_CONFORMANCE_TESTS_SUBFOLDER
246+
fi
247+
248+
cp -R "$suite_folder"/* $NODE_CONFORMANCE_TESTS_SUBFOLDER
249+
250+
# Move to the subfolder with Cypress tests
251+
cd "$NODE_CONFORMANCE_TESTS_SUBFOLDER" 2>/dev/null
252+
253+
if [ $? -ne 0 ]; then
254+
printf "Error: conformance tests Node folder '$NODE_CONFORMANCE_TESTS_SUBFOLDER' does not exist.\n"
255+
exit $UNRECOVERABLE_ERROR_EXIT_CODE
256+
fi
257+
258+
npm install cypress --save-dev --prefer-offline --no-audit --no-fund --loglevel error | grep -Ev "$NPM_INSTALL_OUTPUT_FILTER"
259+
228260
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
229-
printf "Subfolder does not exist. Creating it...\n"
261+
printf "Running Cypress conformance tests...\n"
230262
fi
231263

232-
mkdir -p $NODE_CONFORMANCE_TESTS_SUBFOLDER
233-
fi
264+
BROWSERSLIST_IGNORE_OLD_DATA=1 npx cypress run --browser=chrome --config video=false 2>/dev/null
265+
cypress_run_result=$?
234266

235-
cp -R $2/* $NODE_CONFORMANCE_TESTS_SUBFOLDER
267+
# Move back to the original directory before the next suite
268+
cd $current_dir
236269

237-
# Move to the subfolder with Cypress tests
238-
cd "$NODE_CONFORMANCE_TESTS_SUBFOLDER" 2>/dev/null
270+
return $cypress_run_result
271+
}
239272

240-
if [ $? -ne 0 ]; then
241-
printf "Error: conformance tests Node folder '$NODE_CONFORMANCE_TESTS_SUBFOLDER' does not exist.\n"
242-
exit $UNRECOVERABLE_ERROR_EXIT_CODE
243-
fi
273+
has_cypress_config() {
274+
ls "$1"/cypress.config.* >/dev/null 2>&1
275+
}
244276

245-
npm install cypress --save-dev --prefer-offline --no-audit --no-fund --loglevel error | grep -Ev "$NPM_INSTALL_OUTPUT_FILTER"
277+
if has_cypress_config "$CONFORMANCE_TESTS_FOLDER"; then
278+
# Single conformance test suite ("$2" is the suite folder itself).
279+
stage_and_run_suite "$CONFORMANCE_TESTS_FOLDER"
280+
cypress_run_result=$?
246281

247-
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
248-
printf "Running Cypress conformance tests...\n"
282+
if [ $cypress_run_result -ne 0 ]; then
283+
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
284+
printf "Error: Cypress conformance tests have failed.\n"
285+
fi
286+
exit 1
287+
fi
288+
289+
exit 0
249290
fi
250291

251-
BROWSERSLIST_IGNORE_OLD_DATA=1 npx cypress run --browser=chrome --config video=false 2>/dev/null
252-
cypress_run_result=$?
292+
# "$2" is a folder of conformance test suites: run every non-hidden subfolder
293+
# that contains a cypress config file.
294+
suites_run=0
295+
aggregated_exit_code=0
253296

254-
if [ $cypress_run_result -ne 0 ]; then
255-
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
256-
printf "Error: Cypress conformance tests have failed.\n"
297+
for suite_folder in "$CONFORMANCE_TESTS_FOLDER"/*/; do
298+
suite_name=$(basename "$suite_folder")
299+
300+
case "$suite_name" in
301+
.*) continue ;;
302+
esac
303+
304+
if ! has_cypress_config "${suite_folder%/}"; then
305+
continue
257306
fi
307+
308+
printf "=== conformance suite: %s ===\n" "$suite_name"
309+
310+
stage_and_run_suite "${suite_folder%/}"
311+
cypress_run_result=$?
312+
313+
suites_run=$((suites_run + 1))
314+
315+
# Keep running the remaining suites so the full set of failures is reported,
316+
# but exit non-zero if any suite failed.
317+
if [ $cypress_run_result -ne 0 ]; then
318+
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
319+
printf "Error: Cypress conformance tests have failed.\n"
320+
fi
321+
aggregated_exit_code=1
322+
fi
323+
done
324+
325+
if [ $suites_run -eq 0 ]; then
326+
printf "\nError: No conformance test suites discovered.\n"
258327
exit 1
259-
fi
328+
fi
329+
330+
exit $aggregated_exit_code

0 commit comments

Comments
 (0)