From a24387bc53f3bd73fd0677db6a54783a222ff011 Mon Sep 17 00:00:00 2001 From: roflmuffin Date: Mon, 7 Sep 2026 01:23:37 +0000 Subject: [PATCH 1/4] feat: smoke test workflow --- .github/scripts/smoke-test.cjs | 132 +++++++++++ .github/scripts/smoke-test.test.cjs | 146 ++++++++++++ .github/workflows/pr-smoke-test.yml | 214 ++++++++++++++++++ CONTRIBUTING.md | 2 + eng/SMOKE-TESTS.md | 138 +++++++++++ eng/run-smoke-tests.ts | 200 ++++++++++++++++ .../ConsoleTestReporterSink.cs | 43 +++- .../NativeTestsPlugin.cs | 55 ++++- .../CounterStrikeSharp.Tests.Native/README.md | 3 + .../ScriptContextBenchmarks.cs | 1 + 10 files changed, 924 insertions(+), 10 deletions(-) create mode 100644 .github/scripts/smoke-test.cjs create mode 100644 .github/scripts/smoke-test.test.cjs create mode 100644 .github/workflows/pr-smoke-test.yml create mode 100644 eng/SMOKE-TESTS.md create mode 100644 eng/run-smoke-tests.ts diff --git a/.github/scripts/smoke-test.cjs b/.github/scripts/smoke-test.cjs new file mode 100644 index 000000000..54ddf45aa --- /dev/null +++ b/.github/scripts/smoke-test.cjs @@ -0,0 +1,132 @@ +const fs = require('node:fs'); + +const checkName = 'Game server smoke test'; +const runUrl = (context) => `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + +async function isMaintainerRequest({ github, context }) { + // Check live repository permissions, not author_association (a contributor + // or organization member is not necessarily a maintainer). Check reruns too. + for (const username of new Set([context.payload.comment.user.login, process.env.TRIGGERING_ACTOR || context.actor])) { + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ ...context.repo, username }); + if (data.permission !== 'admin' && data.permission !== 'maintain' && data.role_name !== 'maintain') { + return false; + } + } + return true; +} + +async function authorize({ github, context, core }) { + const { issue, comment } = context.payload; + if (!issue?.pull_request || comment?.body.trim() !== '/smoke-test' || comment.user.type !== 'User') return; + if (!await isMaintainerRequest({ github, context })) { + core.info('Ignoring smoke-test request: maintain/admin permission required.'); + return; + } + const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: issue.number }); + if (pr.state !== 'open') return; + // Snapshot the latest head ONCE. All builds use this immutable SHA, not a + // mutable branch or refs/pull/N/head. Works for fork PRs as well. + const sha = pr.head.sha; + if (!/^[a-f0-9]{40}$/.test(sha)) throw new Error('Invalid PR head SHA'); + const details_url = runUrl(context); + const { data: check } = await github.rest.checks.create({ + ...context.repo, + name: checkName, + head_sha: sha, + status: 'in_progress', + details_url, + output: { title: 'Preparing smoke test', summary: `Building commit ${sha}. Benchmarks excluded.\n\n[Workflow run](${details_url})` }, + }); + const { data: reply } = await github.rest.issues.createComment({ + ...context.repo, + issue_number: issue.number, + body: `### Game server smoke test\n\nRequested for commit ${sha}. Building, then waiting for the dedicated server.\n\n[Follow the run](${details_url})`, + }); + core.setOutput('sha', sha); + core.setOutput('check_id', check.id); + core.setOutput('comment_id', reply.id); + core.setOutput('approved', 'true'); +} + +function readJson(path) { + // Artifacts and server files are untrusted data, never code or Markdown. + if (fs.statSync(path).size > 10 * 1024 * 1024) throw new Error('Report too large'); + return JSON.parse(fs.readFileSync(path, 'utf8')); +} + +function resultSummary(report) { + const keys = ['total', 'passed', 'failed', 'skipped']; + if (!keys.every((key) => Number.isSafeInteger(report[key]) && report[key] >= 0) || + report.total !== report.passed + report.failed + report.skipped || + !Array.isArray(report.errors) || !Array.isArray(report.tests) || report.tests.length !== report.total || + ['passed', 'failed', 'skipped'].some((outcome) => report.tests.filter((test) => test?.outcome === outcome).length !== report[outcome])) { + throw new Error('Invalid test report'); + } + return { + success: report.passed > 0 && report.failed === 0 && report.errors.length === 0, + text: `${report.total} native tests: **${report.passed} passed**, **${report.failed} failed**, **${report.skipped} skipped**.\n\n` + + `Runner/cleanup errors: ${report.errors.length}. Benchmarks excluded.`, + }; +} + +function versionSummary(version) { + // Only render bounded, validated values. Raw steam.inf is also an artifact. + const patterns = { + PatchVersion: /^\d+(\.\d+)+$/, + ServerVersion: /^\d+$/, + ClientVersion: /^\d+$/, + SourceRevision: /^\d+$/, + VersionDate: /^[a-zA-Z0-9 ,/-]+$/, + VersionTime: /^[0-9:]+$/, + }; + if (!Object.entries(patterns).every(([key, pattern]) => + typeof version[key] === 'string' && version[key].length <= 100 && pattern.test(version[key]))) { + throw new Error('Invalid CS2 version'); + } + return `**CS2:** ${version.PatchVersion} (server ${version.ServerVersion}, client ${version.ClientVersion})\n\n` + + `**Source revision:** ${version.SourceRevision} — ${version.VersionDate} ${version.VersionTime}`; +} + +async function report({ github, context, core }) { + const env = process.env; + const jobs = [env.NATIVE_RESULT, env.MANAGED_RESULT, env.SMOKE_RESULT]; + let success = jobs.every((result) => result === 'success'); + let text = ''; + try { + const result = resultSummary(readJson('smoke-results/smoke-results.json')); + success = success && result.success; + text += result.text; + } catch { + success = false; + text += 'No valid, complete native test report was received. See the workflow logs for build, deployment, or timeout errors.'; + } + try { + text += `\n\n${versionSummary(readJson('smoke-results/server-version.json'))}`; + } catch { + success = false; + text += '\n\nCS2 version unavailable (the server may not have reached the test stage).'; + } + const conclusion = success ? 'success' : jobs.includes('cancelled') ? 'cancelled' : 'failure'; + const title = `${checkName}: ${conclusion === 'success' ? 'passed' : conclusion}`; + const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: context.payload.issue.number }); + let summary = `**Tested commit:** ${env.TESTED_SHA}\n\n${text}\n\n` + + `Native build: ${env.NATIVE_RESULT}. Managed build/unit tests: ${env.MANAGED_RESULT}. Server run: ${env.SMOKE_RESULT}.\n\n` + + `[Logs and artifacts (JSON, Markdown, steam.inf, unit-test TRX)](${runUrl(context)})`; + if (pr.head.sha !== env.TESTED_SHA) summary += '\n\n⚠️ The PR has newer commits. This result does **not** cover the latest head; comment `/smoke-test` again to test it.'; + await github.rest.checks.update({ + ...context.repo, + check_run_id: Number(env.CHECK_ID), + status: 'completed', + conclusion, + completed_at: new Date().toISOString(), + output: { title, summary }, + }); + await github.rest.issues.updateComment({ + ...context.repo, + comment_id: Number(env.COMMENT_ID), + body: `### ${title}\n\n${summary}`, + }); + await core.summary.addRaw(`### ${title}\n\n${summary}`).write(); +} + +module.exports = { authorize, isMaintainerRequest, report, resultSummary, versionSummary }; diff --git a/.github/scripts/smoke-test.test.cjs b/.github/scripts/smoke-test.test.cjs new file mode 100644 index 000000000..e6ec0cc87 --- /dev/null +++ b/.github/scripts/smoke-test.test.cjs @@ -0,0 +1,146 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { authorize, report, resultSummary, versionSummary } = require('./smoke-test.cjs'); + +const sha = 'a'.repeat(40); +function request({ permission = 'write', role = 'maintain', body = '/smoke-test', state = 'open', pullRequest = true } = {}) { + const calls = []; + const outputs = {}; + const context = { + repo: { owner: 'owner', repo: 'repo' }, actor: 'maintainer', serverUrl: 'https://github.com', runId: 123, + payload: { + issue: { number: 42, pull_request: pullRequest ? {} : undefined }, + comment: { body, user: { login: 'maintainer', type: 'User' } }, + }, + }; + const github = { rest: { + repos: { getCollaboratorPermissionLevel: async (args) => { + calls.push(['permission', args]); + return { data: { permission, role_name: role } }; + } }, + pulls: { get: async () => ({ data: { state, head: { sha } } }) }, + checks: { create: async (args) => { calls.push(['check', args]); return { data: { id: 1 } }; } }, + issues: { createComment: async (args) => { calls.push(['comment', args]); return { data: { id: 2 } }; } }, + } }; + const core = { info() {}, setOutput: (key, value) => { outputs[key] = value; } }; + return { github, context, core, calls, outputs }; +} + +test('maintainer command snapshots the latest head and attaches the check to it', async () => { + const input = request(); + await authorize(input); + assert.equal(input.outputs.approved, 'true'); + assert.equal(input.outputs.sha, sha); + assert.equal(input.calls.find(([name]) => name === 'check')[1].head_sha, sha); +}); + +test('admin can trigger', async () => { + const input = request({ permission: 'admin', role: 'admin' }); + await authorize(input); + assert.equal(input.outputs.approved, 'true'); +}); + +for (const role of ['write', 'read', 'triage', 'none']) { + test(`${role} permission cannot trigger a server run`, async () => { + const input = request({ permission: role, role }); + await authorize(input); + assert.equal(input.outputs.approved, undefined); + assert.ok(input.calls.every(([name]) => name === 'permission')); + }); +} + +for (const options of [{ body: '/smoke-test something' }, { state: 'closed' }, { pullRequest: false }]) { + test(`ignores invalid request ${JSON.stringify(options)}`, async () => { + const input = request(options); + await authorize(input); + assert.equal(input.outputs.approved, undefined); + assert.ok(!input.calls.some(([name]) => name === 'check')); + }); +} + +test('an unauthorized rerun actor cannot reuse a maintainer request', async () => { + const input = request(); + // context.actor is used when running locally without TRIGGERING_ACTOR. + input.context.actor = 'other'; + input.github.rest.repos.getCollaboratorPermissionLevel = async ({ username }) => ({ + data: { permission: 'write', role_name: username === 'maintainer' ? 'maintain' : 'write' }, + }); + await authorize(input); + assert.equal(input.outputs.approved, undefined); +}); + +const passing = { total: 2, passed: 1, failed: 0, skipped: 1, errors: [], tests: [{ outcome: 'passed' }, { outcome: 'skipped' }] }; +test('summarizes passing results including skips', () => { + assert.equal(resultSummary(passing).success, true); + assert.match(resultSummary(passing).text, /1 passed/); +}); +test('test failures, runner errors, zero tests and all-skipped runs cannot pass', () => { + assert.equal(resultSummary({ ...passing, errors: ['cleanup failure'] }).success, false); + assert.equal(resultSummary({ ...passing, passed: 0, failed: 1, tests: [{ outcome: 'failed' }, { outcome: 'skipped' }] }).success, false); + assert.equal(resultSummary({ total: 0, passed: 0, failed: 0, skipped: 0, errors: [], tests: [] }).success, false); + assert.equal(resultSummary({ total: 1, passed: 0, failed: 0, skipped: 1, errors: [], tests: [{ outcome: 'skipped' }] }).success, false); +}); +test('rejects malformed or inconsistent artifact counts', () => { + for (const report of [{}, { ...passing, total: 99 }, { ...passing, passed: '1' }, { ...passing, tests: [] }]) { + assert.throws(() => resultSummary(report)); + } +}); +const version = { + ClientVersion: '2000899', ServerVersion: '2000899', PatchVersion: '1.41.7.8', + SourceRevision: '10948930', VersionDate: 'Aug 28 2026', VersionTime: '13:05:38', +}; +test('reports the CS2 version from steam.inf fields', () => { + assert.match(versionSummary(version), /1\.41\.7\.8/); + assert.match(versionSummary(version), /server 2000899/); + assert.match(versionSummary(version), /10948930 — Aug 28 2026 13:05:38/); +}); +test('never renders arbitrary server-controlled Markdown', () => { + assert.throws(() => versionSummary({ ...version, PatchVersion: '[click](https://example.org)' })); + assert.throws(() => versionSummary({ ...version, VersionDate: '@everyone' })); +}); + +for (const scenario of ['success', 'new-head', 'build-failure', 'timeout', 'cancelled']) { + test(`publishes check and PR comment: ${scenario}`, async () => { + const originalCwd = process.cwd(); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'smoke-report-')); + const values = { + TESTED_SHA: sha, CHECK_ID: '1', COMMENT_ID: '2', NATIVE_RESULT: 'success', + MANAGED_RESULT: scenario === 'build-failure' ? 'failure' : 'success', + SMOKE_RESULT: scenario === 'cancelled' ? 'cancelled' : scenario === 'timeout' ? 'failure' : 'success', + }; + const originalEnv = Object.fromEntries(Object.keys(values).map((key) => [key, process.env[key]])); + try { + process.chdir(directory); + Object.assign(process.env, values); + fs.mkdirSync('smoke-results'); + if (!['build-failure', 'timeout', 'cancelled'].includes(scenario)) { + fs.writeFileSync('smoke-results/smoke-results.json', JSON.stringify(passing)); + fs.writeFileSync('smoke-results/server-version.json', JSON.stringify(version)); + } + const input = request(); + const published = {}; + input.github.rest.pulls.get = async () => ({ data: { head: { sha: scenario === 'new-head' ? 'b'.repeat(40) : sha } } }); + input.github.rest.checks.update = async (args) => { published.check = args; }; + input.github.rest.issues.updateComment = async (args) => { published.comment = args; }; + input.core.summary = { addRaw(text) { published.summary = text; return this; }, async write() {} }; + await report(input); + assert.equal(published.check.status, 'completed'); + assert.equal(published.check.conclusion, ['success', 'new-head'].includes(scenario) ? 'success' : scenario === 'cancelled' ? 'cancelled' : 'failure'); + assert.match(published.comment.body, new RegExp(sha)); + assert.match(published.comment.body, /actions\/runs\/123/); + if (scenario === 'new-head') assert.match(published.comment.body, /does \*\*not\*\* cover the latest head/); + if (scenario === 'success') assert.match(published.comment.body, /CS2:\*\* 1\.41\.7\.8/); + if (scenario === 'timeout') assert.match(published.comment.body, /No valid, complete native test report/); + } finally { + process.chdir(originalCwd); + fs.rmSync(directory, { recursive: true, force: true }); + for (const [key, value] of Object.entries(originalEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + }); +} diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml new file mode 100644 index 000000000..ab9c10fcd --- /dev/null +++ b/.github/workflows/pr-smoke-test.yml @@ -0,0 +1,214 @@ +name: PR smoke test + +on: + issue_comment: + types: [created] + +permissions: {} + +jobs: + authorize: + if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/smoke-test') + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: write + checks: write + outputs: + approved: ${{ steps.request.outputs.approved }} + sha: ${{ steps.request.outputs.sha }} + check_id: ${{ steps.request.outputs.check_id }} + comment_id: ${{ steps.request.outputs.comment_id }} + steps: + # issue_comment runs the workflow from the default branch, never the PR. + - uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/github-script@v7 + id: request + env: + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + with: + script: | + await require('./.github/scripts/smoke-test.cjs').authorize({ github, context, core }); + + build_native: + needs: authorize + if: needs.authorize.outputs.approved == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + container: + image: registry.gitlab.steamos.cloud/steamrt/sniper/sdk:latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.authorize.outputs.sha }} + submodules: recursive + persist-credentials: false + - name: Build the approved native commit (no server secrets) + env: + PR_SHA: ${{ needs.authorize.outputs.sha }} + run: | + export GITHUB_SHA_SHORT="${PR_SHA:0:7}" + export SEMVER=0.0.0-smoke + cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release + cmake --build build --config Release -- -j2 + shell: bash + - uses: actions/upload-artifact@v4 + with: + name: smoke-native + overwrite: true + path: build/addons/ + if-no-files-found: error + + build_managed: + needs: authorize + if: needs.authorize.outputs.approved == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.authorize.outputs.sha }} + persist-credentials: false + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - name: Build API and native test plugin (no server secrets) + run: | + dotnet build managed/CounterStrikeSharp.API -c Release + dotnet build managed/CounterStrikeSharp.Tests.Native/NativeTestsPlugin.csproj -c Release + - name: Run managed unit tests too + run: dotnet test managed/CounterStrikeSharp.API.Tests/CounterStrikeSharp.API.Tests.csproj -c Release --logger trx --results-directory TestResults/Unit + - uses: actions/upload-artifact@v4 + if: always() + with: + name: smoke-unit-results + overwrite: true + path: TestResults/Unit/ + - uses: actions/upload-artifact@v4 + with: + name: smoke-api + overwrite: true + path: managed/CounterStrikeSharp.API/bin/Release/net10.0/ + if-no-files-found: error + - uses: actions/upload-artifact@v4 + with: + name: smoke-plugin + overwrite: true + path: managed/CounterStrikeSharp.Tests.Native/bin/Release/net10.0/ + if-no-files-found: error + + smoke: + needs: [authorize, build_native, build_managed] + runs-on: ubuntu-latest + timeout-minutes: 25 + permissions: + contents: read + environment: smoke-test + # One dedicated server: never deploy two PRs at once, or cancel an active run + # just because a new request arrives. GitHub may replace older pending jobs. + concurrency: + group: game-server-smoke-test + cancel-in-progress: false + steps: + - name: Check out trusted automation, NOT the PR + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + # A failed-job rerun may reuse authorize's outputs, so recheck permissions + # here before exposing secrets or touching the server. + - uses: actions/github-script@v7 + env: + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + with: + script: | + const { isMaintainerRequest } = require('./.github/scripts/smoke-test.cjs'); + if (!await isMaintainerRequest({ github, context })) { + throw new Error('Only maintainers/admins may deploy a smoke test, including reruns.'); + } + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + - name: Install deployment client + run: sudo apt-get update && sudo apt-get install -y lftp + - uses: actions/download-artifact@v4 + with: + name: smoke-native + path: payload/native/addons + - uses: actions/download-artifact@v4 + with: + name: smoke-api + path: payload/api + - uses: actions/download-artifact@v4 + with: + name: smoke-plugin + path: payload/plugin + - name: Deploy and run the non-benchmark suite + env: + SMOKE_COMMIT: ${{ needs.authorize.outputs.sha }} + GS_HOST: ${{ secrets.GS_HOST }} + GS_PORT: ${{ secrets.GS_PORT }} + GS_PASS: ${{ secrets.GS_PASS }} + SFTP_HOST: ${{ secrets.SFTP_HOST }} + SFTP_USER: ${{ secrets.SFTP_USER }} + SFTP_PASS: ${{ secrets.SFTP_PASS }} + SFTP_KNOWN_HOSTS: ${{ secrets.SFTP_KNOWN_HOSTS }} + PTERO_URL: ${{ secrets.PTERO_URL }} + PTERO_API_KEY: ${{ secrets.PTERO_API_KEY }} + PTERO_SERVER_ID: ${{ secrets.PTERO_SERVER_ID }} + GS_ADDON_DIR: ${{ vars.GS_ADDON_DIR || '/game/csgo/addons/counterstrikesharp' }} + GS_STEAM_INF: ${{ vars.GS_STEAM_INF || '/game/csgo/steam.inf' }} + GS_RESTART_TIMEOUT: ${{ vars.GS_RESTART_TIMEOUT || '120' }} + GS_TEST_TIMEOUT: ${{ vars.GS_TEST_TIMEOUT || '600' }} + run: | + test -n "$SFTP_KNOWN_HOSTS" || { echo 'SFTP_KNOWN_HOSTS is required'; exit 1; } + install -d -m 700 ~/.ssh + printf '%s\n' "$SFTP_KNOWN_HOSTS" > ~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + chmod +x eng/rcon + deno run --allow-run --allow-read --allow-env --allow-net --allow-write eng/run-smoke-tests.ts payload TestResults/Smoke + - uses: actions/upload-artifact@v4 + if: always() + with: + name: smoke-results + overwrite: true + path: TestResults/Smoke/ + + report: + needs: [authorize, build_native, build_managed, smoke] + if: always() && needs.authorize.outputs.approved == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + checks: write + issues: write + pull-requests: read + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: smoke-results + path: smoke-results + - uses: actions/github-script@v7 + env: + TESTED_SHA: ${{ needs.authorize.outputs.sha }} + CHECK_ID: ${{ needs.authorize.outputs.check_id }} + COMMENT_ID: ${{ needs.authorize.outputs.comment_id }} + NATIVE_RESULT: ${{ needs.build_native.result }} + MANAGED_RESULT: ${{ needs.build_managed.result }} + SMOKE_RESULT: ${{ needs.smoke.result }} + with: + script: | + await require('./.github/scripts/smoke-test.cjs').report({ github, context, core }); diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8416a9347..67e8b28f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,8 @@ Before you submit your pull request consider the following guidelines: ``` * Create your patch and run appropriate tests. + Maintainers can optionally run the full non-benchmark suite on a dedicated CS2 + server by commenting `/smoke-test` on the PR. See [PR smoke tests](eng/SMOKE-TESTS.md). * Commit your changes using a descriptive commit message that uses the imperative, present tense: "change" not "changed" nor "changes". ```shell diff --git a/eng/SMOKE-TESTS.md b/eng/SMOKE-TESTS.md new file mode 100644 index 000000000..6b73959fb --- /dev/null +++ b/eng/SMOKE-TESTS.md @@ -0,0 +1,138 @@ +# PR smoke tests + +A repository **maintainer or admin** can post this as a new PR conversation comment: + +```text +/smoke-test +``` + +No SHA is needed. The workflow snapshots the PR's latest head when it accepts the +request, then builds and tests that exact commit (including fork PRs). It updates a +reply on the PR, adds a **Game server smoke test** check to the tested commit, and +writes an Actions job summary. Results include pass/fail/skip counts and the CS2 +patch, server/client versions, source revision and build date/time from the +server's `/game/csgo/steam.inf`. + +Each request gets its own reply and check. If the PR changes during the run, the +reply warns that the result is outdated. Post another `/smoke-test` to test the +new head. Editing a comment does not trigger a run. Plain `write`, `triage`, +organization membership and previous contribution do not grant permission. + +This is optional: **do not add this check to required branch-protection checks**. +The `issue_comment` workflow and trusted scripts must first be merged to the +repository's default branch before the command will work. + +## Repository setup + +1. Provision a **dedicated, disposable Linux CS2 test server** managed by + Pterodactyl, with Metamod, CS#, and the .NET 10 runtime installed. Configure a + playable map, RCON, bots and an actively ticking server (for example, + `sv_hibernate_when_empty 0`). Tests modify entities, players, bots and convars; + do not use a production server. Avoid other plugins and automatic updates or + restarts during tests. CS# must auto-load `NativeTestsPlugin` at startup. +2. Allow the GitHub-hosted runner to reach RCON, SFTP and the panel API. Pin and + verify the SFTP host key; host verification is not disabled. +3. Create a GitHub environment named **`smoke-test`**. Store these environment + secrets (not repository-wide secrets): + + | Secret | Purpose | + | --- | --- | + | `GS_HOST`, `GS_PORT`, `GS_PASS` | RCON host, port and password | + | `SFTP_HOST` | SFTP URL, e.g. `sftp://host:2022` | + | `SFTP_USER`, `SFTP_PASS` | SFTP credentials for the dedicated server | + | `SFTP_KNOWN_HOSTS` | Verified OpenSSH known_hosts entry; use `[host]:port` for a nonstandard port | + | `PTERO_URL` | Panel base URL, e.g. `https://panel.example.com` | + | `PTERO_API_KEY` | Client API key with resources/read and power control for this server only | + | `PTERO_SERVER_ID` | Server identifier for `/api/client/servers/{identifier}` | + + Optional environment variables: + + | Variable | Default | + | --- | --- | + | `GS_ADDON_DIR` | `/game/csgo/addons/counterstrikesharp` | + | `GS_STEAM_INF` | `/game/csgo/steam.inf` | + | `GS_RESTART_TIMEOUT` | `120` seconds per stop/start readiness wait | + | `GS_TEST_TIMEOUT` | `600` seconds for the test report | + + Paths are SFTP-visible absolute paths. If increasing timeouts substantially, + also increase the smoke job's 25-minute timeout. +4. Consider environment required reviewers as an additional approval barrier. + Restrict deployment branches to the default branch: this workflow runs in + default-branch context, not PR context. Allow Actions to create checks and + issue comments; permissions are scoped to the authorization/reporting jobs. + +## What runs + +- Fresh Linux native build (Steam Runtime SDK), API and native test plugin builds. +- The managed `CounterStrikeSharp.API.Tests` unit suite on the Actions runner. +- Stop the game server, deploy the native binaries/configs and replace the API + and test-plugin directories, then start the server and wait for RCON. +- Read `steam.inf` **after startup**, since the panel may update CS2 on startup. +- Run `css_smoke_test ` through RCON. The native plugin discovers + all tests, excluding `Category=Benchmark` and benchmark-named classes/methods. + Add `[Trait("Category", "Benchmark")]` to any new benchmark classes. +- Wait for an atomically published JSON report matching this run's unique ID. + Async/frame-based tests are awaited. Missing/stale/malformed reports, timeouts, + build/deployment errors, failed tests, runner/cleanup errors, zero tests and + all-skipped runs fail the check. Skipped tests are counted and shown. + +Artifacts include individual native test outcomes and failure messages/stack +traces (`smoke-results.json`), a Markdown summary, the original `steam.inf`, parsed +`server-version.json`, and managed unit-test TRX results. The PR links to the run +for logs and artifacts, including when a build fails or the server crashes. + +The shared server is serialized across PRs with `cancel-in-progress: false`. +GitHub keeps only one pending job per concurrency group, so a newer queued job +can replace an older pending one. A failed-job rerun reuses the approved commit; +post a fresh comment to request the latest head. Live maintainer permissions are +checked again before deployment, including on reruns. + +## Trust and operational boundaries + +PR code is built on isolated GitHub-hosted jobs with no server secrets and no +write-enabled repository token. The deployment and reporting jobs use automation +from the default-branch commit, never scripts from the PR. Downloaded artifacts +are treated as data on these runners; reports are validated rather than rendered +as arbitrary Markdown. + +**A maintainer command authorizes arbitrary PR code to execute on the game +server.** Review the PR before requesting a run. Keep that server/container and +its network isolated from production and other tenants; use least-privilege, +server-scoped panel/SFTP credentials. Code running there can access the server's +files and RCON configuration. This is not a sandbox for hostile plugins. + +The runner does not restore the previous installation: the tested build is left +on the dedicated server, and an upload failure can leave it stopped. Reprovision +it after testing suspicious changes or before relying on a clean baseline. Do +not run the local benchmark runner concurrently with this workflow. + +## Local development + +`eng/run-smoke-tests.ts` takes **prebuilt** artifacts and reads configuration from +the process environment (it deliberately does not load `.env`): + +```text +payload/native/addons/counterstrikesharp/ # CMake output including native .so/configs +payload/api/ # API bin/Release/net10.0 contents +payload/plugin/ # native test plugin bin/Release/net10.0 contents +``` + +With RCON/SFTP/panel configuration exported and SSH known_hosts configured: + +```sh +deno run --allow-run --allow-read --allow-env --allow-net --allow-write \ + eng/run-smoke-tests.ts payload TestResults/Smoke +``` + +Automation validation (no game server needed): + +```sh +node --test .github/scripts/smoke-test.test.cjs +deno check eng/run-smoke-tests.ts +deno lint eng/run-smoke-tests.ts +``` + +`eng/run-benchmarks.ts`, `css_itest ` and `css_run_tests` retain their +existing behavior. For a manual non-benchmark server run, use +`css_smoke_test manual-1` from the server console/RCON and read +`NativeTestsPlugin/smoke-results.json`. diff --git a/eng/run-smoke-tests.ts b/eng/run-smoke-tests.ts new file mode 100644 index 000000000..0d7cf3a60 --- /dev/null +++ b/eng/run-smoke-tests.ts @@ -0,0 +1,200 @@ +#!/usr/bin/env -S deno run --allow-run --allow-read --allow-env --allow-net --allow-write + +// Run this TRUSTED script against prebuilt artifacts; never build/execute PR code +// on the runner that holds server credentials. Unlike run-benchmarks.ts, no .env +// is loaded and a fresh native build is mandatory. +import { z } from "https://deno.land/x/zod@v3.22.4/mod.ts"; + +const timeout = z.coerce.number().int().positive().max(1800); +const remotePath = z.string().regex(/^\/[a-zA-Z0-9_./-]+$/); +const config = z.object({ + GS_HOST: z.string().min(1), + GS_PORT: z.string().regex(/^\d+$/), + GS_PASS: z.string().min(1), + SFTP_HOST: z.string().startsWith("sftp://"), + SFTP_USER: z.string().min(1), + SFTP_PASS: z.string().min(1), + GS_ADDON_DIR: remotePath.default("/game/csgo/addons/counterstrikesharp"), + GS_STEAM_INF: remotePath.default("/game/csgo/steam.inf"), + PTERO_URL: z.string().url(), + PTERO_API_KEY: z.string().min(1), + PTERO_SERVER_ID: z.string().regex(/^[a-zA-Z0-9-]+$/), + GS_RESTART_TIMEOUT: timeout.default(120), + GS_TEST_TIMEOUT: timeout.default(600), + SMOKE_COMMIT: z.string().regex(/^[a-f0-9]{40}$/).optional(), +}).parse(Deno.env.toObject()); + +const [artifactDir, resultsDir] = Deno.args; +if (!artifactDir || !resultsDir) { + throw new Error("Usage: run-smoke-tests.ts "); +} +const root = await Deno.realPath(artifactDir); +await Deno.mkdir(resultsDir, { recursive: true }); +const results = await Deno.realPath(resultsDir); +const runId = crypto.randomUUID(); +const addon = config.GS_ADDON_DIR; +const plugin = `${addon}/plugins/NativeTestsPlugin`; +const rcon = new URL("./rcon", import.meta.url).pathname; + +// Never include command arguments or stderr in errors: they can contain secrets. +async function command(program: string, args: string[]) { + const result = await new Deno.Command(program, { + args, + stdout: "piped", + stderr: "piped", + }).output(); + if (!result.success) throw new Error(`${program.split("/").pop()} failed (exit ${result.code})`); + return new TextDecoder().decode(result.stdout); +} +const quote = (value: string) => '"' + value.replaceAll("\\", "\\\\").replaceAll('"', '\\"') + '"'; +const sftp = (commands: string) => + command("lftp", [ + "-u", + `${config.SFTP_USER},${config.SFTP_PASS}`, + config.SFTP_HOST, + "-e", + `set cmd:fail-exit yes; set net:timeout 15; set net:max-retries 1; set xfer:clobber on; ${commands}; bye`, + ]); +const consoleCommand = (text: string) => + command(rcon, [ + "-a", + `${config.GS_HOST}:${config.GS_PORT}`, + "-p", + config.GS_PASS, + "-T", + "15s", + text, + ]); + +async function panel(path: string, signal?: string) { + const response = await fetch(`${config.PTERO_URL.replace(/\/$/, "")}/api/client/servers/${config.PTERO_SERVER_ID}/${path}`, { + method: signal ? "POST" : "GET", + headers: { + Authorization: `Bearer ${config.PTERO_API_KEY}`, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: signal ? JSON.stringify({ signal }) : undefined, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) throw new Error(`Panel ${path} failed (HTTP ${response.status})`); + return response; +} + +async function poll(seconds: number, label: string, check: () => Promise) { + const deadline = Date.now() + seconds * 1000; + while (Date.now() < deadline) { + try { + await check(); + return; + } catch { + console.log(`${label}…`); + await new Promise((resolve) => setTimeout(resolve, 5000)); + } + } + throw new Error(`${label}: timed out after ${seconds}s`); +} + +// Fail before touching the server if any build is missing. +for ( + const file of [ + "native/addons/counterstrikesharp/bin/linuxsteamrt64/counterstrikesharp.so", + "api/CounterStrikeSharp.API.dll", + "plugin/NativeTestsPlugin.dll", + ] +) { + if (!(await Deno.stat(`${root}/${file}`)).isFile) throw new Error(`Missing build: ${file}`); +} + +console.log("Stopping the dedicated test server…"); +await panel("power", "stop"); +await poll(config.GS_RESTART_TIMEOUT, "Waiting for server to stop", async () => { + const response = await (await panel("resources")).json(); + if (response.attributes.current_state !== "offline") throw new Error("Not offline"); +}); + +console.log("Deploying native binaries, configs, API and test plugin…"); +// Do not delete the server's runtime or unrelated addons. API/plugin directories +// are replaced completely to avoid accidentally using assemblies from an old PR. +await sftp( + `mirror -R ${quote(`${root}/native/addons/counterstrikesharp`)} ${quote(addon)}; ` + + `mkdir -p ${quote(`${addon}/api`)} ${quote(plugin)}; ` + + `mirror -R --delete ${quote(`${root}/api`)} ${quote(`${addon}/api`)}; ` + + `mirror -R --delete ${quote(`${root}/plugin`)} ${quote(plugin)}`, +); + +console.log("Starting the server…"); +await panel("power", "start"); +await poll(config.GS_RESTART_TIMEOUT, "Waiting for RCON", async () => { + if (!(await consoleCommand("status")).trim()) throw new Error("Empty RCON response"); +}); +// Allow plugin startup and map initialization to finish. +await new Promise((resolve) => setTimeout(resolve, 10_000)); +// Read after startup: the hosting panel may update CS2 when starting it. +await sftp(`get ${quote(config.GS_STEAM_INF)} -o ${quote(`${results}/steam.inf`)}`); +const steamInfo = Object.fromEntries( + (await Deno.readTextFile(`${results}/steam.inf`)) + .split(/\r?\n/).filter((line) => line.includes("=")).map((line) => { + const separator = line.indexOf("="); + return [line.slice(0, separator).trim(), line.slice(separator + 1).trim()]; + }), +); +const counterStrike = z.object({ + ClientVersion: z.string().regex(/^\d+$/), + ServerVersion: z.string().regex(/^\d+$/), + PatchVersion: z.string().regex(/^\d+(\.\d+)+$/), + SourceRevision: z.string().regex(/^\d+$/), + VersionDate: z.string().regex(/^[a-zA-Z0-9 ,/-]+$/), + VersionTime: z.string().regex(/^[0-9:]+$/), +}).parse(steamInfo); +await Deno.writeTextFile(`${results}/server-version.json`, JSON.stringify(counterStrike, null, 2) + "\n"); + +console.log("Running all non-benchmark native tests…"); +try { + await consoleCommand(`css_smoke_test ${runId}`); +} catch { + // Synchronous tests can delay RCON's reply. A matching, complete report is + // authoritative; a crash or a missing plugin will instead time out below. + console.log("RCON did not return cleanly; waiting for the test report."); +} + +const count = z.number().int().nonnegative(); +const reportSchema = z.object({ + runId: z.literal(runId), + total: count, + passed: count, + failed: count, + skipped: count, + errors: z.array(z.string()), + tests: z.array(z.object({ outcome: z.enum(["passed", "failed", "skipped"]) }).passthrough()), +}).superRefine((report, ctx) => { + if ( + report.total !== report.passed + report.failed + report.skipped || + report.tests.length !== report.total || + ["passed", "failed", "skipped"].some((outcome) => + report.tests.filter((test) => test.outcome === outcome).length !== report[outcome as "passed" | "failed" | "skipped"] + ) + ) ctx.addIssue({ code: "custom", message: "Inconsistent test counts" }); +}); + +let report: z.infer | undefined; +await poll(config.GS_TEST_TIMEOUT, "Waiting for a complete report", async () => { + await sftp(`get ${quote(`${plugin}/smoke-results.json`)} -o ${quote(`${results}/smoke-results.pending.json`)}`); + report = reportSchema.parse(JSON.parse(await Deno.readTextFile(`${results}/smoke-results.pending.json`))); +}); +if (!report) throw new Error("No report received"); +await Deno.writeTextFile( + `${results}/smoke-results.json`, + JSON.stringify({ ...report, testedCommit: config.SMOKE_COMMIT, counterStrike }, null, 2) + "\n", +); +await Deno.remove(`${results}/smoke-results.pending.json`); +const success = report.passed > 0 && report.failed === 0 && report.errors.length === 0; +const summary = `### Native smoke test: ${success ? "passed" : "failed"}\n\n` + + (config.SMOKE_COMMIT ? `**Tested commit:** ${config.SMOKE_COMMIT}\n\n` : "") + + `${report.total} tests: **${report.passed} passed**, **${report.failed} failed**, **${report.skipped} skipped**.\n\n` + + `Runner/cleanup errors: ${report.errors.length}. Benchmarks excluded.\n\n` + + `**CS2:** ${counterStrike.PatchVersion} (server ${counterStrike.ServerVersion}, client ${counterStrike.ClientVersion})\n\n` + + `**Source revision:** ${counterStrike.SourceRevision} — ${counterStrike.VersionDate} ${counterStrike.VersionTime}\n`; +await Deno.writeTextFile(`${results}/smoke-results.md`, summary); +console.log(summary); +if (!success) Deno.exit(1); diff --git a/managed/CounterStrikeSharp.Tests.Native/ConsoleTestReporterSink.cs b/managed/CounterStrikeSharp.Tests.Native/ConsoleTestReporterSink.cs index e430739dc..bfd9f71f9 100644 --- a/managed/CounterStrikeSharp.Tests.Native/ConsoleTestReporterSink.cs +++ b/managed/CounterStrikeSharp.Tests.Native/ConsoleTestReporterSink.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; +using System.IO; using System.Text; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Spectre.Console; @@ -16,6 +19,32 @@ public class ConsoleTestReporterSink : LongLivedMarshalByRefObject, IMessageSink private int _failed = 0; private int _skipped = 0; private readonly object _lock = new(); + private readonly List _tests = new(); + private readonly List _errors = new(); + private int _expectedTotal; + + public void WriteReport(string path, string runId, Exception? error = null) + { + lock (_lock) + { + if (error != null) _errors.Add(error.ToString()); + if (_expectedTotal != _passed + _failed + _skipped) + _errors.Add("The assembly test count did not match the reported results."); + var report = new + { + runId, + total = _passed + _failed + _skipped, + passed = _passed, + failed = _failed, + skipped = _skipped, + errors = _errors, + tests = _tests + }; + // Publish only complete reports; the remote runner polls for this file. + File.WriteAllText(path + ".tmp", JsonSerializer.Serialize(report)); + File.Move(path + ".tmp", path, overwrite: true); + } + } public bool OnMessage(IMessageSinkMessage message) { @@ -26,12 +55,15 @@ public bool OnMessage(IMessageSinkMessage message) // A test has passed case ITestPassed passed: Interlocked.Increment(ref _passed); + _tests.Add(new { name = passed.Test.DisplayName, outcome = "passed", durationSeconds = passed.ExecutionTime }); AnsiConsole.MarkupLineInterpolated($"[underline green][[PASS]][/] [green]{passed.Test.DisplayName}[/]"); break; // A test has failed case ITestFailed failed: Interlocked.Increment(ref _failed); + _tests.Add(new { name = failed.Test.DisplayName, outcome = "failed", durationSeconds = failed.ExecutionTime, + messages = failed.Messages, stackTraces = failed.StackTraces }); AnsiConsole.MarkupLineInterpolated($"[underline red][[FAIL]][/] [red]{failed.Test.DisplayName}[/]"); AnsiConsole.WriteLine($"\tReason: {failed.ExceptionTypes[0]} - {failed.Messages[0]}"); AnsiConsole.WriteLine(IndentStackTrace(failed.StackTraces[0] ?? "No stack trace available.")); @@ -40,14 +72,21 @@ public bool OnMessage(IMessageSinkMessage message) // A test was skipped (e.g., using [Fact(Skip = "...")]) case ITestSkipped skipped: Interlocked.Increment(ref _skipped); + _tests.Add(new { name = skipped.Test.DisplayName, outcome = "skipped", reason = skipped.Reason }); AnsiConsole.MarkupLineInterpolated($"[underline yellow][[SKIP]][/] [yellow]{skipped.Test.DisplayName}[/]"); AnsiConsole.MarkupLineInterpolated($"[yellow]\tReason: {skipped.Reason}[/]"); break; // This message indicates the entire test run for the assembly is complete. - case ITestAssemblyFinished: + case ITestAssemblyFinished finished: + _expectedTotal = finished.TestsRun; // We signal the main thread that it can stop waiting now. - Finished.SetResult(true); + Finished.TrySetResult(true); + break; + + // Includes fatal runner errors and collection/class/fixture cleanup failures. + case IFailureInformation failure: + _errors.Add(string.Join(Environment.NewLine, failure.Messages)); break; } } diff --git a/managed/CounterStrikeSharp.Tests.Native/NativeTestsPlugin.cs b/managed/CounterStrikeSharp.Tests.Native/NativeTestsPlugin.cs index d0a442441..2f2857c5f 100644 --- a/managed/CounterStrikeSharp.Tests.Native/NativeTestsPlugin.cs +++ b/managed/CounterStrikeSharp.Tests.Native/NativeTestsPlugin.cs @@ -16,7 +16,9 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using CounterStrikeSharp.API; @@ -39,6 +41,7 @@ public class NativeTestsPlugin : BasePlugin public override string ModuleDescription => "A an automated test plugin."; public static int gameThreadId; + private bool _running; public static NativeTestsPlugin Instance { get; private set; } = null!; @@ -67,8 +70,25 @@ public void OnCommandTest(CCSPlayerController? player, CommandInfo command) RunTests(filter); } - public async Task RunTests(string? filter = null) + [ConsoleCommand("css_smoke_test", "Run all non-benchmark tests and export a JSON report.")] + public void OnCommandSmokeTest(CCSPlayerController? player, CommandInfo command) { + // Only the server console/RCON may start an automated run. + var runId = command.GetArg(1); + if (player != null || !Regex.IsMatch(runId, @"\A[a-zA-Z0-9-]{1,80}\z")) return; + _ = RunTests(smokeRunId: runId); + } + + public async Task RunTests(string? filter = null, string? smokeRunId = null) + { + if (_running) + { + Console.WriteLine($"[{ModuleName}] A test run is already in progress."); + return; + } + _running = true; + using var reporter = new ConsoleTestReporterSink(); + Exception? runError = null; Console.WriteLine("*****************************************************************"); if (!string.IsNullOrWhiteSpace(filter)) { @@ -83,7 +103,6 @@ public async Task RunTests(string? filter = null) try { - using var reporter = new ConsoleTestReporterSink(); using var controller = new XunitFrontController(AppDomainSupport.IfAvailable, this.ModulePath); var executionOptions = TestFrameworkOptions.ForExecution(); @@ -94,7 +113,7 @@ public async Task RunTests(string? filter = null) var discoveryOptions = TestFrameworkOptions.ForDiscovery(); - if (!string.IsNullOrWhiteSpace(filter)) + if (smokeRunId != null || !string.IsNullOrWhiteSpace(filter)) { // Discover all tests first var discoverySink = new TestDiscoverySink(); @@ -108,8 +127,15 @@ public async Task RunTests(string? filter = null) var testClassName = testCase.TestMethod?.TestClass?.Class?.Name ?? ""; var testMethodName = testCase.TestMethod?.Method?.Name ?? ""; - if (testClassName.Contains(filter, StringComparison.OrdinalIgnoreCase) || - testMethodName.Contains(filter, StringComparison.OrdinalIgnoreCase)) + var isBenchmark = testCase.Traits.Any(trait => + trait.Key.Equals("Category", StringComparison.OrdinalIgnoreCase) && + trait.Value.Any(value => value.Equals("Benchmark", StringComparison.OrdinalIgnoreCase))) || + testClassName.Contains("Benchmark", StringComparison.OrdinalIgnoreCase) || + testMethodName.Contains("Benchmark", StringComparison.OrdinalIgnoreCase); + + if (smokeRunId != null ? !isBenchmark : + testClassName.Contains(filter!, StringComparison.OrdinalIgnoreCase) || + testMethodName.Contains(filter!, StringComparison.OrdinalIgnoreCase)) { filteredTests.Add(testCase); } @@ -117,11 +143,11 @@ public async Task RunTests(string? filter = null) if (filteredTests.Count == 0) { - Console.WriteLine($"[{ModuleName}] No tests matched filter: {filter}"); + Console.WriteLine($"[{ModuleName}] No tests selected (filter: {filter ?? "non-benchmark suite"})."); return; } - Console.WriteLine($"[{ModuleName}] Found {filteredTests.Count} test(s) matching filter."); + Console.WriteLine($"[{ModuleName}] Selected {filteredTests.Count} test(s)."); // Run only the filtered tests controller.RunTests(filteredTests, reporter, executionOptions); @@ -138,15 +164,28 @@ public async Task RunTests(string? filter = null) Console.WriteLine("*****************************************************************"); // Export benchmark results if any were collected - ScriptContextBenchmarks.ExportResults(); + if (smokeRunId == null) ScriptContextBenchmarks.ExportResults(); } catch (Exception ex) { + runError = ex; Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine($"[{ModuleName}] A critical error occurred during the test run setup: {ex.Message}"); Console.WriteLine(ex.StackTrace); Console.ResetColor(); } + finally + { + try + { + if (smokeRunId != null) + reporter.WriteReport(Path.Combine(ModuleDirectory, "smoke-results.json"), smokeRunId, runError); + } + finally + { + _running = false; + } + } } } diff --git a/managed/CounterStrikeSharp.Tests.Native/README.md b/managed/CounterStrikeSharp.Tests.Native/README.md index 9d9ee104f..8ccca42b8 100644 --- a/managed/CounterStrikeSharp.Tests.Native/README.md +++ b/managed/CounterStrikeSharp.Tests.Native/README.md @@ -1,2 +1,5 @@ # Native Tests This plugin is intended to be ran inside a running CS2 server running a version of CS# and runs tests against the exposed `NativeAPI` methods. + +Maintainers can request the non-benchmark suite on a PR with `/smoke-test`. +See [PR smoke tests](../../eng/SMOKE-TESTS.md) for setup, results and server requirements. diff --git a/managed/CounterStrikeSharp.Tests.Native/ScriptContextBenchmarks.cs b/managed/CounterStrikeSharp.Tests.Native/ScriptContextBenchmarks.cs index 20ac589fc..b0af31879 100644 --- a/managed/CounterStrikeSharp.Tests.Native/ScriptContextBenchmarks.cs +++ b/managed/CounterStrikeSharp.Tests.Native/ScriptContextBenchmarks.cs @@ -32,6 +32,7 @@ public class BenchmarkReport public List Results { get; set; } = new(); } +[Trait("Category", "Benchmark")] public class ScriptContextBenchmarks { private const int Iterations = 1_000_000; From b3fb045fc0f677672e6fd924907be902c914d619 Mon Sep 17 00:00:00 2001 From: roflmuffin Date: Mon, 7 Sep 2026 01:30:20 +0000 Subject: [PATCH 2/4] feat: add manual mode --- .github/scripts/smoke-test.cjs | 28 +++++++++--- .github/scripts/smoke-test.test.cjs | 67 +++++++++++++++++++++++++---- .github/workflows/pr-smoke-test.yml | 13 ++++-- eng/SMOKE-TESTS.md | 39 ++++++++++++++--- 4 files changed, 124 insertions(+), 23 deletions(-) diff --git a/.github/scripts/smoke-test.cjs b/.github/scripts/smoke-test.cjs index 54ddf45aa..640ec0b8b 100644 --- a/.github/scripts/smoke-test.cjs +++ b/.github/scripts/smoke-test.cjs @@ -3,10 +3,26 @@ const fs = require('node:fs'); const checkName = 'Game server smoke test'; const runUrl = (context) => `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; +function requestPullNumber(context) { + if (context.eventName === 'workflow_dispatch') { + const value = context.payload.inputs?.pr_number; + if (typeof value !== 'string' || !/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(Number(value))) { + throw new Error('pr_number must be a positive integer'); + } + return Number(value); + } + const { issue, comment } = context.payload; + if (context.eventName === 'issue_comment' && issue?.pull_request && + comment?.body.trim() === '/smoke-test' && comment.user.type === 'User') return issue.number; + return undefined; +} + async function isMaintainerRequest({ github, context }) { // Check live repository permissions, not author_association (a contributor // or organization member is not necessarily a maintainer). Check reruns too. - for (const username of new Set([context.payload.comment.user.login, process.env.TRIGGERING_ACTOR || context.actor])) { + const requester = context.eventName === 'workflow_dispatch' ? context.actor : context.payload.comment?.user.login; + if (!requester) return false; + for (const username of new Set([requester, process.env.TRIGGERING_ACTOR || context.actor])) { const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ ...context.repo, username }); if (data.permission !== 'admin' && data.permission !== 'maintain' && data.role_name !== 'maintain') { return false; @@ -16,13 +32,13 @@ async function isMaintainerRequest({ github, context }) { } async function authorize({ github, context, core }) { - const { issue, comment } = context.payload; - if (!issue?.pull_request || comment?.body.trim() !== '/smoke-test' || comment.user.type !== 'User') return; + const pullNumber = requestPullNumber(context); + if (!pullNumber) return; if (!await isMaintainerRequest({ github, context })) { core.info('Ignoring smoke-test request: maintain/admin permission required.'); return; } - const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: issue.number }); + const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: pullNumber }); if (pr.state !== 'open') return; // Snapshot the latest head ONCE. All builds use this immutable SHA, not a // mutable branch or refs/pull/N/head. Works for fork PRs as well. @@ -39,7 +55,7 @@ async function authorize({ github, context, core }) { }); const { data: reply } = await github.rest.issues.createComment({ ...context.repo, - issue_number: issue.number, + issue_number: pullNumber, body: `### Game server smoke test\n\nRequested for commit ${sha}. Building, then waiting for the dedicated server.\n\n[Follow the run](${details_url})`, }); core.setOutput('sha', sha); @@ -108,7 +124,7 @@ async function report({ github, context, core }) { } const conclusion = success ? 'success' : jobs.includes('cancelled') ? 'cancelled' : 'failure'; const title = `${checkName}: ${conclusion === 'success' ? 'passed' : conclusion}`; - const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: context.payload.issue.number }); + const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: requestPullNumber(context) }); let summary = `**Tested commit:** ${env.TESTED_SHA}\n\n${text}\n\n` + `Native build: ${env.NATIVE_RESULT}. Managed build/unit tests: ${env.MANAGED_RESULT}. Server run: ${env.SMOKE_RESULT}.\n\n` + `[Logs and artifacts (JSON, Markdown, steam.inf, unit-test TRX)](${runUrl(context)})`; diff --git a/.github/scripts/smoke-test.test.cjs b/.github/scripts/smoke-test.test.cjs index e6ec0cc87..05bc6f74d 100644 --- a/.github/scripts/smoke-test.test.cjs +++ b/.github/scripts/smoke-test.test.cjs @@ -3,15 +3,17 @@ const assert = require('node:assert/strict'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); -const { authorize, report, resultSummary, versionSummary } = require('./smoke-test.cjs'); +const { authorize, isMaintainerRequest, report, resultSummary, versionSummary } = require('./smoke-test.cjs'); const sha = 'a'.repeat(40); -function request({ permission = 'write', role = 'maintain', body = '/smoke-test', state = 'open', pullRequest = true } = {}) { +function request({ permission = 'write', role = 'maintain', body = '/smoke-test', state = 'open', pullRequest = true, + eventName = 'issue_comment', prNumber = '42' } = {}) { const calls = []; const outputs = {}; const context = { repo: { owner: 'owner', repo: 'repo' }, actor: 'maintainer', serverUrl: 'https://github.com', runId: 123, - payload: { + eventName, + payload: eventName === 'workflow_dispatch' ? { inputs: { pr_number: prNumber } } : { issue: { number: 42, pull_request: pullRequest ? {} : undefined }, comment: { body, user: { login: 'maintainer', type: 'User' } }, }, @@ -21,7 +23,7 @@ function request({ permission = 'write', role = 'maintain', body = '/smoke-test' calls.push(['permission', args]); return { data: { permission, role_name: role } }; } }, - pulls: { get: async () => ({ data: { state, head: { sha } } }) }, + pulls: { get: async (args) => { calls.push(['pull', args]); return { data: { state, head: { sha } } }; } }, checks: { create: async (args) => { calls.push(['check', args]); return { data: { id: 1 } }; } }, issues: { createComment: async (args) => { calls.push(['comment', args]); return { data: { id: 2 } }; } }, } }; @@ -72,6 +74,52 @@ test('an unauthorized rerun actor cannot reuse a maintainer request', async () = assert.equal(input.outputs.approved, undefined); }); +test('manual dispatch targets the input PR and snapshots its latest head', async () => { + const input = request({ eventName: 'workflow_dispatch', prNumber: '123' }); + await authorize(input); + assert.equal(input.outputs.approved, 'true'); + assert.equal(input.outputs.sha, sha); + assert.equal(input.calls.find(([name]) => name === 'pull')[1].pull_number, 123); + assert.equal(input.calls.find(([name]) => name === 'comment')[1].issue_number, 123); + assert.equal(input.calls.find(([name]) => name === 'check')[1].head_sha, sha); +}); + +test('manual dispatch does not bypass maintainer permissions', async () => { + const input = request({ eventName: 'workflow_dispatch', role: 'write' }); + await authorize(input); + assert.equal(input.outputs.approved, undefined); + assert.ok(input.calls.every(([name]) => name === 'permission')); +}); + +test('manual dispatch ignores closed PRs', async () => { + const input = request({ eventName: 'workflow_dispatch', state: 'closed' }); + await authorize(input); + assert.equal(input.outputs.approved, undefined); +}); + +for (const prNumber of ['', '0', '-1', '1.5', '1e2', '123; echo unsafe', '9007199254740992', null]) { + test(`manual dispatch rejects invalid PR number: ${JSON.stringify(prNumber)}`, async () => { + const input = request({ eventName: 'workflow_dispatch', prNumber }); + await assert.rejects(authorize(input), /positive integer/); + assert.equal(input.calls.length, 0); + }); +} + +test('deployment permission recheck rejects an unauthorized manual rerun actor', async () => { + const input = request({ eventName: 'workflow_dispatch' }); + const previous = process.env.TRIGGERING_ACTOR; + try { + process.env.TRIGGERING_ACTOR = 'other'; + input.github.rest.repos.getCollaboratorPermissionLevel = async ({ username }) => ({ + data: { permission: 'write', role_name: username === 'maintainer' ? 'maintain' : 'write' }, + }); + assert.equal(await isMaintainerRequest(input), false); + } finally { + if (previous === undefined) delete process.env.TRIGGERING_ACTOR; + else process.env.TRIGGERING_ACTOR = previous; + } +}); + const passing = { total: 2, passed: 1, failed: 0, skipped: 1, errors: [], tests: [{ outcome: 'passed' }, { outcome: 'skipped' }] }; test('summarizes passing results including skips', () => { assert.equal(resultSummary(passing).success, true); @@ -102,7 +150,7 @@ test('never renders arbitrary server-controlled Markdown', () => { assert.throws(() => versionSummary({ ...version, VersionDate: '@everyone' })); }); -for (const scenario of ['success', 'new-head', 'build-failure', 'timeout', 'cancelled']) { +for (const scenario of ['success', 'manual-dispatch', 'new-head', 'build-failure', 'timeout', 'cancelled']) { test(`publishes check and PR comment: ${scenario}`, async () => { const originalCwd = process.cwd(); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'smoke-report-')); @@ -120,15 +168,18 @@ for (const scenario of ['success', 'new-head', 'build-failure', 'timeout', 'canc fs.writeFileSync('smoke-results/smoke-results.json', JSON.stringify(passing)); fs.writeFileSync('smoke-results/server-version.json', JSON.stringify(version)); } - const input = request(); + const input = request(scenario === 'manual-dispatch' ? { eventName: 'workflow_dispatch', prNumber: '123' } : {}); const published = {}; - input.github.rest.pulls.get = async () => ({ data: { head: { sha: scenario === 'new-head' ? 'b'.repeat(40) : sha } } }); + input.github.rest.pulls.get = async ({ pull_number }) => { + assert.equal(pull_number, scenario === 'manual-dispatch' ? 123 : 42); + return { data: { head: { sha: scenario === 'new-head' ? 'b'.repeat(40) : sha } } }; + }; input.github.rest.checks.update = async (args) => { published.check = args; }; input.github.rest.issues.updateComment = async (args) => { published.comment = args; }; input.core.summary = { addRaw(text) { published.summary = text; return this; }, async write() {} }; await report(input); assert.equal(published.check.status, 'completed'); - assert.equal(published.check.conclusion, ['success', 'new-head'].includes(scenario) ? 'success' : scenario === 'cancelled' ? 'cancelled' : 'failure'); + assert.equal(published.check.conclusion, ['success', 'manual-dispatch', 'new-head'].includes(scenario) ? 'success' : scenario === 'cancelled' ? 'cancelled' : 'failure'); assert.match(published.comment.body, new RegExp(sha)); assert.match(published.comment.body, /actions\/runs\/123/); if (scenario === 'new-head') assert.match(published.comment.body, /does \*\*not\*\* cover the latest head/); diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index ab9c10fcd..a752206d2 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -1,6 +1,12 @@ name: PR smoke test on: + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to test (latest head commit)' + required: true + type: string issue_comment: types: [created] @@ -8,7 +14,7 @@ permissions: {} jobs: authorize: - if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/smoke-test') + if: github.event_name == 'workflow_dispatch' || (github.event.issue.pull_request && startsWith(github.event.comment.body, '/smoke-test')) runs-on: ubuntu-latest permissions: contents: read @@ -21,7 +27,8 @@ jobs: check_id: ${{ steps.request.outputs.check_id }} comment_id: ${{ steps.request.outputs.comment_id }} steps: - # issue_comment runs the workflow from the default branch, never the PR. + # Comments use default-branch automation; manual dispatch uses the + # maintainer-selected workflow ref. Neither follows the PR head for scripts. - uses: actions/checkout@v7 with: ref: ${{ github.sha }} @@ -118,7 +125,7 @@ jobs: group: game-server-smoke-test cancel-in-progress: false steps: - - name: Check out trusted automation, NOT the PR + - name: Check out the workflow commit, NOT the target PR uses: actions/checkout@v7 with: ref: ${{ github.sha }} diff --git a/eng/SMOKE-TESTS.md b/eng/SMOKE-TESTS.md index 6b73959fb..a100e4242 100644 --- a/eng/SMOKE-TESTS.md +++ b/eng/SMOKE-TESTS.md @@ -22,6 +22,31 @@ This is optional: **do not add this check to required branch-protection checks** The `issue_comment` workflow and trusted scripts must first be merged to the repository's default branch before the command will work. +## Manual runs and testing workflow changes + +Maintainers/admins can also use **Actions → PR smoke test → Run workflow**, choose +an automation branch, and enter the PR number. Or use GitHub CLI: + +```sh +gh workflow run pr-smoke-test.yml --ref my-smoke-test-branch -f pr_number=123 +``` + +The selected branch supplies the **workflow and deployment/reporting scripts**; +the PR-number input independently selects the **latest PR head to build/test**. +Manual runs use the same permission checks, server lock, checks and PR replies. + +GitHub requires the dispatch workflow to be registered on the default branch. +This does not bypass the initial merge requirement for a brand-new workflow: +first merge the automation separately, or put it on a fork's default branch and +test a PR against that fork. Once registered, select a development branch to test +subsequent workflow changes before merging them. + +**Only select an automation branch you trust.** Its scripts run with deployment +secrets and reporting permissions. If the `smoke-test` environment restricts +branches to the default branch, explicitly allow the trusted development branch +for this test (and remove that exception afterward). Prefer required reviewers; +do not broadly allow arbitrary PR branches to use this environment. + ## Repository setup 1. Provision a **dedicated, disposable Linux CS2 test server** managed by @@ -57,9 +82,10 @@ repository's default branch before the command will work. Paths are SFTP-visible absolute paths. If increasing timeouts substantially, also increase the smoke job's 25-minute timeout. 4. Consider environment required reviewers as an additional approval barrier. - Restrict deployment branches to the default branch: this workflow runs in - default-branch context, not PR context. Allow Actions to create checks and - issue comments; permissions are scoped to the authorization/reporting jobs. + Restrict deployment branches to the default branch for normal comment runs; + manual runs use the selected automation branch (see above). Allow Actions to + create checks and issue comments; permissions are scoped to the + authorization/reporting jobs. ## What runs @@ -91,9 +117,10 @@ checked again before deployment, including on reruns. PR code is built on isolated GitHub-hosted jobs with no server secrets and no write-enabled repository token. The deployment and reporting jobs use automation -from the default-branch commit, never scripts from the PR. Downloaded artifacts -are treated as data on these runners; reports are validated rather than rendered -as arbitrary Markdown. +from the default-branch commit for comment runs, or the explicitly selected +workflow commit for manual runs. They do not follow the target PR head for +scripts. Downloaded artifacts are treated as data on these runners; reports are +validated rather than rendered as arbitrary Markdown. **A maintainer command authorizes arbitrary PR code to execute on the game server.** Review the PR before requesting a run. Keep that server/container and From 9bddc1595b4680d2b8e2958e5f4e50b8544b7be5 Mon Sep 17 00:00:00 2001 From: roflmuffin Date: Mon, 7 Sep 2026 01:34:05 +0000 Subject: [PATCH 3/4] feat: cause sync --- .github/workflows/pr-smoke-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index a752206d2..8c61e4305 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -4,11 +4,13 @@ on: workflow_dispatch: inputs: pr_number: - description: 'PR number to test (latest head commit)' + description: "PR number to test (latest head commit)" required: true type: string issue_comment: types: [created] + pull_request: + types: [opened, synchronize, reopened] permissions: {} @@ -86,7 +88,7 @@ jobs: persist-credentials: false - uses: actions/setup-dotnet@v4 with: - dotnet-version: '10.0.x' + dotnet-version: "10.0.x" - name: Build API and native test plugin (no server secrets) run: | dotnet build managed/CounterStrikeSharp.API -c Release From cdaf58bb04a6c02ffcff8896710da58062feb99c Mon Sep 17 00:00:00 2001 From: roflmuffin Date: Mon, 7 Sep 2026 01:36:44 +0000 Subject: [PATCH 4/4] fix: permissions --- .github/workflows/pr-smoke-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index 8c61e4305..c6ff510f8 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - pull-requests: read + pull-requests: write issues: write checks: write outputs: @@ -199,7 +199,7 @@ jobs: contents: read checks: write issues: write - pull-requests: read + pull-requests: write steps: - uses: actions/checkout@v7 with: