From 9bf33ebceb4b63462e32d80b5f34a7f70ddf80c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=85=83=E5=9E=9A?= <146716153+2002yy@users.noreply.github.com> Date: Sun, 19 Jul 2026 21:15:40 +0800 Subject: [PATCH 1/2] ci: add manual real-provider RAG replay workflow --- .github/workflows/rag-provider-replay.yml | 178 ++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 .github/workflows/rag-provider-replay.yml diff --git a/.github/workflows/rag-provider-replay.yml b/.github/workflows/rag-provider-replay.yml new file mode 100644 index 0000000..b9fa572 --- /dev/null +++ b/.github/workflows/rag-provider-replay.yml @@ -0,0 +1,178 @@ +name: RAG Provider Replay + +on: + workflow_dispatch: + inputs: + provider_profile: + description: "Study Agent provider profile" + required: true + default: "openai" + type: choice + options: + - openai + - deepseek + - openrouter + - siliconflow + model_profile: + description: "Study Agent model profile" + required: true + default: "pro" + type: choice + options: + - pro + - flash + model_name: + description: "Exact provider model name used for this replay" + required: true + type: string + base_url: + description: "Optional OpenAI-compatible base URL override" + required: false + type: string + +permissions: + contents: read + +concurrency: + group: rag-provider-replay + cancel-in-progress: false + +jobs: + replay: + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + SELECTED_PROVIDER: ${{ inputs.provider_profile }} + SELECTED_MODEL_PROFILE: ${{ inputs.model_profile }} + SELECTED_MODEL_NAME: ${{ inputs.model_name }} + SELECTED_BASE_URL: ${{ inputs.base_url }} + KEY_OPENAI: ${{ secrets.OPENAI_API_KEY }} + KEY_DEEPSEEK: ${{ secrets.DEEPSEEK_API_KEY }} + KEY_OPENROUTER: ${{ secrets.OPENROUTER_API_KEY }} + KEY_SILICONFLOW: ${{ secrets.SILICONFLOW_API_KEY }} + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + - name: Install dependencies + run: pip install -r requirements-dev.txt + + - name: Resolve selected provider configuration + shell: bash + run: | + set -euo pipefail + + case "$SELECTED_PROVIDER" in + openai) + selected_key="$KEY_OPENAI" + default_base_url="https://api.openai.com/v1" + ;; + deepseek) + selected_key="$KEY_DEEPSEEK" + default_base_url="https://api.deepseek.com/v1" + ;; + openrouter) + selected_key="$KEY_OPENROUTER" + default_base_url="https://openrouter.ai/api/v1" + ;; + siliconflow) + selected_key="$KEY_SILICONFLOW" + default_base_url="https://api.siliconflow.cn/v1" + ;; + *) + echo "Unsupported provider profile." >&2 + exit 2 + ;; + esac + + if [ -z "$selected_key" ]; then + echo "The selected provider secret is not configured for this repository." >&2 + exit 2 + fi + + resolved_base_url="$SELECTED_BASE_URL" + if [ -z "$resolved_base_url" ]; then + resolved_base_url="$default_base_url" + fi + + echo "LLM_PROVIDER_PROFILE=$SELECTED_PROVIDER" >> "$GITHUB_ENV" + echo "OPENAI_API_KEY=$selected_key" >> "$GITHUB_ENV" + echo "OPENAI_BASE_URL=$resolved_base_url" >> "$GITHUB_ENV" + echo "MODEL_FLASH_NAME=$SELECTED_MODEL_NAME" >> "$GITHUB_ENV" + echo "MODEL_PRO_NAME=$SELECTED_MODEL_NAME" >> "$GITHUB_ENV" + + - name: Run real-provider K1e replay + id: replay + shell: bash + run: | + set +e + python tools/run_rag_provider_replay.py \ + --provider-profile "$SELECTED_PROVIDER" \ + --model-profile "$SELECTED_MODEL_PROFILE" \ + --output output/rag-provider-replay.json + replay_status=$? + echo "exit_code=$replay_status" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Validate completed real-provider provenance + shell: python + run: | + import json + from pathlib import Path + + report_path = Path("output/rag-provider-replay.json") + if not report_path.is_file(): + raise SystemExit("Replay report was not created.") + + report = json.loads(report_path.read_text(encoding="utf-8")) + if report.get("replay_kind") != "real_provider": + raise SystemExit("Replay provenance is not real_provider.") + if report.get("status") != "completed": + raise SystemExit(f"Replay did not complete: {report.get('status')}") + if report.get("completed_cases") != report.get("cases"): + raise SystemExit("Not all requested replay cases completed.") + if not isinstance(report.get("answer_quality"), dict): + raise SystemExit("Completed replay is missing answer-quality metrics.") + + provider = report.get("provider") or {} + print( + json.dumps( + { + "status": report.get("status"), + "cases": report.get("cases"), + "provider_profile": provider.get("provider_profile"), + "model_name": provider.get("model_name"), + "corpus_fingerprint": report.get("corpus_fingerprint"), + "answer_quality": report.get("answer_quality"), + "usage": report.get("usage"), + "latency": report.get("latency"), + }, + ensure_ascii=False, + indent=2, + ) + ) + + - name: Upload replay artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: rag-provider-replay-${{ github.run_id }} + path: output/rag-provider-replay.json + if-no-files-found: warn + retention-days: 14 + + - name: Enforce replay command result + if: always() + shell: bash + run: | + if [ "${{ steps.replay.outputs.exit_code }}" != "0" ]; then + echo "Provider replay command failed with exit code ${{ steps.replay.outputs.exit_code }}." >&2 + exit 1 + fi From 83bbd8934c4dc7d3be9e25133895cf8b25825e35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=85=83=E5=9E=9A?= <146716153+2002yy@users.noreply.github.com> Date: Sun, 19 Jul 2026 21:18:20 +0800 Subject: [PATCH 2/2] security: scope replay credentials to provider execution step --- .github/workflows/rag-provider-replay.yml | 42 +++++++++++++++-------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/.github/workflows/rag-provider-replay.yml b/.github/workflows/rag-provider-replay.yml index b9fa572..c9b7490 100644 --- a/.github/workflows/rag-provider-replay.yml +++ b/.github/workflows/rag-provider-replay.yml @@ -46,10 +46,6 @@ jobs: SELECTED_MODEL_PROFILE: ${{ inputs.model_profile }} SELECTED_MODEL_NAME: ${{ inputs.model_name }} SELECTED_BASE_URL: ${{ inputs.base_url }} - KEY_OPENAI: ${{ secrets.OPENAI_API_KEY }} - KEY_DEEPSEEK: ${{ secrets.DEEPSEEK_API_KEY }} - KEY_OPENROUTER: ${{ secrets.OPENROUTER_API_KEY }} - KEY_SILICONFLOW: ${{ secrets.SILICONFLOW_API_KEY }} steps: - name: Check out repository @@ -64,11 +60,30 @@ jobs: - name: Install dependencies run: pip install -r requirements-dev.txt - - name: Resolve selected provider configuration + - name: Run real-provider K1e replay + id: replay shell: bash + env: + KEY_OPENAI: ${{ secrets.OPENAI_API_KEY }} + KEY_DEEPSEEK: ${{ secrets.DEEPSEEK_API_KEY }} + KEY_OPENROUTER: ${{ secrets.OPENROUTER_API_KEY }} + KEY_SILICONFLOW: ${{ secrets.SILICONFLOW_API_KEY }} run: | set -euo pipefail + case "$SELECTED_MODEL_NAME" in + *$'\n'*|*$'\r'*) + echo "Model name must be a single line." >&2 + exit 2 + ;; + esac + case "$SELECTED_BASE_URL" in + *$'\n'*|*$'\r'*) + echo "Base URL must be a single line." >&2 + exit 2 + ;; + esac + case "$SELECTED_PROVIDER" in openai) selected_key="$KEY_OPENAI" @@ -93,7 +108,7 @@ jobs: esac if [ -z "$selected_key" ]; then - echo "The selected provider secret is not configured for this repository." >&2 + echo "The selected provider credential is not configured for this repository." >&2 exit 2 fi @@ -102,22 +117,19 @@ jobs: resolved_base_url="$default_base_url" fi - echo "LLM_PROVIDER_PROFILE=$SELECTED_PROVIDER" >> "$GITHUB_ENV" - echo "OPENAI_API_KEY=$selected_key" >> "$GITHUB_ENV" - echo "OPENAI_BASE_URL=$resolved_base_url" >> "$GITHUB_ENV" - echo "MODEL_FLASH_NAME=$SELECTED_MODEL_NAME" >> "$GITHUB_ENV" - echo "MODEL_PRO_NAME=$SELECTED_MODEL_NAME" >> "$GITHUB_ENV" + export LLM_PROVIDER_PROFILE="$SELECTED_PROVIDER" + export OPENAI_API_KEY="$selected_key" + export OPENAI_BASE_URL="$resolved_base_url" + export MODEL_FLASH_NAME="$SELECTED_MODEL_NAME" + export MODEL_PRO_NAME="$SELECTED_MODEL_NAME" - - name: Run real-provider K1e replay - id: replay - shell: bash - run: | set +e python tools/run_rag_provider_replay.py \ --provider-profile "$SELECTED_PROVIDER" \ --model-profile "$SELECTED_MODEL_PROFILE" \ --output output/rag-provider-replay.json replay_status=$? + set -e echo "exit_code=$replay_status" >> "$GITHUB_OUTPUT" exit 0