diff --git a/.github/workflows/job-deploy-linux.yml b/.github/workflows/job-deploy-linux.yml index aaf3f801..9eb7d98f 100644 --- a/.github/workflows/job-deploy-linux.yml +++ b/.github/workflows/job-deploy-linux.yml @@ -18,14 +18,6 @@ on: IMAGE_TAG: required: true type: string - AZURE_ENV_GPT_MODEL_CAPACITY: - required: false - type: string - default: '10' - AZURE_NONAI_LOCATIONS: - required: false - type: string - default: '' BUILD_DOCKER_IMAGE: required: true type: string @@ -253,8 +245,6 @@ jobs: INPUT_AZURE_LOCATION: ${{ inputs.AZURE_LOCATION }} INPUT_RESOURCE_GROUP_NAME: ${{ inputs.RESOURCE_GROUP_NAME }} INPUT_IMAGE_TAG: ${{ inputs.IMAGE_TAG }} - INPUT_AZURE_ENV_GPT_MODEL_CAPACITY: ${{ inputs.AZURE_ENV_GPT_MODEL_CAPACITY }} - INPUT_AZURE_NONAI_LOCATIONS: ${{ inputs.AZURE_NONAI_LOCATIONS }} INPUT_BUILD_DOCKER_IMAGE: ${{ inputs.BUILD_DOCKER_IMAGE }} INPUT_EXP: ${{ inputs.EXP }} INPUT_AZURE_ENV_EXISTING_LOG_ANALYTICS_WORKSPACE_RID: ${{ inputs.AZURE_ENV_EXISTING_LOG_ANALYTICS_WORKSPACE_RID }} @@ -275,7 +265,6 @@ jobs: azd env set AZURE_LOCATION="$INPUT_AZURE_LOCATION" azd env set AZURE_RESOURCE_GROUP="$INPUT_RESOURCE_GROUP_NAME" azd env set AZURE_ENV_IMAGETAG="$INPUT_IMAGE_TAG" - azd env set AZURE_ENV_GPT_MODEL_CAPACITY="${INPUT_AZURE_ENV_GPT_MODEL_CAPACITY:-10}" if [[ "$INPUT_BUILD_DOCKER_IMAGE" == "true" ]]; then ACR_NAME="${{ vars.ACR_TEST_LOGIN_SERVER }}" @@ -308,57 +297,8 @@ jobs: echo "❌ EXP DISABLED - Skipping EXP parameters" fi - FALLBACK_REGIONS=() - if [[ -n "${INPUT_AZURE_NONAI_LOCATIONS:-}" ]]; then - IFS=',' read -ra RAW_REGIONS <<< "$INPUT_AZURE_NONAI_LOCATIONS" - for raw in "${RAW_REGIONS[@]}"; do - region=$(echo "$raw" | xargs) - if [[ -n "$region" ]]; then - FALLBACK_REGIONS+=("$region") - fi - done - fi - - # Build ordered retry list: preferred selected region first, then other fallback regions. - RETRY_REGIONS=("$INPUT_AZURE_LOCATION") - for region in "${FALLBACK_REGIONS[@]}"; do - if [[ "$region" != "$INPUT_AZURE_LOCATION" ]]; then - RETRY_REGIONS+=("$region") - fi - done - - DEPLOYMENT_SUCCESS=false - for region in "${RETRY_REGIONS[@]}"; do - echo "" - echo "🚀 Attempting deployment with AZURE_LOCATION='$region'" - azd env set AZURE_LOCATION="$region" - - set +e - azd up --no-prompt 2>&1 | tee azd-up-output.log - AZD_EXIT_CODE=${PIPESTATUS[0]} - set -e - - if [[ "$AZD_EXIT_CODE" -eq 0 ]]; then - DEPLOYMENT_SUCCESS=true - echo "✅ Deployment succeeded in region '$region'" - break - fi - - if grep -q "InsufficientResourcesAvailable" azd-up-output.log; then - echo "⚠ī¸ Region '$region' failed with InsufficientResourcesAvailable." - echo "## ⚠ī¸ Deployment Region Retry" >> "$GITHUB_STEP_SUMMARY" - echo "Deployment attempt failed in region \`$region\` with InsufficientResourcesAvailable. Trying next fallback region if available." >> "$GITHUB_STEP_SUMMARY" - continue - fi - - echo "❌ Deployment failed in region '$region' with non-retryable error." - exit "$AZD_EXIT_CODE" - done - - if [[ "$DEPLOYMENT_SUCCESS" != "true" ]]; then - echo "❌ Deployment failed after trying all fallback regions: ${RETRY_REGIONS[*]}" - exit 1 - fi + azd up --no-prompt + echo "✅ Deployment succeeded" WEBAPP_URL=$(azd env get-value WEB_APP_URL) echo "WEBAPP_URL=${WEBAPP_URL}" >> $GITHUB_ENV diff --git a/.github/workflows/job-deploy-windows.yml b/.github/workflows/job-deploy-windows.yml index f89f035c..5594805e 100644 --- a/.github/workflows/job-deploy-windows.yml +++ b/.github/workflows/job-deploy-windows.yml @@ -18,14 +18,6 @@ on: IMAGE_TAG: required: true type: string - AZURE_ENV_GPT_MODEL_CAPACITY: - required: false - type: string - default: '10' - AZURE_NONAI_LOCATIONS: - required: false - type: string - default: '' BUILD_DOCKER_IMAGE: required: true type: string @@ -253,8 +245,6 @@ jobs: INPUT_AZURE_LOCATION: ${{ inputs.AZURE_LOCATION }} INPUT_RESOURCE_GROUP_NAME: ${{ inputs.RESOURCE_GROUP_NAME }} INPUT_IMAGE_TAG: ${{ inputs.IMAGE_TAG }} - INPUT_AZURE_ENV_GPT_MODEL_CAPACITY: ${{ inputs.AZURE_ENV_GPT_MODEL_CAPACITY }} - INPUT_AZURE_NONAI_LOCATIONS: ${{ inputs.AZURE_NONAI_LOCATIONS }} INPUT_BUILD_DOCKER_IMAGE: ${{ inputs.BUILD_DOCKER_IMAGE }} INPUT_EXP: ${{ inputs.EXP }} INPUT_AZURE_ENV_EXISTING_LOG_ANALYTICS_WORKSPACE_RID: ${{ inputs.AZURE_ENV_EXISTING_LOG_ANALYTICS_WORKSPACE_RID }} @@ -278,8 +268,6 @@ jobs: azd env set AZURE_LOCATION="$env:INPUT_AZURE_LOCATION" azd env set AZURE_RESOURCE_GROUP="$env:INPUT_RESOURCE_GROUP_NAME" azd env set AZURE_ENV_IMAGETAG="$env:INPUT_IMAGE_TAG" - $gptCapacity = if ($env:INPUT_AZURE_ENV_GPT_MODEL_CAPACITY) { $env:INPUT_AZURE_ENV_GPT_MODEL_CAPACITY } else { '10' } - azd env set AZURE_ENV_GPT_MODEL_CAPACITY="$gptCapacity" # Set ACR name only when building Docker image if ("$env:INPUT_BUILD_DOCKER_IMAGE" -eq "true") { @@ -314,52 +302,9 @@ jobs: Write-Host "EXP DISABLED - Skipping EXP parameters" } - # Build ordered retry list: preferred selected region first, then other fallback regions. - $retryRegions = @("$env:INPUT_AZURE_LOCATION") - if ($env:INPUT_AZURE_NONAI_LOCATIONS) { - $fallbackRegions = $env:INPUT_AZURE_NONAI_LOCATIONS.Split(',') | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne '' } - foreach ($region in $fallbackRegions) { - if ($region -ne $env:INPUT_AZURE_LOCATION) { - $retryRegions += $region - } - } - } - - $deploymentSucceeded = $false - foreach ($region in $retryRegions) { - Write-Host "" - Write-Host "🚀 Attempting deployment with AZURE_LOCATION='$region'" - azd env set AZURE_LOCATION="$region" - - azd up --no-prompt 2>&1 | Tee-Object -FilePath azd-up-output.log - $azdExitCode = $LASTEXITCODE - - if ($azdExitCode -eq 0) { - $deploymentSucceeded = $true - Write-Host "✅ Deployment succeeded in region '$region'." - break - } - - $outputText = "" - if (Test-Path "azd-up-output.log") { - $outputText = Get-Content "azd-up-output.log" -Raw - } - - if ($outputText -match 'InsufficientResourcesAvailable') { - Write-Host "⚠ī¸ Region '$region' failed with InsufficientResourcesAvailable." - "## ⚠ī¸ Deployment Region Retry" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Encoding utf8 -Append - "Deployment attempt failed in region `$region` with InsufficientResourcesAvailable. Trying next fallback region if available." | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Encoding utf8 -Append - continue - } - - Write-Host "❌ Deployment failed in region '$region' with non-retryable error." - exit $azdExitCode - } - - if (-not $deploymentSucceeded) { - Write-Host "❌ Deployment failed after trying all fallback regions: $($retryRegions -join ', ')" - exit 1 - } + # Deploy using azd up + azd up --no-prompt + Write-Host "✅ Deployment succeeded." $WEBAPP_URL = azd env get-value WEB_APP_URL "WEBAPP_URL=$WEBAPP_URL" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append diff --git a/.github/workflows/job-deploy.yml b/.github/workflows/job-deploy.yml index 3d33615f..956b8fc9 100644 --- a/.github/workflows/job-deploy.yml +++ b/.github/workflows/job-deploy.yml @@ -87,9 +87,6 @@ on: AZURE_ENV_AI_SERVICE_LOCATION: description: "Azure AI Service Location" value: ${{ jobs.azure-setup.outputs.AZURE_ENV_AI_SERVICE_LOCATION }} - AZURE_ENV_GPT_MODEL_CAPACITY: - description: "GPT Model Capacity" - value: ${{ jobs.azure-setup.outputs.AZURE_ENV_GPT_MODEL_CAPACITY }} IMAGE_TAG: description: "Docker Image Tag Used" value: ${{ jobs.azure-setup.outputs.IMAGE_TAG }} @@ -98,8 +95,8 @@ on: value: ${{ jobs.azure-setup.outputs.QUOTA_FAILED }} env: - GPT_MIN_CAPACITY: ${{ vars.GPT_MIN_CAPACITY || 10 }} - GPT_REALTIME_MIN_CAPACITY: ${{ vars.GPT_REALTIME_MIN_CAPACITY || 1 }} + GPT_MIN_CAPACITY: 50 + GPT_REALTIME_MIN_CAPACITY: 1 BRANCH_NAME: ${{ github.event.workflow_run.head_branch || github.head_ref || github.ref_name }} WAF_ENABLED: ${{ inputs.trigger_type == 'workflow_dispatch' && (inputs.waf_enabled || false) || false }} EXP: ${{ inputs.trigger_type == 'workflow_dispatch' && (inputs.EXP || false) || false }} @@ -119,7 +116,6 @@ jobs: ENV_NAME: ${{ steps.generate_env_name.outputs.ENV_NAME }} AZURE_LOCATION: ${{ steps.set_region.outputs.AZURE_LOCATION }} AZURE_ENV_AI_SERVICE_LOCATION: ${{ steps.set_region.outputs.AZURE_ENV_AI_SERVICE_LOCATION }} - AZURE_ENV_GPT_MODEL_CAPACITY: ${{ steps.set_region.outputs.AZURE_ENV_GPT_MODEL_CAPACITY }} IMAGE_TAG: ${{ steps.determine_image_tag.outputs.IMAGE_TAG }} QUOTA_FAILED: ${{ steps.quota_failure_output.outputs.QUOTA_FAILED }} @@ -282,40 +278,21 @@ jobs: - name: Run Quota Check id: quota-check - shell: bash env: AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - INPUT_TRIGGER_TYPE: ${{ inputs.trigger_type }} - INPUT_AZURE_LOCATION: ${{ inputs.azure_location }} - AZURE_REGIONS_FALLBACK: ${{ vars.AZURE_REGIONS }} - AZURE_NONAI_LOCATIONS_FALLBACK: ${{ vars.AZURE_NONAI_LOCATIONS }} + AZURE_REGIONS: ${{ vars.AZURE_REGIONS }} GPT_MIN_CAPACITY: ${{ env.GPT_MIN_CAPACITY }} GPT_REALTIME_MIN_CAPACITY: ${{ env.GPT_REALTIME_MIN_CAPACITY }} run: | - set -euo pipefail - if [[ "$INPUT_TRIGGER_TYPE" == "workflow_dispatch" && -n "$INPUT_AZURE_LOCATION" ]]; then - if [[ -n "${AZURE_REGIONS_FALLBACK:-}" ]]; then - export AZURE_REGIONS="$AZURE_REGIONS_FALLBACK" - else - export AZURE_REGIONS="eastus2,francecentral,swedencentral" - fi - echo "📋 Manual trigger: selected Non-AI region is '$INPUT_AZURE_LOCATION'" - echo "📋 Manual trigger: validating AI quota across AI-allowed regions '$AZURE_REGIONS'" - else - if [[ -n "${AZURE_REGIONS_FALLBACK:-}" ]]; then - export AZURE_REGIONS="$AZURE_REGIONS_FALLBACK" - echo "📋 Automatic trigger: validating AI quota across configured regions '$AZURE_REGIONS'" - elif [[ -n "${AZURE_NONAI_LOCATIONS_FALLBACK:-}" ]]; then - export AZURE_REGIONS="$AZURE_NONAI_LOCATIONS_FALLBACK" - echo "⚠ī¸ AZURE_REGIONS is empty. Falling back to AZURE_NONAI_LOCATIONS for AI quota validation: '$AZURE_REGIONS'" - else - echo "❌ ERROR: No region lists configured. Set GitHub variable AZURE_REGIONS (preferred) or AZURE_NONAI_LOCATIONS." - exit 1 - fi - fi chmod +x infra/scripts/checkquota.sh - infra/scripts/checkquota.sh + if ! infra/scripts/checkquota.sh; then + # If quota check fails due to insufficient quota, set the flag + if grep -q "No region with sufficient quota found" infra/scripts/checkquota.sh; then + echo "QUOTA_FAILED=true" >> $GITHUB_ENV + fi + exit 1 # Fail the pipeline if any other failure occurs + fi - name: Set Quota Failure Output id: quota_failure_output @@ -337,8 +314,6 @@ jobs: INPUT_AZURE_LOCATION: ${{ inputs.azure_location }} TRIGGER_TYPE: ${{ inputs.trigger_type }} ALLOWED_AZURE_LOCATIONS: ${{ vars.AZURE_NONAI_LOCATIONS }} - AI_VALID_REGION: ${{ env.VALID_REGION }} - AZURE_REGIONS_FALLBACK: ${{ vars.AZURE_REGIONS }} run: | # Determine search tier based on WAF (scalability) setting # WAF enabled → standard tier, WAF disabled → basic tier @@ -375,29 +350,20 @@ jobs: } # Parse allowed regions list (used by automatic triggers AND as fallback report for manual triggers) - # Fallback order: - # 1) AZURE_NONAI_LOCATIONS (preferred) - # 2) VALID_REGION from AI quota check - # 3) AZURE_REGIONS ALLOWED_REGIONS=() if [[ -n "$ALLOWED_AZURE_LOCATIONS" ]]; then IFS=', ' read -r -a ALLOWED_REGIONS <<< "$ALLOWED_AZURE_LOCATIONS" - elif [[ -n "${AI_VALID_REGION:-}" ]]; then - ALLOWED_REGIONS=("$AI_VALID_REGION") - echo "⚠ī¸ AZURE_NONAI_LOCATIONS is empty. Falling back to AI quota-selected region '$AI_VALID_REGION' for Search quota validation." - elif [[ -n "${AZURE_REGIONS_FALLBACK:-}" ]]; then - IFS=', ' read -r -a ALLOWED_REGIONS <<< "$AZURE_REGIONS_FALLBACK" - echo "⚠ī¸ AZURE_NONAI_LOCATIONS is empty. Falling back to AZURE_REGIONS for Search quota validation: '${ALLOWED_REGIONS[*]}'" fi SEARCH_VALID_REGION="" CHECKED_REGIONS=() - MANUAL_USER_REGION_FALLBACK_USED=false + MANUAL_USER_REGION_FAILED=false if [[ "$TRIGGER_TYPE" == "workflow_dispatch" && -n "$INPUT_AZURE_LOCATION" ]]; then # ─── MANUAL TRIGGER ─── - # 1) Prefer user-selected region - # 2) If it fails, automatically fall back to next available allowed region + # 1) Check ONLY the user-selected region to decide pass/fail + # 2) If it fails, still scan other allowed regions for reporting purposes + # and FAIL the deployment (do NOT fall back) echo "📋 Manual trigger: checking user-selected region '$INPUT_AZURE_LOCATION'" echo "" echo "----------------------------------------" @@ -409,16 +375,17 @@ jobs: CHECKED_REGIONS+=("$INPUT_AZURE_LOCATION: ✅ available=$TIER_AVAILABLE (user-selected)") else echo " ❌ Insufficient quota in user-selected region '$INPUT_AZURE_LOCATION'" + MANUAL_USER_REGION_FAILED=true if [ "$TIER_AVAILABLE" -lt 0 ]; then CHECKED_REGIONS+=("$INPUT_AZURE_LOCATION: ⚠ī¸ skipped (user-selected)") else CHECKED_REGIONS+=("$INPUT_AZURE_LOCATION: ❌ available=$TIER_AVAILABLE (user-selected)") fi - # Scan other allowed regions and fall back to first available + # Informational scan of other allowed regions (does NOT override failure) if [ ${#ALLOWED_REGIONS[@]} -gt 0 ]; then echo "" - echo "ℹī¸ User-selected region is constrained. Scanning fallback regions..." + echo "ℹī¸ Scanning other allowed regions for informational reporting..." for REGION in "${ALLOWED_REGIONS[@]}"; do if [[ "$REGION" == "$INPUT_AZURE_LOCATION" ]]; then continue @@ -430,10 +397,6 @@ jobs: CHECKED_REGIONS+=("$REGION: ⚠ī¸ skipped") elif [ "$TIER_AVAILABLE" -ge 1 ]; then CHECKED_REGIONS+=("$REGION: ✅ available=$TIER_AVAILABLE") - SEARCH_VALID_REGION="$REGION" - MANUAL_USER_REGION_FALLBACK_USED=true - echo "✅ Falling back to '$REGION' for Non-AI services deployment." - break else CHECKED_REGIONS+=("$REGION: ❌ available=$TIER_AVAILABLE") fi @@ -473,13 +436,44 @@ jobs: echo "" echo "========================================" - # ─── Manual trigger: fallback used → continue with selected fallback region ─── - if [[ "$TRIGGER_TYPE" == "workflow_dispatch" && "$MANUAL_USER_REGION_FALLBACK_USED" == "true" ]]; then - echo "## ⚠ī¸ Azure Search Region Fallback Applied" >> $GITHUB_STEP_SUMMARY + # ─── Manual trigger: user-selected region failed → FAIL with report ─── + if [[ "$MANUAL_USER_REGION_FAILED" == "true" ]]; then + echo "❌ User-selected region '$INPUT_AZURE_LOCATION' does not have sufficient Azure Search $SEARCH_TIER_DISPLAY tier quota." + echo "" + echo "Regions scanned:" + for entry in "${CHECKED_REGIONS[@]}"; do + echo " - $entry" + done + + echo "## ❌ Azure Search Quota Check Failed" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "Requested Non-AI region **\`$INPUT_AZURE_LOCATION\`** did not have enough Azure Search **$SEARCH_TIER_DISPLAY** quota." >> $GITHUB_STEP_SUMMARY - echo "Deployment will continue with fallback region **\`$SEARCH_VALID_REGION\`**." >> $GITHUB_STEP_SUMMARY + echo "User-selected region **\`$INPUT_AZURE_LOCATION\`** does not have sufficient Azure Search **$SEARCH_TIER_DISPLAY** tier quota." >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY + echo "Deployment has been failed because the user-selected region does not meet quota. Please re-trigger the workflow with one of the available regions listed below." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Filter to only regions with available quota + AVAILABLE_REGIONS=() + for entry in "${CHECKED_REGIONS[@]}"; do + if [[ "$entry" == *"✅"* ]]; then + AVAILABLE_REGIONS+=("$entry") + fi + done + + if [ ${#AVAILABLE_REGIONS[@]} -gt 0 ]; then + echo "### ✅ Regions with available Azure Search $SEARCH_TIER_DISPLAY quota" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Region | Status |" >> $GITHUB_STEP_SUMMARY + echo "|--------|--------|" >> $GITHUB_STEP_SUMMARY + for entry in "${AVAILABLE_REGIONS[@]}"; do + region="${entry%%:*}" + status="${entry#*: }" + echo "| \`$region\` | $status |" >> $GITHUB_STEP_SUMMARY + done + else + echo "### ⚠ī¸ No other allowed regions currently have available Azure Search $SEARCH_TIER_DISPLAY quota." >> $GITHUB_STEP_SUMMARY + fi + exit 1 fi # ─── Automatic trigger: nothing available → FAIL with report ─── @@ -517,11 +511,8 @@ jobs: INPUT_AZURE_LOCATION: ${{ inputs.azure_location }} run: | echo "Selected AI Service Region (QuotaRegions from GPT quota check): $VALID_REGION" - echo "Available GPT Capacity in $VALID_REGION: ${AVAILABLE_CAPACITY:-10}" echo "AZURE_ENV_AI_SERVICE_LOCATION=$VALID_REGION" >> $GITHUB_ENV echo "AZURE_ENV_AI_SERVICE_LOCATION=$VALID_REGION" >> $GITHUB_OUTPUT - echo "AZURE_ENV_GPT_MODEL_CAPACITY=${AVAILABLE_CAPACITY:-10}" >> $GITHUB_ENV - echo "AZURE_ENV_GPT_MODEL_CAPACITY=${AVAILABLE_CAPACITY:-10}" >> $GITHUB_OUTPUT # AZURE_LOCATION = region validated by Azure Search service quota check echo "Selected Deployment Region (from Search quota check): $SEARCH_VALID_REGION" @@ -529,60 +520,11 @@ jobs: echo "AZURE_LOCATION=$SEARCH_VALID_REGION" >> $GITHUB_OUTPUT if [[ "$INPUT_TRIGGER_TYPE" == "workflow_dispatch" && -n "$INPUT_AZURE_LOCATION" ]]; then - if [[ "$SEARCH_VALID_REGION" == "$INPUT_AZURE_LOCATION" ]]; then - echo "ℹī¸ Manual trigger: AZURE_LOCATION resolved to user-selected region '$INPUT_AZURE_LOCATION'" - else - echo "ℹī¸ Manual trigger: user-selected region '$INPUT_AZURE_LOCATION' had constrained Search quota; using fallback '$SEARCH_VALID_REGION'" - fi + echo "ℹī¸ Manual trigger: AZURE_LOCATION resolved to user-selected region '$INPUT_AZURE_LOCATION'" else echo "ℹī¸ Automatic trigger: AZURE_LOCATION resolved from allowed regions list" fi - - name: Check Subscription-Wide Search Service Quota - id: search-subscription-quota - shell: bash - env: - WAF_ENABLED: ${{ env.WAF_ENABLED }} - run: | - set -euo pipefail - if [[ "$WAF_ENABLED" == "true" ]]; then - SEARCH_TIER="standard" - else - SEARCH_TIER="basic" - fi - - echo "🔍 Checking subscription-wide $SEARCH_TIER tier Azure Search service count..." - TOTAL_SERVICES=$(az search service list --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} \ - --query "[?sku.name=='${SEARCH_TIER}'] | length(@)" -o tsv 2>/dev/null || echo "0") - - # Subscription limits: basic=12, standard=12 (typical defaults) - # Use 12 as safe upper bound for both tiers - SUBSCRIPTION_LIMIT=12 - REMAINING=$((SUBSCRIPTION_LIMIT - TOTAL_SERVICES)) - - echo " $SEARCH_TIER tier: In use=$TOTAL_SERVICES | Subscription limit=$SUBSCRIPTION_LIMIT | Remaining=$REMAINING" - - if [ "$REMAINING" -le 0 ]; then - echo "" - echo "❌ SUBSCRIPTION SEARCH QUOTA EXHAUSTED" - echo "The subscription has reached its limit of $SUBSCRIPTION_LIMIT $SEARCH_TIER tier Azure Search services." - echo "To resolve: delete unused Search services (from previous test deployments) in the Azure Portal:" - echo " https://portal.azure.com/#view/Microsoft_Azure_Search/SearchService.ReactView" - echo "" - EXISTING_SERVICES=$(az search service list --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} \ - --query "[?sku.name=='${SEARCH_TIER}'].[name, resourceGroup, location]" -o tsv 2>/dev/null || echo "Unable to list") - echo "Existing $SEARCH_TIER tier Search services:" - echo "$EXISTING_SERVICES" - echo "" - echo "## ❌ Azure Search Subscription Quota Exhausted" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "The subscription has **$TOTAL_SERVICES / $SUBSCRIPTION_LIMIT** $SEARCH_TIER tier Search services in use." >> $GITHUB_STEP_SUMMARY - echo "Delete unused services from previous test deployments before re-running." >> $GITHUB_STEP_SUMMARY - exit 1 - fi - - echo "✅ Subscription has $REMAINING $SEARCH_TIER tier Search service slots available." - - name: Generate Resource Group Name id: generate_rg_name shell: bash @@ -733,8 +675,6 @@ jobs: AZURE_LOCATION: ${{ needs.azure-setup.outputs.AZURE_LOCATION }} RESOURCE_GROUP_NAME: ${{ needs.azure-setup.outputs.RESOURCE_GROUP_NAME }} IMAGE_TAG: ${{ needs.azure-setup.outputs.IMAGE_TAG }} - AZURE_ENV_GPT_MODEL_CAPACITY: ${{ needs.azure-setup.outputs.AZURE_ENV_GPT_MODEL_CAPACITY }} - AZURE_NONAI_LOCATIONS: ${{ vars.AZURE_NONAI_LOCATIONS }} BUILD_DOCKER_IMAGE: ${{ inputs.build_docker_image || 'false' }} EXP: ${{ inputs.EXP || 'false' }} WAF_ENABLED: ${{ inputs.waf_enabled == true && 'true' || 'false' }} @@ -754,8 +694,6 @@ jobs: AZURE_LOCATION: ${{ needs.azure-setup.outputs.AZURE_LOCATION }} RESOURCE_GROUP_NAME: ${{ needs.azure-setup.outputs.RESOURCE_GROUP_NAME }} IMAGE_TAG: ${{ needs.azure-setup.outputs.IMAGE_TAG }} - AZURE_ENV_GPT_MODEL_CAPACITY: ${{ needs.azure-setup.outputs.AZURE_ENV_GPT_MODEL_CAPACITY }} - AZURE_NONAI_LOCATIONS: ${{ vars.AZURE_NONAI_LOCATIONS }} BUILD_DOCKER_IMAGE: ${{ inputs.build_docker_image || 'false' }} EXP: ${{ inputs.EXP || 'false' }} WAF_ENABLED: ${{ inputs.waf_enabled == true && 'true' || 'false' }} diff --git a/.github/workflows/job-send-notification.yml b/.github/workflows/job-send-notification.yml index 6ca72e3e..aa257ed3 100644 --- a/.github/workflows/job-send-notification.yml +++ b/.github/workflows/job-send-notification.yml @@ -68,7 +68,7 @@ on: type: string env: - GPT_MIN_CAPACITY: ${{ vars.GPT_MIN_CAPACITY || 10 }} + GPT_MIN_CAPACITY: 50 BRANCH_NAME: ${{ github.event.workflow_run.head_branch || github.head_ref || github.ref_name }} WAF_ENABLED: ${{ inputs.trigger_type == 'workflow_dispatch' && (inputs.waf_enabled || false) || false }} EXP: ${{ inputs.trigger_type == 'workflow_dispatch' && (inputs.EXP || false) || false }} diff --git a/infra/scripts/checkquota.sh b/infra/scripts/checkquota.sh index 0cf1c7d2..9addb594 100644 --- a/infra/scripts/checkquota.sh +++ b/infra/scripts/checkquota.sh @@ -3,38 +3,17 @@ # Function to trim leading and trailing whitespace trim() { local var="$*" + # Remove leading whitespace var="${var#"${var%%[![:space:]]*}"}" + # Remove trailing whitespace var="${var%"${var##*[![:space:]]}"}" printf '%s' "$var" } -# Valid Azure AI service regions for this accelerator. -# Must match infra/main.bicep @allowed values for azureAiServiceLocation. -ALLOWED_REGIONS=( - "eastus2" - "francecentral" - "swedencentral" -) - -# Get requested regions from the environment variable and keep only supported regions. -REGIONS=() -if [[ -n "${AZURE_REGIONS:-}" ]]; then - IFS=',' read -ra REQUESTED_REGIONS <<< "$AZURE_REGIONS" - for req_region in "${REQUESTED_REGIONS[@]}"; do - req_region=$(trim "$req_region") - for allowed in "${ALLOWED_REGIONS[@]}"; do - if [[ "$req_region" == "$allowed" ]]; then - REGIONS+=("$req_region") - break - fi - done - done -fi -if [[ ${#REGIONS[@]} -eq 0 ]]; then - echo "⚠ī¸ WARNING: No valid regions found in AZURE_REGIONS. Using built-in supported regions list." - REGIONS=("${ALLOWED_REGIONS[@]}") -fi +# List of Azure regions to check for quota (update as needed) +IFS=', ' read -ra REGIONS <<< "$AZURE_REGIONS" +# Trim whitespace from environment variables to avoid issues with leading/trailing spaces SUBSCRIPTION_ID=$(trim "${AZURE_SUBSCRIPTION_ID}") GPT_MIN_CAPACITY="${GPT_MIN_CAPACITY:-10}" EMBEDDING_MIN_CAPACITY="${EMBEDDING_MIN_CAPACITY:-10}" @@ -49,8 +28,10 @@ fi echo "✅ Azure CLI is authenticated." echo "🔄 Validating required environment variables..." -if [[ -z "$SUBSCRIPTION_ID" ]]; then - echo "❌ ERROR: Missing required environment variable AZURE_SUBSCRIPTION_ID." +if [[ -z "$SUBSCRIPTION_ID" || -z "$REGIONS" ]]; then + echo "❌ ERROR: Missing required environment variables." + echo "Required: AZURE_SUBSCRIPTION_ID, AZURE_REGIONS" + echo "Optional: GPT_MIN_CAPACITY (default: 10), EMBEDDING_MIN_CAPACITY (default: 10)" exit 1 fi @@ -61,93 +42,53 @@ if ! az account set --subscription "$SUBSCRIPTION_ID"; then fi echo "✅ Azure subscription set successfully." +# Define models and their minimum required capacities +# Based on infrastructure analysis: +# - gpt-4.1-mini (version 2024-07-18) for chat completion +# - text-embedding-3-small for embeddings declare -A MIN_CAPACITY=( ["OpenAI.GlobalStandard.gpt4.1-mini"]="${GPT_MIN_CAPACITY}" ["OpenAI.GlobalStandard.text-embedding-3-small"]="${EMBEDDING_MIN_CAPACITY}" ["OpenAI.GlobalStandard.gpt-realtime-mini"]="${GPT_REALTIME_MIN_CAPACITY}" ) -# Check subscription-level GlobalStandard quotas FIRST -echo "----------------------------------------" -echo "🔍 Checking subscription-level GlobalStandard quota..." -GLOBALSTANDARD_QUOTA_FAILED=false - -# Query subscription-level quota usage for GlobalStandard models -QUOTA_RESPONSE=$(az rest --method GET \ - --url "https://management.azure.com/subscriptions/${SUBSCRIPTION_ID}/providers/Microsoft.Quota/quotas?api-version=2023-02-01" \ - -o json 2>/dev/null) - -if [[ -n "$QUOTA_RESPONSE" ]]; then - # Look for GlobalStandard quotas and check if any are exhausted - echo "$QUOTA_RESPONSE" | grep -i "GlobalStandard" > /dev/null 2>&1 - if [[ $? -eq 0 ]]; then - # Extract quota entries and check usage vs limit - while IFS= read -r quota_entry; do - if [[ -n "$quota_entry" && "$quota_entry" == *"GlobalStandard"* ]]; then - echo " 📊 GlobalStandard quota entry found" - # Check usageValue and limit fields - USAGE=$(echo "$quota_entry" | grep -o '"usageValue"[^,]*' | grep -o '[0-9]*$' || echo "0") - LIMIT=$(echo "$quota_entry" | grep -o '"limit"[^,]*' | grep -o '[0-9]*$' || echo "0") - - if [[ "$USAGE" -ge "$LIMIT" && "$LIMIT" -gt 0 ]]; then - echo " ❌ GlobalStandard quota exhausted: $USAGE/$LIMIT TPM" - GLOBALSTANDARD_QUOTA_FAILED=true - fi - fi - done < <(echo "$QUOTA_RESPONSE" | tr ',' '\n') - fi -fi - -if [[ "$GLOBALSTANDARD_QUOTA_FAILED" == "true" ]]; then - echo "❌ ERROR: Subscription-level GlobalStandard quota is exhausted." - echo " Please request a quota increase in the Azure Portal or use a different model deployment strategy." - echo "QUOTA_FAILED=true" >> "$GITHUB_ENV" - exit 0 -fi - VALID_REGION="" -VALID_REGION_AVAILABLE_CAPACITY="" for REGION in "${REGIONS[@]}"; do echo "----------------------------------------" echo "🔍 Checking region: $REGION" - # Check if Search service provider is available in this region - echo " 📋 Checking Search service availability in $REGION..." - SEARCH_AVAILABLE=$(az search service list-skus --region "$REGION" -o json 2>/dev/null | grep -c '"name"' || echo "0") - if [[ "$SEARCH_AVAILABLE" -le 0 ]]; then - echo " ⚠ī¸ WARNING: Search service is not available in $REGION. Trying next region..." - continue - fi - echo " ✅ Search service is available in $REGION" - - USAGE_COUNT=$(az cognitiveservices usage list --location "$REGION" --query 'length(@)' -o tsv 2>/dev/null || echo "0") - if [[ -z "$USAGE_COUNT" || "$USAGE_COUNT" == "0" ]]; then + QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json) + if [ -z "$QUOTA_INFO" ]; then echo "⚠ī¸ WARNING: Failed to retrieve quota for region $REGION. Skipping." continue fi INSUFFICIENT_QUOTA=false - REGION_GPT_AVAILABLE="" for MODEL in "${!MIN_CAPACITY[@]}"; do - CURRENT_VALUE=$(az cognitiveservices usage list --location "$REGION" --query "[?name.value=='$MODEL'].currentValue | [0]" -o tsv 2>/dev/null) - LIMIT=$(az cognitiveservices usage list --location "$REGION" --query "[?name.value=='$MODEL'].limit | [0]" -o tsv 2>/dev/null) + MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL\"" ' + BEGIN { RS="},"; FS="," } + $0 ~ model { print $0 } + ') - if [[ -z "$CURRENT_VALUE" || -z "$LIMIT" || "$CURRENT_VALUE" == "None" || "$LIMIT" == "None" ]]; then - echo "⚠ī¸ WARNING: No quota information found for model: $MODEL in $REGION." + if [ -z "$MODEL_INFO" ]; then + echo "⚠ī¸ WARNING: No quota information found for model: $MODEL in $REGION. Skipping." INSUFFICIENT_QUOTA=true - break + continue fi - CURRENT_VALUE=${CURRENT_VALUE%.*} - LIMIT=${LIMIT%.*} + CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentValue"/ {print $2}' | tr -d ',' | tr -d ' ') + LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ') + + CURRENT_VALUE=${CURRENT_VALUE:-0} + LIMIT=${LIMIT:-0} + + CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1) + LIMIT=$(echo "$LIMIT" | cut -d'.' -f1) + AVAILABLE=$((LIMIT - CURRENT_VALUE)) echo "✅ Model: $MODEL | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE" - if [[ "$MODEL" == "OpenAI.GlobalStandard.gpt4.1-mini" ]]; then - REGION_GPT_AVAILABLE="$AVAILABLE" - fi - if [ "$AVAILABLE" -lt "${MIN_CAPACITY[$MODEL]}" ]; then echo "❌ ERROR: $MODEL in $REGION has insufficient quota." echo " Required: ${MIN_CAPACITY[$MODEL]}, Available: $AVAILABLE" @@ -158,9 +99,9 @@ for REGION in "${REGIONS[@]}"; do if [ "$INSUFFICIENT_QUOTA" = false ]; then VALID_REGION="$REGION" - VALID_REGION_AVAILABLE_CAPACITY="$REGION_GPT_AVAILABLE" break fi + done if [ -z "$VALID_REGION" ]; then @@ -173,12 +114,10 @@ if [ -z "$VALID_REGION" ]; then exit 0 else echo "✅ Final Region: $VALID_REGION" - echo "✅ Available GPT Capacity: $VALID_REGION_AVAILABLE_CAPACITY" echo "All required models have sufficient quota:" for MODEL in "${!MIN_CAPACITY[@]}"; do echo " ✅ $MODEL: ${MIN_CAPACITY[$MODEL]} capacity available" done echo "VALID_REGION=$VALID_REGION" >> "$GITHUB_ENV" - echo "AVAILABLE_CAPACITY=$VALID_REGION_AVAILABLE_CAPACITY" >> "$GITHUB_ENV" exit 0 -fi +fi \ No newline at end of file