ci: fix broken OpenAPI generation

.azure-pipelines/generation-templates/capture-metadata.yml

@@ -166,9 +166,9 @@ steps:
 
 # Checkin clean metadata into metadata repo or make it an artifact.
 - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
-
-  displayName: push clean ${{ parameters.endpoint }} metadata to msgraph-metadata repo
+  displayName: push clean ${{ parameters.endpoint }} CSDL metadata to msgraph-metadata repo
   env:
+    CreatePR: False
     EndpointVersion: ${{ parameters.endpoint }}
     PublishChanges: $(publishChanges)
   workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'

.azure-pipelines/generation-templates/capture-openapi.yml

@@ -1,3 +1,4 @@
+# capture-openapi.yml
 # Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the project root for license information.
 # The following template creates multiple artifacts from a matrix, this is the intended behavior to follow the same workflow before the governance migration
 
@@ -21,6 +22,7 @@ jobs:
     persistCredentials: true
 
   - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
+
   - pwsh: |
       $dirPath = "./conversion-settings"
       $endpoint = "${{ parameters.endpoint }}"
@@ -37,8 +39,10 @@ jobs:
       Write-Host "##vso[task.setvariable variable=targets;isOutput=true]$json"
     name: setTargets
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
+
   - script: echo $(setTargets.targets)
     displayName: "Print settings"
+
 - job: convert_openapi
   dependsOn: get_conversion_settings
   displayName: Convert
@@ -58,28 +62,33 @@ jobs:
     persistCredentials: true
 
   - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
-
   # required for the hidi to run
   - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '8.x'
-
-# required for the hidi installation validation
+    displayName: use dotnet sdk 8.x
+  # required for the hidi installation validation
   - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '9.x'
+    displayName: use dotnet sdk 9.x
 
   - pwsh: dotnet tool install --global Microsoft.OpenApi.Hidi --version 1.6.24
     displayName: install hidi
 
   - pwsh: |
-      git fetch origin master
-      git switch master
+      git fetch origin master | Write-Host -ForegroundColor Yellow
+      git switch master | Write-Host -ForegroundColor Yellow
     displayName: ensure the generation happens from master latest
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
-  - pwsh: '$(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"'
-    displayName: 'update ${{ parameters.endpoint }} open API description'
+  - pwsh: |
+      Write-Host "`nGet status before generation:" -ForegroundColor Green
+      git status | Write-Host -ForegroundColor Yellow
+      $(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"
+      Write-Host "`nGet status after generation:" -ForegroundColor Green
+      git status | Write-Host -ForegroundColor Yellow
+    displayName: 'generate ${{ parameters.endpoint }} open API description'
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
   # publish metadata as an artifact
@@ -88,16 +97,16 @@ jobs:
       sourceFolder: ${{ parameters.outputPath }}
       contents: '**/$(Name).yaml'
       targetFolder: '$(Build.ArtifactStagingDirectory)/$(Name)'
-    displayName: Copy generated metadata
+    displayName: Copy generated OpenAPI yaml file to ArtifactStagingDirectory
+
   - pwsh: |
       ./scripts/run-openapi-validation.ps1 -repoDirectory (Get-Location).Path -version "${{ parameters.endpoint }}" -platformName "$(Name)"
-    displayName: ensure that OpenAPI docs can be parsed
+    displayName: Validate that OpenAPI docs can be parsed
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
-
 - job: publish_openapi
   dependsOn: convert_openapi
-  displayName: Publish
+  displayName: Publish OpenAPI files
   ## If there's new settings added please add them here too
   templateContext:
     inputs:
@@ -123,35 +132,58 @@ jobs:
     displayName: checkout generator
     fetchDepth: 1
     persistCredentials: true
-    # Copy files from the maxtrix artifacts to a single folder
+
+  #  Note that msgraph-metadata repo has detached HEAD
+  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self    
+
+  - template: /.azure-pipelines/generation-templates/set-user-config.yml@self  
+
+  # Copy OpenAPI files from the matrix artifacts to the ArtifactStagingDirectory
+  # This is used for library generation later
   - task: CopyFiles@2
     inputs:
       sourceFolder: '$(Build.SourcesDirectory)/artifacts'
       contents: '**/*.yaml'
       targetFolder: '$(Build.ArtifactStagingDirectory)'
-      displayName: Copy artifact metadata
-    # Copy files from the target path where artifacts should be downloaded
+    displayName: Copy OpenAPI input artifacts to artifact staging directory
+
+  # Copy OpenAPI files from ArtifactStagingDirectory to local msgraph-metadata repo
   - task: CopyFiles@2
     inputs:
       sourceFolder: '$(Build.ArtifactStagingDirectory)'
       contents: '**/*.yaml'
       targetFolder: '$(Build.SourcesDirectory)/msgraph-metadata/openapi/${{ parameters.endpoint }}'
-    displayName: Copy downloaded metadata
+    displayName: Copy OpenAPI files to local msgraph-metadata repo
 
-    # publish metadata as an artifact
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: ${{ parameters.outputPath }}
-      contents: '**/*.yaml'
-      targetFolder: '$(Build.ArtifactStagingDirectory)'
-    displayName: Copy downloaded metadata
-  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
-  - template: /.azure-pipelines/generation-templates/set-user-config.yml@self
+  # Push changes to msgraph-metadata repo and create PR
   - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
-
-    displayName: push clean ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo
+    displayName: PR for ${{ parameters.endpoint }} OpenAPI description in msgraph-metadata repo
     env:
+      CreatePR: True
       EndpointVersion: ${{ parameters.endpoint }}
       PublishChanges: $(publishChanges)
     workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'
     enabled: true
+
+  # Create PR - note that this PR is not used for gating. It's just for discovery purposes. 
+  # Library generation PRs will still be created based on the OpenAPI files.
+
+  - task: AzureKeyVault@2
+    displayName: "Azure Key Vault: Get Secrets"
+    inputs:
+      azureSubscription: "Federated AKV Managed Identity Connection"
+      KeyVaultName: akv-prod-eastus
+      SecretsFilter: "microsoft-graph-devx-bot-appid,microsoft-graph-devx-bot-privatekey"
+
+  - pwsh: '$(scriptsDirectory)/create-pull-request.ps1'
+    displayName: 'Create Pull Request for the generated OpenAPI files for msgraph-metadata'
+    env:
+      BaseBranch: master
+      GeneratePullRequest: true
+      GhAppId: $(microsoft-graph-devx-bot-appid)
+      GhAppKey: $(microsoft-graph-devx-bot-privatekey)
+      OverrideSkipCI: false
+      RepoName: 'microsoftgraph/msgraph-metadata'
+      ScriptsDirectory: $(scriptsDirectory)
+      Version: ''
+    workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'

scripts/create-pull-request.ps1

@@ -9,6 +9,8 @@ if (($env:GeneratePullRequest -eq $False)) { # Skip CI if manually running this
     return;
 }
 
+$version = $env:Version
+
 # Special case for beta typings as it uses a non-conforming preview versioning.
 if ($env:RepoName.Contains("msgraph-beta-typescript-typings"))
 {
@@ -18,7 +20,6 @@ else {
     $title = "Generated $version models and request builders"
 }
 
-$version = $env:Version
 $body = ":bangbang:**_Important_**:bangbang: <br> Check for unexpected deletions or changes in this PR and ensure relevant CI checks are passing. <br><br> **Note:** This pull request was automatically created by Azure pipelines."
 $baseBranchParameter = ""
 

scripts/git-push-cleanmetadata.ps1

@@ -1,31 +1,81 @@
+# git-push-cleanmetadata.ps1
+# Copyright (c) Microsoft Corporation.  All Rights Reserved.  Licensed under the MIT License.  See License in the 
+# project root for license information.
+
+# Referenced by:
+# capture-metadata.yml
+# capture-openapi.yml
+
+# This script stashes any changes, checks out the latest master branch, applies the stashed changes, commits, and 
+# pushes the changes back to the remote repository.
+
 if ($env:PublishChanges -eq $False)
 {
-    Write-Host "Not publishing changes per the run parameter!" -ForegroundColor Green
+    Write-Host "Not publishing changes per the run parameter!" -ForegroundColor Red
     return;
 }
 
-Write-Host "About to add clean $env:EndpointVersion metadata file....."
+Write-Host "`nGet status:" -ForegroundColor Green
+git status | Write-Host -ForegroundColor Yellow
 
-git fetch origin master
+Write-Host "`nStash the update metadata files.....`nRunning: git stash" -ForegroundColor Green
+git stash | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nFetching latest master branch to ensure we are up to date..." -ForegroundColor Green
+git fetch origin master | Write-Host -ForegroundColor Yellow
 # checkout master to move from detached HEAD mode
-git switch master
+git switch master | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nGet status:" -ForegroundColor Green
+git status | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nApply stashed metadata files...`nRunning: git stash pop" -ForegroundColor Green
+git stash pop | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nGet status:" -ForegroundColor Green
+git status | Write-Host -ForegroundColor Yellow
+
+if ($env:CreatePR -eq $True)
+{
+    Write-Host "`nCreate branch: $env:BUILD_BUILDID/updateOpenAPI" -ForegroundColor Green
+    git checkout -B $env:BUILD_BUILDID/updateOpenAPI | Write-Host -ForegroundColor Yellow
+}
+
+Write-Host "`nStaging clean $env:EndpointVersion metadata files....." -ForegroundColor Green
+git add . | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nGet status:" -ForegroundColor Green
+git status | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nAttempting to commit clean $env:EndpointVersion metadata files....." -ForegroundColor Green
 
-git add . | Write-Host
 if ($env:BUILD_REASON -eq 'Manual') # Skip CI if manually running this pipeline.
 {
-    git commit -m "Update clean metadata file with $env:BUILD_BUILDID [skip ci]" | Write-Host
+    git commit -m "Update clean metadata file with $env:BUILD_BUILDID [skip ci]" | Write-Host -ForegroundColor Yellow
 }
 else
 {
-    git commit -m "Update clean metadata file with $env:BUILD_BUILDID" | Write-Host
+    git commit -m "Update clean metadata file with $env:BUILD_BUILDID" | Write-Host -ForegroundColor Yellow
 }
 
-Write-Host "Added and commited cleaned $env:EndpointVersion metadata." -ForegroundColor Green
+Write-Host "`nGet status:" -ForegroundColor Green
+git status | Write-Host -ForegroundColor Yellow
 
-# sync branch before pushing
-# this is especially important while running v1 and beta in parallel
-# and one process goes out of sync because of the other's check-in
-git pull origin master --rebase
+if ($env:CreatePR -eq $True)
+{
+    Write-Host "`nPushing branch for PR creation" -ForegroundColor Green
+
+    Write-Host "`ngit push --set-upstream origin $env:BUILD_BUILDID/updateOpenAPI:" -ForegroundColor Green
+    git push --set-upstream origin $env:BUILD_BUILDID/updateOpenAPI | Write-Host -ForegroundColor Yellow
+}
+else # original behavior: push to master
+{
+    Write-Host "`nRunning: git pull origin master --rebase..." -ForegroundColor Green
+    # sync branch before pushing
+    # this is especially important while running v1 and beta in parallel
+    # and one process goes out of sync because of the other's check-in
+    git pull origin master --rebase | Write-Host -ForegroundColor Yellow
 
-git push --set-upstream origin master | Write-Host
-Write-Host "Pushed the results of the build $env:BUILD_BUILDID to the master branch." -ForegroundColor Green
+    Write-Host "`nRunning: git push --set-upstream origin master ..." -ForegroundColor Green
+    git push --set-upstream origin master | Write-Host -ForegroundColor Yellow
+}