diff --git a/.azure-pipelines/common-templates/checkout.yml b/.azure-pipelines/common-templates/checkout.yml new file mode 100644 index 0000000000..8c79b44211 --- /dev/null +++ b/.azure-pipelines/common-templates/checkout.yml @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +steps: + - checkout: self + clean: true + fetchDepth: 1 + persistCredentials: true + + - task: PowerShell@2 + displayName: "Configure user" + inputs: + targetType: "inline" + script: | + git config --global user.email "GraphTooling@service.microsoft.com" + git config --global user.name "Microsoft Graph DevX Tooling" + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 + displayName: "Run CredScan" + inputs: + debugMode: false diff --git a/.azure-pipelines/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml new file mode 100644 index 0000000000..d6d9447deb --- /dev/null +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -0,0 +1,137 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +parameters: + - name: Branch + type: string + default: "WeeklyOpenApiDocsDownload" + - name: BaseBranch + type: string + default: "dev" + +jobs: + - job: GetLatestDocs + displayName: Download OpenApiDocs + pool: MsGraphDevXAzureAgents + steps: + - template: ./checkout.yml + + - template: ./install-tools.yml + + - task: PowerShell@2 + name: "ComputeBranch" + displayName: "Compute weekly branch name" + inputs: + targetType: inline + script: | + $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm) + Write-Host "##vso[task.setvariable variable=WeeklyBranch;isOutput=true]$branch" + + - task: Bash@3 + displayName: "Create weekly branch" + inputs: + targetType: inline + script: | + git status + git fetch --all + git checkout $(BaseBranch) + git branch $(ComputeBranch.WeeklyBranch) + git checkout $(ComputeBranch.WeeklyBranch) + git status + + - task: PowerShell@2 + displayName: Download v1.0 OpenApi docs + continueOnError: false + inputs: + filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1" + pwsh: true + + - task: PowerShell@2 + displayName: Download beta OpenApi docs + continueOnError: false + inputs: + filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1" + arguments: "-BetaGraphVersion" + pwsh: true + + - task: PowerShell@2 + name: OpenAPIDocDiff + displayName: Get OpenAPI docs diff + inputs: + pwsh: true + targetType: "inline" + script: | + $diff = git diff --name-only + $ModulesWithChanges = @{} + $diff | %{ + if (($_ -match 'openApiDocs\/(v1.0|beta)\/(.*).yml') -and !$ModulesWithChanges.ContainsKey($matches.2)) + { + $ModulesWithChanges.Add($matches.2, $matches.1) + } + } + $ModuleNames = $ModulesWithChanges.Keys + Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" + + - task: PowerShell@2 + displayName: Generate profiles + condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) + continueOnError: false + inputs: + targetType: filePath + pwsh: true + filePath: $(System.DefaultWorkingDirectory)/tools/GenerateProfiles.ps1 + + - task: PowerShell@2 + name: CalculateAndBumpModuleVersion + displayName: Calculate and bump module version + condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) + inputs: + pwsh: true + targetType: inline + script: | + # Calculate meta-module version + $MetaModule = Find-Module "Microsoft.Graph" -Repository PSGallery + $MetaModuleVersion = [System.Version]($MetaModule.Version) + $NewMetaModuleVersion = "$($MetaModuleVersion.Major).$($MetaModuleVersion.Minor + 1).$($MetaModuleVersion.Build)" + # Bump meta-module minor version + Write-Host "Bumping Microsoft.Graph to $NewMetaModuleVersion." + & "$(System.DefaultWorkingDirectory)\tools\SetMetaModuleVersion.ps1" -VersionNumber $NewMetaModuleVersion + + # Calculate existing service module version + "$(OpenAPIDocDiff.ModulesWithChanges)" -split " " | ForEach-Object { + try { + $Module = Find-Module "Microsoft.Graph.$_" -Repository PSGallery -ErrorAction Stop + $ModuleVersion = [System.Version]($Module.Version) + $NewModuleVersion = "$($ModuleVersion.Major).$($ModuleVersion.Minor + 1).$($ModuleVersion.Build)" + Write-Host "Bumping $_ to $NewModuleVersion." + . "$(System.DefaultWorkingDirectory)\tools\SetServiceModuleVersion.ps1" -VersionNumber $NewModuleVersion -Modules $_ + } catch { + if ($_.Exception.Message -like "No match*") { + Write-Warning "$_. Version will be set to $NewMetaModuleVersion." + } + } + } + + # Calculate new service module version + $NewModuleReadMePath = Join-Path $(System.DefaultWorkingDirectory) "/tools/Templates/readme.md" + . "$(System.DefaultWorkingDirectory)\tools\WriteToModuleReadMe.ps1" -ReadMePath $NewModuleReadMePath -FieldName "module-version" -NewFieldValue $NewMetaModuleVersion + + - task: Bash@3 + displayName: Commit downloaded files + condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) + env: + GITHUB_TOKEN: $(GITHUB_TOKEN) + inputs: + targetType: inline + script: | + git status + git add . + git commit -m 'Weekly OpenApiDocs Download' + git status + git push --set-upstream origin $(ComputeBranch.WeeklyBranch) + git status + +# References +# [0] https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables +# [1] https://hub.github.com/hub-pull-request.1.html +# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token diff --git a/.azure-pipelines/common-templates/install-tools.yml b/.azure-pipelines/common-templates/install-tools.yml new file mode 100644 index 0000000000..a3a73d8528 --- /dev/null +++ b/.azure-pipelines/common-templates/install-tools.yml @@ -0,0 +1,79 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +steps: + - task: UseDotNet@2 + displayName: "Use .NET Core SDK 2.x" + inputs: + debugMode: false + version: 2.x + + - task: NuGetToolInstaller@1 + displayName: Install Nuget 5.7 + inputs: + versionSpec: 5.7.0 + checkLatest: false # Optional + + - task: NuGetAuthenticate@0 + displayName: Authenticate NuGet + + - task: PowerShell@2 + displayName: Install Powershell core + inputs: + targetType: inline + script: | + dotnet tool update --global PowerShell + pwsh + + - task: PowerShell@2 + displayName: Version check + inputs: + targetType: inline + pwsh: true + script: | + Write-Host $PSVersionTable.PSVersion + Write-Host $host.Version + Write-Host (Get-Host).Version + + - task: NodeTool@0 + displayName: Install NodeJs 14.11.0 + inputs: + versionSpec: "14.11.0" + checkLatest: true # Optional + + - task: Npm@1 + displayName: Install AutoRest + inputs: + command: "custom" + customCommand: "install -g autorest@latest" + + - task: PowerShell@2 + displayName: Install PowerShell dependencies + inputs: + targetType: inline + pwsh: true + errorActionPreference: "continue" + script: | + Install-Module "powershell-yaml" -Repository PSGallery -Force + + - task: PowerShell@2 + displayName: Register PS repository + enabled: false + inputs: + targetType: inline + pwsh: true + errorActionPreference: "continue" + script: | + $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Continue + Get-PSRepository + $patToken = '$(NUGETFEEDKEY)' | ConvertTo-SecureString -AsPlainText -Force + $nugetFeed = '$(NUGETFEED)' + $user = '$(NUGETBUILDUSER)' + $credsAzureDevopsServices = New-Object System.Management.Automation.PSCredential($user, $patToken) + UnRegister-PackageSource -Name 'LocalNugetPackageSource' -ErrorAction Continue + UnRegister-PSRepository -Name 'LocalNugetFeed' -ErrorAction Continue + Register-PackageSource -Name 'LocalNugetPackageSource' -Location $nugetFeed -SkipValidate -Trusted -Verbose -ProviderName 'Nuget' -ErrorAction Continue + Register-PSRepository -Name 'LocalNugetFeed' -SourceLocation $nugetFeed -PublishLocation $nugetFeed -InstallationPolicy Trusted -Credential $credsAzureDevopsServices -PackageManagementProvider 'Nuget' -ErrorAction Continue + Get-PSRepository + Find-Module -Name Microsoft.Graph.Authentication -AllowPrerelease -Credential $credsAzureDevopsServices -AllVersions -Repository 'LocalNugetFeed' + Find-Module -Name Microsoft.Graph.Authentication -AllowPrerelease -Repository 'LocalNugetFeed' diff --git a/.azure-pipelines/download-openapidocs-template.yml b/.azure-pipelines/download-openapidocs-template.yml deleted file mode 100644 index 451d266653..0000000000 --- a/.azure-pipelines/download-openapidocs-template.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -parameters: -- name: BUILDNUMBER - displayName: 'Build Number' - type: string - default: $[format('{0:yyMMddHH}', pipeline.startTime)] - -jobs: -- job: MsGraphDownloadOpenApiDocs - displayName: Microsoft Graph PowerShell SDK Download OpenApiDocs Module Generation - pool: MsGraphDevXAzureAgents - timeoutInMinutes: 600 - - steps: - - checkout: self - persistCredentials: true - - - template: ./install-tools-template.yml - - - task: PowerShell@2 - displayName: 'Download v1.0 OpenApiDocs' - continueOnError: true - condition: and(succeeded(), eq(eq(variables['Build.SourceBranch'], 'refs/heads/master'), false)) - inputs: - targetType: 'filePath' - pwsh: true - filePath: '$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1' - - - - task: PowerShell@2 - displayName: 'Download beta OpenApiDocs' - continueOnError: false - condition: and(succeeded(), eq(eq(variables['Build.SourceBranch'], 'refs/heads/master'), false)) - inputs: - targetType: 'filePath' - pwsh: true - filePath: '$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1' - arguments: > - -BetaGraphVersion - - - task: PowerShell@2 - displayName: "Configure User" - condition: and(succeeded(), eq(eq(variables['Build.SourceBranch'], 'refs/heads/master'), false)) - inputs: - targetType: 'inline' - pwsh: true - script: | - git config --global user.email 'GraphTooling@service.microsoft.com' - git config --global user.name 'Microsoft Graph DevX Tooling' - - - task: PowerShell@2 - condition: and(succeeded(), eq(eq(variables['Build.SourceBranch'], 'refs/heads/master'), false)) - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) - inputs: - targetType: 'inline' - script: | - $date = Get-Date -Format yyyyMMddHH - $docsBranch = "{0}{1}" -f $date, "buildDocsDownload" - git status - git checkout '$(Build.SourceBranchName)' - git branch $docsBranch - git checkout $docsBranch - git status - git add . - git commit -m '$(BUILDNUMBER): Build OpenApiDocs Download [skip ci]' - git status - git push --set-upstream origin $docsBranch - git status \ No newline at end of file diff --git a/.azure-pipelines/download-openapidocs.yml b/.azure-pipelines/download-openapidocs.yml deleted file mode 100644 index 791c457dca..0000000000 --- a/.azure-pipelines/download-openapidocs.yml +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Generates a release build artifact (nuget) from HEAD of master for auth module. -name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) - -pool: - vmImage: "windows-latest" - -variables: - BRANCH: 'weeklyOpenApiDocsDownload' - GitUserEmail: 'GraphTooling@service.microsoft.com' - GitUserName: 'Microsoft Graph DevX Tooling' - BaseBranch: 'dev' - -schedules: - - cron: "0 0 * * WED" # Run Every Wednesday - displayName: "Weekly OpenApiDocs Download and PR" - branches: - include: - - dev - always: true - -steps: - - checkout: self - persistCredentials: true - clean: true - fetchDepth: 1 - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 - displayName: 'Run CredScan' - inputs: - debugMode: false - - - task: PowerShell@2 - displayName: "Compute Branch" - inputs: - targetType: inline - script: | - $branch = "{0}.{1}" -f "weeklyOpenApiDocsDownload", (Get-Date -Format yyyyMMdd) - Write-Host "##vso[task.setvariable variable=BRANCH;]$branch" - - - task: PowerShell@2 - displayName: "Configure User" - inputs: - targetType: 'inline' - script: | - git config --global user.email '$(GitUserEmail)' - git config --global user.name '$(GitUserName)' - - - task: PowerShell@2 - displayName: "Show Directory" - inputs: - targetType: 'inline' - script: | - ls $(System.DefaultWorkingDirectory) - ls $(System.DefaultWorkingDirectory)/tools - - - task: PowerShell@2 - displayName: Download v1.0 OpenApiDocs - continueOnError: false - inputs: - filePath: '$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1' - pwsh: true - - - task: PowerShell@2 - displayName: Download beta OpenApiDocs - continueOnError: false - inputs: - filePath: '$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1' - arguments: '-BetaGraphVersion' - pwsh: true - - - task: Bash@3 - displayName : "Create PR $(BRANCH)" - inputs: - targetType: 'inline' - script: | - git status - git checkout $(BaseBranch) - git branch $(BRANCH) - git checkout $(BRANCH) - git status - - - task: Bash@3 - displayName : "Commit Downloaded Files" - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) - inputs: - targetType: 'inline' - script: | - git status - git add . - git commit -m 'Weekly OpenApiDocs Download' - git status - git push --set-upstream origin $(BRANCH) - git status - -# References -# [0] https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables -# [1] https://hub.github.com/hub-pull-request.1.html -# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token \ No newline at end of file diff --git a/.azure-pipelines/downloadopenapidocs-pipeline.yml b/.azure-pipelines/downloadopenapidocs-pipeline.yml deleted file mode 100644 index ca568ab115..0000000000 --- a/.azure-pipelines/downloadopenapidocs-pipeline.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) - -pool: - vmImage: "windows-latest" - -variables: - BRANCH: 'weeklyOpenApiDocsDownload' - GitUserEmail: 'GraphTooling@service.microsoft.com' - GitUserName: 'Microsoft Graph DevX Tooling' - BaseBranch: 'dev' - -schedules: - - cron: "0 0 * * WED" # Run Every Wednesday - displayName: "Weekly OpenApiDocs Download and PR" - branches: - include: - - dev - always: true - -stages: -- stage: DownloadOpenApiDocs - displayName: 'Download Open Api Docs from DevX API' - jobs: - - job: DownloadOpenAPiDocs - steps: - - template: ./download-openapidocs-template.yml - parameters: - GitUserEmail: $(GitUserEmail) - GitUserName: $(GitUserName) - BaseBranch: $(BaseBranch) \ No newline at end of file diff --git a/.azure-pipelines/generation-templates/generate-service-modules.yml b/.azure-pipelines/generation-templates/generate-service-modules.yml new file mode 100644 index 0000000000..09292d1f57 --- /dev/null +++ b/.azure-pipelines/generation-templates/generate-service-modules.yml @@ -0,0 +1,276 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +parameters: + - name: ModulesToGenerate + type: string + - name: AuthModulePath + type: string + - name: AuthModuleDllPattern + type: string + - name: ServiceModulePath + type: string + - name: ModulePrefix + type: string + - name: Branch + type: string + - name: EnableSigning + type: boolean + default: false + - name: PublishToFeed + type: boolean + default: false +jobs: + - job: GenerateServiceModules + displayName: Service module generation + pool: MsGraphDevXAzureAgents + timeoutInMinutes: 360 + condition: and(succeeded(), ne(stageDependencies.DownloadOpenAPIDocs.GetLatestDocs.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) + variables: + Branch: ${{ parameters.Branch }} + ModulesToGenerate: ${{ parameters.ModulesToGenerate }} + AuthModulePath: ${{ parameters.AuthModulePath }} + AuthModuleDllPattern: ${{ parameters.AuthModuleDllPattern }} + ServiceModulePath: ${{ parameters.ServiceModulePath }} + ModulePrefix: ${{ parameters.ModulePrefix }} + EnableSigning: ${{ parameters.EnableSigning }} + PublishToFeed: ${{ parameters.PublishToFeed }} + steps: + - template: ../common-templates/install-tools.yml + + - task: Bash@3 + displayName: "Switch branch to $(Branch)" + inputs: + targetType: inline + script: | + git status + git fetch --all + git checkout $(Branch) + git pull + git status + + - task: PowerShell@2 + displayName: Build auth module + inputs: + targetType: inline + pwsh: true + script: | + [bool]$EnableSigning = if ("$(EnableSigning)" -eq "true") { $true } else { $false } + . $(System.DefaultWorkingDirectory)/tools/GenerateAuthenticationModule.ps1 -ArtifactsLocation $(Build.ArtifactStagingDirectory) -Build -EnableSigning:$EnableSigning -BuildWhenEqual -RepositoryName "LocalNugetFeed" + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ESRP DLL strong name (Auth Module) + condition: and(succeeded(), eq('${{ parameters.EnableSigning }}', true)) + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: $(AuthModulePath) + Pattern: $(AuthModuleDllPattern) + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-233863-SN", + "operationSetCode": "StrongNameSign", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-233863-SN", + "operationSetCode": "StrongNameVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: "ESRP DLL code sign (Auth Module)" + condition: and(succeeded(), eq('${{ parameters.EnableSigning }}', true)) + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: $(AuthModulePath) + Pattern: $(AuthModuleDllPattern) + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd \"SHA256\"" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: PowerShell@2 + displayName: Generate and build service modules + inputs: + targetType: inline + pwsh: true + script: | + [bool]$EnableSigning = if ("$(EnableSigning)" -eq "true") { $true } else { $false } + $Modules = "$(ModulesToGenerate)" -split " " + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -Build -Test -UpdateAutoRest -EnableSigning:$EnableSigning -ModulesToGenerate $Modules + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ESRP DLL strong name (Service Modules) + condition: and(succeeded(), eq('${{ parameters.EnableSigning }}', true)) + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: $(ServiceModulePath) + Pattern: "$(ModulePrefix).*.private.dll" + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-233863-SN", + "operationSetCode": "StrongNameSign", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-233863-SN", + "operationSetCode": "StrongNameVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: "ESRP DLL code sign (Service Modules)" + condition: and(succeeded(), eq('${{ parameters.EnableSigning }}', true)) + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: $(ServiceModulePath) + Pattern: "$(ModulePrefix).*.private.dll, $(ModulePrefix).*.psm1, $(ModulePrefix).*.format.ps1xml, ProxyCmdletDefinitions.ps1, load-dependency.ps1" + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd \"SHA256\"" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 160 + + - task: PowerShell@2 + displayName: Pack service modules + inputs: + targetType: inline + pwsh: true + script: | + $ModuleMappingConfigPath = '$(System.DefaultWorkingDirectory)/config/ModulesMapping.jsonc' + [HashTable] $ModuleMapping = Get-Content $ModuleMappingConfigPath | ConvertFrom-Json -AsHashTable + $ModuleMapping.Keys | ForEach-Object { + $ModuleName = $_ + $ModuleProjectDir = "$(System.DefaultWorkingDirectory)/src/$ModuleName/$ModuleName" + & $(System.DefaultWorkingDirectory)/tools/PackModule.ps1 -Module $ModuleName -ArtifactsLocation $(Build.ArtifactStagingDirectory)\ + } + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ESRP NuGet code sign (Service Modules) + condition: and(succeeded(), eq('${{ parameters.EnableSigning }}', true)) + inputs: + ConnectedServiceName: "microsoftgraph ESRP CodeSign DLL and NuGet (AKV)" + FolderPath: '$(Build.ArtifactStagingDirectory)\' + Pattern: "*.nupkg" + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetSign", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetVerify", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 20 + + - task: PublishBuildArtifacts@1 + displayName: Publish service modules build artifacts + inputs: + PathtoPublish: "$(Build.ArtifactStagingDirectory)/" + ArtifactName: "drop" + publishLocation: "Container" + + - task: NuGetCommand@2 + displayName: Publish NuGet to local build feed + condition: and(succeeded(), eq('${{ parameters.PublishToFeed }}', true)) + inputs: + command: push + packagesToPush: "$(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg" + publishVstsFeed: "0985d294-5762-4bc2-a565-161ef349ca3e/edc337b9-e5ea-49dd-a2cb-e8d66668ca57" + allowPackageConflicts: true diff --git a/.azure-pipelines/integrated-pipeline.yml b/.azure-pipelines/integrated-pipeline.yml index 6af676623e..60e5e82eb2 100644 --- a/.azure-pipelines/integrated-pipeline.yml +++ b/.azure-pipelines/integrated-pipeline.yml @@ -52,13 +52,6 @@ stages: jobs: - template: ./security-prechecks-template.yml -- stage: DownloadOpenApiDocs - displayName: 'Download Open Api Docs from DevX API' - jobs: - - template: ./download-openapidocs-template.yml - parameters: - BUILDNUMBER: $(BUILDNUMBER) - - stage: GenerateAuthModule displayName: 'Generate Authentication Module (Microsoft.Graph.Authentication)' jobs: diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml new file mode 100644 index 0000000000..eb9951d278 --- /dev/null +++ b/.azure-pipelines/weekly-generation.yml @@ -0,0 +1,41 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) +pool: MsGraphDevXAzureAgents + +variables: + Branch: "WeeklyOpenApiDocsDownload" + BaseBranch: "dev" +trigger: none +pr: none +schedules: + - cron: "0 12 * * WED" # Run every wednesday at noon UTC + displayName: "Weekly PS SDK generation" + branches: + include: + - dev + always: true + +stages: + - stage: DownloadOpenAPIDocs + displayName: Download OpenAPI docs + jobs: + - template: common-templates/download-openapi-docs.yml + parameters: + Branch: $(Branch) + BaseBranch: $(BaseBranch) + + - stage: GenerateServiceModules + displayName: Generate service modules + jobs: + - template: generation-templates/generate-service-modules.yml + parameters: + ModulesToGenerate: $[ stageDependencies.DownloadOpenAPIDocs.GetLatestDocs.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] + Branch: $[ stageDependencies.DownloadOpenAPIDocs.GetLatestDocs.outputs['ComputeBranch.WeeklyBranch'] ] + AuthModulePath: "src/Authentication/Authentication/bin/" + AuthModuleDllPattern: "Microsoft.Graph.Authentication.dll" + ServiceModulePath: "src/" + ModulePrefix: "Microsoft.Graph" + EnableSigning: false + PublishToFeed: false diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4c9ea2c5ef..1d3735ee47 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ name: WeeklyOpenApiDocsDownload on: push: branches: - - 'weeklyOpenApiDocsDownload.*' + - 'WeeklyOpenApiDocsDownload/*' # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: diff --git a/tools/DownloadOpenApiDoc.ps1 b/tools/DownloadOpenApiDoc.ps1 index 3b1e7cfd08..55ca298de5 100644 --- a/tools/DownloadOpenApiDoc.ps1 +++ b/tools/DownloadOpenApiDoc.ps1 @@ -17,14 +17,50 @@ if ($ForceRefresh.IsPresent) { $OpenApiServiceUrl = "$OpenApiServiceUrl&forceRefresh=true" } Write-Host -ForegroundColor Green "[$RequestCount] Downloading OpenAPI doc for '$ModuleName' module: $OpenApiServiceUrl" -try { - Invoke-WebRequest $OpenApiServiceUrl -OutFile "$OpenApiDocOutput\$ModuleName.yml" -} -catch { - # Get the Http Error Message from DevX Api, Rethrow Error to be handled Upstream - $ErrorMessage = $_.Exception.Message - Write-Host -ForegroundColor Red "[$RequestCount] Request Failed for $ModuleName Error Message: $ErrorMessage" - if ($GraphVersion -eq "beta") { - throw +$Retries = 3 +$Delay = 3 +$Retrycount = 0 +$Completed = $false +while (-not $Completed) { + try { + Invoke-WebRequest $OpenApiServiceUrl -OutFile "$OpenApiDocOutput\$ModuleName.yml" + $Completed = $true + } + catch { + $Exception = $_.Exception + switch ($Exception.Response.StatusCode.value__) { + { @(429, 503, 504) -contains $_ } { + if ($Retrycount -ge $Retries) { + Write-Warning "Request to $OpenApiServiceUrl failed the maximum number of $Retrycount times." + throw + } + else { + [ref]$RetryAfterHeader = $null + if ($Exception.Response.Headers.TryGetValues("Retry-After", $RetryAfterHeader)) { + # Use Retry-After response header + $DelayInSeconds = $RetryAfterHeader.Value + } + else { + # Use exponential backoff + $mPow = [math]::Pow(2, $Retrycount) + $DelayInSeconds = $mPow * $Delay + } + + Write-Warning "Request to $OpenApiServiceUrl failed. Retrying in $DelayInSeconds seconds." + Start-Sleep $DelayInSeconds + $Retrycount++ + } + } + 404 { + Write-Warning "Request to $OpenApiServiceUrl returned 404. Download will be skipped." + $Completed = $true + } + default { + # Get HTTP error message from DevX api, Re-throw error to be handled Upstream + $ErrorMessage = $Exception.Message + Write-Warning "[$RequestCount] Request for $ModuleName failed with error message: $ErrorMessage" + throw + } + } } } \ No newline at end of file diff --git a/tools/GenerateModules.ps1 b/tools/GenerateModules.ps1 index 28b45e1f09..8f2871bd6e 100644 --- a/tools/GenerateModules.ps1 +++ b/tools/GenerateModules.ps1 @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. Param( + $ModulesToGenerate = @(), [string] $RepositoryApiKey, [string] $RepositoryName = "PSGallery", [int] $ModulePreviewNumber = -1, @@ -50,7 +51,7 @@ if (-not (Test-Path $ModuleMappingConfigPath)) { } $AllowPreRelease = $true -if($ModulePreviewNumber -eq -1) { +if ($ModulePreviewNumber -eq -1) { $AllowPreRelease = $false } # Install module locally in order to specify it as a dependency for other modules down the generation pipeline. @@ -60,7 +61,7 @@ Write-Host -ForegroundColor Green "Auth Module: $($ExistingAuthModule.Name), $($ if (!(Get-Module -Name $ExistingAuthModule.Name -ListAvailable)) { Install-Module $ExistingAuthModule.Name -Repository $RepositoryName -Force -AllowClobber -AllowPrerelease:$AllowPreRelease } -if($ExistingAuthModule.Version -like '*preview*' ) { +if ($ExistingAuthModule.Version -like '*preview*' ) { $version = $ExistingAuthModule.Version.Remove($ExistingAuthModule.Version.IndexOf('-')) Write-Warning "Required Version: $ModulePrefix.$RequiredModule Version: $version" $RequiredGraphModules += @{ ModuleName = $ExistingAuthModule.Name ; ModuleVersion = $version } @@ -73,9 +74,13 @@ if ($UpdateAutoRest) { # Update AutoRest. & autorest --reset } -[HashTable] $ModuleMapping = Get-Content $ModuleMappingConfigPath | ConvertFrom-Json -AsHashTable -$ModuleMapping.Keys | ForEach-Object -ThrottleLimit $ModuleMapping.Keys.Count -Parallel { +if ($ModulesToGenerate.Count -eq 0) { + [HashTable] $ModuleMapping = Get-Content $ModuleMappingConfigPath | ConvertFrom-Json -AsHashTable + $ModulesToGenerate = $ModuleMapping.Keys +} + +$ModulesToGenerate | ForEach-Object -ThrottleLimit $ModulesToGenerate.Count -Parallel { enum VersionState { Invalid Valid @@ -87,7 +92,7 @@ $ModuleMapping.Keys | ForEach-Object -ThrottleLimit $ModuleMapping.Keys.Count -P $FullyQualifiedModuleName = "$using:ModulePrefix.$ModuleName" Write-Host -ForegroundColor Green "Generating '$FullyQualifiedModuleName' module..." $ModuleProjectDir = Join-Path $Using:ModulesOutputDir "$ModuleName\$ModuleName" - + # Copy AutoRest readme.md config is none exists. if (-not (Test-Path "$ModuleProjectDir\readme.md")) { New-Item -Path $ModuleProjectDir -Type Directory -Force @@ -144,7 +149,7 @@ $ModuleMapping.Keys | ForEach-Object -ThrottleLimit $ModuleMapping.Keys.Count -P # Get profiles for generated modules. $ModuleExportsPath = Join-Path $ModuleProjectDir "\exports" - $Profiles = Get-ChildItem -Path $ModuleExportsPath -Directory | %{ $_.Name} + $Profiles = Get-ChildItem -Path $ModuleExportsPath -Directory | % { $_.Name } # Update module manifest wiht profiles. $ModuleManifestPath = Join-Path $ModuleProjectDir "$FullyQualifiedModuleName.psd1" @@ -153,12 +158,13 @@ $ModuleMapping.Keys | ForEach-Object -ThrottleLimit $ModuleMapping.Keys.Count -P # Update module psm1 with Graph session profile name. $ModulePsm1 = Join-Path $ModuleProjectDir "/$FullyQualifiedModuleName.psm1" - (Get-Content -Path $ModulePsm1) | ForEach-Object{ + (Get-Content -Path $ModulePsm1) | ForEach-Object { if ($_ -match '\$instance = \[Microsoft.Graph.PowerShell.Module\]::Instance') { # Update main psm1 with Graph session profile name and module name. $_ ' $instance.ProfileName = [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.SelectedProfile' - } else { + } + else { # Rename all Azure instances in psm1 to `Microsoft Graph`. $updatedLine = $_ -replace 'Azure', 'Microsoft Graph' # Replace all 'instance.Name' declarations with fully qualified module name. @@ -171,7 +177,7 @@ $ModuleMapping.Keys | ForEach-Object -ThrottleLimit $ModuleMapping.Keys.Count -P # Address AutoREST bug where it looks for exports in the wrong directory. $InternalModulePsm1 = Join-Path $ModuleProjectDir "/internal/$FullyQualifiedModuleName.internal.psm1" - (Get-Content -Path $InternalModulePsm1) | ForEach-Object{ + (Get-Content -Path $InternalModulePsm1) | ForEach-Object { $updatedLine = $_ # Address AutoREST bug where it looks for exports in the wrong directory. if ($_ -match '\$exportsPath = \$PSScriptRoot') { diff --git a/tools/GenerateProfiles.ps1 b/tools/GenerateProfiles.ps1 index 208faf13eb..fdc79b04b7 100644 --- a/tools/GenerateProfiles.ps1 +++ b/tools/GenerateProfiles.ps1 @@ -51,8 +51,8 @@ try { } # Get crawl data. Write-Host "Crawling '$moduleName' paths for resources and operations ..." -ForegroundColor Green - $crawlResult = @{resources= @(); operations = @{}} - foreach ($path in $allPaths) { + $crawlResult = @{resources= @(); operations = [ordered]@{}} + foreach ($path in ($allPaths | Sort-Object -Property endpoint)) { $crawlResult.operations[$path.endpoint] = (@{apiVersion = $path.apiVersion; originalLocation = $path.originalLocation}) } $telemetryDir = Join-Path $ModuleProfilesDirectory "crawl-log-$profileName.json" @@ -60,11 +60,11 @@ try { Write-Host "Telemetry written at $telemetryDir" -ForegroundColor Blue # Get profile. - $profile = @{resources = @{}; operations = @{}} + $profile = @{resources = @{}; operations = [ordered]@{}} foreach ($operation in $crawlResult.operations.keys) { $profile.operations[$operation] = $crawlResult.operations[$operation].apiVersion } - $profilesNode = @{profiles = @{ $profileName = $profile}} + $profilesNode = @{profiles = [ordered]@{ $profileName = $profile}} $profilesInYaml = $profilesNode | ConvertTo-Yaml $profileReadMeContent = @" # Microsoft Graph $profileName Profile diff --git a/tools/SetMetaModuleVersion.ps1 b/tools/SetMetaModuleVersion.ps1 new file mode 100644 index 0000000000..ef20547cc3 --- /dev/null +++ b/tools/SetMetaModuleVersion.ps1 @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +Param( + [string] $VersionNumber, + [string] $ReleaseNotes = "See https://aka.ms/GraphPowerShell-Release." +) + +$ModuleMetadataPath = Join-Path $PSScriptRoot "..\config\ModuleMetadata.json" -Resolve +$ModuleContent = (Get-Content $ModuleMetadataPath -Raw | ConvertFrom-Json) + +# Update meta-module +$ModuleContent.Version = $VersionNumber +$ModuleContent.ReleaseNotes = $ReleaseNotes + +Set-Content -Value (ConvertTo-Json $ModuleContent) -Path $ModuleMetadataPath diff --git a/tools/SetVersionToAllModules.ps1 b/tools/SetServiceModuleVersion.ps1 similarity index 54% rename from tools/SetVersionToAllModules.ps1 rename to tools/SetServiceModuleVersion.ps1 index d3ec060982..ae439a2de9 100644 --- a/tools/SetVersionToAllModules.ps1 +++ b/tools/SetServiceModuleVersion.ps1 @@ -2,12 +2,23 @@ # Licensed under the MIT License. Param( [string] $VersionNumber, - [string] $ReleaseNotes = "See https://aka.ms/GraphPowerShell-Release." + [string] $ReleaseNotes = "See https://aka.ms/GraphPowerShell-Release.", + [string[]] $Modules = $null ) $WriteToModuleReadMe = Join-Path $PSScriptRoot ".\WriteToModuleReadMe.ps1" -Resolve -$SrcPath = Join-Path $PSScriptRoot "..\src\*\*" -# Get all module readme.md -Get-Item "$SrcPath\readme.md" | ForEach-Object { + +$ModulesReadme = @() +if ($null -ne $Modules) { + $Modules | ForEach-Object { + $ReadMePath = Join-Path $PSScriptRoot "..\src\$_\$_\readme.md" + $ModulesReadme += Get-Item $ReadMePath + } +} else { + $ReadMePath = Join-Path $PSScriptRoot "..\src\*\*\readme.md" + # Get all module readme.md + $ModulesReadme = Get-Item $ReadMePath +} +$ModulesReadme | ForEach-Object { # Set readme values. & $WriteToModuleReadMe -ReadMePath $_.FullName -FieldName "module-version" -NewFieldValue $VersionNumber & $WriteToModuleReadMe -ReadMePath $_.FullName -FieldName "release-notes" -NewFieldValue $ReleaseNotes diff --git a/tools/UpdateOpenApi.ps1 b/tools/UpdateOpenApi.ps1 index 46d66a43a3..f02fa04695 100644 --- a/tools/UpdateOpenApi.ps1 +++ b/tools/UpdateOpenApi.ps1 @@ -33,7 +33,7 @@ $ModuleMapping.Keys | ForEach-Object -Begin { $RequestCount = 0 } -End { Write-H $ModuleName = $_ $ForceRefresh = $false # Check whether ForceRefresh is required, Only required for the First Request. - if ($RequestCount -eq 0){ + if ($RequestCount -eq 0) { $ForceRefresh = $true } @@ -47,5 +47,4 @@ $ModuleMapping.Keys | ForEach-Object -Begin { $RequestCount = 0 } -End { Write-H $RequestCount++ } - Write-Host -ForegroundColor Green "-------------Done-------------" \ No newline at end of file