Abstractions Package - Pipeline Changes (#3628)

Author: paulmedynski

build.proj

@@ -450,7 +450,6 @@
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "bin", SearchOption.AllDirectories))' />
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", ".nuget", SearchOption.AllDirectories))' />
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "obj", SearchOption.AllDirectories))' />
-    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "packages", SearchOption.AllDirectories))' />
 </Target>
 
   <!-- AKV Targets ========================================================= -->

eng/pipelines/akv-official-pipeline.yml

@@ -105,15 +105,16 @@ extends:
             roslyn:
                 enabled: ${{ parameters.runSdlTasks }}
                 break: true
-                # Requires RoslynAnalyzers task to be added after build task
+                # Requires RoslynAnalyzers task to be added somewhere in
+                # the build stage.
 
             publishLogs:
                 enabled: ${{ parameters.runSdlTasks }}
 
             sbom:
                 enabled: ${{ parameters.runSdlTasks }}
                 packageName: 'Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider'
-                packageVersion: ${{ variables.nugetPackageVersion }}
+                packageVersion: ${{ variables.akvPackageVersion }}
 
             tsa:
                 # OneBranch publishes all sdl results to TSA. If TSA is disabled all SDL tools will
@@ -127,11 +128,11 @@ extends:
               jobs:
                   - template: /eng/pipelines/jobs/build-akv-official-job.yml@self
                     parameters:
+                        akvAssemblyFileVersion: '${{ variables.assemblyFileVersion }}'
+                        akvPackageVersion: '${{ variables.akvPackageVersion }}'
                         apiScanDllPath: '${{ variables.apiScanDllPath }}'
                         apiScanPdbPath: '${{ variables.apiScanPdbPath }}'
-                        assemblyFileVersion: '${{ variables.assemblyFileVersion }}'
                         buildConfiguration: '${{ parameters.buildConfiguration }}'
-                        nugetPackageVersion: '${{ variables.nugetPackageVersion }}'
                         mdsPackageVersion: '${{ variables.mdsPackageVersion }}'
                         publishSymbols: '${{ parameters.publishSymbols }}'
                         signingAppRegistrationClientId: '$(SigningAppRegistrationClientId)'

eng/pipelines/common/templates/jobs/build-signed-package-job.yml

@@ -27,8 +27,10 @@ jobs:
   variables:
   - template: ../../../libraries/variables.yml@self
   - ${{ if parameters.isPreview }}:
-    - name: NugetPackageVersion
-      value: $(PreviewNugetPackageVersion)
+    - name: abstractionsPackageVersion
+      value: $(abstractionsPackagePreviewVersion)
+    - name: mdsPackageVersion
+      value: $(previewMdsPackageVersion)
 
   steps:
   - script: SET
@@ -37,25 +39,59 @@ jobs:
   - powershell: |
       Write-Host "##vso[task.setvariable variable=CDP_BUILD_TYPE_COPY;isOutput=true]$($env:CDP_BUILD_TYPE)"
     name: GetBuildType
+
+  # Build our tooling, which is required by the analysis step below, but
+  # shouldn't be analyzed itself.
+  - task: MSBuild@1
+    displayName: 'Build Tooling'
+    inputs:
+      solution: '**/build.proj'
+      configuration: $(Configuration)
+      msbuildArguments: -t:BuildTools
+
+  # Perform analysis before building, since this step will clobber build output
+  - template: ../steps/code-analyze-step.yml@self
 
-  - template: ../steps/build-all-configurations-signed-dlls-step.yml@self
+  # Update the root NuGet.config to use the packages/ directory as a source.
+  # When we build MDS in Package mode, it depends on the Abstractions package,
+  # which we will build and package into packages/.
+  - template: ../steps/update-nuget-config-local-feed-step.yml
     parameters:
-      buildConfiguration: Release
+      packagePath: $(REPOROOT)/packages
 
-  - template: ../steps/code-analyze-step.yml@self
+  # Build the Abstractions package.
+  - task: MSBuild@1
+    displayName: Build Abstractions
+    inputs:
+      solution: '**/build.proj'
+      configuration: $(Configuration)
+      msbuildArguments: -t:BuildAbstractions
+
+  # Build MDS in Package mode, producing signed DLLs.
+  - template: ../steps/build-all-configurations-signed-dlls-step.yml@self
     parameters:
-      analyzeType: all
-  
+      # These variables are sourced from common-variables.yml.
+      abstractionsAssemblyFileVersion: $(abstractionsAssemblyFileVersion)
+      abstractionsPackageVersion: $(abstractionsPackageVersion)
+      configuration: $(Configuration)
+      mdsAssemblyFileVersion: $(mdsAssemblyFileVersion)
+      mdsPackageVersion: $(mdsPackageVersion)
+      referenceType: Package
+
   - template: ../steps/esrp-code-signing-step.yml@self
     parameters:
       artifactType: dll
 
   - template: ../steps/generate-nuget-package-step.yml@self
     parameters:
       buildConfiguration: Release
-      OutputDirectory: $(artifactDirectory)
+      displayName: 'Create MDS NuGet Package'
       installNuget: false
+      nuspecPath: $(nuspecPath)
+      outputDirectory: $(artifactDirectory)
+      packageVersion: $(mdsPackageVersion)
       properties: 'AbstractionsPackageVersion=$(abstractionsPackageVersion)'
+      referenceType: Package
 
   - template: ../steps/esrp-code-signing-step.yml@self
     parameters:
@@ -71,4 +107,4 @@ jobs:
     parameters:
       buildConfiguration: Release
       publishSymbols: ${{ parameters['PublishSymbols'] }}
-      symbolsArtifactName: mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId)
+      symbolsArtifactName: mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(mdsPackageVersion)_$(System.TimelineId)

eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml

@@ -4,6 +4,17 @@
 # See the LICENSE file in the project root for more information.                #
 #################################################################################
 parameters:
+
+  - name: 'debug'
+    type: boolean
+    default: false
+  
+  - name: referenceType
+    type: string
+    values:
+      - Package
+      - Project
+
   - name: poolName
     type: string
     default: $(ci_var_defaultPoolName)
@@ -12,9 +23,13 @@ parameters:
     type: string
     default: ADO-MMS22-SQL19
 
-  - name: artifactName
+  - name: abstractionsArtifactName
     type: string
-    default: Artifacts
+    default: Abstractions.Artifact
+
+  - name: mdsArtifactName
+    type: string
+    default: MDS.Artifact
 
   - name: platform
     type: string
@@ -30,8 +45,15 @@ parameters:
     type: stepList
     default: []
 
+  - name: abstractionsPackageVersion
+    type: string
+
+  - name: mdsPackageVersion
+    type: string
+
 jobs:
-- job: build_nugets
+- job: build_mds_akv_packages_job
+  displayName: Build MDS & AKV Packages
 
   pool:
     name: ${{parameters.poolName }}
@@ -46,36 +68,65 @@ jobs:
   - ${{ if ne(parameters.prebuildSteps, '') }}:
     - ${{ parameters.prebuildSteps }} # extra steps to run before the build like downloading sni and the required configuration
 
+  # If we're testing in Package mode, download the Abstractions package artifacts into packages/,
+  # and then setup the top-level NuGet.config to look in packages/ for local
+  # NuGet dependencies.
+  - ${{ if eq(parameters.referenceType, 'Package') }}:
+    - task: DownloadPipelineArtifact@2
+      displayName: Download Abstractions Package Artifact
+      inputs:
+        artifactName: ${{ parameters.abstractionsArtifactName }}
+        targetPath: $(packagePath)
+    # Note that packages/ will have been created by the above step, which is a
+    # pre-requisite for configuring NuGet.
+    - template: ../steps/ci-prebuild-step.yml@self
+      parameters:
+        debug: ${{ parameters.debug }}
+        referenceType: ${{ parameters.referenceType }}
+
   - template: ../steps/ci-project-build-step.yml@self
     parameters:
       platform: ${{ parameters.platform }}
       buildConfiguration: ${{ parameters.buildConfiguration }}
+      referenceType: ${{ parameters.referenceType }}
       operatingSystem: Windows
-      build: all
+      build: MDS
+      abstractionsPackageVersion: ${{parameters.abstractionsPackageVersion}}
 
   - template: ../steps/generate-nuget-package-step.yml@self
     parameters:
-      NugetPackageVersion: $(NugetPackageVersion)
       buildConfiguration: ${{ parameters.buildConfiguration }}
-      nuspecPath: 'tools/specs/Microsoft.Data.SqlClient.nuspec'
-      OutputDirectory: $(packagePath)
+      displayName: 'Create MDS NuGet Package'
       generateSymbolsPackage: false
-      properties: 'AbstractionsPackageVersion=$(abstractionsPackageVersion)'
-      displayName: 'Generate NuGet package M.D.SqlClient'
+      packageVersion: ${{ parameters.mdsPackageVersion }}
+      nuspecPath: 'tools/specs/Microsoft.Data.SqlClient.nuspec'
+      outputDirectory: $(packagePath)
+      properties: 'AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }}'
+      referenceType: ${{ parameters.referenceType }}
+
+  - template: ../steps/ci-project-build-step.yml@self
+    parameters:
+      platform: ${{ parameters.platform }}
+      buildConfiguration: ${{ parameters.buildConfiguration }}
+      referenceType: ${{ parameters.referenceType }}
+      operatingSystem: Windows
+      build: AKV
+      mdsPackageVersion: ${{parameters.mdsPackageVersion}}
 
   - template: ../steps/generate-nuget-package-step.yml@self
     parameters:
-      NugetPackageVersion: $(NugetPackageVersion)
       buildConfiguration: ${{ parameters.buildConfiguration }}
-      nuspecPath: 'tools/specs/add-ons/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider.nuspec'
-      OutputDirectory: $(packagePath)
+      displayName: 'Create AKV NuGet Package'
       generateSymbolsPackage: false
-      properties: 'MdsPackageVersion=$(mdsPackageVersion)'
       installNuget: false
-      displayName: 'Generate NuGet package AKV Provider'
+      nuspecPath: 'tools/specs/add-ons/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider.nuspec'
+      outputDirectory: $(packagePath)
+      packageVersion: $(akvPackageVersion)
+      properties: 'MdsPackageVersion=${{ parameters.mdsPackageVersion }}'
+      referenceType: ${{ parameters.referenceType }}
 
-  - task: PublishBuildArtifacts@1
-    displayName: 'Publish Artifact: Artifacts'
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish Pipeline Artifact'
     inputs:
-      PathtoPublish: $(packagePath)
-      ArtifactName: ${{ parameters.artifactName }}
+      targetPath: $(packagePath)
+      artifactName: ${{ parameters.mdsArtifactName }}