Abstractions Package - C# project changes (#3626)

- Fixed expansion of split expression in version numbers.
- Removed addition of PDBs from <AllowedOutputExtensionsInPackageBuildOutputFolder>.
- Removed MDS package ref dependency on Abstractions until pipelines are ready.
- Renamed AbstractionsPackage to Abstractions in targets.
- Updated BUILDGUIDE based on project ref behaviour.
- Added feature branches to CI pipeline triggers.
- Added missing/incomplete paths to the trigger.
- Added dev/* branches to the CI triggers so PRs that target other dev/ branches can run CI.
- Added missing MdsPackageVersion property to signed build pipeline job.
- Commented-out failing NuGet tool installer task.
- Re-ordered Guardian analysis step _before_ build step to avoid clobbering versioned DLLs.
- Added MDS package version to AKV build/package steps.
- Restored AKV nuspec ReferenceType property for AKV Official builds.
- Explicitly building tooling before analysis.
- Fixing validation steps to match XML props files.
- Added PR automation triggers, and documented other pipeline sections.
- Added ReferenceType throughout the MDS/AKV CI build steps.
- Added NuGet.config update to main CI build step for Package reference mode.
- Swapped Abstractions download and NuGet.config update to ensure packages/ exists before attempting to re-configure NuGet.
- Clean target no longer removes packages/
- Uncommented package refs to Abstractions.
- Added separate MDS and AKV project builds to support Package mode.
- Added $ReferenceType$ property to MDS .nuspec like it is for AKV.

Author: paulmedynski

build.proj

@@ -443,7 +443,6 @@
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "bin", SearchOption.AllDirectories))' />
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", ".nuget", SearchOption.AllDirectories))' />
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "obj", SearchOption.AllDirectories))' />
-    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "packages", SearchOption.AllDirectories))' />
 </Target>
 
   <!-- AKV Targets ========================================================= -->

eng/pipelines/akv-official-pipeline.yml

@@ -113,7 +113,7 @@ extends:
             sbom:
                 enabled: ${{ parameters.runSdlTasks }}
                 packageName: 'Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider'
-                packageVersion: ${{ variables.nugetPackageVersion }}
+                packageVersion: ${{ variables.akvPackageVersion }}
 
             tsa:
                 # OneBranch publishes all sdl results to TSA. If TSA is disabled all SDL tools will
@@ -131,7 +131,7 @@ extends:
                         apiScanPdbPath: '${{ variables.apiScanPdbPath }}'
                         assemblyFileVersion: '${{ variables.assemblyFileVersion }}'
                         buildConfiguration: '${{ parameters.buildConfiguration }}'
-                        nugetPackageVersion: '${{ variables.nugetPackageVersion }}'
+                        akvPackageVersion: '${{ variables.akvPackageVersion }}'
                         mdsPackageVersion: '${{ variables.mdsPackageVersion }}'
                         publishSymbols: '${{ parameters.publishSymbols }}'
                         signingAppRegistrationClientId: '$(SigningAppRegistrationClientId)'

eng/pipelines/common/templates/jobs/build-signed-package-job.yml

@@ -27,8 +27,10 @@ jobs:
   variables:
   - template: ../../../libraries/variables.yml@self
   - ${{ if parameters.isPreview }}:
-    - name: NugetPackageVersion
-      value: $(PreviewNugetPackageVersion)
+    - name: abstrationsPackageVersion
+      value: $(abstractionsPackagePreviewVersion)
+    - name: mdsPackageVersion
+      value: $(previewMdsPackageVersion)
 
   steps:
   - script: SET
@@ -37,20 +39,43 @@ jobs:
   - powershell: |
       Write-Host "##vso[task.setvariable variable=CDP_BUILD_TYPE_COPY;isOutput=true]$($env:CDP_BUILD_TYPE)"
     name: GetBuildType
-  
-  - template: ../steps/build-all-configurations-signed-dlls-step.yml@self
-  
+
+  # Build our tooling, which is required by the analysis step below, but
+  # shouldn't be analyzed itself.
+  - task: MSBuild@1
+    displayName: 'Build Tooling'
+    inputs:
+      solution: '**/build.proj'
+      configuration: $(Configuration)
+      msbuildArguments: -t:BuildTools
+
+  # GOTCHA: This analysis step must run _before_ the build (below) step because
+  # it builds DLLs that would otherwise clobber the properly-versioned DLLs from
+  # the build step!
   - template: ../steps/code-analyze-step.yml@self
     parameters:
       analyzeType: all
 
+  - template: ../steps/build-all-configurations-signed-dlls-step.yml@self
+    parameters:
+      # These values are sourced from common-variables.yml.
+      configuration: $(Configuration)
+      abstractionsPackageVersion: $(abstractionsPackageVersion)
+      abstractionsAssemblyFileVersion: $(abstractionsAssemblyFileVersion)
+      mdsPackageVersion: $(mdsPackageVersion)
+      mdsAssemblyFileVersion: $(mdsAssemblyFileVersion)
+  
   - template: ../steps/esrp-code-signing-step.yml@self
     parameters:
       artifactType: dll
 
   - template: ../steps/generate-nuget-package-step.yml@self
     parameters:
-      OutputDirectory: $(artifactDirectory)
+      nuspecPath: $(nuspecPath)
+      packageVersion: $(mdsPackageVersion)
+      outputDirectory: $(artifactDirectory)
+      displayName: 'Create MDS NuGet Package'
+      referenceType: Package
       properties: 'AbstractionsPackageVersion=$(abstractionsPackageVersion)'
 
   - template: ../steps/esrp-code-signing-step.yml@self
@@ -65,4 +90,4 @@ jobs:
   - template: ../steps/publish-symbols-step.yml@self
     parameters:
       publishSymbols: ${{ parameters['PublishSymbols'] }}
-      symbolsArtifactName: mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId)
+      symbolsArtifactName: mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(mdsPackageVersion)_$(System.TimelineId)

eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml

@@ -4,6 +4,17 @@
 # See the LICENSE file in the project root for more information.                #
 #################################################################################
 parameters:
+
+  - name: 'debug'
+    type: boolean
+    default: false
+  
+  - name: referenceType
+    type: string
+    values:
+      - Package
+      - Project
+
   - name: poolName
     type: string
     default: $(ci_var_defaultPoolName)
@@ -12,9 +23,13 @@ parameters:
     type: string
     default: ADO-MMS22-SQL19
 
-  - name: artifactName
+  - name: abstractionsArtifactName
     type: string
-    default: Artifacts
+    default: Abstractions.Artifact
+
+  - name: mdsArtifactName
+    type: string
+    default: MDS.Artifact
 
   - name: platform
     type: string
@@ -28,8 +43,15 @@ parameters:
     type: stepList
     default: []
 
+  - name: abstractionsPackageVersion
+    type: string
+
+  - name: mdsPackageVersion
+    type: string
+
 jobs:
-- job: build_nugets
+- job: build_mds_akv_packages_job
+  displayName: Build MDS & AKV Packages
 
   pool:
     name: ${{parameters.poolName }}
@@ -44,36 +66,65 @@ jobs:
   - ${{ if ne(parameters.prebuildSteps, '') }}:
     - ${{ parameters.prebuildSteps }} # extra steps to run before the build like downloading sni and the required configuration
 
+  # If we're testing in Package mode, download the Abstractions package artifacts into packages/,
+  # and then setup the top-level NuGet.config to look in packages/ for local
+  # NuGet dependencies.
+  - ${{ if eq(parameters.referenceType, 'Package') }}:
+    - task: DownloadPipelineArtifact@2
+      displayName: Download Abstractions Package Artifact
+      inputs:
+        artifactName: ${{ parameters.abstractionsArtifactName }}
+        targetPath: $(packagePath)
+    # Note that packages/ will have been created by the above step, which is a
+    # pre-requisite for configuring NuGet.
+    - template: ../steps/ci-prebuild-step.yml@self
+      parameters:
+        debug: ${{ parameters.debug }}
+        referenceType: ${{ parameters.referenceType }}
+
   - template: ../steps/ci-project-build-step.yml@self
     parameters:
       platform: ${{ parameters.platform }}
       configuration: ${{ parameters.configuration }}
+      referenceType: ${{ parameters.referenceType }}
       operatingSystem: Windows
-      build: all
+      build: MDS
+      abstractionsPackageVersion: ${{parameters.abstractionsPackageVersion}}
 
   - template: ../steps/generate-nuget-package-step.yml@self
     parameters:
-      NugetPackageVersion: $(NugetPackageVersion)
+      packageVersion: ${{ parameters.mdsPackageVersion }}
       configuration: $(Configuration)
       nuspecPath: 'tools/specs/Microsoft.Data.SqlClient.nuspec'
-      OutputDirectory: $(packagePath)
+      outputDirectory: $(packagePath)
       generateSymbolsPackage: false
-      properties: 'AbstractionsPackageVersion=$(abstractionsPackageVersion)'
-      displayName: 'Generate NuGet package M.D.SqlClient'
+      referenceType: ${{ parameters.referenceType }}
+      properties: 'AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }}'
+      displayName: 'Create MDS NuGet Package'
+
+  - template: ../steps/ci-project-build-step.yml@self
+    parameters:
+      platform: ${{ parameters.platform }}
+      configuration: ${{ parameters.configuration }}
+      referenceType: ${{ parameters.referenceType }}
+      operatingSystem: Windows
+      build: AKV
+      mdsPackageVersion: ${{parameters.mdsPackageVersion}}
 
   - template: ../steps/generate-nuget-package-step.yml@self
     parameters:
-      NugetPackageVersion: $(NugetPackageVersion)
+      packageVersion: $(akvPackageVersion)
       configuration: $(Configuration)
       nuspecPath: 'tools/specs/add-ons/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider.nuspec'
-      OutputDirectory: $(packagePath)
+      outputDirectory: $(packagePath)
       generateSymbolsPackage: false
-      properties: 'MdsPackageVersion=$(mdsPackageVersion)'
+      referenceType: ${{ parameters.referenceType }}
+      properties: 'MdsPackageVersion=${{ parameters.mdsPackageVersion }}'
       installNuget: false
-      displayName: 'Generate NuGet package AKV Provider'
+      displayName: 'Create AKV NuGet Package'
 
-  - task: PublishBuildArtifacts@1
-    displayName: 'Publish Artifact: Artifacts'
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish Pipeline Artifact'
     inputs:
-      PathtoPublish: $(packagePath)
-      ArtifactName: ${{ parameters.artifactName }}
+      targetPath: $(packagePath)
+      artifactName: ${{ parameters.mdsArtifactName }}