Abstractions Package - C# project changes (#3626)

Abstractions Package - Pipeline Changes (#3628)
Move AAD/Entra Authentication into new Azure package (#3680)
User Story 39839: Move existing MDS tests to Azure package

- Identified and moved tests specific to ActiveDirectoryAuthenticationProvider into the Azure Test project.
- Updated some MDS tests that unnecessarily used ActiveDirectoryAuthenticationProvider.
- Added tests to Abstractions Test project to ensure AAD/Entra auth fails when the Azure package isn't present.
- Various build and pipeline changes to support the test changes.
- Fixed project-based builds so we can restore MDS netcore and netfx projects at the same time.
- Added caching to the managed identity provider for the ManualTest project.
- Moved username/password Entra auth provider into its own file for sharing across tests.
- Added ADO service connection for test tasks that require access to Azure resources.
- Added workload identity federation tests.
- Solved the mystery of SNI DLLs missing for .NET Framework project-reference-based builds.
- Added ADO-CI-1ES pool jobs to test the Azure package on Linux and Windows.
- Fixed Azure package pipelines to support pools in the ADO.Net and Public projects.
- Added some new pipeline variables to handle the Workload Identity Federation test.

Author: paulmedynski

build.proj

@@ -515,7 +515,7 @@
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "bin", SearchOption.AllDirectories))' />
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", ".nuget", SearchOption.AllDirectories))' />
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "obj", SearchOption.AllDirectories))' />
-</Target>
+  </Target>
 
   <!-- AKV Targets ========================================================= -->
   <Target Name="BuildAkv">

eng/pipelines/common/templates/jobs/build-signed-package-job.yml

@@ -73,7 +73,7 @@ jobs:
       # These variables are sourced from common-variables.yml.
       abstractionsAssemblyFileVersion: $(abstractionsAssemblyFileVersion)
       abstractionsPackageVersion: $(abstractionsPackageVersion)
-      configuration: $(Configuration)
+      buildConfiguration: $(Configuration)
       mdsAssemblyFileVersion: $(mdsAssemblyFileVersion)
       mdsPackageVersion: $(mdsPackageVersion)
       referenceType: Package

eng/pipelines/common/templates/jobs/ci-run-tests-job.yml

@@ -96,6 +96,10 @@ parameters:
     type: boolean
     default: false
 
+  - name: usemanagedSNI
+    type: boolean
+    default: false
+
 jobs:
 - job: ${{ format('{0}', coalesce(parameters.jobDisplayName, parameters.image, 'unknown_image')) }}
 

eng/pipelines/common/templates/steps/ensure-dotnet-version.yml

@@ -10,6 +10,13 @@
 # Reason for not using UseDotNet task:
 #   [BUG]: UseDotNet task installs x86 build on Windows arm64
 #   https://github.com/microsoft/azure-pipelines-tasks/issues/20300
+#
+# A possible workaround is discussed here:
+#
+#   https://github.com/microsoft/azure-pipelines-tasks/issues/16501
+#
+# TODO: See if we can eliminate this template and just use the above workaround
+# for the Windows x86 builds.
 
 parameters:
     - # Directory where dotnet binaries should be installed. If not specified, defaults to the

eng/pipelines/common/templates/steps/update-config-file-step.yml

@@ -124,6 +124,10 @@ parameters:
     type: boolean
     default: true
 
+  - name: WorkloadIdentityFederationServiceConnectionId
+    type: string
+    default: ''
+
 steps:
 # All properties should be added here, and this template should be used for any manipulation of the config.json file.
 - pwsh: |
@@ -180,6 +184,7 @@ steps:
         $p.IsDNSCachingSupportedCR=[System.Convert]::ToBoolean("${{parameters.IsDNSCachingSupportedCR }}")
         $p.TracingEnabled=[System.Convert]::ToBoolean("${{parameters.TracingEnabled }}")
         $p.EnclaveEnabled=[System.Convert]::ToBoolean("${{parameters.EnclaveEnabled }}")
+        $p.WorkloadIdentityFederationServiceConnectionId="${{parameters.WorkloadIdentityFederationServiceConnectionId }}"
     }
     $jdata | ConvertTo-Json | Set-Content "config.json"
   workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities
@@ -196,4 +201,4 @@ steps:
           }
       }
     workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities
-    displayName: 'Read config.json [debug]'
+    displayName: '[Debug] Read config.json'

eng/pipelines/dotnet-sqlclient-ci-core.yml

@@ -93,6 +93,17 @@ parameters:
   type: boolean
   default: true
 
+- name: dotnetVerbosity
+  displayName: dotnet CLI Verbosity
+  type: string
+  default: normal
+  values:
+    - quiet
+    - minimal
+    - normal
+    - detailed
+    - diagnostic
+
 variables:
   - template: libraries/ci-build-variables.yml@self
 
@@ -114,11 +125,11 @@ stages:
   # under the given artifact name.
   - template: stages/build-abstractions-package-ci-stage.yml@self
     parameters:
-      buildConfiguration: ${{ parameters.buildConfiguration }}
+      abstractionsArtifactName: $(abstractionsArtifactName)
       abstractionsPackageVersion: $(abstractionsPackageVersion)
-      artifactName: $(abstractionsArtifactName)
-      ${{if eq(parameters.debug, 'true')}}:
-        verbosity: diagnostic
+      buildConfiguration: ${{ parameters.buildConfiguration }}
+      debug: ${{ parameters.debug }}
+      dotnetVerbosity: ${{ parameters.dotnetVerbosity }}
 
   # Build MDS and its NuGet packages.
   - stage: build_mds_akv_packages_stage
@@ -157,25 +168,29 @@ stages:
       azureArtifactName: $(azureArtifactName)
       azurePackageVersion: $(azurePackageVersion)
       buildConfiguration: ${{ parameters.buildConfiguration }}
+      debug: ${{ parameters.debug }}
+      mdsArtifactName: $(mdsArtifactName)
+      mdsPackageVersion: $(mdsPackageVersion)
       # When building via package references, we must depend on the Abstractions
-      # package.
+      # and MDS packages
       ${{ if eq(parameters.referenceType, 'Package') }}:
         dependsOn:
           - build_abstractions_package_stage
+          - build_mds_akv_packages_stage
       referenceType: ${{ parameters.referenceType }}
-      ${{if eq(parameters.debug, 'true')}}:
-        verbosity: diagnostic
+      dotnetVerbosity: ${{ parameters.dotnetVerbosity }}
 
   # Run the stress tests, if desired.
   - ${{ if eq(parameters.enableStressTests, true) }}:
     - template: stages/stress-tests-ci-stage.yml@self
       parameters:
         buildConfiguration: ${{ parameters.buildConfiguration }}
-        dependsOn: [build_mds_akv_packages_stage]
+        dependsOn:
+          - build_mds_akv_packages_stage
+          - build_azure_package_stage
         pipelineArtifactName: $(artifactName)
         mdsPackageVersion: $(mdsPackageVersion)
-        ${{ if eq(parameters.debug, 'true') }}:
-          verbosity: 'detailed'
+        dotnetVerbosity: ${{ parameters.dotnetVerbosity }}
 
   # Run the MDS and AKV tests.
   - template: common/templates/stages/ci-run-tests-stage.yml@self
@@ -190,9 +205,6 @@ stages:
       mdsArtifactName: $(mdsArtifactName)
       mdsPackageVersion: $(mdsPackageVersion)
       testJobTimeout: ${{ parameters.testJobTimeout }}
-      ${{ if eq(parameters.buildType, 'Package') }}:
-        dependsOn: build_nugets
-
       # When testing MDS via packages, we must depend on the Abstractions,
       # Azure, and MDS packages.
       ${{ if eq(parameters.referenceType, 'Package') }}:

eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml

@@ -149,6 +149,18 @@ parameters:
     type: object
     default: [false, true]
 
+  # Dotnet CLI verbosity level.
+  - name: dotnetVerbosity
+    displayName: dotnet CLI Verbosity
+    type: string
+    default: normal
+    values:
+      - quiet
+      - minimal
+      - normal
+      - detailed
+      - diagnostic
+
 extends:
   template: dotnet-sqlclient-ci-core.yml@self
   parameters:
@@ -157,6 +169,7 @@ extends:
     referenceType: Package
     codeCovTargetFrameworks: ${{ parameters.codeCovTargetFrameworks }}
     debug: ${{ parameters.debug }}
+    dotnetVerbosity: ${{ parameters.dotnetVerbosity }}
     enableStressTests: ${{ parameters.enableStressTests }}
     targetFrameworks: ${{ parameters.targetFrameworks }}
     targetFrameworksLinux: ${{ parameters.targetFrameworksLinux }}

eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml

@@ -149,6 +149,18 @@ parameters:
     type: object
     default: [false, true]
 
+  # Dotnet CLI verbosity level.
+  - name: dotnetVerbosity
+    displayName: dotnet CLI Verbosity
+    type: string
+    default: normal
+    values:
+      - quiet
+      - minimal
+      - normal
+      - detailed
+      - diagnostic
+
 extends:
   template: dotnet-sqlclient-ci-core.yml@self
   parameters:
@@ -157,6 +169,7 @@ extends:
     referenceType: Project
     codeCovTargetFrameworks: ${{ parameters.codeCovTargetFrameworks }}
     debug: ${{ parameters.debug }}
+    dotnetVerbosity: ${{ parameters.dotnetVerbosity }}
     enableStressTests: ${{ parameters.enableStressTests }}
     targetFrameworks: ${{ parameters.targetFrameworks }}
     targetFrameworksLinux: ${{ parameters.targetFrameworksLinux }}

eng/pipelines/jobs/pack-abstractions-package-ci-job.yml

@@ -12,29 +12,34 @@
 
 parameters:
 
-  # The version to apply to the Abstractions NuGet package and its assemblies.
-  - name: abstractionsPackageVersion
-    type: string
-
   # The name to apply to the published pipeline artifact.
-  - name: artifactName
+  - name: abstractionsArtifactName
     type: string
     default: Abstractions.Artifact
 
+  # The version to apply to the Abstractions NuGet package and its assemblies.
+  - name: abstractionsPackageVersion
+    type: string
+
   # The type of build to test (Release or Debug)
   - name: buildConfiguration
     type: string
     values:
     - Release
     - Debug
+
+  # True to enable extra debug steps and logging.
+  - name: debug
+    type: boolean
+    default: false
 
   # The list of upstream jobs to depend on.
   - name: dependsOn
     type: object
     default: []
 
   # The verbosity level for the dotnet CLI commands.
-  - name: verbosity
+  - name: dotnetVerbosity
     type: string
     default: normal
     values:
@@ -69,13 +74,14 @@ jobs:
       # dotnet CLI arguments common to all commands.
       - name: commonArguments
         value: >-
-          --verbosity ${{ parameters.verbosity }}
+          --verbosity ${{ parameters.dotnetVerbosity }}
 
       # dotnet CLI arguments for build/test/pack commands
       - name: buildArguments
         value: >-
           $(commonArguments)
           --configuration ${{ parameters.buildConfiguration }}
+          -p:ForceMdsAssemblyNameSuffix=true
           -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }}
 
       # Explicitly unset the $PLATFORM environment variable that is set by the
@@ -102,6 +108,11 @@ jobs:
 
     steps:
 
+      # Emit environment variables if debug is enabled.
+      - ${{ if eq(parameters.debug, true) }}:
+        - pwsh: 'Get-ChildItem Env: | Sort-Object Name'
+          displayName: '[Debug] Print Environment Variables'
+
       # Install the .NET 9.0 SDK.
       - task: UseDotNet@2
         displayName: Install .NET 9.0 SDK
@@ -112,18 +123,18 @@ jobs:
       # We use the 'custom' command because the DotNetCoreCLI@2 task doesn't
       # support all of our argument combinations for the different build steps.
 
-      # Restore the solution.
+      # Restore the project.
       - task: DotNetCoreCLI@2
-        displayName: Restore Solution
+        displayName: Restore Project
         inputs:
           command: custom
           custom: restore
           projects: $(project)
           arguments: $(commonArguments)
 
-      # Build the solution.
+      # Build the project.
       - task: DotNetCoreCLI@2
-        displayName: Build Solution
+        displayName: Build Project
         inputs:
           command: custom
           custom: build
@@ -144,5 +155,5 @@ jobs:
         displayName: Publish Pipeline Artifact
         inputs:
           targetPath: $(dotnetPackagesDir)
-          artifactName: ${{ parameters.artifactName }}
+          artifactName: ${{ parameters.abstractionsArtifactName }}
           publishLocation: pipeline

eng/pipelines/jobs/pack-azure-package-ci-job.yml

@@ -40,12 +40,28 @@ parameters:
     values:
     - Release
     - Debug
+
+  # True to enable extra debug steps and logging.
+  - name: debug
+    type: boolean
+    default: false
 
   # The list of upstream jobs to depend on.
   - name: dependsOn
     type: object
     default: []
 
+  # The verbosity level for the dotnet CLI commands.
+  - name: dotnetVerbosity
+    type: string
+    default: normal
+    values:
+    - quiet
+    - minimal
+    - normal
+    - detailed
+    - diagnostic
+
   # The reference type to use:
   #
   # Project - dependent projects are referenced directly.
@@ -57,17 +73,6 @@ parameters:
     - Package
     - Project
 
-  # The verbosity level for the dotnet CLI commands.
-  - name: verbosity
-    type: string
-    default: normal
-    values:
-    - quiet
-    - minimal
-    - normal
-    - detailed
-    - diagnostic
-
 jobs:
 
   - job: pack_azure_package_job
@@ -93,8 +98,9 @@ jobs:
       # dotnet CLI arguments common to all commands.
       - name: commonArguments
         value: >-
-          --verbosity ${{ parameters.verbosity }}
+          --verbosity ${{ parameters.dotnetVerbosity }}
           -p:ReferenceType=${{ parameters.referenceType }}
+          -p:ForceMdsAssemblyNameSuffix=true
           -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }}
 
       # dotnet CLI arguments for build/test/pack commands
@@ -128,6 +134,11 @@ jobs:
 
     steps:
 
+      # Emit environment variables if debug is enabled.
+      - ${{ if eq(parameters.debug, true) }}:
+        - pwsh: 'Get-ChildItem Env: | Sort-Object Name'
+          displayName: '[Debug] Print Environment Variables'
+
       # We have a few extra steps for Package reference builds.
       - ${{ if eq(parameters.referenceType, 'Package') }}:
 
@@ -152,18 +163,18 @@ jobs:
       # We use the 'custom' command because the DotNetCoreCLI@2 task doesn't
       # support all of our argument combinations for the different build steps.
 
-      # Restore the solution.
+      # Restore the project.
       - task: DotNetCoreCLI@2
-        displayName: Restore Solution
+        displayName: Restore Project
         inputs:
           command: custom
           custom: restore
           projects: $(project)
           arguments: $(commonArguments)
 
-      # Build the solution.
+      # Build the project.
       - task: DotNetCoreCLI@2
-        displayName: Build Solution
+        displayName: Build Project
         inputs:
           command: custom
           custom: build