diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index deb9fb7b21bbe9..a42011c11afee7 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -15,7 +15,7 @@ ] }, "microsoft.dotnet.xharness.cli": { - "version": "8.0.0-prerelease.24208.4", + "version": "8.0.0-prerelease.24229.2", "commands": [ "xharness" ] diff --git a/eng/Subsets.props b/eng/Subsets.props index a35f99ef759b7d..c27a284127733e 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -342,7 +342,7 @@ - + @@ -507,6 +507,7 @@ + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 99589f6331a8d2..da4282cc396150 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -104,9 +104,9 @@ 6ed73280a6d70f7e7ac39c86f2abe8c10983f0bb - + https://github.com/dotnet/source-build-externals - 908177a58a41532b3302c17f1e1a8cf1c1234545 + 4f2151df120194f0268944f1b723c14820738fc8 @@ -185,57 +185,57 @@ https://github.com/dotnet/arcade e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 https://github.com/dotnet/llvm-project @@ -322,17 +322,17 @@ https://github.com/dotnet/runtime edbd5c769a19798b6955050baccf99e6797d3208 - + https://github.com/dotnet/xharness - 9d21162000c444b2da3d6cdd805d43e1af51453a + aacfb6328fdef17e572617bbb551431bb9cb1ff2 - + https://github.com/dotnet/xharness - 9d21162000c444b2da3d6cdd805d43e1af51453a + aacfb6328fdef17e572617bbb551431bb9cb1ff2 - + https://github.com/dotnet/xharness - 9d21162000c444b2da3d6cdd805d43e1af51453a + aacfb6328fdef17e572617bbb551431bb9cb1ff2 https://github.com/dotnet/arcade @@ -354,13 +354,13 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-optimization 67613417f5e1af250e6ddfba79f8f2885d8e90fb - + https://github.com/dotnet/hotreload-utils - 61f137aacabdbd8f279415287a2dd70e150f5eb1 + c804541158619aae93105f54698ca7f149d28232 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 https://github.com/dotnet/roslyn diff --git a/eng/Versions.props b/eng/Versions.props index c723e90c1bf1c3..7528b6bfbf306d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -143,20 +143,20 @@ 4.5.0 8.0.0-rc.1.23406.6 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 1.0.0-prerelease.23566.3 1.0.0-prerelease.23566.3 @@ -165,7 +165,7 @@ 1.0.0-prerelease.23566.3 1.0.0-prerelease.23566.3 - 17.8.7-beta1.24113.1 + 17.10.0-beta1.24272.1 2.0.0-beta4.23307.1 3.0.3 2.1.0 @@ -183,10 +183,10 @@ 1.1.0 17.4.0-preview-20220707-01 - 8.0.0-prerelease.24208.4 - 8.0.0-prerelease.24208.4 - 8.0.0-prerelease.24208.4 - 8.0.0-alpha.0.24229.2 + 8.0.0-prerelease.24229.2 + 8.0.0-prerelease.24229.2 + 8.0.0-prerelease.24229.2 + 8.0.0-alpha.0.24271.1 2.4.2 1.0.0 2.4.5 @@ -260,4 +260,12 @@ 8.0.101 $(MicrosoftDotnetSdkInternalVersion) + + + true + diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index cdda8c99dc519c..86cea9fbd98a20 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -30,9 +30,10 @@ parameters: extraVariablesTemplates: [] isManualCodeQLBuild: false preBuildSteps: [] + templatePath: 'templates' jobs: -- template: /eng/common/templates/job/job.yml +- template: /eng/common/${{ parameters.templatePath }}/job/job.yml parameters: ${{ if eq(parameters.hostedOs, '') }}: name: ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }} @@ -141,6 +142,7 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{ variable }} + steps: - ${{ if eq(parameters.osGroup, 'windows') }}: - template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml @@ -181,7 +183,7 @@ jobs: path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' - ${{ if eq(parameters.isSourceBuild, true) }}: - - template: /eng/common/templates/steps/source-build.yml + - template: /eng/common/${{ parameters.templatePath }}/steps/source-build.yml parameters: platform: baseOS: ${{ parameters.baseOS }} @@ -237,10 +239,11 @@ jobs: - task: CodeQL3000Init@0 displayName: Initialize CodeQL (manually-injected) - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci $(_archParameter) $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) - displayName: Build product - ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: - continueOnError: ${{ parameters.shouldContinueOnError }} + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: ${{ parameters.buildArgs }} + useContinueOnErrorDuringBuild: ${{ parameters.useContinueOnErrorDuringBuild }} + shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - ${{ if eq(parameters.isManualCodeQLBuild, true) }}: - task: CodeQL3000Finalize@0 @@ -284,14 +287,16 @@ jobs: displayName: Collect vslogs on exit condition: always() - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/' - PublishLocation: Container - ${{ if notin(parameters.osGroup, 'browser', 'wasi') }}: - ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} - ${{ if in(parameters.osGroup, 'browser', 'wasi') }}: - ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-build-artifacts.yml + parameters: + isOfficialBuild: ${{ parameters.isOfficialBuild }} + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/' + PublishLocation: Container + ${{ if notin(parameters.osGroup, 'browser', 'wasi') }}: + ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} + ${{ if in(parameters.osGroup, 'browser', 'wasi') }}: + ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} + continueOnError: true + condition: always() diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml new file mode 100644 index 00000000000000..7f38a9fd118405 --- /dev/null +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -0,0 +1,12 @@ +parameters: + buildArgs: '' + useContinueOnErrorDuringBuild: false + shouldContinueOnError: false + archParameter: $(_archParameter) + displayName: Build product + +steps: + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) + displayName: ${{ parameters.displayName }} + ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: + continueOnError: ${{ parameters.shouldContinueOnError }} diff --git a/eng/pipelines/common/templates/pipeline-with-resources.yml b/eng/pipelines/common/templates/pipeline-with-resources.yml index edba5767b612c5..f5fd4abbd25bf8 100644 --- a/eng/pipelines/common/templates/pipeline-with-resources.yml +++ b/eng/pipelines/common/templates/pipeline-with-resources.yml @@ -1,113 +1,124 @@ parameters: - name: stages type: stageList - -resources: - containers: - - container: linux_arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm - env: - ROOTFS_DIR: /crossrootfs/arm - - - container: linux_armv6 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10 - env: - ROOTFS_DIR: /crossrootfs/armv6 - - - container: linux_arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 - env: - ROOTFS_DIR: /crossrootfs/arm64 - - - container: linux_musl_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: linux_musl_arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine - env: - ROOTFS_DIR: /crossrootfs/arm - - - container: linux_musl_arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine - env: - ROOTFS_DIR: /crossrootfs/arm64 - - # This container contains all required toolsets to build for Android and for Linux with bionic libc. - - container: linux_bionic - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64 - - # This container contains all required toolsets to build for Android as well as tooling to build docker images. - - container: android_docker - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker - - - container: linux_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: linux_x86 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86 - env: - ROOTFS_DIR: /crossrootfs/x86 - - - container: linux_x64_dev_innerloop - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - - # We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9. - - container: SourceBuild_centos_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - - # AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8. - - container: SourceBuild_linux_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build - - - container: linux_s390x - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x - env: - ROOTFS_DIR: /crossrootfs/s390x - - - container: linux_ppc64le - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le - env: - ROOTFS_DIR: /crossrootfs/ppc64le - - - container: linux_riscv64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64 - env: - ROOTFS_DIR: /crossrootfs/riscv64 - - - container: debian-12-gcc13-amd64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64 - - - container: linux_x64_llvmaot - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - - - container: browser_wasm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: wasi_wasm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: freebsd_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-freebsd-12 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: tizen_armel - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-armel-tizen - env: - ROOTFS_DIR: /crossrootfs/armel - - - container: debpkg - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - - - container: rpmpkg - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - -stages: ${{ parameters.stages }} + - name: isOfficialBuild + type: boolean + default: false + +extends: + template: templateDispatch.yml + parameters: + ${{ if parameters.isOfficialBuild }}: + templatePath: template1es.yml + ${{ else }}: + templatePath: templatePublic.yml + + stages: ${{ parameters.stages }} + + containers: + linux_arm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm + env: + ROOTFS_DIR: /crossrootfs/arm + + linux_armv6: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10 + env: + ROOTFS_DIR: /crossrootfs/armv6 + + linux_arm64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-biarch-amd64-arm64 + env: + ROOTFS_HOST_DIR: /crossrootfs/x64 + ROOTFS_DIR: /crossrootfs/arm64 + + linux_musl_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine + env: + ROOTFS_DIR: /crossrootfs/x64 + + linux_musl_arm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine + env: + ROOTFS_DIR: /crossrootfs/arm + + linux_musl_arm64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine + env: + ROOTFS_DIR: /crossrootfs/arm64 + + # This container contains all required toolsets to build for Android and for Linux with bionic libc. + linux_bionic: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64 + + # This container contains all required toolsets to build for Android as well as tooling to build docker images. + android_docker: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker + + linux_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 + env: + ROOTFS_DIR: /crossrootfs/x64 + + linux_x86: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86 + env: + ROOTFS_DIR: /crossrootfs/x86 + + linux_x64_dev_innerloop: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + + # We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9. + SourceBuild_centos_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + + # AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8. + SourceBuild_linux_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build + + linux_s390x: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x + env: + ROOTFS_DIR: /crossrootfs/s390x + + linux_ppc64le: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le + env: + ROOTFS_DIR: /crossrootfs/ppc64le + + linux_riscv64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64 + env: + ROOTFS_DIR: /crossrootfs/riscv64 + + debian-12-gcc13-amd64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64 + + linux_x64_llvmaot: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + + browser_wasm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 + env: + ROOTFS_DIR: /crossrootfs/x64 + + wasi_wasm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 + env: + ROOTFS_DIR: /crossrootfs/x64 + + freebsd_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-freebsd-13 + env: + ROOTFS_DIR: /crossrootfs/x64 + + tizen_armel: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-armel-tizen + env: + ROOTFS_DIR: /crossrootfs/armel + + debpkg: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + + rpmpkg: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm \ No newline at end of file diff --git a/eng/pipelines/common/templates/publish-build-artifacts.yml b/eng/pipelines/common/templates/publish-build-artifacts.yml new file mode 100644 index 00000000000000..b9b263c361f890 --- /dev/null +++ b/eng/pipelines/common/templates/publish-build-artifacts.yml @@ -0,0 +1,22 @@ +parameters: + - name: isOfficialBuild + type: boolean + - name: displayName + type: string + - name: inputs + type: object + - name: condition + type: string + default: '' + +steps: + - ${{ if parameters.isOfficialBuild }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} + condition: ${{ parameters.condition }} + - ${{ else }}: + - task: PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/publish-pipeline-artifacts.yml b/eng/pipelines/common/templates/publish-pipeline-artifacts.yml new file mode 100644 index 00000000000000..81f292ec5528ce --- /dev/null +++ b/eng/pipelines/common/templates/publish-pipeline-artifacts.yml @@ -0,0 +1,17 @@ +parameters: +- name: displayName + type: string +- name: inputs + type: object +- name: isOfficialBuild + type: boolean + +steps: + - ${{ if parameters.isOfficialBuild }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} + - ${{ else }}: + - task: PublishPipelineArtifact@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index e249e8ac922a7f..2809d1487700f6 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -14,6 +14,7 @@ parameters: dependsOn: [] dependOnEvaluatePaths: false crossBuild: false + isOfficialBuild: false ### Build managed test components (native components are getting built as part ### of the product build job). @@ -142,12 +143,13 @@ jobs: artifactName: $(microsoftNetSdkIlArtifactName) displayName: 'Microsoft.NET.Sdk.IL package' - # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}' - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Publish Logs + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + ArtifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}' + continueOnError: true + condition: always() diff --git a/eng/pipelines/common/templates/runtimes/xplat-job.yml b/eng/pipelines/common/templates/runtimes/xplat-job.yml index f4ac7e82957123..625d88d63d3e97 100644 --- a/eng/pipelines/common/templates/runtimes/xplat-job.yml +++ b/eng/pipelines/common/templates/runtimes/xplat-job.yml @@ -20,11 +20,12 @@ parameters: enableMicrobuild: '' gatherAssetManifests: false disableComponentGovernance: false + templatePath: 'templates' variables: {} ## any extra variables to add to the defaults defined below jobs: -- template: /eng/common/templates/job/job.yml +- template: /eng/common/${{ parameters.templatePath }}/job/job.yml parameters: name: ${{ parameters.name }} @@ -77,15 +78,6 @@ jobs: - name: buildConfig value: ${{ parameters.buildConfig }} - - name: archType - value: ${{ parameters.archType }} - - - name: osGroup - value: ${{ parameters.osGroup }} - - - name: osSubgroup - value: ${{ parameters.osSubgroup }} - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - name: _HelixSource value: official/dotnet/runtime/$(Build.SourceBranch) diff --git a/eng/pipelines/common/templates/template1es.yml b/eng/pipelines/common/templates/template1es.yml new file mode 100644 index 00000000000000..0770e37d6bd0ed --- /dev/null +++ b/eng/pipelines/common/templates/template1es.yml @@ -0,0 +1,31 @@ + + +parameters: + - name: templatePath + type: string + default: 'templates-official' + - name: stages + type: stageList + - name: containers + type: object + + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + containers: + ${{ parameters.containers }} + + stages: ${{ parameters.stages }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/templateDispatch.yml b/eng/pipelines/common/templates/templateDispatch.yml new file mode 100644 index 00000000000000..1860af47aeefff --- /dev/null +++ b/eng/pipelines/common/templates/templateDispatch.yml @@ -0,0 +1,13 @@ +parameters: + - name: templatePath + type: string + - name: stages + type: stageList + - name: containers + type: object + +extends: + template: ${{ parameters.templatePath }} + parameters: + stages: ${{ parameters.stages }} + containers: ${{ parameters.containers }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/templatePublic.yml b/eng/pipelines/common/templates/templatePublic.yml new file mode 100644 index 00000000000000..cd7c02720167d5 --- /dev/null +++ b/eng/pipelines/common/templates/templatePublic.yml @@ -0,0 +1,21 @@ + +parameters: + - name: templatePath + type: string + default: 'templates' + - name: stages + type: stageList + - name: containers + type: object + +resources: + containers: + - ${{ each container_pair in parameters.containers }}: + - ${{ if container_pair.value.image }}: + - container: ${{ container_pair.key }} + ${{ each pair in container_pair.value }}: + ${{ if notIn(pair.key, 'tenantId', 'identityType', 'registry') }}: + ${{ pair.key }}: ${{ pair.value }} + + +stages: ${{ parameters.stages }} \ No newline at end of file diff --git a/eng/pipelines/common/upload-artifact-step.yml b/eng/pipelines/common/upload-artifact-step.yml index 249da066c7aae6..d4091a7cc192f5 100644 --- a/eng/pipelines/common/upload-artifact-step.yml +++ b/eng/pipelines/common/upload-artifact-step.yml @@ -7,6 +7,7 @@ parameters: artifactName: '' displayName: '' condition: succeeded() + isOfficialBuild: false steps: # Zip Artifact @@ -20,9 +21,11 @@ steps: includeRootFolder: ${{ parameters.includeRootFolder }} condition: ${{ parameters.condition }} - - task: PublishBuildArtifacts@1 - displayName: 'Publish ${{ parameters.displayName }}' - inputs: - pathtoPublish: $(Build.StagingDirectory)/${{ parameters.artifactName }}${{ parameters.archiveExtension }} - artifactName: ${{ parameters.artifactName }} - condition: ${{ parameters.condition }} + - template: /eng/pipelines/common/templates/publish-build-artifacts.yml + parameters: + isOfficialBuild: ${{ parameters.isOfficialBuild }} + displayName: 'Publish ${{ parameters.displayName }}' + inputs: + PathtoPublish: $(Build.StagingDirectory)/${{ parameters.artifactName }}${{ parameters.archiveExtension }} + artifactName: ${{ parameters.artifactName }} + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/pipelines/common/upload-intermediate-artifacts-step.yml b/eng/pipelines/common/upload-intermediate-artifacts-step.yml index bde6c61a0a04ad..da9b1ef0b627fd 100644 --- a/eng/pipelines/common/upload-intermediate-artifacts-step.yml +++ b/eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -25,9 +25,11 @@ steps: TargetFolder: '$(Build.StagingDirectory)/IntermediateArtifacts/${{ parameters.name }}' CleanTargetFolder: true -- task: PublishBuildArtifacts@1 - displayName: Publish intermediate artifacts - inputs: - pathToPublish: '$(Build.StagingDirectory)/IntermediateArtifacts' - artifactName: IntermediateArtifacts - artifactType: container +- template: /eng/pipelines/common/templates/publish-build-artifacts.yml + parameters: + isOfficialBuild: true + displayName: Publish intermediate artifacts + inputs: + PathtoPublish: '$(Build.StagingDirectory)/IntermediateArtifacts' + ArtifactName: IntermediateArtifacts + ArtifactType: container diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml index 2a757f82572ad3..075dea178ab623 100644 --- a/eng/pipelines/common/variables.yml +++ b/eng/pipelines/common/variables.yml @@ -1,3 +1,8 @@ +parameters: + - name: templatePath + type: string + default: 'templates' + variables: # These values enable longer delays, configurable number of retries, and special understanding of TCP hang-up @@ -54,3 +59,5 @@ variables: eq(variables['isRollingBuild'], true))) ] - template: /eng/pipelines/common/perf-variables.yml + +- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml \ No newline at end of file diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index eb19570aeecac2..794a23bb218b5e 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -22,7 +22,7 @@ jobs: dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-community', 'runtime-extra-platforms', 'runtime-wasm', 'runtime-wasm-libtests', 'runtime-wasm-non-libtests', 'dotnet-linker-tests', 'runtime-dev-innerloop', 'runtime-coreclr superpmi-replay', 'runtime-coreclr superpmi-diffs')) }} variables: - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/${{ coalesce(parameters.jobParameters.templatePath, 'templates') }}/variables/pool-providers.yml # Disable component governance in our CI builds. These builds are not shipping nor # are they a service. Also the component governance jobs issue lots of inconsequential # warnings and errors into our build timelines that make it hard to track down @@ -45,6 +45,15 @@ jobs: - name: _BuildConfig value: $(buildConfigUpper) + + - name: archType + value: ${{ parameters.archType }} + + - name: osGroup + value: ${{ parameters.osGroup }} + + - name: osSubgroup + value: ${{ parameters.osSubgroup }} - name: _runSmokeTestsOnlyArg value: '/p:RunSmokeTestsOnly=$(isRunSmokeTestsOnly)' @@ -159,12 +168,19 @@ jobs: # Official Build Linux Pool ${{ if and(or(in(parameters.osGroup, 'linux', 'freebsd', 'android', 'tizen'), eq(parameters.jobParameters.hostedOs, 'linux')), ne(variables['System.TeamProject'], 'public')) }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + demands: ImageOverride -equals 1es-ubuntu-2204 + os: linux - # OSX Build Pool (we don't have on-prem OSX BuildPool). - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: + # OSX Public Build Pool (we don't have on-prem OSX BuildPool). + ${{ if and(in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator'), eq(variables['System.TeamProject'], 'public')) }}: vmImage: 'macos-12' + # OSX Internal Pool + ${{ if and(in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator'), ne(variables['System.TeamProject'], 'public')) }}: + name: "Azure Pipelines" + vmImage: 'macOS-12' + os: macOS + # Official Build Windows Pool ${{ if and(or(eq(parameters.osGroup, 'windows'), eq(parameters.jobParameters.hostedOs, 'windows')), ne(variables['System.TeamProject'], 'public')) }}: name: $(DncEngInternalBuildPool) diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index e995e626a56991..ebaeae5e124042 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -231,27 +231,6 @@ jobs: artifactName: $(buildProductArtifactName) displayName: 'product build' - - ${{ if and(in(parameters.osGroup, 'windows', 'linux'), ne(parameters.archType, 'x86')) }}: - - template: /eng/pipelines/coreclr/templates/crossdac-build.yml - parameters: - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - isOfficialBuild: ${{ parameters.signBinaries }} - ${{ if eq(parameters.archType, 'arm') }}: - hostArchType: x86 - ${{ else }}: - hostArchType: x64 - - - ${{ if and(in(parameters.osGroup, 'windows'), eq(parameters.archType, 'x86')) }}: - - template: /eng/pipelines/coreclr/templates/crossdac-build.yml - parameters: - archType: arm - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - isOfficialBuild: ${{ parameters.signBinaries }} - hostArchType: x86 - - ${{ if and(ne(parameters.testGroup, ''), ne(parameters.disableClrTest, true)) }}: # Publish test native components for consumption by test execution. - ${{ if ne(parameters.isOfficialBuild, true) }}: diff --git a/eng/pipelines/coreclr/templates/crossdac-build.yml b/eng/pipelines/coreclr/templates/crossdac-build.yml deleted file mode 100644 index a8accb9d2e3d28..00000000000000 --- a/eng/pipelines/coreclr/templates/crossdac-build.yml +++ /dev/null @@ -1,80 +0,0 @@ -parameters: - archType: '' - isOfficialBuild: false - osGroup: '' - osSubgroup: '' - hostArchType: '' - -steps: - # Always build the crossdac, that way we know in CI/PR if things break to build. - - ${{ if and(eq(parameters.osGroup, 'windows'), notin(parameters.archType, 'x86')) }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) ${{ parameters.archType }} -hostarch ${{ parameters.hostArchType }} -ci -os linux -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" -ninja $(officialBuildIdArg) -component crosscomponents - displayName: Build Cross OS Linux DAC for Windows - - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) ${{ parameters.archType }} -hostarch ${{ parameters.hostArchType }} -ci -os alpine -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" -ninja $(officialBuildIdArg) -component crosscomponents - displayName: Build Cross OS Linux-musl DAC for Windows - - - powershell: | - function CopyAndVerifyCrossOsAssets { - [CmdletBinding()] - param ( - [Parameter(Mandatory)][string]$crossDacDir, - [Parameter(Mandatory)][string]$targetDir - ) - - $crossDacDir = Join-Path $crossDacDir -ChildPath '${{ parameters.hostArchType }}' - - $availableFiles = ls -File $crossDacDir - - Write-Host "Probed for files in ""$crossDacDir"", found:" - $availableFiles | fl - - if (-not ("mscordaccore.dll" -in $availableFiles.Name ` - -and "mscordaccore.pdb" -in $availableFiles.Name ` - -and "mscordbi.dll" -in $availableFiles.Name ` - -and "mscordbi.pdb" -in $availableFiles.Name` - )) - { - Write-Error "Couldn't find one of the expected crossdac files." - } - - New-Item $targetDir -ItemType 'Directory' -Force -ea 0 - $availableFiles | %{ cp $_.FullName $targetDir -v } - } - - $buildMuslDacRootFolderPath = "$(Build.SourcesDirectory)/artifacts/bin/coreclr/alpine.${{ parameters.archType }}.$(buildConfigUpper)" - $buildMuslStagingPath = "$(crossDacArtifactPath)/Linux_musl.${{ parameters.archType }}.$(buildConfigUpper)/${{ parameters.hostArchType }}" - - $buildLinuxDacRootFolderPath = "$(Build.SourcesDirectory)/artifacts/bin/coreclr/Linux.${{ parameters.archType }}.$(buildConfigUpper)" - $buildLinuxDacStagingPath = "$(crossDacArtifactPath)/Linux.${{ parameters.archType }}.$(buildConfigUpper)/${{ parameters.hostArchType }}" - - - CopyAndVerifyCrossOsAssets -CrossDacDir $buildMuslDacRootFolderPath -TargetDir $buildMuslStagingPath - CopyAndVerifyCrossOsAssets -CrossDacDir $buildLinuxDacRootFolderPath -TargetDir $buildLinuxDacStagingPath - - Write-Host "Final directory contents:" - ls -R $(crossDacArtifactPath) - - displayName: Gather CrossDac Artifacts - - - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml - parameters: - basePath: $(crossDacArtifactPath) - isOfficialBuild: ${{ parameters.isOfficialBuild }} - timeoutInMinutes: 30 - - - ${{ if eq(parameters.osGroup, 'linux') }}: - - task: CopyFiles@2 - displayName: Gather runtime for CrossDac - inputs: - SourceFolder: $(coreClrProductRootFolderPath) - Contents: libcoreclr.so - TargetFolder: '$(crossDacArtifactPath)/${{ parameters.osGroup }}${{ parameters.osSubgroup }}.$(archType).$(buildConfigUpper)/${{ parameters.hostArchType }}' - - # Make the assets available in a single container for the packaging job. - - task: PublishBuildArtifacts@1 - displayName: Publish runtime for CrossDac - inputs: - pathtoPublish: $(crossDacArtifactPath) - PublishLocation: Container - artifactName: $(buildCrossDacArtifactName) diff --git a/eng/pipelines/coreclr/templates/crossdac-hostarch.yml b/eng/pipelines/coreclr/templates/crossdac-hostarch.yml new file mode 100644 index 00000000000000..e100217a48151a --- /dev/null +++ b/eng/pipelines/coreclr/templates/crossdac-hostarch.yml @@ -0,0 +1,9 @@ +parameters: + archType: '' + +variables: + - name: crossDacHostArch + value: x64 + - ${{ if eq(parameters.archType, 'arm') }}: + - name: crossDacHostArch + value: x86 \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/crossdac-pack.yml b/eng/pipelines/coreclr/templates/crossdac-pack.yml deleted file mode 100644 index 520eb65fb337e6..00000000000000 --- a/eng/pipelines/coreclr/templates/crossdac-pack.yml +++ /dev/null @@ -1,75 +0,0 @@ -parameters: - archType: '' - buildConfig: '' - container: '' - crossDacPlatforms: {} - dependOnEvaluatePaths: false - isOfficialBuild: false - osGroup: '' - osSubgroup: '' - platform: '' - pool: '' - runtimeVariant: '' - testGroup: '' - timeoutInMinutes: '' - variables: {} - -jobs: -- template: xplat-pipeline-job.yml - parameters: - archType: ${{ parameters.archType }} - buildConfig: ${{ parameters.buildConfig }} - container: ${{ parameters.container }} - condition: ${{ parameters.isOfficialBuild }} - helixType: 'build/product/' - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - pool: ${{ parameters.pool }} - runtimeVariant: ${{ parameters.runtimeVariant }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - - name: crossdacpack - displayName: CrossDac Packaging - - variables: - - name: officialBuildIdArg - value: '' - - name: crossDacArgs - value: '/p:CrossDacArtifactsDir=$(crossDacArtifactPath)/$(buildCrossDacArtifactName)' - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - - name: officialBuildIdArg - value: '/p:OfficialBuildId=$(Build.BuildNumber)' - - name: SignType - value: $[ coalesce(variables.OfficialSignType, 'real') ] - - ${{ parameters.variables }} - - dependsOn: - - ${{ if ne(parameters.crossDacPlatforms, '') }}: - - ${{ each platform in parameters.crossDacPlatforms }}: - - ${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_product_build_${{ platform }}_${{ parameters.buildConfig }} - - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download CrossDac artifacts - inputs: - artifactName: $(buildCrossDacArtifactName) - downloadPath: $(crossDacArtifactPath) - checkDownloadedFiles: true - - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset crossdacpack -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) $(crossDacArgs) -ci - displayName: Build crossdac packaging - - # Save packages using the prepare-signed-artifacts format. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: ${{ parameters.platform }} - - # Upload to artifacts to be signed - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'CrossDacPackagingLogs_Attempt$(System.JobAttempt)' - continueOnError: true - condition: always() diff --git a/eng/pipelines/coreclr/templates/sign-diagnostic-files.yml b/eng/pipelines/coreclr/templates/sign-diagnostic-files.yml index 89cbbec6390df9..ed1f8523925846 100644 --- a/eng/pipelines/coreclr/templates/sign-diagnostic-files.yml +++ b/eng/pipelines/coreclr/templates/sign-diagnostic-files.yml @@ -12,10 +12,15 @@ steps: version: '6.0.x' installationPath: '$(Agent.TempDirectory)/dotnet' - - task: EsrpCodeSigning@1 + - task: EsrpCodeSigning@5 displayName: Sign Diagnostic Binaries inputs: - ConnectedServiceName: 'dotnetesrp-diagnostics-dnceng' + ConnectedServiceName: 'diagnostics-esrp-kvcertuser' + AppRegistrationClientId: '2234cdec-a13f-4bb2-aa63-04c57fd7a1f9' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'clrdiag-esrp-id' + AuthCertName: 'dotnetesrp-diagnostics-aad-ssl-cert' + AuthSignCertName: 'dotnet-diagnostics-esrp-pki-onecert' FolderPath: ${{ parameters.basePath }} Pattern: | **/mscordaccore*.dll @@ -48,6 +53,7 @@ steps: SessionTimeout: ${{ parameters.timeoutInMinutes }} MaxConcurrency: '50' MaxRetryAttempts: '5' + PendingAnalysisWaitTimeoutMinutes: '5' env: DOTNET_MULTILEVEL_LOOKUP: 0 DOTNET_ROOT: '$(Agent.TempDirectory)/dotnet' diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 162f5e9f0ed1c8..82d6346a60d7a0 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -11,6 +11,7 @@ parameters: liveLibrariesBuildConfig: '' strategy: '' pool: '' + templatePath: 'templates' # arcade-specific parameters condition: true @@ -28,6 +29,7 @@ parameters: jobs: - template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: + templatePath: ${{ parameters.templatePath }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} @@ -69,12 +71,6 @@ jobs: - name: buildProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' - - name: buildCrossDacArtifactName - value: CoreCLRCrossDacArtifacts - - - name: crossDacArtifactPath - value: $(Build.SourcesDirectory)/artifacts/$(buildCrossDacArtifactName) - # We need this because both mono and coreclr build currently depends on CoreClr - name: coreClrProductArtifactName value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml index ee481119ad9607..e2e95c6bac5739 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml @@ -355,40 +355,41 @@ jobs: # Mono CoreCLR runtime Test executions using live libraries and LLVM Full AOT # Only when Mono is changed # -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - buildConfig: Release - runtimeFlavor: mono - platforms: - - linux_x64 - # - linux_arm64 - variables: - - name: timeoutPerTestInMinutes - value: 60 - - name: timeoutPerTestCollectionInMinutes - value: 180 - jobParameters: - testGroup: innerloop - nameSuffix: AllSubsets_Mono_LLVMFullAot_RuntimeTests - runtimeVariant: llvmfullaot - buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true - timeoutInMinutes: 300 +# Disabled due to OOM errors: https://github.com/dotnet/runtime/issues/90427 +# - template: /eng/pipelines/common/platform-matrix.yml +# parameters: +# jobTemplate: /eng/pipelines/common/global-build-job.yml +# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml +# buildConfig: Release +# runtimeFlavor: mono +# platforms: +# - linux_x64 +# # - linux_arm64 +# variables: +# - name: timeoutPerTestInMinutes +# value: 60 +# - name: timeoutPerTestCollectionInMinutes +# value: 180 +# jobParameters: +# testGroup: innerloop +# nameSuffix: AllSubsets_Mono_LLVMFullAot_RuntimeTests +# runtimeVariant: llvmfullaot +# buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true +# timeoutInMinutes: 300 - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - postBuildSteps: - - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - parameters: - creator: dotnet-bot - llvmAotStepContainer: linux_x64_llvmaot - testRunNamePrefixSuffix: Mono_Release - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/runtimes/test-variables.yml +# condition: >- +# or( +# eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), +# eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), +# eq(variables['isRollingBuild'], true)) +# postBuildSteps: +# - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml +# parameters: +# creator: dotnet-bot +# llvmAotStepContainer: linux_x64_llvmaot +# testRunNamePrefixSuffix: Mono_Release +# extraVariablesTemplates: +# - template: /eng/pipelines/common/templates/runtimes/test-variables.yml # # Mono CoreCLR runtime Test executions using live libraries in interpreter mode diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml index 6f80e6ad104525..8b6291cd0db0bc 100644 --- a/eng/pipelines/global-build.yml +++ b/eng/pipelines/global-build.yml @@ -30,6 +30,7 @@ variables: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: + isOfficialBuild: false stages: - stage: Build jobs: @@ -177,3 +178,97 @@ extends: timeoutInMinutes: 95 condition: eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true) + + # + # Build CoreCLR as a non-portable build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - tizen_armel + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR_NonPortable + buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) /p:PortableBuild=false + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build CoreCLR with no R2R + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - linux_x86 + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR_NoR2R + buildArgs: -s clr.runtime+clr.jit+clr.iltools+clr.spmi+clr.corelib -c $(_BuildConfig) + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build CoreCLR release + # Always as they are needed by Installer and we always build and test the Installer. + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + buildConfig: release + platforms: + - freebsd_x64 + jobParameters: + testGroup: innerloop + # Mono/runtimetests also need this, but skip for wasm + condition: + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: + - freebsd_x64 + jobParameters: + testScope: innerloop + condition: + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/installer/jobs/build-job.yml + buildConfig: Release + platforms: + - freebsd_x64 + jobParameters: + liveRuntimeBuildConfig: release + liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runOnlyIfDependenciesSucceeded: true + condition: + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index 1c45b437c8013f..43f19c69ccacfc 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -339,32 +339,9 @@ jobs: df -h displayName: Disk Usage before Build - # Build the default subset non-MacOS platforms - - ${{ if ne(parameters.osGroup, 'osx') }}: - - script: $(BaseJobBuildCommand) - displayName: Build - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} - - # Build corehost, sign and add entitlements to MacOS binaries - - ${{ if eq(parameters.osGroup, 'osx') }}: - - script: $(BaseJobBuildCommand) -subset host.native - displayName: Build CoreHost - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/pipelines/common/macos-sign-with-entitlements.yml - parameters: - filesToSign: - - name: dotnet - path: $(Build.SourcesDirectory)/artifacts/bin/osx-${{ parameters.archType }}.$(_BuildConfig)/corehost - entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist - - name: apphost - path: $(Build.SourcesDirectory)/artifacts/bin/osx-${{ parameters.archType }}.$(_BuildConfig)/corehost - entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist - - - script: $(BaseJobBuildCommand) -subset host.pkg+host.tools+host.tests+packs - displayName: Build and Package - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} + - script: $(BaseJobBuildCommand) + displayName: Build + continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log diff --git a/eng/pipelines/installer/jobs/steps/build-linux-package.yml b/eng/pipelines/installer/jobs/steps/build-linux-package.yml index a5645af3d9ccd3..102eab770c278f 100644 --- a/eng/pipelines/installer/jobs/steps/build-linux-package.yml +++ b/eng/pipelines/installer/jobs/steps/build-linux-package.yml @@ -4,6 +4,7 @@ parameters: packageStepDescription: null packagingArgs: '' subsetArg: '' + condition: succeeded() steps: ## Run NuGet Authentication for each of the side containers @@ -20,6 +21,7 @@ steps: /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.packageType }} target: ${{ parameters.target }} + condition: ${{ parameters.condition }} # Broken symbolic links break the SBOM processing # We make some symlinks during the installer generation process, # but they aren't always valid on disk afterwards. Some of our tooling, diff --git a/eng/pipelines/mono/templates/generate-offsets.yml b/eng/pipelines/mono/templates/generate-offsets.yml index c68adfe67a9e6b..909ca9a99c54d4 100644 --- a/eng/pipelines/mono/templates/generate-offsets.yml +++ b/eng/pipelines/mono/templates/generate-offsets.yml @@ -9,11 +9,13 @@ parameters: pool: '' condition: true isOfficialBuild: false + templatePath: 'templates' ### Product build jobs: - template: xplat-pipeline-job.yml parameters: + templatePath: ${{ parameters.templatePath }} buildConfig: ${{ parameters.buildConfig }} osGroup: ${{ parameters.osGroup }} osSubGroup: ${{ parameters.osSubGroup }} @@ -76,17 +78,21 @@ jobs: contents: '**/offsets-*.h' targetFolder: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles/' - - task: PublishPipelineArtifact@1 - displayName: Upload offset files - inputs: - targetPath: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' - artifactName: 'Mono_Offsets_$(osGroup)$(osSubGroup)' + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Upload offset files + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' + artifactName: 'Mono_Offsets_$(osGroup)$(osSubGroup)' # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'BuildLogs_Attempt$(System.JobAttempt)_Mono_Offsets_$(osGroup)$(osSubGroup)' - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Publish Logs + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: 'BuildLogs_Attempt$(System.JobAttempt)_Mono_Offsets_$(osGroup)$(osSubGroup)' + continueOnError: true + condition: always() diff --git a/eng/pipelines/mono/templates/workloads-build.yml b/eng/pipelines/mono/templates/workloads-build.yml index 4a50ed665e24ee..a10bf343fa4f1f 100644 --- a/eng/pipelines/mono/templates/workloads-build.yml +++ b/eng/pipelines/mono/templates/workloads-build.yml @@ -12,11 +12,13 @@ parameters: runtimeVariant: '' testGroup: '' timeoutInMinutes: '' + templatePath: 'templates' variables: {} jobs: - template: xplat-pipeline-job.yml parameters: + templatePath: ${{ parameters.templatePath }} archType: ${{ parameters.archType }} buildConfig: ${{ parameters.buildConfig }} container: ${{ parameters.container }} @@ -92,13 +94,15 @@ jobs: name: workloads # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'WorkloadLogs_Attempt$(System.JobAttempt)' - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Publish Logs + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: 'WorkloadLogs_Attempt$(System.JobAttempt)' + continueOnError: true + condition: always() # Delete wixpdb files before they are uploaded to artifacts - task: DeleteFiles@1 diff --git a/eng/pipelines/mono/templates/xplat-pipeline-job.yml b/eng/pipelines/mono/templates/xplat-pipeline-job.yml index 1ca84d9caac1bb..67b43722e0c57d 100644 --- a/eng/pipelines/mono/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/mono/templates/xplat-pipeline-job.yml @@ -12,6 +12,7 @@ parameters: pool: '' runtimeVariant: '' liveRuntimeBuildConfig: 'release' + templatePath: 'templates' # arcade-specific parameters condition: true @@ -28,6 +29,7 @@ parameters: jobs: - template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: + templatePath: ${{ parameters.templatePath }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml index 908f2b64c71c22..24fd2df48d74be 100644 --- a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml +++ b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml @@ -20,6 +20,14 @@ jobs: - name: SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish BuildLogs' + condition: succeededOrFailed() + targetPath: '$(Build.StagingDirectory)\BuildLogs' + artifactName: ${{ parameters.logArtifactName }} + steps: - checkout: self clean: true @@ -65,11 +73,4 @@ jobs: **/*.binlog TargetFolder: '$(Build.StagingDirectory)\BuildLogs' continueOnError: true - condition: succeededOrFailed() - - - task: PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)\BuildLogs' - artifactName: ${{ parameters.logArtifactName }} - condition: succeededOrFailed() + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/pipelines/official/stages/publish.yml b/eng/pipelines/official/stages/publish.yml index d23afa7003b25d..9553baae305f2f 100644 --- a/eng/pipelines/official/stages/publish.yml +++ b/eng/pipelines/official/stages/publish.yml @@ -7,7 +7,7 @@ stages: - stage: PrepareForPublish displayName: Prepare for Publish variables: - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/templates-official/variables/pool-providers.yml jobs: # Prep artifacts: sign them and upload pipeline artifacts expected by stages-based publishing. - template: /eng/pipelines/official/jobs/prepare-signed-artifacts.yml @@ -15,7 +15,7 @@ stages: PublishRidAgnosticPackagesFromPlatform: ${{ parameters.PublishRidAgnosticPackagesFromPlatform }} # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact. - - template: /eng/common/templates/job/publish-build-assets.yml + - template: /eng/common/templates-official/job/publish-build-assets.yml parameters: publishUsingPipelines: true publishAssetsImmediately: true @@ -26,7 +26,7 @@ stages: symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' # Stages-based publishing entry point -- template: /eng/common/templates/post-build/post-build.yml +- template: /eng/common/templates-official/post-build/post-build.yml parameters: publishingInfraVersion: ${{ parameters.publishingInfraVersion }} validateDependsOn: diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 3138f18ebd734b..7cadf6a800f675 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -23,6 +23,8 @@ pr: none variables: - template: /eng/pipelines/common/variables.yml + parameters: + templatePath: 'templates-official' - template: /eng/pipelines/common/internal-variables.yml parameters: teamName: dotnet-core-acquisition @@ -33,6 +35,7 @@ variables: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: + isOfficialBuild: true stages: - stage: Build jobs: @@ -41,64 +44,267 @@ extends: # Localization build # - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/8.0') }}: - - template: /eng/common/templates/job/onelocbuild.yml - parameters: - MirrorRepo: runtime - MirrorBranch: release/8.0 - LclSource: lclFilesfromPackage - LclPackageId: 'LCL-JUNO-PROD-RUNTIME' + - template: /eng/common/templates-official/job/onelocbuild.yml + parameters: + MirrorRepo: runtime + MirrorBranch: main + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-RUNTIME' # # Source Index Build # - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: - - template: /eng/common/templates/job/source-index-stage1.yml + - template: /eng/common/templates-official/job/source-index-stage1.yml parameters: sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os linux -ci /p:SkipLibrariesNativeRuntimePackages=true # - # Build CoreCLR + # Build CoreCLR runtime packs + # Windows x64/arm64 + # Sign diagnostic files after native build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x64 + - windows_arm64 + jobParameters: + templatePath: 'templates-official' + buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + nameSuffix: CoreCLR + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml + parameters: + basePath: $(Build.SourcesDirectory)/artifacts/bin/coreclr + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 30 + # Now that we've signed the diagnostic files, do the rest of the build. + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+libs+host+packs -c $(_BuildConfig) + displayName: Build managed CoreCLR components, all libraries, hosts, and packs + + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + + + # + # Build CoreCLR runtime packs + # Windows x86 + # No NativeAOT as NativeAOT is not supported on x86 + # Sign diagnostic files after native build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x86 + jobParameters: + templatePath: 'templates-official' + buildArgs: -s clr.runtime+clr.alljits -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + nameSuffix: CoreCLR + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml + parameters: + basePath: $(Build.SourcesDirectory)/artifacts/bin/coreclr + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 30 + # Now that we've signed the diagnostic files, do the rest of the build. + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.corelib+clr.nativecorelib+clr.tools+clr.packages+libs+host+packs -c $(_BuildConfig) + displayName: Build managed CoreCLR components, all libraries, hosts, and packs + + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + # + # Build CoreCLR runtime packs + # Mac x64/arm64 + # Sign and entitle createdump and corerun after native build. # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - osx_arm64 - osx_x64 + jobParameters: + templatePath: 'templates-official' + buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime+host.native -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + nameSuffix: CoreCLR + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 120 + postBuildSteps: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/pipelines/common/macos-sign-with-entitlements.yml + parameters: + filesToSign: + - name: createdump + path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/createdump-entitlements.plist + - name: corerun + path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist + - name: dotnet + path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist + - name: apphost + path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist + + - task: CopyFiles@2 + displayName: 'Copy signed createdump to sharedFramework' + inputs: + contents: createdump + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + targetFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig)/sharedFramework + overWrite: true + + # Now that we've entitled and signed createdump, we can build the rest. + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+libs+host.tools+host.pkg+packs -c $(_BuildConfig) + displayName: Build managed CoreCLR and host components, all libraries, and packs + + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + + # + # Build CoreCLR runtime packs + # Linux and Linux_musl + # CoreCLR runtime for CrossDac packaging + # Create Linux installers + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: - linux_x64 - linux_arm - linux_arm64 - linux_musl_x64 - linux_musl_arm - linux_musl_arm64 - - windows_x86 - - windows_x64 - - windows_arm64 jobParameters: + templatePath: 'templates-official' + buildArgs: -s clr.runtime+clr.alljits+clr.corelib+clr.nativecorelib+clr.tools+clr.aot+clr.packages+libs+host+packs -c $(_BuildConfig) + nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} - signBinaries: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 + postBuildSteps: + # Upload libcoreclr.so for CrossDac packaging + - task: CopyFiles@2 + displayName: Gather runtime for CrossDac + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + Contents: libcoreclr.so + TargetFolder: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts/$(osGroup)$(osSubgroup).$(archType).$(_BuildConfig)/$(crossDacHostArch) + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish runtime for CrossDac + inputs: + PathtoPublish: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts + PublishLocation: Container + ArtifactName: CoreCLRCrossDacArtifacts + # Create RPMs and DEBs + - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml + parameters: + packageType: deb + target: debpkg + packageStepDescription: Runtime Deps, Runtime, Framework Packs Deb installers + subsetArg: -s packs.installers + packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true + condition: and(succeeded(), eq(variables.osSubgroup, ''), eq(variables.archType, 'x64')) + - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml + parameters: + packageType: rpm + target: rpmpkg + packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers + subsetArg: -s packs.installers + packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true + condition: and(succeeded(), eq(variables.osSubgroup, ''), in(variables.archType, 'x64', 'arm64')) + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/crossdac-hostarch.yml + + # + # Build and Pack CrossDac + # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/crossdac-pack.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' + buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) + nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 - crossDacPlatforms: - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - linux_musl_arm - - linux_musl_arm64 - - windows_x64 - - windows_arm64 + preBuildSteps: + - task: DownloadBuildArtifacts@0 + displayName: Download Runtimes for CrossDac packaging + inputs: + artifactName: $(crossDacArtifactsContainer) + downloadPath: $(crossDacArtifactsBasePath) + checkDownloadedFiles: true + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s linuxdac+alpinedac -c $(_BuildConfig) + archParameter: -arch x64,x86,arm,arm64 + - task: CopyFiles@2 + displayName: Gather CrossDacs + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + Contents: | + ** + !**\sharedFramework\** + TargetFolder: $(crossDacArtifactsPath) + - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml + parameters: + basePath: $(crossDacArtifactsPath) + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 30 + postBuildSteps: + # Save packages using the prepare-signed-artifacts format. + # CrossDac packages are expected to be in the windows_x64 folder. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: windows_x64 + dependsOn: + - build_linux_x64_release_CoreCLR + - build_linux_arm_release_CoreCLR + - build_linux_arm64_release_CoreCLR + - build_linux_musl_x64_release_CoreCLR + - build_linux_musl_arm_release_CoreCLR + - build_linux_musl_arm64_release_CoreCLR + variables: + - name: crossDacArtifactsContainer + value: CoreCLRCrossDacArtifacts + - name: crossDacArtifactsBasePath + value: $(Build.StagingDirectory)/CrossDac + - name: crossDacArtifactsPath + value: $(crossDacArtifactsBasePath)/$(crossDacArtifactsContainer) # # Build NativeAOT runtime packs @@ -121,8 +327,9 @@ extends: - linux_bionic_arm64 - linux_bionic_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.nativeaotlibs+clr.nativeaotruntime+libs+packs -c $(_BuildConfig) /p:BuildNativeAOTRuntimePack=true /p:SkipLibrariesNativeRuntimePackages=true - nameSuffix: AllSubsets_NativeAOT + nameSuffix: NativeAOT isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -164,8 +371,9 @@ extends: - windows_x86 # - windows_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:BuildMonoAOTCrossCompiler=false - nameSuffix: AllSubsets_Mono + nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -181,8 +389,9 @@ extends: - browser_wasm - wasi_wasm jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) - nameSuffix: AllSubsets_Mono + nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -197,8 +406,9 @@ extends: platforms: - browser_wasm jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoWasmBuildVariant=multithread /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) - nameSuffix: AllSubsets_Mono_multithread + nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread postBuildSteps: @@ -219,6 +429,7 @@ extends: - ios_arm64 - maccatalyst_x64 jobParameters: + templatePath: 'templates-official' isOfficialBuild: ${{ variables.isOfficialBuild }} # @@ -235,6 +446,7 @@ extends: - linux_arm64 - linux_musl_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+packs -c $(_BuildConfig) /p:MonoCrossAOTTargetOS=android+browser /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true nameSuffix: CrossAOT_Mono @@ -259,6 +471,7 @@ extends: platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+packs -c $(_BuildConfig) /p:MonoCrossAOTTargetOS=android+browser /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true nameSuffix: CrossAOT_Mono @@ -284,6 +497,7 @@ extends: - osx_x64 - osx_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+packs -c $(_BuildConfig) /p:MonoCrossAOTTargetOS=android+browser+tvos+ios+maccatalyst /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true nameSuffix: CrossAOT_Mono @@ -327,9 +541,10 @@ extends: buildConfig: release runtimeFlavor: mono jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false - nameSuffix: AllSubsets_Mono_LLVMJIT + nameSuffix: Mono_LLVMJIT runtimeVariant: LLVMJIT isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: @@ -341,9 +556,10 @@ extends: buildConfig: release runtimeFlavor: mono jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true - nameSuffix: AllSubsets_Mono_LLVMAOT + nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: @@ -352,49 +568,27 @@ extends: name: MonoRuntimePacks # - # Build libraries using live CoreLib from CoreCLR + # Build libraries AllConfigurations for packages # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: Release platforms: - - osx_arm64 - - osx_x64 - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - linux_musl_arm - - linux_musl_arm64 - - windows_x86 - windows_x64 - - windows_arm64 jobParameters: + templatePath: 'templates-official' + buildArgs: -s tools+libs -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + nameSuffix: Libraries_AllConfigurations isOfficialBuild: ${{ variables.isOfficialBuild }} - liveRuntimeBuildConfig: release - # Official builds don't run tests, locally or on Helix - runTests: false - useHelix: false - + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: Libraries_AllConfigurations + timeoutInMinutes: 95 # - # Build libraries AllConfigurations for packages + # Build SourceBuild packages # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - windows_x64 - jobParameters: - framework: allConfigurations - isOfficialBuild: ${{ variables.isOfficialBuild }} - isOfficialAllConfigurations: true - liveRuntimeBuildConfig: release - # Official builds don't run tests, locally or on Helix - runTests: false - useHelix: false - - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml @@ -403,37 +597,15 @@ extends: platforms: - SourceBuild_linux_x64 jobParameters: + templatePath: 'templates-official' nameSuffix: PortableSourceBuild + isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml parameters: name: SourceBuildPackages timeoutInMinutes: 95 - # - # Installer Build - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/installer/jobs/build-job.yml - buildConfig: Release - jobParameters: - liveRuntimeBuildConfig: release - liveLibrariesBuildConfig: Release - isOfficialBuild: ${{ variables.isOfficialBuild }} - platforms: - - osx_arm64 - - osx_x64 - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - linux_musl_arm - - linux_musl_arm64 - - windows_x86 - - windows_x64 - - windows_arm64 - # # Build PGO Instrumented CoreCLR Release # @@ -449,12 +621,14 @@ extends: - windows_arm64 - linux_arm64 jobParameters: - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument + templatePath: 'templates-official' + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: PGO + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: PGO timeoutInMinutes: 95 # @@ -467,27 +641,28 @@ extends: platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 dependsOn: - - Build_android_arm_release_AllSubsets_Mono - - Build_android_arm64_release_AllSubsets_Mono - - Build_android_x86_release_AllSubsets_Mono - - Build_android_x64_release_AllSubsets_Mono - - Build_browser_wasm_Linux_release_AllSubsets_Mono - - Build_wasi_wasm_linux_release_AllSubsets_Mono - - Build_ios_arm64_release_AllSubsets_Mono - - Build_iossimulator_x64_release_AllSubsets_Mono - - Build_iossimulator_arm64_release_AllSubsets_Mono - - Build_maccatalyst_arm64_release_AllSubsets_Mono - - Build_maccatalyst_x64_release_AllSubsets_Mono - - Build_tvos_arm64_release_AllSubsets_Mono - - Build_tvossimulator_arm64_release_AllSubsets_Mono - - Build_tvossimulator_x64_release_AllSubsets_Mono + - Build_android_arm_release_Mono + - Build_android_arm64_release_Mono + - Build_android_x86_release_Mono + - Build_android_x64_release_Mono + - Build_browser_wasm_Linux_release_Mono + - Build_wasi_wasm_linux_release_Mono + - Build_ios_arm64_release_Mono + - Build_iossimulator_x64_release_Mono + - Build_iossimulator_arm64_release_Mono + - Build_maccatalyst_arm64_release_Mono + - Build_maccatalyst_x64_release_Mono + - Build_tvos_arm64_release_Mono + - Build_tvossimulator_arm64_release_Mono + - Build_tvossimulator_x64_release_Mono - Build_windows_x64_release_CrossAOT_Mono - - installer_coreclr__windows_x64_Release_ - - installer_coreclr__windows_x86_Release_ - - installer_coreclr__windows_arm64_Release_ + - Build_windows_x64_release_CoreCLR + - Build_windows_x86_release_CoreCLR + - Build_windows_arm64_release_CoreCLR - ${{ if eq(variables.isOfficialBuild, true) }}: - template: /eng/pipelines/official/stages/publish.yml diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index c83a76993e60d6..8f001c4903003e 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -52,6 +52,7 @@ variables: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: + isOfficialBuild: false stages: - stage: Build jobs: @@ -155,7 +156,6 @@ extends: - windows_x64 - windows_x86 - windows_arm64 - - freebsd_x64 jobParameters: testGroup: innerloop # Mono/runtimetests also need this, but skip for wasm @@ -187,48 +187,6 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_jit.containsChange'], true), eq(variables['isRollingBuild'], true))) - # - # Build CoreCLR with no R2R - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - buildConfig: checked - runtimeFlavor: coreclr - platforms: - - linux_x86 - jobParameters: - testScope: innerloop - nameSuffix: CoreCLR_NoR2R - buildArgs: -s clr.runtime+clr.jit+clr.iltools+clr.spmi+clr.corelib -c $(_BuildConfig) - timeoutInMinutes: 120 - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Build CoreCLR as a non-portable build - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - buildConfig: checked - runtimeFlavor: coreclr - platforms: - - tizen_armel - jobParameters: - testScope: innerloop - nameSuffix: CoreCLR_NonPortable - buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) /p:PortableBuild=false - timeoutInMinutes: 120 - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(variables['isRollingBuild'], true)) - # # CoreCLR NativeAOT debug build and smoke tests # Only when CoreCLR is changed @@ -400,6 +358,27 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_tools_illink.containsChange'], true), eq(variables['isRollingBuild'], true)) + # + # Build CrossDacs + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x64 + variables: + - name: _archParameter + value: -arch x64,x86,arm,arm64 + jobParameters: + buildArgs: -s linuxdac+alpinedac -c Checked,$(_BuildConfig) + nameSuffix: CrossDac + isOfficialBuild: false + timeoutInMinutes: 60 + postBuildSteps: + - publish: $(Build.SourcesDirectory)/artifacts/bin/coreclr + displayName: Publish CrossDacs for diagnostics + artifact: CoreCLRCrossDacArtifacts # Build Mono AOT offset headers once, for consumption elsewhere # Only when mono changed @@ -407,6 +386,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/mono/templates/generate-offsets.yml + templatePath: 'templates' buildConfig: release platforms: - android_x64 @@ -415,7 +395,7 @@ extends: - ios_arm64 - maccatalyst_x64 jobParameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: false # needed by crossaot condition: >- or( @@ -1038,7 +1018,6 @@ extends: - osx_arm64 - osx_x64 - windows_x64 - - freebsd_x64 jobParameters: testScope: innerloop condition: @@ -1162,7 +1141,6 @@ extends: - linux_arm64 - linux_musl_x64 - windows_x64 - - freebsd_x64 jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} diff --git a/eng/targetingpacks.targets b/eng/targetingpacks.targets index b3ad8560d63b1c..e9ca4e11615267 100644 --- a/eng/targetingpacks.targets +++ b/eng/targetingpacks.targets @@ -35,7 +35,7 @@ LatestRuntimeFrameworkVersion="$(ProductVersion)" RuntimeFrameworkName="$(LocalFrameworkOverrideName)" RuntimePackNamePatterns="$(LocalFrameworkOverrideName).Runtime.**RID**" - RuntimePackRuntimeIdentifiers="linux-arm;linux-arm64;linux-musl-arm64;linux-musl-x64;linux-x64;osx-x64;rhel.6-x64;tizen.4.0.0-armel;tizen.5.0.0-armel;win-arm64;win-x64;win-x86;linux-musl-arm;osx-arm64;maccatalyst-x64;maccatalyst-arm64;linux-s390x;linux-bionic-arm;linux-bionic-arm64;linux-bionic-x64;linux-bionic-x86" + RuntimePackRuntimeIdentifiers="linux-arm;linux-arm64;linux-musl-arm64;linux-musl-x64;linux-x64;osx-x64;rhel.6-x64;tizen.4.0.0-armel;tizen.5.0.0-armel;win-arm64;win-x64;win-x86;linux-musl-arm;osx-arm64;maccatalyst-x64;maccatalyst-arm64;linux-s390x;linux-bionic-arm;linux-bionic-arm64;linux-bionic-x64;linux-bionic-x86;freebsd-x64;freebsd-arm64" TargetFramework="$(NetCoreAppCurrent)" TargetingPackName="$(LocalFrameworkOverrideName).Ref" TargetingPackVersion="$(ProductVersion)" @@ -104,6 +104,10 @@ Condition="'$(UsePackageDownload)' == 'true' and $([System.String]::Copy('%(Identity)').StartsWith('$(LocalFrameworkOverrideName).Runtime'))" /> + + - - + diff --git a/src/coreclr/debug/daccess/CMakeLists.txt b/src/coreclr/debug/daccess/CMakeLists.txt index 9a867c078019dd..5332e957c9eca1 100644 --- a/src/coreclr/debug/daccess/CMakeLists.txt +++ b/src/coreclr/debug/daccess/CMakeLists.txt @@ -53,7 +53,7 @@ if(CLR_CMAKE_HOST_FREEBSD OR CLR_CMAKE_HOST_NETBSD OR CLR_CMAKE_HOST_SUNOS) DEPENDS coreclr VERBATIM COMMAND_EXPAND_LISTS - COMMAND ${CLR_DIR}/pal/tools/gen-dactable-rva.sh ${args} + COMMAND ${CMAKE_COMMAND} -E env NM=${CMAKE_NM} ${CLR_DIR}/pal/tools/gen-dactable-rva.sh ${args} COMMENT Generating ${GENERATED_INCLUDE_DIR}/dactablerva.h ) diff --git a/src/coreclr/gc/env/gcenv.os.h b/src/coreclr/gc/env/gcenv.os.h index c38892b0cadc20..753b751f798c63 100644 --- a/src/coreclr/gc/env/gcenv.os.h +++ b/src/coreclr/gc/env/gcenv.os.h @@ -417,7 +417,7 @@ class GCToOSInterface // Remarks: // If a process runs with a restricted memory limit, it returns the limit. If there's no limit // specified, it returns amount of actual physical memory. - static uint64_t GetPhysicalMemoryLimit(bool* is_restricted=NULL); + static uint64_t GetPhysicalMemoryLimit(bool* is_restricted=NULL, bool refresh=false); // Get memory status // Parameters: diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 86022c70550a50..1cf604da8a87bc 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -52808,7 +52808,7 @@ int gc_heap::refresh_memory_limit() size_t old_heap_hard_limit_poh = heap_hard_limit_oh[poh]; bool old_hard_limit_config_p = hard_limit_config_p; - total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit (&is_restricted_physical_mem); + total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit (&is_restricted_physical_mem, true); bool succeed = true; diff --git a/src/coreclr/gc/unix/gcenv.unix.cpp b/src/coreclr/gc/unix/gcenv.unix.cpp index dd591a54426351..0dc9790e0662fd 100644 --- a/src/coreclr/gc/unix/gcenv.unix.cpp +++ b/src/coreclr/gc/unix/gcenv.unix.cpp @@ -1121,14 +1121,13 @@ size_t GCToOSInterface::GetVirtualMemoryLimit() // Remarks: // If a process runs with a restricted memory limit, it returns the limit. If there's no limit // specified, it returns amount of actual physical memory. -uint64_t GCToOSInterface::GetPhysicalMemoryLimit(bool* is_restricted) +uint64_t GCToOSInterface::GetPhysicalMemoryLimit(bool* is_restricted, bool refresh) { size_t restricted_limit; if (is_restricted) *is_restricted = false; - // The limit was not cached - if (g_RestrictedPhysicalMemoryLimit == 0) + if (g_RestrictedPhysicalMemoryLimit == 0 || refresh) { restricted_limit = GetRestrictedPhysicalMemoryLimit(); VolatileStore(&g_RestrictedPhysicalMemoryLimit, restricted_limit); diff --git a/src/coreclr/gc/windows/gcenv.windows.cpp b/src/coreclr/gc/windows/gcenv.windows.cpp index f12a64d7ed1ab3..0d2af7904c89c9 100644 --- a/src/coreclr/gc/windows/gcenv.windows.cpp +++ b/src/coreclr/gc/windows/gcenv.windows.cpp @@ -960,7 +960,7 @@ size_t GCToOSInterface::GetVirtualMemoryLimit() // Remarks: // If a process runs with a restricted memory limit, it returns the limit. If there's no limit // specified, it returns amount of actual physical memory. -uint64_t GCToOSInterface::GetPhysicalMemoryLimit(bool* is_restricted) +uint64_t GCToOSInterface::GetPhysicalMemoryLimit(bool* is_restricted, bool refresh) { if (is_restricted) *is_restricted = false; diff --git a/src/coreclr/pal/src/safecrt/vsprintf.cpp b/src/coreclr/pal/src/safecrt/vsprintf.cpp index b8ff745f563ceb..360222d5dc6798 100644 --- a/src/coreclr/pal/src/safecrt/vsprintf.cpp +++ b/src/coreclr/pal/src/safecrt/vsprintf.cpp @@ -95,7 +95,7 @@ DLLEXPORT int __cdecl _vsnprintf_s ( retvalue = vsnprintf(string, sizeInBytes, format, ap); string[sizeInBytes - 1] = '\0'; /* we allow truncation if count == _TRUNCATE */ - if (retvalue > (int)sizeInBytes && count == _TRUNCATE) + if (retvalue >= (int)sizeInBytes && count == _TRUNCATE) { if (errno == ERANGE) { diff --git a/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp b/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp index fb5ab3a2d7af41..62b725208769c2 100644 --- a/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp +++ b/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp @@ -49,6 +49,18 @@ PALTEST(c_runtime__vsnprintf_s_test1_paltest_vsnprintf_test1, "c_runtime/_vsnpri Fail("ERROR: expected %s (up to %d chars), got %s\n", checkstr, 8, buf); } + char buf8[8] = {0}; + + ret = Testvsnprintf(buf8, 8, "abcdefgh"); + if (ret >= 0) + { + Fail("ERROR: expected negative return value, got %d", ret); + } + if (memcmp(buf8, "abcdefg\0", 8) != 0) + { + Fail("ERROR: Expected 7 chars + null terminator"); + } + PAL_Terminate(); return PASS; } diff --git a/src/coreclr/tools/aot/ILCompiler/ILCompiler.csproj b/src/coreclr/tools/aot/ILCompiler/ILCompiler.csproj index 7c25444f209c33..c0a239225fa3e7 100644 --- a/src/coreclr/tools/aot/ILCompiler/ILCompiler.csproj +++ b/src/coreclr/tools/aot/ILCompiler/ILCompiler.csproj @@ -12,6 +12,7 @@ $(RuntimeBinDir)ilc-published/ false + false false true diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2.csproj index ef1b82f537fbfd..6f8578b26b9691 100644 --- a/src/coreclr/tools/aot/crossgen2/crossgen2.csproj +++ b/src/coreclr/tools/aot/crossgen2/crossgen2.csproj @@ -1,113 +1,14 @@ - - - - + - $(RuntimeBinDir)crossgen2 - - false - - true - linux-x64;linux-musl-x64;linux-arm;linux-musl-arm;linux-arm64;linux-musl-arm64;freebsd-x64;freebsd-arm64;osx-x64;osx-arm64;win-x64;win-x86;win-arm64 - $(PackageRID) - false - true + $(RuntimeBinDir)/crossgen2 + false + + $(NetCoreAppToolCurrent) - - - - true - true - - false - - false - true - - - - - - $(CoreCLRILCompilerDir) - $(CoreCLRCrossILCompilerDir) - $(ROOTFS_DIR) - $(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll - $(CoreCLRAotSdkDir) - $(MicrosoftNetCoreAppRuntimePackRidLibTfmDir) - $(MicrosoftNetCoreAppRuntimePackNativeDir) - false - - .dwarf - --flat - - - - - - - - - - - - - - - - - $(MicrosoftNetCoreAppRuntimePackDir) - - - - - - - - - $(RuntimeIdentifier) - - - x86_64 - aarch64 - arm64 - - - $(CrossCompileArch)-linux-gnu - $(CrossCompileArch)-alpine-linux-musl - $(CrossCompileArch)-unknown-freebsd12 - - - - - - - - - - clang - - - - - - - - $(_CC_LDFLAGS.SubString(0, $(_CC_LDFLAGS.IndexOf(';')))) - <_LDFLAGS>$(_CC_LDFLAGS.SubString($([MSBuild]::Add($(_CC_LDFLAGS.IndexOf(';')), 1)))) - lld - - - diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2.props b/src/coreclr/tools/aot/crossgen2/crossgen2.props index 0f2f954c6a1544..4d5cc30c3bb604 100644 --- a/src/coreclr/tools/aot/crossgen2/crossgen2.props +++ b/src/coreclr/tools/aot/crossgen2/crossgen2.props @@ -3,7 +3,6 @@ crossgen2 true Exe - $(NetCoreAppToolCurrent) 8002,NU1701 x64;x86;arm64;arm;loongarch64 AnyCPU diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2_crossarch.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2_crossarch.csproj deleted file mode 100644 index 544b4271117b5f..00000000000000 --- a/src/coreclr/tools/aot/crossgen2/crossgen2_crossarch.csproj +++ /dev/null @@ -1,8 +0,0 @@ - - - $(BuildArchitecture) - $(RuntimeBinDir)/$(CrossHostArch)/crossgen2 - false - - - diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj new file mode 100644 index 00000000000000..75766fd75eabde --- /dev/null +++ b/src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj @@ -0,0 +1,9 @@ + + + $(BuildArchitecture) + $(RuntimeBinDir)/$(BuildArchitecture)/crossgen2 + false + $(NetCoreAppToolCurrent) + + + diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj new file mode 100644 index 00000000000000..d5105eccff36de --- /dev/null +++ b/src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj @@ -0,0 +1,105 @@ + + + + + + false + false + true + $(PackageRID) + true + $(NetCoreAppCurrent) + true + true + + + + + + true + true + + false + + false + true + + + + + + + + + + + + + $(CoreCLRILCompilerDir) + $(CoreCLRCrossILCompilerDir) + $(ROOTFS_DIR) + $(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll + $(CoreCLRAotSdkDir) + $(MicrosoftNetCoreAppRuntimePackRidLibTfmDir) + $(MicrosoftNetCoreAppRuntimePackNativeDir) + false + + .dwarf + --flat + + + + + + + + + + + + + + + $(RuntimeIdentifier) + + + x86_64 + aarch64 + arm64 + + + $(CrossCompileArch)-linux-gnu + $(CrossCompileArch)-alpine-linux-musl + $(CrossCompileArch)-unknown-freebsd12 + + + + + + + + + + clang + + + + + + + + $(_CC_LDFLAGS.SubString(0, $(_CC_LDFLAGS.IndexOf(';')))) + <_LDFLAGS>$(_CC_LDFLAGS.SubString($([MSBuild]::Add($(_CC_LDFLAGS.IndexOf(';')), 1)))) + lld + + + + diff --git a/src/coreclr/vm/arm/thunktemplates.S b/src/coreclr/vm/arm/thunktemplates.S index 0686bb2ed4b735..8744c8ebb63264 100644 --- a/src/coreclr/vm/arm/thunktemplates.S +++ b/src/coreclr/vm/arm/thunktemplates.S @@ -11,7 +11,7 @@ PAGE_SIZE = 4096 -#define DATA_SLOT(stub, field) stub##Code + PAGE_SIZE + stub##Data__##field +#define DATA_SLOT(stub, field) . - (. - stub##Code) + PAGE_SIZE + stub##Data__##field LEAF_ENTRY StubPrecodeCode ldr r12, DATA_SLOT(StubPrecode, MethodDesc) diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index b88d6a959bb321..9fe9376fd58efa 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -6696,14 +6696,6 @@ VEH_ACTION WINAPI CLRVectoredExceptionHandlerPhase3(PEXCEPTION_POINTERS pExcepti VEH_ACTION WINAPI CLRVectoredExceptionHandler(PEXCEPTION_POINTERS pExceptionInfo) { - // It is not safe to execute code inside VM after we shutdown EE. One example is DisablePreemptiveGC - // will block forever. - if (g_fForbidEnterEE) - { - return VEH_CONTINUE_SEARCH; - } - - // // DO NOT USE CONTRACTS HERE AS THIS ROUTINE MAY NEVER RETURN. You can use // static contracts, but currently this is all WRAPPER_NO_CONTRACT. @@ -7385,12 +7377,6 @@ LONG WINAPI CLRVectoredExceptionHandlerShim(PEXCEPTION_POINTERS pExceptionInfo) // WARNING WARNING WARNING WARNING WARNING WARNING WARNING // - // If runtime have been disabled, then simply return. - if (g_fForbidEnterEE) - { - return EXCEPTION_CONTINUE_SEARCH; - } - // WARNING // // We must preserve this so that GCStress=4 eh processing doesnt kill last error. diff --git a/src/coreclr/vm/threadstatics.cpp b/src/coreclr/vm/threadstatics.cpp index e3430acf8e4ff2..0216ae054a181a 100644 --- a/src/coreclr/vm/threadstatics.cpp +++ b/src/coreclr/vm/threadstatics.cpp @@ -52,13 +52,22 @@ void ThreadLocalBlock::FreeTLM(SIZE_T i, BOOL isThreadShuttingdown) ThreadLocalModule::CollectibleDynamicEntry *entry = (ThreadLocalModule::CollectibleDynamicEntry*)pThreadLocalModule->m_pDynamicClassTable[k].m_pDynamicEntry; PTR_LoaderAllocator pLoaderAllocator = entry->m_pLoaderAllocator; - if (entry->m_hGCStatics != NULL) - { - pLoaderAllocator->FreeHandle(entry->m_hGCStatics); - } - if (entry->m_hNonGCStatics != NULL) + // LoaderAllocator may be collected when the thread is shutting down. + // We enter coop mode to ensure that we get a valid value of the exposed object and + // can safely clean up handles if it is not yet collected. + GCX_COOP(); + + LOADERALLOCATORREF loaderAllocator = pLoaderAllocator->GetExposedObject(); + if (loaderAllocator != NULL) { - pLoaderAllocator->FreeHandle(entry->m_hNonGCStatics); + if (entry->m_hGCStatics != NULL) + { + pLoaderAllocator->FreeHandle(entry->m_hGCStatics); + } + if (entry->m_hNonGCStatics != NULL) + { + pLoaderAllocator->FreeHandle(entry->m_hNonGCStatics); + } } } delete pThreadLocalModule->m_pDynamicClassTable[k].m_pDynamicEntry; diff --git a/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props b/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props index 1d51a78880ae2d..7cbe74757e28f5 100644 --- a/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props +++ b/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props @@ -2,7 +2,6 @@ - diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj index e8e8591bbaf8b5..a95eb81b9317fb 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj @@ -28,36 +28,22 @@ + + + + - - - - - - - - - - <_CrossgenPublishFiles Include="@(_RawCrossgenPublishFiles->'%(OutputPath)')" KeepMetadata="REMOVE_ALL" /> diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets b/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets index 0b82891e6c42e4..6b24c54f4c1eb6 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets @@ -14,9 +14,7 @@ - - $(BuildArchitecture) - $(CoreCLRArtifactsPath)\$(CrossDir)\crossgen2\crossgen2.dll + $(CoreCLRArtifactsPath)\$(BuildArchitecture)\crossgen2\crossgen2.dll true @(PublishReadyToRunCrossgen2ExtraArgsList) diff --git a/src/installer/pkg/sfx/installers.proj b/src/installer/pkg/sfx/installers.proj index 7f4ce6b9c1c409..2f947e6d9e291e 100644 --- a/src/installer/pkg/sfx/installers.proj +++ b/src/installer/pkg/sfx/installers.proj @@ -22,6 +22,10 @@ + + + + + + true + false + netstandard-targeting-pack-2.1 + NETStandard.Library.Ref + NETStandard.Library.Ref 2.1.0 + false + ToolPack + true + netstandard-targeting-pack-2.1.0-x64.rpm + https://dotnetcli.blob.core.windows.net/dotnet/Runtime/3.1.0/$(OriginalNETStandard21PkgFilename) + $([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', 'download')) + $(NETStandard21TempDir)$(OriginalNETStandard21PkgFilename) + $(NETStandard21TempDir)netstandard21.semaphore + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs index c526f37b0b9fbf..110d4714ba5d12 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs @@ -787,7 +787,7 @@ private static unsafe void RemoveSessionCallback(IntPtr ctx, IntPtr session) IntPtr name = Ssl.SessionGetHostname(session); Debug.Assert(name != IntPtr.Zero); - ctxHandle.RemoveSession(name); + ctxHandle.RemoveSession(name, session); } #if DEBUG diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.SslCtx.cs b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.SslCtx.cs index 5baa776d9d5a8a..d92e15e940e65e 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.SslCtx.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.SslCtx.cs @@ -39,6 +39,9 @@ internal static partial class Ssl [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslCtxSetCaching")] internal static unsafe partial int SslCtxSetCaching(SafeSslContextHandle ctx, int mode, int cacheSize, int contextIdLength, Span contextId, delegate* unmanaged neewSessionCallback, delegate* unmanaged removeSessionCallback); + [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslCtxRemoveSession")] + internal static unsafe partial void SslCtxRemoveSession(SafeSslContextHandle ctx, IntPtr session); + internal static bool AddExtraChainCertificates(SafeSslContextHandle ctx, ReadOnlyCollection chain) { // send pre-computed list of intermediates. @@ -142,27 +145,38 @@ internal bool TryAddSession(IntPtr namePtr, IntPtr session) // This will use strdup() so it is safe to pass in raw pointer. Interop.Ssl.SessionSetHostname(session, namePtr); + IntPtr oldSession = IntPtr.Zero; + lock (_sslSessions) { if (!_sslSessions.TryAdd(targetName, session)) { - if (_sslSessions.Remove(targetName, out IntPtr oldSession)) - { - Interop.Ssl.SessionFree(oldSession); - } - + // session to this target host exists, replace it + _sslSessions.Remove(targetName, out oldSession); bool added = _sslSessions.TryAdd(targetName, session); Debug.Assert(added); } } + if (oldSession != IntPtr.Zero) + { + // remove old session also from the internal OpenSSL cache + // and drop reference count. Since SSL_CTX_remove_session + // will call session_remove_cb, we need to do this outside + // of _sslSessions lock to avoid deadlock with another thread + // which could be holding SSL_CTX lock and trying to acquire + // _sslSessions lock. + Interop.Ssl.SslCtxRemoveSession(this, oldSession); + Interop.Ssl.SessionFree(oldSession); + } + return true; } return false; } - internal void RemoveSession(IntPtr namePtr) + internal void RemoveSession(IntPtr namePtr, IntPtr session) { Debug.Assert(_sslSessions != null); @@ -171,11 +185,14 @@ internal void RemoveSession(IntPtr namePtr) if (_sslSessions != null && targetName != null) { - IntPtr oldSession; - bool removed; + IntPtr oldSession = IntPtr.Zero; + bool removed = false; lock (_sslSessions) { - removed = _sslSessions.Remove(targetName, out oldSession); + if (_sslSessions.TryGetValue(targetName, out IntPtr existingSession) && existingSession == session) + { + removed = _sslSessions.Remove(targetName, out oldSession); + } } if (removed) @@ -209,7 +226,6 @@ internal bool TrySetSession(SafeSslHandle sslHandle, string name) // This will increase reference count on the session as needed. // We need to hold lock here to prevent session being deleted before the call is done. Interop.Ssl.SslSetSession(sslHandle, session); - return true; } } diff --git a/src/libraries/Common/tests/System/Net/Security/FakeNtlmServer.cs b/src/libraries/Common/tests/System/Net/Security/FakeNtlmServer.cs index cb7a3a785e7faf..1117b3412f35a4 100644 --- a/src/libraries/Common/tests/System/Net/Security/FakeNtlmServer.cs +++ b/src/libraries/Common/tests/System/Net/Security/FakeNtlmServer.cs @@ -42,6 +42,8 @@ public FakeNtlmServer(NetworkCredential expectedCredential) public bool IsAuthenticated { get; private set; } public bool IsMICPresent { get; private set; } public string? ClientSpecifiedSpn { get; private set; } + public Flags InitialClientFlags { get; private set; } + public Flags NegotiatedFlags => _negotiatedFlags; private NetworkCredential _expectedCredential; @@ -83,7 +85,7 @@ private enum MessageType : uint } [Flags] - private enum Flags : uint + public enum Flags : uint { NegotiateUnicode = 0x00000001, NegotiateOEM = 0x00000002, @@ -177,17 +179,17 @@ private static ReadOnlySpan GetField(ReadOnlySpan payload, int field case MessageType.Negotiate: // We don't negotiate, we just verify Assert.True(incomingBlob.Length >= 32); - Flags flags = (Flags)BinaryPrimitives.ReadUInt32LittleEndian(incomingBlob.AsSpan(12, 4)); - Assert.Equal(_requiredFlags, (flags & _requiredFlags)); - Assert.True((flags & (Flags.NegotiateOEM | Flags.NegotiateUnicode)) != 0); - if (flags.HasFlag(Flags.NegotiateDomainSupplied)) + InitialClientFlags = (Flags)BinaryPrimitives.ReadUInt32LittleEndian(incomingBlob.AsSpan(12, 4)); + Assert.Equal(_requiredFlags, (InitialClientFlags & _requiredFlags)); + Assert.True((InitialClientFlags & (Flags.NegotiateOEM | Flags.NegotiateUnicode)) != 0); + if (InitialClientFlags.HasFlag(Flags.NegotiateDomainSupplied)) { string domain = Encoding.ASCII.GetString(GetField(incomingBlob, 16)); Assert.Equal(_expectedCredential.Domain, domain); } _expectedMessageType = MessageType.Authenticate; _negotiateMessage = incomingBlob; - return _challengeMessage = GenerateChallenge(flags); + return _challengeMessage = GenerateChallenge(InitialClientFlags); case MessageType.Authenticate: // Validate the authentication! diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs index 3d415f8e3690f1..7a5ff6136cae94 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs @@ -469,7 +469,7 @@ internal void WriteCentralDirectoryFileHeader() bool zip64Needed = false; - if (SizesTooLarge() + if (AreSizesTooLarge #if DEBUG_FORCE_ZIP64 || _archive._forceZip64 #endif @@ -490,7 +490,7 @@ internal void WriteCentralDirectoryFileHeader() } - if (_offsetOfLocalHeader > uint.MaxValue + if (IsOffsetTooLarge #if DEBUG_FORCE_ZIP64 || _archive._forceZip64 #endif @@ -797,7 +797,11 @@ private bool IsOpenable(bool needToUncompress, bool needToLoadIntoMemory, out st return true; } - private bool SizesTooLarge() => _compressedSize > uint.MaxValue || _uncompressedSize > uint.MaxValue; + private bool AreSizesTooLarge => _compressedSize > uint.MaxValue || _uncompressedSize > uint.MaxValue; + + private bool IsOffsetTooLarge => _offsetOfLocalHeader > uint.MaxValue; + + private bool ShouldUseZIP64 => AreSizesTooLarge || IsOffsetTooLarge; // return value is true if we allocated an extra field for 64 bit headers, un/compressed size private bool WriteLocalFileHeader(bool isEmptyFile) @@ -813,6 +817,9 @@ private bool WriteLocalFileHeader(bool isEmptyFile) bool zip64Used = false; uint compressedSizeTruncated, uncompressedSizeTruncated; + // save offset + _offsetOfLocalHeader = writer.BaseStream.Position; + // if we already know that we have an empty file don't worry about anything, just do a straight shot of the header if (isEmptyFile) { @@ -840,7 +847,7 @@ private bool WriteLocalFileHeader(bool isEmptyFile) { // We are in seekable mode so we will not need to write a data descriptor _generalPurposeBitFlag &= ~BitFlagValues.DataDescriptor; - if (SizesTooLarge() + if (ShouldUseZIP64 #if DEBUG_FORCE_ZIP64 || (_archive._forceZip64 && _archive.Mode == ZipArchiveMode.Update) #endif @@ -865,9 +872,6 @@ private bool WriteLocalFileHeader(bool isEmptyFile) } } - // save offset - _offsetOfLocalHeader = writer.BaseStream.Position; - // calculate extra field. if zip64 stuff + original extraField aren't going to fit, dump the original extraField, because this is more important int bigExtraFieldLength = (zip64Used ? zip64ExtraField.TotalSize : 0) + (_lhUnknownExtraFields != null ? ZipGenericExtraField.TotalSize(_lhUnknownExtraFields) : 0); @@ -964,7 +968,7 @@ private void WriteCrcAndSizesInLocalHeader(bool zip64HeaderUsed) long finalPosition = _archive.ArchiveStream.Position; BinaryWriter writer = new BinaryWriter(_archive.ArchiveStream); - bool zip64Needed = SizesTooLarge() + bool zip64Needed = ShouldUseZIP64 #if DEBUG_FORCE_ZIP64 || _archive._forceZip64 #endif @@ -1048,7 +1052,7 @@ private void WriteDataDescriptor() writer.Write(ZipLocalFileHeader.DataDescriptorSignature); writer.Write(_crc32); - if (SizesTooLarge()) + if (AreSizesTooLarge) { writer.Write(_compressedSize); writer.Write(_uncompressedSize); diff --git a/src/libraries/System.IO.Compression/tests/ZipArchive/zip_LargeFiles.cs b/src/libraries/System.IO.Compression/tests/ZipArchive/zip_LargeFiles.cs index d240a176b2b7a2..b4623e653af84d 100644 --- a/src/libraries/System.IO.Compression/tests/ZipArchive/zip_LargeFiles.cs +++ b/src/libraries/System.IO.Compression/tests/ZipArchive/zip_LargeFiles.cs @@ -1,48 +1,129 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Generic; -using System.Linq; +using System.Reflection; using Xunit; -namespace System.IO.Compression.Tests +namespace System.IO.Compression.Tests; + +[Collection(nameof(DisableParallelization))] +public class zip_LargeFiles : ZipFileTestBase { - [Collection(nameof(DisableParallelization))] - public class zip_LargeFiles : ZipFileTestBase + [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes + [OuterLoop("It requires almost 12 GB of free disk space")] + public static void UnzipOver4GBZipFile() { - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes - [OuterLoop("It requires almost 12 GB of free disk space")] - public static void UnzipOver4GBZipFile() + byte[] buffer = GC.AllocateUninitializedArray(1_000_000_000); // 1 GB + + string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); + DirectoryInfo tempDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "over4GB")); + + try + { + for (byte i = 0; i < 6; i++) + { + File.WriteAllBytes(Path.Combine(tempDir.FullName, $"{i}.test"), buffer); + } + + ZipFile.CreateFromDirectory(tempDir.FullName, zipArchivePath, CompressionLevel.NoCompression, includeBaseDirectory: false); + + using ZipArchive zipArchive = ZipFile.OpenRead(zipArchivePath); + foreach (ZipArchiveEntry entry in zipArchive.Entries) + { + using Stream entryStream = entry.Open(); + + Assert.True(entryStream.CanRead); + Assert.Equal(buffer.Length, entryStream.Length); + } + } + finally { - byte[] buffer = GC.AllocateUninitializedArray(1_000_000_000); // 1 GB + File.Delete(zipArchivePath); + + tempDir.Delete(recursive: true); + } + } + + private static void FillWithHardToCompressData(byte[] buffer) + { + Random.Shared.NextBytes(buffer); + } - string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); - DirectoryInfo tempDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "over4GB")); + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes + [OuterLoop("It requires 5~6 GB of free disk space and a lot of CPU time for compressed tests")] + [InlineData(false)] + [InlineData(true)] + public static void CheckZIP64VersionIsSet_ForSmallFilesAfterBigFiles(bool isCompressed) + { + // issue #94899 + + CompressionLevel compressLevel = isCompressed ? CompressionLevel.Optimal : CompressionLevel.NoCompression; + byte[] smallBuffer = GC.AllocateUninitializedArray(1000); + byte[] largeBuffer = GC.AllocateUninitializedArray(1_000_000_000); // ~1 GB + string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); + string LargeFileName = "largefile"; + string SmallFileName = "smallfile"; + uint ZipLocalFileHeader_OffsetToVersionFromHeaderStart = 4; + ushort Zip64Version = 45; - try + try + { + using FileStream fs = File.Open(zipArchivePath, FileMode.Create, FileAccess.ReadWrite); + + // Create + using (ZipArchive archive = new(fs, ZipArchiveMode.Create, true)) { - for (byte i = 0; i < 6; i++) + ZipArchiveEntry file = archive.CreateEntry(LargeFileName, compressLevel); + + using (Stream stream = file.Open()) { - File.WriteAllBytes(Path.Combine(tempDir.FullName, $"{i}.test"), buffer); + // Write 5GB of data + for (var i = 0; i < 5; i++) + { + if (isCompressed) + { + FillWithHardToCompressData(largeBuffer); + } + + stream.Write(largeBuffer); + } } - ZipFile.CreateFromDirectory(tempDir.FullName, zipArchivePath, CompressionLevel.NoCompression, includeBaseDirectory: false); + file = archive.CreateEntry(SmallFileName, compressLevel); - using ZipArchive zipArchive = ZipFile.OpenRead(zipArchivePath); - foreach (ZipArchiveEntry entry in zipArchive.Entries) + using (Stream stream = file.Open()) { - using Stream entryStream = entry.Open(); - - Assert.True(entryStream.CanRead); - Assert.Equal(buffer.Length, entryStream.Length); + stream.Write(smallBuffer); } } - finally + + fs.Position = 0; + + // Validate + using (ZipArchive archive = new(fs, ZipArchiveMode.Read)) { - File.Delete(zipArchivePath); + using var reader = new BinaryReader(fs); - tempDir.Delete(recursive: true); + FieldInfo offsetOfLHField = typeof(ZipArchiveEntry).GetField("_offsetOfLocalHeader", BindingFlags.NonPublic | BindingFlags.Instance); + + if (offsetOfLHField is null || offsetOfLHField.FieldType != typeof(long)) + { + Assert.Fail("Cannot find the private field of _offsetOfLocalHeader in ZipArchiveEntry or the type is not long. Code may be changed after the test is written."); + } + + foreach (ZipArchiveEntry entry in archive.Entries) + { + fs.Position = (long)offsetOfLHField.GetValue(entry) + ZipLocalFileHeader_OffsetToVersionFromHeaderStart; + ushort versionNeeded = reader.ReadUInt16(); + + // Version is not ZIP64 for files with Local Header at >4GB offset. + Assert.Equal(Zip64Version, versionNeeded); + } } } + finally + { + File.Delete(zipArchivePath); + } } } diff --git a/src/libraries/System.IO.Packaging/tests/LargeFilesTests.Net.cs b/src/libraries/System.IO.Packaging/tests/LargeFilesTests.Net.cs new file mode 100644 index 00000000000000..c073c09b8f92f7 --- /dev/null +++ b/src/libraries/System.IO.Packaging/tests/LargeFilesTests.Net.cs @@ -0,0 +1,97 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO.Compression; +using System.Net.Mime; +using System.Reflection; +using Xunit; + +namespace System.IO.Packaging.Tests; + +public partial class LargeFilesTests +{ + private static void FillWithHardToCompressData(byte[] buffer) + { + Random.Shared.NextBytes(buffer); + } + + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes + [InlineData(false)] + [InlineData(true)] + [OuterLoop("It requires 5~6 GB of free disk space and a lot of CPU time for compressed tests")] + public static void CheckZIP64VersionIsSet_ForSmallFilesAfterBigFiles(bool isCompressed) + { + // issue #94899 + + CompressionOption compressionOption = isCompressed ? CompressionOption.Normal : CompressionOption.NotCompressed; + byte[] smallBuffer = GC.AllocateUninitializedArray(1000); + byte[] largeBuffer = GC.AllocateUninitializedArray(1_000_000_000); // ~1 GB + string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); + Uri largePartUri = PackUriHelper.CreatePartUri(new Uri("large.bin", UriKind.Relative)); + Uri smallPartUri = PackUriHelper.CreatePartUri(new Uri("small.bin", UriKind.Relative)); + uint ZipLocalFileHeader_OffsetToVersionFromHeaderStart = 4; + ushort Zip64Version = 45; + + try + { + using FileStream fs = File.Open(zipArchivePath, FileMode.Create, FileAccess.ReadWrite); + + // Create + using (Package package = Package.Open(fs, FileMode.Create, FileAccess.Write)) + { + PackagePart partLarge = package.CreatePart(largePartUri, MediaTypeNames.Application.Octet, compressionOption); + + using (Stream streamLarge = partLarge.GetStream()) + { + // Write 5GB of data + + for (var i = 0; i < 5; i++) + { + if (isCompressed) + { + FillWithHardToCompressData(largeBuffer); + } + + streamLarge.Write(largeBuffer); + } + } + + PackagePart partSmall = package.CreatePart(smallPartUri, MediaTypeNames.Application.Octet, compressionOption); + + using (Stream streamSmall = partSmall.GetStream()) + { + streamSmall.Write(smallBuffer); + } + } + + + fs.Position = 0; + + // Validate + using (ZipArchive archive = new ZipArchive(fs, ZipArchiveMode.Read)) + { + using var reader = new BinaryReader(fs); + + FieldInfo offsetOfLHField = typeof(ZipArchiveEntry).GetField("_offsetOfLocalHeader", BindingFlags.NonPublic | BindingFlags.Instance); + + if (offsetOfLHField is null || offsetOfLHField.FieldType != typeof(long)) + { + Assert.Fail("Cannot find the private field of _offsetOfLocalHeader in ZipArchiveEntry or the type is not long. Code may be changed after the test is written."); + } + + foreach (ZipArchiveEntry entry in archive.Entries) + { + fs.Position = (long)offsetOfLHField.GetValue(entry) + ZipLocalFileHeader_OffsetToVersionFromHeaderStart; + ushort versionNeeded = reader.ReadUInt16(); + + // Version is not ZIP64 for files with Local Header at >4GB offset. + Assert.Equal(Zip64Version, versionNeeded); + } + } + } + finally + { + File.Delete(zipArchivePath); + } + } +} diff --git a/src/libraries/System.IO.Packaging/tests/LargeFilesTests.cs b/src/libraries/System.IO.Packaging/tests/LargeFilesTests.cs new file mode 100644 index 00000000000000..e5e2350d7632bd --- /dev/null +++ b/src/libraries/System.IO.Packaging/tests/LargeFilesTests.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace System.IO.Packaging.Tests; + +[Collection(nameof(DisableParallelization))] +public partial class LargeFileTests +{ + [Fact] + [OuterLoop] + public void VeryLargePart() + { + // FileAccess.Write is important, this tells ZipPackage to open the underlying ZipArchive in + // ZipArchiveMode.Create mode as opposed to ZipArchiveMode.Update + // When ZipArchive is opened in Create it will write entries directly to the zip stream + // When ZipArchive is opened in Update it will write uncompressed data to memory until + // the archive is closed. + using (Stream stream = new MemoryStream()) + { + Uri partUri = PackUriHelper.CreatePartUri(new Uri("test.bin", UriKind.Relative)); + + // should compress *very well* + byte[] buffer = new byte[1024 * 1024]; + for (int i = 0; i < buffer.Length; i++) + { + buffer[i] = (byte)(i % 2); + } + + const long SizeInMb = 6 * 1024; // 6GB + long totalLength = SizeInMb * buffer.Length; + + // issue on .NET Framework we cannot use FileAccess.Write on a ZipArchive + using (Package package = Package.Open(stream, FileMode.Create, PlatformDetection.IsNetFramework ? FileAccess.ReadWrite : FileAccess.Write)) + { + PackagePart part = package.CreatePart(partUri, + System.Net.Mime.MediaTypeNames.Application.Octet, + CompressionOption.Fast); + + + using (Stream partStream = part.GetStream()) + { + for (long i = 0; i < SizeInMb; i++) + { + partStream.Write(buffer, 0, buffer.Length); + } + } + } + + // reopen for read and make sure we can get the part length & data matches + stream.Seek(0, SeekOrigin.Begin); + using (Package readPackage = Package.Open(stream)) + { + PackagePart part = readPackage.GetPart(partUri); + + using (Stream partStream = part.GetStream()) + { + Assert.Equal(totalLength, partStream.Length); + byte[] readBuffer = new byte[buffer.Length]; + for (long i = 0; i < SizeInMb; i++) + { + int totalRead = 0; + while (totalRead < readBuffer.Length) + { + int actualRead = partStream.Read(readBuffer, totalRead, readBuffer.Length - totalRead); + Assert.InRange(actualRead, 1, readBuffer.Length - totalRead); + totalRead += actualRead; + } + + Assert.Equal(readBuffer.Length, totalRead); + Assert.True(buffer.AsSpan().SequenceEqual(readBuffer)); + } + } + } + } + } +} diff --git a/src/libraries/System.IO.Packaging/tests/System.IO.Packaging.Tests.csproj b/src/libraries/System.IO.Packaging/tests/System.IO.Packaging.Tests.csproj index 78bc14a789b086..c7bdadf60c902c 100644 --- a/src/libraries/System.IO.Packaging/tests/System.IO.Packaging.Tests.csproj +++ b/src/libraries/System.IO.Packaging/tests/System.IO.Packaging.Tests.csproj @@ -3,10 +3,14 @@ $(NetCoreAppCurrent);$(NetFrameworkCurrent) + + + + diff --git a/src/libraries/System.IO.Packaging/tests/Tests.cs b/src/libraries/System.IO.Packaging/tests/Tests.cs index 4d547db5e1c199..c3d8c6d3a39963 100644 --- a/src/libraries/System.IO.Packaging/tests/Tests.cs +++ b/src/libraries/System.IO.Packaging/tests/Tests.cs @@ -3771,74 +3771,6 @@ private void ForEachPartWithFileName(Package package, Actiontrue true true + true + 1 Provides a message handler for HttpClient based on the WinHTTP interface of Windows. While similar to HttpClientHandler, it provides developers more granular control over the application's HTTP communication than the HttpClientHandler. Commonly Used Types: diff --git a/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestState.cs b/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestState.cs index a0af9a557f5809..f8b01f140b75ec 100644 --- a/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestState.cs +++ b/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestState.cs @@ -157,6 +157,7 @@ public WinHttpTransportContext TransportContext public long CurrentBytesRead { get; set; } private GCHandle _cachedReceivePinnedBuffer; + private GCHandle _cachedSendPinnedBuffer; public void PinReceiveBuffer(byte[] buffer) { @@ -171,6 +172,19 @@ public void PinReceiveBuffer(byte[] buffer) } } + public void PinSendBuffer(byte[] buffer) + { + if (!_cachedSendPinnedBuffer.IsAllocated || _cachedSendPinnedBuffer.Target != buffer) + { + if (_cachedSendPinnedBuffer.IsAllocated) + { + _cachedSendPinnedBuffer.Free(); + } + + _cachedSendPinnedBuffer = GCHandle.Alloc(buffer, GCHandleType.Pinned); + } + } + #region IDisposable Members private void Dispose(bool disposing) { @@ -193,12 +207,18 @@ private void Dispose(bool disposing) { // This method only gets called when the WinHTTP request handle is fully closed and thus all // async operations are done. So, it is safe at this point to unpin the buffers and release - // the strong GCHandle for this object. + // the strong GCHandle for the pinned buffers. if (_cachedReceivePinnedBuffer.IsAllocated) { _cachedReceivePinnedBuffer.Free(); _cachedReceivePinnedBuffer = default(GCHandle); } + + if (_cachedSendPinnedBuffer.IsAllocated) + { + _cachedSendPinnedBuffer.Free(); + _cachedSendPinnedBuffer = default(GCHandle); + } #if DEBUG Interlocked.Increment(ref s_dbg_operationHandleFree); #endif diff --git a/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestStream.cs b/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestStream.cs index ebbb43eaeedb46..64288c537fa674 100644 --- a/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestStream.cs +++ b/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestStream.cs @@ -24,8 +24,6 @@ internal sealed class WinHttpRequestStream : Stream private readonly SafeWinHttpHandle _requestHandle; private readonly WinHttpChunkMode _chunkedMode; - private GCHandle _cachedSendPinnedBuffer; - internal WinHttpRequestStream(WinHttpRequestState state, WinHttpChunkMode chunkedMode) { _state = state; @@ -182,15 +180,7 @@ internal async Task EndUploadAsync(CancellationToken token) protected override void Dispose(bool disposing) { - if (!_disposed) - { - _disposed = true; - if (_cachedSendPinnedBuffer.IsAllocated) - { - _cachedSendPinnedBuffer.Free(); - } - } - + _disposed = true; base.Dispose(disposing); } @@ -234,16 +224,7 @@ private Task InternalWriteDataAsync(byte[] buffer, int offset, int count, { Debug.Assert(count > 0); - if (!_cachedSendPinnedBuffer.IsAllocated || _cachedSendPinnedBuffer.Target != buffer) - { - if (_cachedSendPinnedBuffer.IsAllocated) - { - _cachedSendPinnedBuffer.Free(); - } - - _cachedSendPinnedBuffer = GCHandle.Alloc(buffer, GCHandleType.Pinned); - } - + _state.PinSendBuffer(buffer); _state.TcsInternalWriteDataToRequestStream = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/ThrowHelper.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/ThrowHelper.cs index ec677f9f4e58d6..e99f5fda640e1d 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/ThrowHelper.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/ThrowHelper.cs @@ -27,13 +27,27 @@ internal static QuicException GetOperationAbortedException(string? message = nul return new QuicException(QuicError.OperationAborted, null, message ?? SR.net_quic_operationaborted); } - internal static bool TryGetStreamExceptionForMsQuicStatus(int status, [NotNullWhen(true)] out Exception? exception) + internal static bool TryGetStreamExceptionForMsQuicStatus(int status, [NotNullWhen(true)] out Exception? exception, bool streamWasSuccessfullyStarted = true, string? message = null) { if (status == QUIC_STATUS_ABORTED) { - // If status == QUIC_STATUS_ABORTED, we will receive an event later, which will complete the task source. - exception = null; - return false; + // Connection has been closed by the peer (either at transport or application level), + if (streamWasSuccessfullyStarted) + { + // we will receive an event later, which will complete the stream with concrete + // information why the connection was aborted. + exception = null; + return false; + } + else + { + // we won't be receiving any event callback for shutdown on this stream, so we don't + // necessarily know which error to report. So we throw an exception which we can distinguish + // at the caller (ConnectionAborted normally has App error code) and throw the correct + // exception from there. + exception = new QuicException(QuicError.ConnectionAborted, null, ""); + return true; + } } else if (status == QUIC_STATUS_INVALID_STATE) { @@ -43,13 +57,16 @@ internal static bool TryGetStreamExceptionForMsQuicStatus(int status, [NotNullWh } else if (StatusFailed(status)) { - exception = GetExceptionForMsQuicStatus(status); + exception = GetExceptionForMsQuicStatus(status, message: message); return true; } exception = null; return false; } + // see TryGetStreamExceptionForMsQuicStatus for explanation + internal static bool IsConnectionAbortedWhenStartingStreamException(Exception ex) => ex is QuicException qe && qe.QuicError == QuicError.ConnectionAborted && qe.ApplicationErrorCode is null; + internal static Exception GetExceptionForMsQuicStatus(int status, long? errorCode = default, string? message = null) { Exception ex = GetExceptionInternal(status, errorCode, message); diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs index a2ade033afe59f..13351faaa20c0b 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs @@ -98,6 +98,11 @@ static async ValueTask StartConnectAsync(QuicClientConnectionOpt /// private int _disposed; + /// + /// Completed when connection shutdown is initiated. + /// + private TaskCompletionSource _connectionCloseTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly ValueTaskSource _connectedTcs = new ValueTaskSource(); private readonly ValueTaskSource _shutdownTcs = new ValueTaskSource(); @@ -376,16 +381,22 @@ public async ValueTask OpenOutboundStreamAsync(QuicStreamType type, stream = new QuicStream(_handle, type, _defaultStreamErrorCode); await stream.StartAsync(cancellationToken).ConfigureAwait(false); } - catch + catch (Exception ex) { if (stream is not null) { await stream.DisposeAsync().ConfigureAwait(false); } + + // In case of an incoming race when the connection is closed by the peer just before we open the stream, + // we receive QUIC_STATUS_ABORTED from MsQuic, but we don't know how the connection was closed. We throw + // special exception and handle it here where we can determine the shutdown reason. + bool connectionAbortedByPeer = ThrowHelper.IsConnectionAbortedWhenStartingStreamException(ex); + // Propagate connection error if present. - if (_acceptQueue.Reader.Completion.IsFaulted) + if (_connectionCloseTcs.Task.IsFaulted || connectionAbortedByPeer) { - await _acceptQueue.Reader.Completion.ConfigureAwait(false); + await _connectionCloseTcs.Task.ConfigureAwait(false); } throw; } @@ -475,17 +486,21 @@ private unsafe int HandleEventShutdownInitiatedByTransport(ref SHUTDOWN_INITIATE { Exception exception = ExceptionDispatchInfo.SetCurrentStackTrace(ThrowHelper.GetExceptionForMsQuicStatus(data.Status, (long)data.ErrorCode)); _connectedTcs.TrySetException(exception); + _connectionCloseTcs.TrySetException(exception); _acceptQueue.Writer.TryComplete(exception); return QUIC_STATUS_SUCCESS; } private unsafe int HandleEventShutdownInitiatedByPeer(ref SHUTDOWN_INITIATED_BY_PEER_DATA data) { - _acceptQueue.Writer.TryComplete(ExceptionDispatchInfo.SetCurrentStackTrace(ThrowHelper.GetConnectionAbortedException((long)data.ErrorCode))); + Exception exception = ExceptionDispatchInfo.SetCurrentStackTrace(ThrowHelper.GetConnectionAbortedException((long)data.ErrorCode)); + _connectionCloseTcs.TrySetException(exception); + _acceptQueue.Writer.TryComplete(exception); return QUIC_STATUS_SUCCESS; } private unsafe int HandleEventShutdownComplete() { Exception exception = ExceptionDispatchInfo.SetCurrentStackTrace(ThrowHelper.GetOperationAbortedException()); + _connectionCloseTcs.TrySetException(exception); _acceptQueue.Writer.TryComplete(exception); _connectedTcs.TrySetException(exception); _shutdownTcs.TrySetResult(); diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicStream.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicStream.cs index 6165f2085cb5f0..82ee656dc6bdc6 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicStream.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicStream.cs @@ -161,13 +161,18 @@ internal unsafe QuicStream(MsQuicContextSafeHandle connectionHandle, QuicStreamT try { QUIC_HANDLE* handle; - ThrowHelper.ThrowIfMsQuicError(MsQuicApi.Api.StreamOpen( + int status = MsQuicApi.Api.StreamOpen( connectionHandle, type == QuicStreamType.Unidirectional ? QUIC_STREAM_OPEN_FLAGS.UNIDIRECTIONAL : QUIC_STREAM_OPEN_FLAGS.NONE, &NativeCallback, (void*)GCHandle.ToIntPtr(context), - &handle), - "StreamOpen failed"); + &handle); + + if (ThrowHelper.TryGetStreamExceptionForMsQuicStatus(status, out Exception? ex, streamWasSuccessfullyStarted: false, message: "StreamOpen failed")) + { + throw ex; + } + _handle = new MsQuicContextSafeHandle(handle, context, SafeHandleType.Stream, connectionHandle); } catch @@ -241,7 +246,8 @@ internal ValueTask StartAsync(CancellationToken cancellationToken = default) int status = MsQuicApi.Api.StreamStart( _handle, QUIC_STREAM_START_FLAGS.SHUTDOWN_ON_FAIL | QUIC_STREAM_START_FLAGS.INDICATE_PEER_ACCEPT); - if (ThrowHelper.TryGetStreamExceptionForMsQuicStatus(status, out Exception? exception)) + + if (ThrowHelper.TryGetStreamExceptionForMsQuicStatus(status, out Exception? exception, streamWasSuccessfullyStarted: false)) { _startedTcs.TrySetException(exception); } diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs index 51c4279308e56a..5627009287a209 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs @@ -1055,60 +1055,6 @@ public BufferSegment Append(ReadOnlyMemory memory) } } - [Fact] - [OuterLoop("May take several seconds")] - public async Task ByteMixingOrNativeAVE_MinimalFailingTest() - { - const int writeSize = 64 * 1024; - const int NumberOfWrites = 512; - byte[] data1 = new byte[writeSize * NumberOfWrites]; - byte[] data2 = new byte[writeSize * NumberOfWrites]; - Array.Fill(data1, (byte)1); - Array.Fill(data2, (byte)2); - - Task t1 = RunTest(data1); - Task t2 = RunTest(data2); - - async Task RunTest(byte[] data) - { - await RunClientServer( - iterations: 20, - serverFunction: async connection => - { - await using QuicStream stream = await connection.AcceptInboundStreamAsync(); - - byte[] buffer = new byte[data.Length]; - int bytesRead = await ReadAll(stream, buffer); - Assert.Equal(data.Length, bytesRead); - AssertExtensions.SequenceEqual(data, buffer); - - for (int pos = 0; pos < data.Length; pos += writeSize) - { - await stream.WriteAsync(data[pos..(pos + writeSize)]); - } - await stream.WriteAsync(Memory.Empty, completeWrites: true); - }, - clientFunction: async connection => - { - await using QuicStream stream = await connection.OpenOutboundStreamAsync(QuicStreamType.Bidirectional); - - for (int pos = 0; pos < data.Length; pos += writeSize) - { - await stream.WriteAsync(data[pos..(pos + writeSize)]); - } - await stream.WriteAsync(Memory.Empty, completeWrites: true); - - byte[] buffer = new byte[data.Length]; - int bytesRead = await ReadAll(stream, buffer); - Assert.Equal(data.Length, bytesRead); - AssertExtensions.SequenceEqual(data, buffer); - } - ); - } - - await (new[] { t1, t2 }).WhenAllOrAnyFailed(millisecondsTimeout: 1000000); - } - [Fact] public async Task ManagedAVE_MinimalFailingTest() { diff --git a/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs b/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs index 866a754af72922..00f3369e6acec8 100644 --- a/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs +++ b/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs @@ -262,8 +262,14 @@ public override void Dispose() { Debug.Assert(incomingBlob.IsEmpty); + Flags requiredFlags = s_requiredFlags; + if (_protectionLevel == ProtectionLevel.EncryptAndSign) + { + requiredFlags |= Flags.NegotiateSeal; + } + _negotiateMessage = new byte[sizeof(NegotiateMessage)]; - CreateNtlmNegotiateMessage(_negotiateMessage); + CreateNtlmNegotiateMessage(_negotiateMessage, requiredFlags); outgoingBlob = _negotiateMessage; statusCode = NegotiateAuthenticationStatusCode.ContinueNeeded; @@ -278,7 +284,7 @@ public override void Dispose() return outgoingBlob; } - private static unsafe void CreateNtlmNegotiateMessage(Span asBytes) + private static unsafe void CreateNtlmNegotiateMessage(Span asBytes, Flags requiredFlags) { Debug.Assert(HeaderLength == NtlmHeader.Length); Debug.Assert(asBytes.Length == sizeof(NegotiateMessage)); @@ -288,7 +294,7 @@ private static unsafe void CreateNtlmNegotiateMessage(Span asBytes) asBytes.Clear(); NtlmHeader.CopyTo(asBytes); message.Header.MessageType = MessageType.Negotiate; - message.Flags = s_requiredFlags; + message.Flags = requiredFlags; message.Version = s_version; } @@ -416,10 +422,23 @@ private unsafe void WriteChannelBindingHash(Span hashBuffer) { if (_channelBinding != null) { - IntPtr cbtData = _channelBinding.DangerousGetHandle(); - int cbtDataSize = _channelBinding.Size; - int written = MD5.HashData(new Span((void*)cbtData, cbtDataSize), hashBuffer); - Debug.Assert(written == MD5.HashSizeInBytes); + int appDataOffset = sizeof(SecChannelBindings); + IntPtr cbtData = (nint)_channelBinding.DangerousGetHandle() + appDataOffset; + int cbtDataSize = _channelBinding.Size - appDataOffset; + + // Channel bindings are calculated according to RFC 4121, section 4.1.1.2, + // so we need to include zeroed initiator fields and length prefix for the + // application data. + Span prefix = stackalloc byte[sizeof(uint) * 5]; + prefix.Clear(); + BinaryPrimitives.WriteInt32LittleEndian(prefix.Slice(sizeof(uint) * 4), cbtDataSize); + using (var md5 = IncrementalHash.CreateHash(HashAlgorithmName.MD5)) + { + md5.AppendData(prefix); + md5.AppendData(new Span((void*)cbtData, cbtDataSize)); + int written = md5.GetHashAndReset(hashBuffer); + Debug.Assert(written == MD5.HashSizeInBytes); + } } else { @@ -560,6 +579,13 @@ private static byte[] DeriveKey(ReadOnlySpan exportedSessionKey, ReadOnlyS return null; } + // We already negotiate signing, so we only need to check sealing/encryption. + if ((flags & Flags.NegotiateSeal) == 0 && _protectionLevel == ProtectionLevel.EncryptAndSign) + { + statusCode = NegotiateAuthenticationStatusCode.QopNotSupported; + return null; + } + ReadOnlySpan targetInfo = GetField(challengeMessage.TargetInfo, blob); byte[] targetInfoBuffer = ProcessTargetInfo(targetInfo, out DateTime time, out bool hasNbNames); @@ -594,7 +620,7 @@ private static byte[] DeriveKey(ReadOnlySpan exportedSessionKey, ReadOnlyS NtlmHeader.CopyTo(responseAsSpan); response.Header.MessageType = MessageType.Authenticate; - response.Flags = s_requiredFlags; + response.Flags = s_requiredFlags | (flags & Flags.NegotiateSeal); response.Version = s_version; // Calculate hash for hmac - same for lm2 and ntlm2 diff --git a/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.Unix.cs b/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.Unix.cs index ed1fe4e2e91937..4dba6bcfbd86af 100644 --- a/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.Unix.cs +++ b/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.Unix.cs @@ -551,7 +551,7 @@ Interop.NetSecurityNative.Status status } } - private NegotiateAuthenticationStatusCode InitializeSecurityContext( + private unsafe NegotiateAuthenticationStatusCode InitializeSecurityContext( ref SafeGssCredHandle credentialsHandle, ref SafeGssContextHandle? contextHandle, ref SafeGssNameHandle? targetNameHandle, @@ -594,7 +594,7 @@ private NegotiateAuthenticationStatusCode InitializeSecurityContext( { // If a TLS channel binding token (cbt) is available then get the pointer // to the application specific data. - int appDataOffset = Marshal.SizeOf(); + int appDataOffset = sizeof(SecChannelBindings); Debug.Assert(appDataOffset < channelBinding.Size); IntPtr cbtAppData = channelBinding.DangerousGetHandle() + appDataOffset; int cbtAppDataSize = channelBinding.Size - appDataOffset; diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/NegotiateStream.cs b/src/libraries/System.Net.Security/src/System/Net/Security/NegotiateStream.cs index f26a837e0c110f..8026a955635c7c 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/NegotiateStream.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/NegotiateStream.cs @@ -883,7 +883,16 @@ private async Task ReceiveBlobAsync(CancellationToken cancellationTo if (_framer.ReadHeader.MessageId == FrameHeader.HandshakeDoneId) { - _remoteOk = true; + if (HandshakeComplete && message.Length > 0) + { + Debug.Assert(_context != null); + _context.GetOutgoingBlob(message, out NegotiateAuthenticationStatusCode statusCode); + _remoteOk = statusCode is NegotiateAuthenticationStatusCode.Completed; + } + else + { + _remoteOk = true; + } } else if (_framer.ReadHeader.MessageId != FrameHeader.HandshakeId) { diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/Pal.Managed/SafeChannelBindingHandle.cs b/src/libraries/System.Net.Security/src/System/Net/Security/Pal.Managed/SafeChannelBindingHandle.cs index 35daf739b88af8..0b8699fd0eaa04 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/Pal.Managed/SafeChannelBindingHandle.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/Pal.Managed/SafeChannelBindingHandle.cs @@ -11,7 +11,7 @@ namespace System.Net.Security internal sealed class SafeChannelBindingHandle : ChannelBinding { private const int CertHashMaxSize = 128; - private static readonly int s_secChannelBindingSize = Marshal.SizeOf(); + private static unsafe int SecChannelBindingSize => sizeof(SecChannelBindings); private readonly int _cbtPrefixByteArraySize; internal int Length { get; private set; } @@ -36,8 +36,8 @@ internal unsafe SafeChannelBindingHandle(ChannelBindingKind kind) "tls-unique:"u8; _cbtPrefixByteArraySize = cbtPrefix.Length; - handle = Marshal.AllocHGlobal(s_secChannelBindingSize + _cbtPrefixByteArraySize + CertHashMaxSize); - IntPtr cbtPrefixPtr = handle + s_secChannelBindingSize; + handle = Marshal.AllocHGlobal(SecChannelBindingSize + _cbtPrefixByteArraySize + CertHashMaxSize); + IntPtr cbtPrefixPtr = handle + SecChannelBindingSize; cbtPrefix.CopyTo(new Span((byte*)cbtPrefixPtr, cbtPrefix.Length)); CertHashPtr = cbtPrefixPtr + _cbtPrefixByteArraySize; Length = CertHashMaxSize; @@ -46,12 +46,12 @@ internal unsafe SafeChannelBindingHandle(ChannelBindingKind kind) internal void SetCertHashLength(int certHashLength) { int cbtLength = _cbtPrefixByteArraySize + certHashLength; - Length = s_secChannelBindingSize + cbtLength; + Length = SecChannelBindingSize + cbtLength; SecChannelBindings channelBindings = new SecChannelBindings() { ApplicationDataLength = cbtLength, - ApplicationDataOffset = s_secChannelBindingSize + ApplicationDataOffset = SecChannelBindingSize }; Marshal.StructureToPtr(channelBindings, handle, true); } diff --git a/src/libraries/System.Net.Security/tests/UnitTests/NegotiateAuthenticationTests.cs b/src/libraries/System.Net.Security/tests/UnitTests/NegotiateAuthenticationTests.cs index ee909c5b091072..2bb25b13e6aa1d 100644 --- a/src/libraries/System.Net.Security/tests/UnitTests/NegotiateAuthenticationTests.cs +++ b/src/libraries/System.Net.Security/tests/UnitTests/NegotiateAuthenticationTests.cs @@ -190,6 +190,42 @@ public void NtlmIncorrectExchangeTest() Assert.False(fakeNtlmServer.IsAuthenticated); } + [ConditionalFact(nameof(IsNtlmAvailable))] + public void NtlmEncryptionTest() + { + using FakeNtlmServer fakeNtlmServer = new FakeNtlmServer(s_testCredentialRight); + + NegotiateAuthentication ntAuth = new NegotiateAuthentication( + new NegotiateAuthenticationClientOptions + { + Package = "NTLM", + Credential = s_testCredentialRight, + TargetName = "HTTP/foo", + RequiredProtectionLevel = ProtectionLevel.EncryptAndSign + }); + + NegotiateAuthenticationStatusCode statusCode; + byte[]? negotiateBlob = ntAuth.GetOutgoingBlob((byte[])null, out statusCode); + Assert.Equal(NegotiateAuthenticationStatusCode.ContinueNeeded, statusCode); + Assert.NotNull(negotiateBlob); + + byte[]? challengeBlob = fakeNtlmServer.GetOutgoingBlob(negotiateBlob); + Assert.NotNull(challengeBlob); + // Validate that the client sent NegotiateSeal flag + Assert.Equal(FakeNtlmServer.Flags.NegotiateSeal, (fakeNtlmServer.InitialClientFlags & FakeNtlmServer.Flags.NegotiateSeal)); + + byte[]? authenticateBlob = ntAuth.GetOutgoingBlob(challengeBlob, out statusCode); + Assert.Equal(NegotiateAuthenticationStatusCode.Completed, statusCode); + Assert.NotNull(authenticateBlob); + + byte[]? empty = fakeNtlmServer.GetOutgoingBlob(authenticateBlob); + Assert.Null(empty); + Assert.True(fakeNtlmServer.IsAuthenticated); + + // Validate that the NegotiateSeal flag survived the full exchange + Assert.Equal(FakeNtlmServer.Flags.NegotiateSeal, (fakeNtlmServer.NegotiatedFlags & FakeNtlmServer.Flags.NegotiateSeal)); + } + [ConditionalFact(nameof(IsNtlmAvailable))] [ActiveIssue("https://github.com/dotnet/runtime/issues/65678", TestPlatforms.OSX | TestPlatforms.iOS | TestPlatforms.MacCatalyst)] public void NtlmSignatureTest() @@ -218,7 +254,7 @@ public void NtlmSignatureTest() fakeNtlmServer.Unwrap(output.WrittenSpan, temp); Assert.Equal(s_Hello, temp); - // Test creating signature on server side and decoding it with VerifySignature on client side + // Test creating signature on server side and decoding it with VerifySignature on client side byte[] serverSignedMessage = new byte[16 + s_Hello.Length]; fakeNtlmServer.Wrap(s_Hello, serverSignedMessage); output.Clear(); diff --git a/src/libraries/System.Threading.Tasks.Dataflow/src/Base/DataflowBlock.cs b/src/libraries/System.Threading.Tasks.Dataflow/src/Base/DataflowBlock.cs index 63cdedf4b466a6..7007b0c80f5f7a 100644 --- a/src/libraries/System.Threading.Tasks.Dataflow/src/Base/DataflowBlock.cs +++ b/src/libraries/System.Threading.Tasks.Dataflow/src/Base/DataflowBlock.cs @@ -1434,63 +1434,48 @@ public static Task OutputAvailableAsync(this ISourceBlock OutputAvailableAsync( this ISourceBlock source, CancellationToken cancellationToken) { - if (source is null) - { - throw new ArgumentNullException(nameof(source)); - } - - // Fast path for cancellation - if (cancellationToken.IsCancellationRequested) - return Common.CreateTaskFromCancellation(cancellationToken); - - // In a method like this, normally we would want to check source.Completion.IsCompleted - // and avoid linking completely by simply returning a completed task. However, - // some blocks that are completed still have data available, like WriteOnceBlock, - // which completes as soon as it gets a value and stores that value forever. - // As such, OutputAvailableAsync must link from the source so that the source - // can push data to us if it has it, at which point we can immediately unlink. + return + source is null ? throw new ArgumentNullException(nameof(source)) : + cancellationToken.IsCancellationRequested ? Common.CreateTaskFromCancellation(cancellationToken) : + Impl(source, cancellationToken); - // Create a target task that will complete when it's offered a message (but it won't accept the message) - var target = new OutputAvailableAsyncTarget(); - try + static async Task Impl(ISourceBlock source, CancellationToken cancellationToken) { - // Link from the source. If the source propagates a message during or immediately after linking - // such that our target is already completed, just return its task. - target._unlinker = source.LinkTo(target, DataflowLinkOptions.UnlinkAfterOneAndPropagateCompletion); + // In a method like this, normally we would want to check source.Completion.IsCompleted + // and avoid linking completely by simply returning a completed task. However, + // some blocks that are completed still have data available, like WriteOnceBlock, + // which completes as soon as it gets a value and stores that value forever. + // As such, OutputAvailableAsync must link from the source so that the source + // can push data to us if it has it, at which point we can immediately unlink. - // If the task is already completed (an exception may have occurred, or the source may have propagated - // a message to the target during LinkTo or soon thereafter), just return the task directly. - if (target.Task.IsCompleted) - { - return target.Task; - } + // Create a target task that will complete when it's offered a message (but it won't accept the message) + var target = new OutputAvailableAsyncTarget(); - // If cancellation could be requested, hook everything up to be notified of cancellation requests. - if (cancellationToken.CanBeCanceled) + // Link from the source. + using (source.LinkTo(target, DataflowLinkOptions.UnlinkAfterOneAndPropagateCompletion)) { - // When cancellation is requested, unlink the target from the source and cancel the target. - target._ctr = cancellationToken.Register( + CancellationTokenRegistration registration = default; + try + { + // Register for cancellation if the target isn't already completed (the source may have propagated + // a message to the target during LinkTo or soon thereafter). + if (!target.Task.IsCompleted) + { + registration = #if NET6_0_OR_GREATER - OutputAvailableAsyncTarget.CancelAndUnlink, + cancellationToken.UnsafeRegister(static (state, cancellationToken) => ((OutputAvailableAsyncTarget)state!).TrySetCanceled(cancellationToken), target); #else - static state => OutputAvailableAsyncTarget.CancelAndUnlink(state, default), + cancellationToken.Register(static state => ((OutputAvailableAsyncTarget)state!).TrySetCanceled(), target); #endif - target); - } - - return target.Task; - } - catch (Exception exc) - { - // Source.LinkTo could throw, as could cancellationToken.Register if cancellation was already requested - // such that it synchronously invokes the source's unlinker IDisposable, which could throw. - target.TrySetException(exc); - - // Undo the link from the source to the target - target.AttemptThreadSafeUnlink(); + } - // Return the now faulted task - return target.Task; + return await target.Task.ConfigureAwait(false); + } + finally + { + registration.Dispose(); + } + } } } @@ -1504,46 +1489,6 @@ public OutputAvailableAsyncTarget() : { } - /// - /// Cached continuation delegate that unregisters from cancellation and - /// marshals the antecedent's result to the return value. - /// - internal static readonly Func, object?, bool> s_handleCompletion = (antecedent, state) => - { - var target = state as OutputAvailableAsyncTarget; - Debug.Assert(target != null, "Expected non-null target"); - target._ctr.Dispose(); - return antecedent.GetAwaiter().GetResult(); - }; - - /// Cancels the target and unlinks the target from the source. - /// An OutputAvailableAsyncTarget. - /// The token that triggered cancellation - internal static void CancelAndUnlink(object? state, CancellationToken cancellationToken) - { - var target = state as OutputAvailableAsyncTarget; - Debug.Assert(target != null, "Expected a non-null target"); - - target.TrySetCanceled(cancellationToken); - target.AttemptThreadSafeUnlink(); - } - - /// Disposes of _unlinker if the target has been linked. - internal void AttemptThreadSafeUnlink() - { - // A race is possible. Therefore use an interlocked operation. - IDisposable? cachedUnlinker = _unlinker; - if (cachedUnlinker != null && Interlocked.CompareExchange(ref _unlinker, null, cachedUnlinker) == cachedUnlinker) - { - cachedUnlinker.Dispose(); - } - } - - /// The IDisposable used to unlink this target from its source. - internal IDisposable? _unlinker; - /// The registration used to unregister this target from the cancellation token. - internal CancellationTokenRegistration _ctr; - /// Completes the task when offered a message (but doesn't consume the message). DataflowMessageStatus ITargetBlock.OfferMessage(DataflowMessageHeader messageHeader, T messageValue, ISourceBlock? source, bool consumeToAccept) { @@ -1551,14 +1496,12 @@ DataflowMessageStatus ITargetBlock.OfferMessage(DataflowMessageHeader message if (source == null) throw new ArgumentNullException(nameof(source)); TrySetResult(true); + return DataflowMessageStatus.DecliningPermanently; } /// - void IDataflowBlock.Complete() - { - TrySetResult(false); - } + void IDataflowBlock.Complete() => TrySetResult(false); /// void IDataflowBlock.Fault(Exception exception) @@ -1572,13 +1515,13 @@ void IDataflowBlock.Fault(Exception exception) } /// - Task IDataflowBlock.Completion { get { throw new NotSupportedException(SR.NotSupported_MemberNotNeeded); } } + Task IDataflowBlock.Completion => throw new NotSupportedException(SR.NotSupported_MemberNotNeeded); /// The data to display in the debugger display attribute. private object DebuggerDisplayContent => $"{Common.GetNameForDebugger(this)} IsCompleted = {base.Task.IsCompleted}"; /// Gets the data to display in the debugger display attribute for this instance. - object IDebuggerDisplay.Content { get { return DebuggerDisplayContent; } } + object IDebuggerDisplay.Content => DebuggerDisplayContent; } #endregion diff --git a/src/libraries/System.Threading.Tasks.Dataflow/src/System.Threading.Tasks.Dataflow.csproj b/src/libraries/System.Threading.Tasks.Dataflow/src/System.Threading.Tasks.Dataflow.csproj index 9835aa30e3b35a..3a4156130e82f9 100644 --- a/src/libraries/System.Threading.Tasks.Dataflow/src/System.Threading.Tasks.Dataflow.csproj +++ b/src/libraries/System.Threading.Tasks.Dataflow/src/System.Threading.Tasks.Dataflow.csproj @@ -2,6 +2,8 @@ $(NetCoreAppCurrent);$(NetCoreAppPrevious);$(NetCoreAppMinimum);netstandard2.1;netstandard2.0;$(NetFrameworkMinimum) true + true + 1 TPL Dataflow promotes actor/agent-oriented designs through primitives for in-process message passing, dataflow, and pipelining. TDF builds upon the APIs and scheduling infrastructure provided by the Task Parallel Library (TPL), and integrates with the language support for asynchrony provided by C#, Visual Basic, and F#. Commonly Used Types: diff --git a/src/mono/mono.proj b/src/mono/mono.proj index 3122cdf153cd17..a643e5a049e79c 100644 --- a/src/mono/mono.proj +++ b/src/mono/mono.proj @@ -810,6 +810,16 @@ + + + $(ROOTFS_HOST_DIR) + + @@ -859,7 +869,7 @@ <_MonoSkipInitCompiler Condition="'$(CrossBuild)' == 'true'">false <_MonoAotCrossOffsetsCommand Condition="'$(MonoUseCrossTool)' == 'true'">$(PythonCmd) $(MonoProjectRoot)mono/tools/offsets-tool/offsets-tool.py @(MonoAotCrossOffsetsToolParams, ' ') <_MonoAotCMakeConfigureCommand>cmake @(MonoAOTCMakeArgs, ' ') $(MonoCMakeExtraArgs) "$(MonoProjectRoot.TrimEnd('\/'))" - <_MonoAotCMakeConfigureCommand Condition="'$(_MonoSkipInitCompiler)' != 'true' and '$(HostOS)' != 'windows'">sh -c 'build_arch="$(_CompilerTargetArch)" compiler="$(MonoCCompiler)" . "$(RepositoryEngineeringCommonDir)native/init-compiler.sh" && @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand)' + <_MonoAotCMakeConfigureCommand Condition="'$(_MonoSkipInitCompiler)' != 'true' and '$(HostOS)' != 'windows'">sh -c 'build_arch="$(_CompilerTargetArch)" ROOTFS_DIR="$(MonoCrossDir)" compiler="$(MonoCCompiler)" . "$(RepositoryEngineeringCommonDir)native/init-compiler.sh" && @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand)' <_MonoAotCMakeConfigureCommand Condition="'$(_MonoSkipInitCompiler)' == 'true' and '$(HostOS)' != 'windows'">$(_MonoAOTCCOption) $(_MonoAOTCXXOption) @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand) <_MonoAotCMakeConfigureCommand Condition="'$(HostOS)' == 'windows'">call "$(RepositoryEngineeringDir)native\init-vs-env.cmd" $(_CompilerTargetArch) && cd /D "$(MonoObjCrossDir)" && @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand) <_MonoAotCMakeBuildCommand>cmake --build . --target install --config $(Configuration) diff --git a/src/mono/mono/mini/interp/interp-internals.h b/src/mono/mono/mini/interp/interp-internals.h index 4e0be7db043348..79b93dae1fb5f8 100644 --- a/src/mono/mono/mini/interp/interp-internals.h +++ b/src/mono/mono/mini/interp/interp-internals.h @@ -153,6 +153,7 @@ struct InterpMethod { unsigned int hasthis; // boolean MonoProfilerCallInstrumentationFlags prof_flags; InterpMethodCodeType code_type; + int ref_slot_offset; // GC visible pointer slot #ifdef ENABLE_EXPERIMENT_TIERED MiniTieredCounter tiered_counter; #endif diff --git a/src/mono/mono/mini/interp/interp.c b/src/mono/mono/mini/interp/interp.c index 620dc135d50f27..2942fb23d3f82f 100644 --- a/src/mono/mono/mini/interp/interp.c +++ b/src/mono/mono/mini/interp/interp.c @@ -4047,6 +4047,9 @@ mono_interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClause } cmethod = del_imethod; if (!is_multicast) { + int ref_slot_offset = frame->imethod->ref_slot_offset; + if (ref_slot_offset >= 0) + LOCAL_VAR (ref_slot_offset, gpointer) = del; if (cmethod->param_count == param_count + 1) { // Target method is static but the delegate has a target object. We handle // this separately from the case below, because, for these calls, the instance diff --git a/src/mono/mono/mini/interp/transform.c b/src/mono/mono/mini/interp/transform.c index 0d5ef5380de266..bdb2b3f45eaf08 100644 --- a/src/mono/mono/mini/interp/transform.c +++ b/src/mono/mono/mini/interp/transform.c @@ -412,6 +412,17 @@ create_interp_dummy_var (TransformData *td) td->locals [td->dummy_var].flags = INTERP_LOCAL_FLAG_GLOBAL; } +static int alloc_global_var_offset (TransformData *td, int var); + +static void +interp_create_ref_handle_var (TransformData *td) +{ + int var = create_interp_local_explicit (td, m_class_get_byval_arg (mono_defaults.int_class), sizeof (gpointer)); + td->locals [var].flags = INTERP_LOCAL_FLAG_GLOBAL; + alloc_global_var_offset (td, var); + td->ref_handle_var = var; +} + static int get_tos_offset (TransformData *td) { @@ -3798,6 +3809,10 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target td->last_ins->data [0] = get_data_item_index_imethod (td, mono_interp_get_imethod (target_method)); } else { if (is_delegate_invoke) { + // MINT_CALL_DELEGATE will store the delegate object into this slot so it is kept alive + // while the method is invoked + if (td->ref_handle_var == -1) + interp_create_ref_handle_var (td); interp_add_ins (td, MINT_CALL_DELEGATE); interp_ins_set_dreg (td->last_ins, dreg); interp_ins_set_sreg (td->last_ins, MINT_CALL_ARGS_SREG); @@ -4983,7 +4998,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, arg_locals = (guint32*) g_malloc ((!!signature->hasthis + signature->param_count) * sizeof (guint32)); /* Allocate locals to store inlined method args from stack */ for (int i = signature->param_count - 1; i >= 0; i--) { - MonoType *type = td->locals [td->sp [-1].local].type; + MonoType *type = get_type_from_stack (td->sp [-1].type, td->sp [-1].klass); local = create_interp_local (td, type); arg_locals [i + !!signature->hasthis] = local; store_local (td, local); @@ -11149,6 +11164,7 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG td->n_data_items = 0; td->max_data_items = 0; td->dummy_var = -1; + td->ref_handle_var = -1; td->data_items = NULL; td->data_hash = g_hash_table_new (NULL, NULL); #ifdef ENABLE_EXPERIMENT_TIERED @@ -11287,6 +11303,11 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG mono_interp_register_imethod_data_items (rtm->data_items, td->imethod_items); rtm->patchpoint_data = td->patchpoint_data; + if (td->ref_handle_var != -1) + rtm->ref_slot_offset = td->locals [td->ref_handle_var].offset; + else + rtm->ref_slot_offset = -1; + /* Save debug info */ interp_save_debug_info (rtm, header, td, td->line_numbers); diff --git a/src/mono/mono/mini/interp/transform.h b/src/mono/mono/mini/interp/transform.h index 2bfbe42390510e..c9ff2df80a598c 100644 --- a/src/mono/mono/mini/interp/transform.h +++ b/src/mono/mono/mini/interp/transform.h @@ -227,6 +227,7 @@ typedef struct gint32 max_stack_size; InterpLocal *locals; int dummy_var; + int ref_handle_var; int *local_ref_count; unsigned int il_locals_offset; unsigned int il_locals_size; diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest.pkgproj b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest.pkgproj index e327fcf2635a2a..b7d58cbebf6ba2 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest.pkgproj +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest.pkgproj @@ -42,6 +42,7 @@ + <_WorkloadManifestValues Include="NetVersion" Value="net8" /> <_WorkloadManifestValues Include="WorkloadVersion" Value="$(PackageVersion)" /> <_WorkloadManifestValues Include="PackageVersion" Value="$(PackageVersion)" /> <_WorkloadManifestValues Include="PackageVersionNet6" Value="$(PackageVersionNet6)" /> diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in index 076e642d2b6209..a6ca85da25d9bb 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in @@ -5,30 +5,30 @@ }, "workloads": { "wasm-tools": { - "description": ".NET WebAssembly build tools for net8.0", + "description": ".NET WebAssembly build tools for ${NetVersion}.0", "packs": [ - "Microsoft.NET.Runtime.WebAssembly.Sdk", - "Microsoft.NETCore.App.Runtime.Mono.browser-wasm", - "Microsoft.NETCore.App.Runtime.AOT.Cross.browser-wasm" + "Microsoft.NET.Runtime.WebAssembly.Sdk.${NetVersion}", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.browser-wasm", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.browser-wasm" ], "extends": [ "microsoft-net-runtime-mono-tooling", "microsoft-net-sdk-emscripten" ], "platforms": [ "win-x64", "win-arm64", "linux-x64", "linux-arm64", "osx-x64", "osx-arm64"] }, "wasm-experimental": { - "description": ".NET WebAssembly experimental tooling for net8.0", + "description": ".NET WebAssembly experimental tooling for ${NetVersion}.0", "packs": [ - "Microsoft.NET.Runtime.WebAssembly.Templates", - "Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm", + "Microsoft.NET.Runtime.WebAssembly.Templates.${NetVersion}", + "Microsoft.NETCore.App.Runtime.Mono.multithread.${NetVersion}.browser-wasm", ], "extends": [ "wasm-tools" ], "platforms": [ "win-x64", "win-arm64", "linux-x64", "linux-arm64", "osx-x64", "osx-arm64" ] }, "wasi-experimental": { - "description": ".NET WASI experimental for net8.0", + "description": ".NET WASI experimental for ${NetVersion}.0", "packs": [ - "Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk", - "Microsoft.NETCore.App.Runtime.Mono.wasi-wasm", - "Microsoft.NET.Runtime.WebAssembly.Templates" + "Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk.${NetVersion}", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.wasi-wasm", + "Microsoft.NET.Runtime.WebAssembly.Templates.${NetVersion}" ], "extends": [ "microsoft-net-runtime-mono-tooling" ], "platforms": [ "win-x64", "win-arm64", "linux-x64", "linux-arm64", "osx-x64", "osx-arm64" ] @@ -36,7 +36,7 @@ "mobile-librarybuilder": { "description": "Mobile SDK for building a self-contained .NET native library", "packs": [ - "Microsoft.NET.Runtime.LibraryBuilder.Sdk" + "Microsoft.NET.Runtime.LibraryBuilder.Sdk.${NetVersion}" ], "extends": [ "microsoft-net-runtime-android-aot", "microsoft-net-runtime-ios", "microsoft-net-runtime-maccatalyst", "microsoft-net-runtime-tvos" ], "platforms": [ "win-x64", "win-arm64", "osx-x64", "osx-arm64" ] @@ -45,10 +45,10 @@ "abstract": true, "description": "Android Mono Runtime", "packs": [ - "Microsoft.NETCore.App.Runtime.Mono.android-arm", - "Microsoft.NETCore.App.Runtime.Mono.android-arm64", - "Microsoft.NETCore.App.Runtime.Mono.android-x64", - "Microsoft.NETCore.App.Runtime.Mono.android-x86" + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-arm", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-x64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-x86" ], "extends": [ "microsoft-net-runtime-mono-tooling" ], "platforms": [ "win-x64", "win-arm64", "linux-x64", "osx-x64", "osx-arm64" ] @@ -57,10 +57,10 @@ "abstract": true, "description": "Android Mono AOT Workload", "packs": [ - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-x86", - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-x64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-arm", - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-arm64" + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-x86", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-x64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-arm", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-arm64" ], "extends": [ "microsoft-net-runtime-android" ], "platforms": [ "win-x64", "win-arm64", "linux-x64", "osx-x64", "osx-arm64" ] @@ -69,9 +69,9 @@ "abstract": true, "description": "iOS Mono Runtime and AOT Workload", "packs": [ - "Microsoft.NETCore.App.Runtime.AOT.Cross.ios-arm64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.iossimulator-arm64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.iossimulator-x64" + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.ios-arm64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.iossimulator-arm64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.iossimulator-x64" ], "extends": [ "runtimes-ios" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -80,9 +80,9 @@ "abstract": true, "description": "iOS Mono Runtime Packs", "packs": [ - "Microsoft.NETCore.App.Runtime.Mono.ios-arm64", - "Microsoft.NETCore.App.Runtime.Mono.iossimulator-arm64", - "Microsoft.NETCore.App.Runtime.Mono.iossimulator-x64" + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.ios-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.iossimulator-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.iossimulator-x64" ], "extends": [ "microsoft-net-runtime-mono-tooling" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -91,8 +91,8 @@ "abstract": true, "description": "MacCatalyst Mono Runtime and AOT Workload", "packs": [ - "Microsoft.NETCore.App.Runtime.AOT.Cross.maccatalyst-arm64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.maccatalyst-x64" + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.maccatalyst-arm64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.maccatalyst-x64" ], "extends": [ "runtimes-maccatalyst" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -101,8 +101,8 @@ "abstract": true, "description": "MacCatalyst Mono Runtime Packs", "packs": [ - "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-arm64", - "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-x64" + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.maccatalyst-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.maccatalyst-x64" ], "extends": [ "microsoft-net-runtime-mono-tooling" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -111,8 +111,8 @@ "abstract": true, "description": "MacOS CoreCLR and Mono Runtime Workload", "packs": [ - "Microsoft.NETCore.App.Runtime.Mono.osx-arm64", - "Microsoft.NETCore.App.Runtime.Mono.osx-x64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.osx-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.osx-x64", "Microsoft.NETCore.App.Runtime.osx-arm64", "Microsoft.NETCore.App.Runtime.osx-x64" ], @@ -123,9 +123,9 @@ "abstract": true, "description": "tvOS Mono Runtime and AOT Workload", "packs": [ - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvos-arm64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvossimulator-arm64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvossimulator-x64" + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvos-arm64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvossimulator-arm64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvossimulator-x64" ], "extends": [ "runtimes-tvos" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -134,9 +134,9 @@ "abstract": true, "description": "tvOS Mono Runtime Packs", "packs": [ - "Microsoft.NETCore.App.Runtime.Mono.tvos-arm64", - "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-arm64", - "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-x64" + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvos-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvossimulator-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvossimulator-x64" ], "extends": [ "microsoft-net-runtime-mono-tooling" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -154,53 +154,83 @@ "abstract": true, "description": "Shared native build tooling for Mono runtime", "packs": [ - "Microsoft.NET.Runtime.MonoAOTCompiler.Task", - "Microsoft.NET.Runtime.MonoTargets.Sdk" + "Microsoft.NET.Runtime.MonoAOTCompiler.Task.${NetVersion}", + "Microsoft.NET.Runtime.MonoTargets.Sdk.${NetVersion}" ] } }, "packs": { - "Microsoft.NET.Runtime.MonoAOTCompiler.Task": { + "Microsoft.NET.Runtime.MonoAOTCompiler.Task.${NetVersion}": { "kind": "Sdk", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.MonoAOTCompiler.Task" + } }, - "Microsoft.NET.Runtime.MonoTargets.Sdk": { + "Microsoft.NET.Runtime.MonoTargets.Sdk.${NetVersion}": { "kind": "Sdk", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.MonoTargets.Sdk" + } }, - "Microsoft.NET.Runtime.LibraryBuilder.Sdk": { + "Microsoft.NET.Runtime.LibraryBuilder.Sdk.${NetVersion}": { "kind": "Sdk", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.LibraryBuilder.Sdk" + } }, - "Microsoft.NET.Runtime.WebAssembly.Sdk": { + "Microsoft.NET.Runtime.WebAssembly.Sdk.${NetVersion}": { "kind": "Sdk", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.WebAssembly.Sdk" + } }, - "Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk": { + "Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk.${NetVersion}": { "kind": "Sdk", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk" + } }, - "Microsoft.NET.Runtime.WebAssembly.Templates": { + "Microsoft.NET.Runtime.WebAssembly.Templates.${NetVersion}": { "kind": "template", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.WebAssembly.Templates" + } }, - "Microsoft.NETCore.App.Runtime.Mono.android-arm": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-arm": { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.android-arm" + } }, - "Microsoft.NETCore.App.Runtime.Mono.android-arm64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-arm64": { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.android-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.android-x64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-x64": { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.android-x64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.android-x86": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-x86": { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.android-x86" + } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-x86": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-x86": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -212,7 +242,7 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.android-x86" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-x64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-x64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -224,7 +254,7 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.android-x64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-arm": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-arm": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -236,7 +266,7 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.android-arm" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -248,43 +278,64 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.android-arm64" } }, - "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-arm64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.maccatalyst-arm64": { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-x64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.maccatalyst-x64": { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-x64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.osx-arm64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.osx-arm64": { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.osx-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.osx-x64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.osx-x64": { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.osx-x64" + } }, "Microsoft.NETCore.App.Runtime.osx-arm64": { "kind": "framework", - "version": "${PackageVersion}", + "version": "${PackageVersion}" }, "Microsoft.NETCore.App.Runtime.osx-x64": { "kind": "framework", - "version": "${PackageVersion}", + "version": "${PackageVersion}" }, - "Microsoft.NETCore.App.Runtime.Mono.ios-arm64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.ios-arm64" : { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.ios-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.iossimulator-arm64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.iossimulator-arm64" : { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.iossimulator-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.iossimulator-x64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.iossimulator-x64" : { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.iossimulator-x64" + } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvos-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvos-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -292,19 +343,28 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.tvos-arm64", } }, - "Microsoft.NETCore.App.Runtime.Mono.tvos-arm64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvos-arm64" : { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.tvos-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-arm64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvossimulator-arm64" : { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-x64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvossimulator-x64" : { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-x64" + } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.maccatalyst-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.maccatalyst-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -312,7 +372,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.maccatalyst-arm64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.maccatalyst-x64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.maccatalyst-x64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -320,7 +380,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.maccatalyst-x64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvossimulator-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvossimulator-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -328,7 +388,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.tvossimulator-arm64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvossimulator-x64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvossimulator-x64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -344,7 +404,7 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.ios-arm64", } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.iossimulator-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.iossimulator-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -352,7 +412,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.iossimulator-arm64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.iossimulator-x64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.iossimulator-x64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -360,7 +420,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.iossimulator-x64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.browser-wasm": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.browser-wasm": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -372,17 +432,26 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.browser-wasm" } }, - "Microsoft.NETCore.App.Runtime.Mono.browser-wasm" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.browser-wasm" : { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.browser-wasm" + } }, - "Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm" : { + "Microsoft.NETCore.App.Runtime.Mono.multithread.${NetVersion}.browser-wasm" : { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm" + } }, - "Microsoft.NETCore.App.Runtime.Mono.wasi-wasm" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.wasi-wasm" : { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.wasi-wasm" + } }, "Microsoft.NETCore.App.Runtime.win-x64" : { "kind": "framework", diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in index 70658c5c21a1f7..313514d0b6574e 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in @@ -82,70 +82,70 @@ - + - - + + - - + + - - - - + + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - - + + + - - - - - + + + + + - - - - + + + + diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 52b2988f8dd3d3..7aa1203bd11551 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -257,7 +257,7 @@ <_EmccLDFlags Include="$(EmccLinkOptimizationFlag)" /> <_EmccLDFlags Include="@(_EmccCommonFlags)" /> - <_EmccLDFlags Include="-s EXPORT_ES6=1" /> + <_EmccLDFlags Include="-s EXPORT_ES6=1 -lexports.js" /> <_DriverCDependencies Include="$(_WasmPInvokeHPath);$(_WasmICallTablePath)" /> <_DriverCDependencies Include="$(_DriverGenCPath)" Condition="'$(_DriverGenCNeeded)' == 'true'" /> diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 3b75fa9feeb1f0..1790445cbc6726 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -370,7 +370,8 @@ $(CMakeConfigurationEmccFlags) -s ASSERTIONS=1 -O2 - $(CMakeConfigurationLinkFlags) -s EXPORT_ES6=1 + + $(CMakeConfigurationLinkFlags) -s EXPORT_ES6=1 -lexports.js $(CMakeConfigurationLinkFlags) -msimd128 $(CMakeConfigurationLinkFlags) -Wno-pthreads-mem-growth $(CMakeConfigurationLinkFlags) --emit-symbol-map diff --git a/src/native/eventpipe/ds-ipc-pal-namedpipe.c b/src/native/eventpipe/ds-ipc-pal-namedpipe.c index 01a12275a421b1..151b3d95cc4610 100644 --- a/src/native/eventpipe/ds-ipc-pal-namedpipe.c +++ b/src/native/eventpipe/ds-ipc-pal-namedpipe.c @@ -173,6 +173,27 @@ ds_ipc_free (DiagnosticsIpc *ipc) ep_rt_object_free (ipc); } +void +ds_ipc_reset (DiagnosticsIpc *ipc) +{ + if (!ipc) + return; + + if (ipc->pipe != INVALID_HANDLE_VALUE) { + DisconnectNamedPipe (ipc->pipe); + CloseHandle (ipc->pipe); + ipc->pipe = INVALID_HANDLE_VALUE; + } + + if (ipc->overlap.hEvent != INVALID_HANDLE_VALUE) { + CloseHandle (ipc->overlap.hEvent); + } + + memset(&ipc->overlap, 0, sizeof(OVERLAPPED)); // clear the overlapped objects state + ipc->overlap.hEvent = INVALID_HANDLE_VALUE; + ipc->is_listening = false; +} + int32_t ds_ipc_poll ( DiagnosticsIpcPollHandle *poll_handles_data, @@ -192,6 +213,10 @@ ds_ipc_poll ( // SERVER EP_ASSERT (poll_handles_data [i].ipc->mode == DS_IPC_CONNECTION_MODE_LISTEN); handles [i] = poll_handles_data [i].ipc->overlap.hEvent; + if (handles [i] == INVALID_HANDLE_VALUE) { + // Invalid handle, wait will fail. Signal error + poll_handles_data [i].events = DS_IPC_POLL_EVENTS_ERR; + } } else { // CLIENT bool success = true; diff --git a/src/native/eventpipe/ds-ipc-pal-socket.c b/src/native/eventpipe/ds-ipc-pal-socket.c index d93233c506b787..7ad0b0f5d4859c 100644 --- a/src/native/eventpipe/ds-ipc-pal-socket.c +++ b/src/native/eventpipe/ds-ipc-pal-socket.c @@ -1064,6 +1064,11 @@ ds_ipc_free (DiagnosticsIpc *ipc) ep_rt_object_free (ipc); } +void +ds_ipc_reset (DiagnosticsIpc *ipc) +{ +} + int32_t ds_ipc_poll ( DiagnosticsIpcPollHandle *poll_handles_data, diff --git a/src/native/eventpipe/ds-ipc-pal.h b/src/native/eventpipe/ds-ipc-pal.h index 98e0fba180e69b..8e246ed671955c 100644 --- a/src/native/eventpipe/ds-ipc-pal.h +++ b/src/native/eventpipe/ds-ipc-pal.h @@ -35,6 +35,9 @@ ds_ipc_alloc ( void ds_ipc_free (DiagnosticsIpc *ipc); +void +ds_ipc_reset (DiagnosticsIpc *ipc); + // Poll // Parameters: // - IpcPollHandle * poll_handles_data: Array of IpcPollHandles to poll diff --git a/src/native/eventpipe/ds-ipc.c b/src/native/eventpipe/ds-ipc.c index 1256e3b00333cb..d3b7292514dd07 100644 --- a/src/native/eventpipe/ds-ipc.c +++ b/src/native/eventpipe/ds-ipc.c @@ -839,7 +839,11 @@ listen_port_reset ( ds_ipc_error_callback_func callback) { EP_ASSERT (object != NULL); - return; +#ifdef _WIN32 + DiagnosticsListenPort *listen_port = (DiagnosticsListenPort *)object; + ds_ipc_reset (listen_port->port.ipc); + ds_ipc_listen (listen_port->port.ipc, callback); +#endif // _WIN32 } static DiagnosticsPortVtable listen_port_vtable = { diff --git a/src/native/libs/System.Security.Cryptography.Native/entrypoints.c b/src/native/libs/System.Security.Cryptography.Native/entrypoints.c index f69959f36a4b51..1045645f1713af 100644 --- a/src/native/libs/System.Security.Cryptography.Native/entrypoints.c +++ b/src/native/libs/System.Security.Cryptography.Native/entrypoints.c @@ -299,6 +299,7 @@ static const Entry s_cryptoNative[] = DllImportEntry(CryptoNative_IsSslStateOK) DllImportEntry(CryptoNative_SslCtxAddExtraChainCert) DllImportEntry(CryptoNative_SslCtxSetCaching) + DllImportEntry(CryptoNative_SslCtxRemoveSession) DllImportEntry(CryptoNative_SslCtxSetCiphers) DllImportEntry(CryptoNative_SslCtxSetDefaultOcspCallback) DllImportEntry(CryptoNative_SslCtxSetEncryptionPolicy) diff --git a/src/native/libs/System.Security.Cryptography.Native/opensslshim.h b/src/native/libs/System.Security.Cryptography.Native/opensslshim.h index 1f1b1851f098aa..f94ddc01274c44 100644 --- a/src/native/libs/System.Security.Cryptography.Native/opensslshim.h +++ b/src/native/libs/System.Security.Cryptography.Native/opensslshim.h @@ -525,6 +525,7 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len); REQUIRED_FUNCTION(SSL_CTX_new) \ REQUIRED_FUNCTION(SSL_CTX_sess_set_new_cb) \ REQUIRED_FUNCTION(SSL_CTX_sess_set_remove_cb) \ + REQUIRED_FUNCTION(SSL_CTX_remove_session) \ LIGHTUP_FUNCTION(SSL_CTX_set_alpn_protos) \ LIGHTUP_FUNCTION(SSL_CTX_set_alpn_select_cb) \ REQUIRED_FUNCTION(SSL_CTX_set_cipher_list) \ @@ -1040,6 +1041,7 @@ FOR_ALL_OPENSSL_FUNCTIONS #define SSL_CTX_new SSL_CTX_new_ptr #define SSL_CTX_sess_set_new_cb SSL_CTX_sess_set_new_cb_ptr #define SSL_CTX_sess_set_remove_cb SSL_CTX_sess_set_remove_cb_ptr +#define SSL_CTX_remove_session SSL_CTX_remove_session_ptr #define SSL_CTX_set_alpn_protos SSL_CTX_set_alpn_protos_ptr #define SSL_CTX_set_alpn_select_cb SSL_CTX_set_alpn_select_cb_ptr #define SSL_CTX_set_cipher_list SSL_CTX_set_cipher_list_ptr diff --git a/src/native/libs/System.Security.Cryptography.Native/pal_ssl.c b/src/native/libs/System.Security.Cryptography.Native/pal_ssl.c index e6bd41143c1655..e320d1c73d776b 100644 --- a/src/native/libs/System.Security.Cryptography.Native/pal_ssl.c +++ b/src/native/libs/System.Security.Cryptography.Native/pal_ssl.c @@ -701,6 +701,11 @@ int CryptoNative_SslCtxSetCaching(SSL_CTX* ctx, int mode, int cacheSize, int con return retValue; } +int CryptoNative_SslCtxRemoveSession(SSL_CTX* ctx, SSL_SESSION* session) +{ + return SSL_CTX_remove_session(ctx, session); +} + const char* CryptoNative_SslGetServerName(SSL* ssl) { return SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); diff --git a/src/native/libs/System.Security.Cryptography.Native/pal_ssl.h b/src/native/libs/System.Security.Cryptography.Native/pal_ssl.h index 3c63564cc4e59f..9c9d7026119c5c 100644 --- a/src/native/libs/System.Security.Cryptography.Native/pal_ssl.h +++ b/src/native/libs/System.Security.Cryptography.Native/pal_ssl.h @@ -167,6 +167,11 @@ Sets session caching. 0 is disabled. */ PALEXPORT int CryptoNative_SslCtxSetCaching(SSL_CTX* ctx, int mode, int cacheSize, int contextIdLength, uint8_t* contextId, SslCtxNewSessionCallback newSessionCb, SslCtxRemoveSessionCallback removeSessionCb); +/* +Removes a session from internal cache. +*/ +PALEXPORT int CryptoNative_SslCtxRemoveSession(SSL_CTX* ctx, SSL_SESSION* session); + /* Sets callback to log TLS session keys */