Skip to content

Commit 3085962

Browse files
crummelcrummel
authored
Always build portable runtime in addition to non-portable. (#1433)
* Make portable build before non-portable. * Use conditions on portable dependencies * Add portable versions to Version.Details.xml so we get git commit info for them. * Move GatherBuiltPackages after targets import so we override it, and clean up repos after WriteVersionsFile instead of CopyPackage. * ILAsm/ILDasm aren't produced in the always-portable proj. * Try not to break things for ARM. * Add an override for people who don't want to spend the extra time to build portable. * Exclude symbol packages to cut down on space requirements. * Up some timeouts for additional builds * More disk space logging * Try shallow clone and more agressive deletions
1 parent ce346d9 commit 3085962

16 files changed

+414
-254
lines changed

‎.vsts.pipelines/jobs/ci-linux.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
strategy:
1515
matrix: ${{ parameters.matrix }}
1616
pool: ${{ parameters.pool }}
17-
timeoutInMinutes: 270
17+
timeoutInMinutes: 300
1818
variables:
1919
# Prefix to distinguish artifacts from different legs.
2020
artifactName: ${{ format('{0} $(type)', parameters.job) }}
@@ -72,7 +72,7 @@ jobs:
7272
/p:AzDoPat=$(System.AccessToken)
7373
du -h $(rootDirectory) | sort -h | tail -n 50
7474
displayName: Build source-build
75-
timeoutInMinutes: 150
75+
timeoutInMinutes: 180
7676
7777
# Generate prebuilt burndown data
7878
- script: |
@@ -208,7 +208,7 @@ jobs:
208208
$poisonArg
209209
du -h $(rootDirectory) | sort -h | tail -n 50
210210
displayName: Build tarball
211-
timeoutInMinutes: 120
211+
timeoutInMinutes: 150
212212
condition: and(succeeded(), eq(variables['sb.tarball'], true))
213213
214214
# Run smoke tests.

‎.vsts.pipelines/jobs/ci-linux_bootstrap.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ jobs:
3535
type: Production
3636

3737
steps:
38+
- template: ../steps/cleanup-unneeded-files.yml
3839
- template: ../steps/docker-cleanup-linux.yml
3940

4041
- task: DownloadBuildArtifacts@0
@@ -49,7 +50,7 @@ jobs:
4950
set -ex
5051
df -h
5152
$(docker.run) $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
52-
git clone https://github.com/dotnet/source-build-reference-packages'
53+
git clone --depth 1 https://github.com/dotnet/source-build-reference-packages'
5354
du -h $(rootDirectory) | sort -h | tail -n 50
5455
displayName: clone source-build-reference-packages
5556
condition: always()

‎.vsts.pipelines/steps/run-bootstrap.yml

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
1-
# Delete some files from the default image since we're tight on disk space.
2-
# List is from https://github.com/microsoft/azure-pipelines-image-generation/pull/1231#issuecomment-537174621
3-
41
# Extract the tarball fetched from the previous Offline stage
52
steps:
63
- script: |
74
set -ex
5+
mount
86
df -h
97
$(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
108
mkdir -p /bst/sb/tarball
11-
tar -xvf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz" -C /bst/sb/tarball --no-same-owner --strip-components=1
9+
tar -xf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz" -C /bst/sb/tarball --no-same-owner --strip-components=1
1210
rm -rf "/bst/Tarball centos71 Offline"'
11+
df -h
1312
displayName: Unpack Tarball
1413
condition: always()
1514

@@ -19,6 +18,7 @@ steps:
1918
df -h
2019
$(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
2120
/bst/sb/tarball/tool-bootstrapping/bootstrap.sh /bst/bootstrap_dir /bst/sb/tarball /bst/source-build-reference-packages 1'
21+
df -h
2222
displayName: Build stage1
2323
condition: succeeded()
2424

@@ -105,7 +105,8 @@ steps:
105105
set -ex
106106
df -h
107107
$(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
108-
rm -rf /bst/source-build-reference-packages/*
108+
rm -rf /bst/source-build-reference-packages/*
109+
rm -rf /bst/source-build-reference-packages/.git
109110
pushd /bst/bootstrap_dir/reference-packages
110111
echo "Removing everything in reference-packages except artifacts"
111112
find . -type f -not -name '*SourceBuild.ReferencePackages.bootstrap.tar.gz' -delete
@@ -226,4 +227,4 @@ steps:
226227
inputs:
227228
PathtoPublish: $(Build.ArtifactStagingDirectory)/drop/tarball
228229
ArtifactName: Tarball Artifacts $(artifactName)
229-
ArtifactType: Container
230+
ArtifactType: Container

‎eng/Version.Details.xml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@
3232
<Sha>1ed7226e39b7214807318d2cc8ff37492527ffa1</Sha>
3333
<RepoName>coreclr</RepoName>
3434
</Dependency>
35+
<Dependency Name="Microsoft.NETCore.Runtime.CoreCLR" Version="3.1.0-rtm.19564.2" CoherentParentDependency="Microsoft.Private.CoreFx.NETCoreApp">
36+
<Uri>https://github.com/dotnet/coreclr</Uri>
37+
<Sha>1ed7226e39b7214807318d2cc8ff37492527ffa1</Sha>
38+
<RepoName>coreclr-portable</RepoName>
39+
</Dependency>
3540
<Dependency Name="NETStandard.Library" Version="2.1.0" CoherentParentDependency="Microsoft.Dotnet.Sdk.Internal">
3641
<Uri>https://github.com/dotnet/standard</Uri>
3742
<Sha>a5b5f2e1e369972c8ff1e2183979fab6099f52ef</Sha>
@@ -41,11 +46,21 @@
4146
<Sha>0f7f38c4fd323b26da10cce95f857f77f0f09b48</Sha>
4247
<RepoName>corefx</RepoName>
4348
</Dependency>
49+
<Dependency Name="Microsoft.Private.CoreFx.NETCoreApp" Version="4.7.0-rtm.19564.4" CoherentParentDependency="Microsoft.NETCore.App">
50+
<Uri>https://github.com/dotnet/corefx</Uri>
51+
<Sha>0f7f38c4fd323b26da10cce95f857f77f0f09b48</Sha>
52+
<RepoName>corefx-portable</RepoName>
53+
</Dependency>
4454
<Dependency Name="Microsoft.NETCore.App" Version="3.1.0-rtm.19565.2" CoherentParentDependency="Microsoft.Dotnet.Sdk.Internal">
4555
<Uri>https://github.com/dotnet/core-setup</Uri>
4656
<Sha>65f04fb6db7a5e198d05dbebd5c4ad21eb018f89</Sha>
4757
<RepoName>core-setup</RepoName>
4858
</Dependency>
59+
<Dependency Name="Microsoft.NETCore.App" Version="3.1.0-rtm.19565.2" CoherentParentDependency="Microsoft.Dotnet.Sdk.Internal">
60+
<Uri>https://github.com/dotnet/core-setup</Uri>
61+
<Sha>65f04fb6db7a5e198d05dbebd5c4ad21eb018f89</Sha>
62+
<RepoName>core-setup-portable</RepoName>
63+
</Dependency>
4964
<Dependency Name="Microsoft.AspNetCore.Mvc.Razor.Extensions" Version="3.1.0" CoherentParentDependency="Microsoft.AspNetCore.App.Ref">
5065
<Uri>https://github.com/aspnet/AspNetCore-Tooling</Uri>
5166
<Sha>29036777e7cd2172a81c45c69e0dc257e8b8184e</Sha>

‎repos/core-setup-portable.proj

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<PropertyGroup>
4+
<SourceDirectory>core-setup</SourceDirectory>
5+
<PortableBuild>true</PortableBuild>
6+
<BuildingPortableRuntime>true</BuildingPortableRuntime>
7+
</PropertyGroup>
8+
9+
<Import Project="core-setup.common.props" />
10+
11+
<ItemGroup>
12+
<RepositoryReference Include="coreclr" />
13+
<RepositoryReference Include="corefx" />
14+
<RepositoryReference Include="newtonsoft-json" />
15+
<RepositoryReference Include="newtonsoft-json901" />
16+
<RepositoryReference Include="roslyn" />
17+
<RepositoryReference Include="standard" />
18+
</ItemGroup>
19+
20+
<Target Name="CleanupRepoForNonPortableBuild" AfterTargets="RemoveBuiltPackagesFromCache">
21+
<ItemGroup>
22+
<DirsToDelete Include="$(ProjectDirectory)artifacts" Condition="Exists('$(ProjectDirectory)artifacts')" />
23+
<DirsToDelete Include="$(ProjectDirectory)bin" Condition="Exists('$(ProjectDirectory)bin')" />
24+
</ItemGroup>
25+
26+
<RemoveDir Directories="@(DirsToDelete)" />
27+
</Target>
28+
29+
<Import Project="core-setup.common.targets" />
30+
31+
<Target Name="GatherBuiltPackages" BeforeTargets="CopyPackage">
32+
<ItemGroup>
33+
<_BuiltPackages Include="$(ShippingPackagesOutput)/*linux*nupkg" Exclude="$(ShippingPackagesOutput)/*.symbols.nupkg" />
34+
<_BuiltPackages Include="$(NonShippingPackagesOutput)/*linux*nupkg" Exclude="$(NonShippingPackagesOutput)/*.symbols.nupkg" />
35+
</ItemGroup>
36+
</Target>
37+
</Project>

‎repos/core-setup.common.props

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))/dir.props" />
4+
5+
<PropertyGroup>
6+
<!-- core-sdk always wants a portable runtime on OSX and FreeBSD-->
7+
<OverrideTargetRid>$(TargetRid)</OverrideTargetRid>
8+
<OverrideTargetRid Condition="'$(TargetOS)' == 'OSX'">osx-x64</OverrideTargetRid>
9+
<OverrideTargetRid Condition="'$(TargetOS)' == 'FreeBSD'">freebsd-x64</OverrideTargetRid>
10+
<OverrideTargetRid Condition="'$(TargetOS)' == 'Linux' and ('$(PortableBuild)' == 'true' or '$(BuildingPortableRuntime)' == 'true')">linux-$(Platform)</OverrideTargetRid>
11+
<OverridePortable>$(PortableBuild)</OverridePortable>
12+
<OverridePortable Condition="'$(TargetOS)' == 'OSX' or '$(TargetOS)' == 'FreeBSD'">true</OverridePortable>
13+
<OverridePortable Condition="'$(BuildingPortableRuntime)' == 'true'">true</OverridePortable>
14+
15+
<BuildArguments>$(FlagParameterPrefix)restore $(FlagParameterPrefix)build</BuildArguments>
16+
<BuildArguments>$(BuildArguments) $(FlagParameterPrefix)configuration $(Configuration)</BuildArguments>
17+
<BuildArguments>$(BuildArguments) $(FlagParameterPrefix)ci</BuildArguments>
18+
<BuildArguments>$(BuildArguments) /p:PortableBuild=$(OverridePortable)</BuildArguments>
19+
<BuildArguments Condition="$(Platform.Contains('arm')) AND '$(BuildArchitecture)' != 'arm64'">$(BuildArguments) /p:TargetArchitecture=$(Platform) /p:DisableCrossgen=true /p:CrossBuild=true</BuildArguments>
20+
<BuildArguments>$(BuildArguments) /p:BuildDebPackage=false</BuildArguments>
21+
<BuildArguments>$(BuildArguments) /p:BuildAllPackages=true</BuildArguments>
22+
<BuildArguments>$(BuildArguments) /p:RestoreAllBuildRids=false</BuildArguments>
23+
<BuildArguments>$(BuildArguments) /p:OutputRid=$(OverrideTargetRid)</BuildArguments>
24+
<BuildArguments>$(BuildArguments) /p:DotNetOutputBlobFeedDir=$(SourceBuiltBlobFeedDir)</BuildArguments>
25+
<BuildArguments>$(BuildArguments) /m:1</BuildArguments>
26+
27+
<LogVerbosityOptOut>true</LogVerbosityOptOut>
28+
<BuildArguments>$(BuildArguments) $(FlagParameterPrefix)verbosity $(LogVerbosity)</BuildArguments>
29+
<BuildArguments>$(BuildArguments) $(FlagParameterPrefix)binaryLog</BuildArguments>
30+
<BuildArguments>$(BuildArguments) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg)</BuildArguments>
31+
<BuildArguments>$(BuildArguments) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg)</BuildArguments>
32+
33+
<BuildCommand>$(ProjectDirectory)/build$(ShellExtension) $(BuildArguments)</BuildCommand>
34+
<BuildCommand Condition="$(Platform.Contains('arm')) AND '$(BuildArchitecture)' != 'arm64'">$(ArmEnvironmentVariables) $(BuildCommand)</BuildCommand>
35+
36+
<OutputPlacementRepoApiImplemented>false</OutputPlacementRepoApiImplemented>
37+
<ShippingPackagesOutput>$(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/</ShippingPackagesOutput>
38+
<NonShippingPackagesOutput>$(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/</NonShippingPackagesOutput>
39+
40+
<NuGetConfigFile>$(ProjectDirectory)/NuGet.config</NuGetConfigFile>
41+
<DependencyVersionInputRepoApiImplemented>true</DependencyVersionInputRepoApiImplemented>
42+
43+
<EnvironmentRestoreSources>$(SourceBuiltPackagesPath)</EnvironmentRestoreSources>
44+
<EnvironmentRestoreSources>$(EnvironmentRestoreSources)%3B$(ShippingPackagesOutput)</EnvironmentRestoreSources>
45+
<EnvironmentRestoreSources>$(EnvironmentRestoreSources)%3B$(NonShippingPackagesOutput)</EnvironmentRestoreSources>
46+
<EnvironmentRestoreSources Condition="'$(OfflineBuild)' == 'true'">$(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath)</EnvironmentRestoreSources>
47+
<EnvironmentRestoreSources Condition="'$(OfflineBuild)' != 'true'">$(EnvironmentRestoreSources)%3Bhttps://api.nuget.org/v3/index.json</EnvironmentRestoreSources>
48+
<EnvironmentRestoreSources Condition="'$(OfflineBuild)' != 'true'">$(EnvironmentRestoreSources)%3Bhttps://dotnetfeed.blob.core.windows.net/dotnet-core/index.json</EnvironmentRestoreSources>
49+
<EnvironmentRestoreSources Condition="'$(OfflineBuild)' != 'true'">$(EnvironmentRestoreSources)%3Bhttps://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json</EnvironmentRestoreSources>
50+
51+
<!-- core-setup does not destabilize for intermediate servicing builds -->
52+
<IsStable>true</IsStable>
53+
</PropertyGroup>
54+
55+
<ItemGroup>
56+
<UseSourceBuiltSdkOverride Include="@(ArcadeSdkOverride)" />
57+
<UseSourceBuiltSdkOverride Include="@(ArcadePackagingOverride)" />
58+
</ItemGroup>
59+
60+
<ItemGroup>
61+
<EnvironmentVariables Include="RestoreSources=$(EnvironmentRestoreSources)" />
62+
<EnvironmentVariables Include="BuildInParallel=false" />
63+
</ItemGroup>
64+
65+
</Project>

‎repos/core-setup.common.targets

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
4+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))/dir.targets" />
5+
6+
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="PublishCoreSetupBinaries" />
7+
8+
<!-- Need to "publish" the tarballs for cli, until cli respects the assets blob -->
9+
<Target Name="PublishBinaries" AfterTargets="Package">
10+
<ItemGroup>
11+
<CoreSetupTarballs Include="$(SourceBuiltAssetsDir)dotnet-runtime-*$(TarBallExtension)" />
12+
<CoreSetupTarballs Include="$(SourceBuiltAssetsDir)dotnet-hostfxr-*$(TarBallExtension)" />
13+
</ItemGroup>
14+
<PublishCoreSetupBinaries Binaries="@(CoreSetupTarballs)"
15+
DestinationFolderTemplate="$(SourceBuiltRuntimeDir){version}/" >
16+
<Output TaskParameter="PublishedVersion" PropertyName="PublishedVersion" />
17+
</PublishCoreSetupBinaries>
18+
<Message Text="Published core-setup binaries version $(PublishedVersion)" Importance="High" />
19+
</Target>
20+
21+
<Target Name="CopyBinariesToBinFolder" AfterTargets="PublishBinaries">
22+
<ItemGroup>
23+
<_builtRuntimePackages Include="$(SourceBuiltAssetsDir)*.symbols.nupkg" />
24+
<_builtRuntimePackages>
25+
<TransformedFileName>$([System.String]::Copy('%(FileName)').Replace('symbols', 'nupkg'))</TransformedFileName>
26+
</_builtRuntimePackages>
27+
<BinariesToCopy Include="$(SourceBuiltAssetsDir)*.*" Exclude="$(SourceBuiltAssetsDir)*.nupkg;$(SourceBuiltAssetsDir)*.requires_nupkg_signing" />
28+
<BinariesToCopy Include="@(_builtRuntimePackages->'$(SourceBuiltPackagesPath)%(TransformedFileName)')" />
29+
</ItemGroup>
30+
31+
<Copy SourceFiles="@(BinariesToCopy)"
32+
DestinationFolder="$(OutputPath)runtime"
33+
Condition="'@(BinariesToCopy)'!=''" />
34+
</Target>
35+
36+
<Target Name="SetOutputList" AfterTargets="RepoBuild">
37+
<ItemGroup>
38+
<PackagesOutputList Include="$(ShippingPackagesOutput)" />
39+
<PackagesOutputList Include="$(NonShippingPackagesOutput)" />
40+
</ItemGroup>
41+
</Target>
42+
43+
</Project>

0 commit comments

Comments
 (0)