Skip to content

Commit 347ddcb

Browse files
dotnet-maestronatemcmaster
dotnet-maestro
authored and
natemcmaster
committed
Initialize eng/common/ folder from dotnet/arcade
1 parent 200c9e2 commit 347ddcb

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+5694
-0
lines changed

‎eng/common/CIBuild.cmd

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
@echo off
2+
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"

‎eng/common/CheckSymbols.ps1

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
param(
2+
[Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
3+
[Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
4+
[Parameter(Mandatory=$true)][string] $SymbolToolPath # Full path to directory where dotnet symbol-tool was installed
5+
)
6+
7+
Add-Type -AssemblyName System.IO.Compression.FileSystem
8+
9+
function FirstMatchingSymbolDescriptionOrDefault {
10+
param(
11+
[string] $FullPath, # Full path to the module that has to be checked
12+
[string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
13+
[string] $SymbolsPath
14+
)
15+
16+
$FileName = [System.IO.Path]::GetFileName($FullPath)
17+
$Extension = [System.IO.Path]::GetExtension($FullPath)
18+
19+
# Those below are potential symbol files that the `dotnet symbol` might
20+
# return. Which one will be returned depend on the type of file we are
21+
# checking and which type of file was uploaded.
22+
23+
# The file itself is returned
24+
$SymbolPath = $SymbolsPath + "\" + $FileName
25+
26+
# PDB file for the module
27+
$PdbPath = $SymbolPath.Replace($Extension, ".pdb")
28+
29+
# PDB file for R2R module (created by crossgen)
30+
$NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb")
31+
32+
# DBG file for a .so library
33+
$SODbg = $SymbolPath.Replace($Extension, ".so.dbg")
34+
35+
# DWARF file for a .dylib
36+
$DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf")
37+
38+
.\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
39+
40+
if (Test-Path $PdbPath) {
41+
return "PDB"
42+
}
43+
elseif (Test-Path $NGenPdb) {
44+
return "NGen PDB"
45+
}
46+
elseif (Test-Path $SODbg) {
47+
return "DBG for SO"
48+
}
49+
elseif (Test-Path $DylibDwarf) {
50+
return "Dwarf for Dylib"
51+
}
52+
elseif (Test-Path $SymbolPath) {
53+
return "Module"
54+
}
55+
else {
56+
return $null
57+
}
58+
}
59+
60+
function CountMissingSymbols {
61+
param(
62+
[string] $PackagePath # Path to a NuGet package
63+
)
64+
65+
# Ensure input file exist
66+
if (!(Test-Path $PackagePath)) {
67+
throw "Input file does not exist: $PackagePath"
68+
}
69+
70+
# Extensions for which we'll look for symbols
71+
$RelevantExtensions = @(".dll", ".exe", ".so", ".dylib")
72+
73+
# How many files are missing symbol information
74+
$MissingSymbols = 0
75+
76+
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
77+
$PackageGuid = New-Guid
78+
$ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid
79+
$SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols"
80+
81+
[System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
82+
83+
# Makes easier to reference `symbol tool`
84+
Push-Location $SymbolToolPath
85+
86+
Get-ChildItem -Recurse $ExtractPath |
87+
Where-Object {$RelevantExtensions -contains $_.Extension} |
88+
ForEach-Object {
89+
if ($_.FullName -Match "\\ref\\") {
90+
Write-Host "`t Ignoring reference assembly file" $_.FullName
91+
return
92+
}
93+
94+
$SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath
95+
$SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath
96+
97+
Write-Host -NoNewLine "`t Checking file" $_.FullName "... "
98+
99+
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
100+
Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")"
101+
}
102+
else {
103+
$MissingSymbols++
104+
105+
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
106+
Write-Host "No symbols found on MSDL or SymWeb!"
107+
}
108+
else {
109+
if ($SymbolsOnMSDL -eq $null) {
110+
Write-Host "No symbols found on MSDL!"
111+
}
112+
else {
113+
Write-Host "No symbols found on SymWeb!"
114+
}
115+
}
116+
}
117+
}
118+
119+
Pop-Location
120+
121+
return $MissingSymbols
122+
}
123+
124+
function CheckSymbolsAvailable {
125+
if (Test-Path $ExtractPath) {
126+
Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
127+
}
128+
129+
Get-ChildItem "$InputPath\*.nupkg" |
130+
ForEach-Object {
131+
$FileName = $_.Name
132+
133+
# These packages from Arcade-Services include some native libraries that
134+
# our current symbol uploader can't handle. Below is a workaround until
135+
# we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
136+
if ($FileName -Match "Microsoft\.DotNet\.Darc\.") {
137+
Write-Host "Ignoring Arcade-services file: $FileName"
138+
Write-Host
139+
return
140+
}
141+
elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") {
142+
Write-Host "Ignoring Arcade-services file: $FileName"
143+
Write-Host
144+
return
145+
}
146+
147+
Write-Host "Validating $FileName "
148+
$Status = CountMissingSymbols "$InputPath\$FileName"
149+
150+
if ($Status -ne 0) {
151+
Write-Error "Missing symbols for $Status modules in the package $FileName"
152+
}
153+
154+
Write-Host
155+
}
156+
}
157+
158+
CheckSymbolsAvailable

‎eng/common/PublishToPackageFeed.proj

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
2+
<Project Sdk="Microsoft.NET.Sdk">
3+
<!--
4+
This MSBuild file is intended to be used as the body of the default
5+
publishing release pipeline. The release pipeline will use this file
6+
to invoke the PushToStaticFeed task that will read the build asset
7+
manifest and publish the assets described in the manifest to
8+
informed target feeds.
9+
-->
10+
11+
<PropertyGroup>
12+
<TargetFramework>netcoreapp2.1</TargetFramework>
13+
</PropertyGroup>
14+
15+
<Import Project="$(MSBuildThisFileDirectory)DefaultVersions.props" Condition="Exists('$(MSBuildThisFileDirectory)DefaultVersions.props')" />
16+
17+
<!--
18+
This won't be necessary once we solve this issue:
19+
https://github.com/dotnet/arcade/issues/2266
20+
-->
21+
<Import Project="$(MSBuildThisFileDirectory)ArtifactsCategory.props" Condition="Exists('$(MSBuildThisFileDirectory)ArtifactsCategory.props')" />
22+
23+
<Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
24+
25+
<Target Name="PublishToFeed">
26+
<Error Condition="'$(ArtifactsCategory)' == ''" Text="ArtifactsCategory: The artifacts' category produced by the build wasn't provided." />
27+
<Error Condition="'$(AccountKeyToStaticFeed)' == ''" Text="AccountKeyToStaticFeed: Account key for target feed wasn't provided." />
28+
<Error Condition="'$(ManifestsBasePath)' == ''" Text="Full path to asset manifests directory wasn't provided." />
29+
<Error Condition="'$(BlobBasePath)' == '' AND '$(PackageBasePath)' == ''" Text="A valid full path to BlobBasePath of PackageBasePath is required." />
30+
31+
<ItemGroup>
32+
<!-- Include all manifests found in the manifest folder. -->
33+
<ManifestFiles Include="$(ManifestsBasePath)*.xml" />
34+
</ItemGroup>
35+
36+
<Error Condition="'@(ManifestFiles)' == ''" Text="No manifest file was found in the provided path: $(ManifestsBasePath)" />
37+
38+
<!--
39+
For now the type of packages being published will be informed for the whole build.
40+
Eventually this will be specified on a per package basis:
41+
TODO: https://github.com/dotnet/arcade/issues/2266
42+
-->
43+
<PropertyGroup>
44+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == '.NETCORE'">https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json</TargetStaticFeed>
45+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == '.NETCOREVALIDATION'">https://dotnetfeed.blob.core.windows.net/arcade-validation/index.json</TargetStaticFeed>
46+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETCORE'">https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json</TargetStaticFeed>
47+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETCORETOOLING'">https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json</TargetStaticFeed>
48+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ENTITYFRAMEWORKCORE'">https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json</TargetStaticFeed>
49+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETEXTENSIONS'">https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json</TargetStaticFeed>
50+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'CORECLR'">https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json</TargetStaticFeed>
51+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'CORESDK'">https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json</TargetStaticFeed>
52+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'TOOLSINTERNAL'">https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json</TargetStaticFeed>
53+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'TOOLSET'">https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json</TargetStaticFeed>
54+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'WINDOWSDESKTOP'">https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json</TargetStaticFeed>
55+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'NUGETCLIENT'">https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json</TargetStaticFeed>
56+
</PropertyGroup>
57+
58+
<Error
59+
Condition="'$(TargetStaticFeed)' == ''"
60+
Text="'$(ArtifactsCategory)' wasn't recognized as a valid artifact category. Valid categories are: '.NetCore' and '.NetCoreValidation'" />
61+
62+
<!-- Iterate publishing assets from each manifest file. -->
63+
<PushArtifactsInManifestToFeed
64+
ExpectedFeedUrl="$(TargetStaticFeed)"
65+
AccountKey="$(AccountKeyToStaticFeed)"
66+
BARBuildId="$(BARBuildId)"
67+
MaestroApiEndpoint="$(MaestroApiEndpoint)"
68+
BuildAssetRegistryToken="$(BuildAssetRegistryToken)"
69+
Overwrite="$(OverrideAssetsWithSameName)"
70+
PassIfExistingItemIdentical="$(PassIfExistingItemIdentical)"
71+
MaxClients="$(MaxParallelUploads)"
72+
UploadTimeoutInMinutes="$(MaxUploadTimeoutInMinutes)"
73+
AssetManifestPath="%(ManifestFiles.Identity)"
74+
BlobAssetsBasePath="$(BlobBasePath)"
75+
PackageAssetsBasePath="$(PackageBasePath)"/>
76+
</Target>
77+
78+
<ItemGroup>
79+
<PackageReference Include="Microsoft.DotNet.Build.Tasks.Feed" Version="$(MicrosoftDotNetBuildTasksFeedVersion)" />
80+
</ItemGroup>
81+
</Project>

‎eng/common/PublishToSymbolServers.proj

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
2+
<Project Sdk="Microsoft.NET.Sdk">
3+
<!--
4+
This MSBuild file is intended to be used as the body of the default
5+
publishing release pipeline. The release pipeline will use this file
6+
to invoke the PublishSymbols tasks to publish symbols to MSDL and SymWeb.
7+
8+
Parameters:
9+
10+
- PDBArtifactsDirectory : Full path to directory containing PDB files to be published.
11+
- BlobBasePath : Full path containing *.symbols.nupkg packages to be published.
12+
- DotNetSymbolServerTokenMsdl : PAT to access MSDL.
13+
- DotNetSymbolServerTokenSymWeb : PAT to access SymWeb.
14+
- DotNetSymbolExpirationInDays : Expiration days for published packages. Default is 3650.
15+
-->
16+
17+
<PropertyGroup>
18+
<TargetFramework>netcoreapp2.1</TargetFramework>
19+
</PropertyGroup>
20+
21+
<Import Project="$(NuGetPackageRoot)microsoft.symboluploader.build.task\$(SymbolUploaderVersion)\build\PublishSymbols.targets" />
22+
23+
<Target Name="PublishSymbols">
24+
<ItemGroup>
25+
<FilesToPublishToSymbolServer Include="$(PDBArtifactsDirectory)\*.pdb"/>
26+
<PackagesToPublishToSymbolServer Include="$(BlobBasePath)\*.symbols.nupkg"/>
27+
28+
<!--
29+
These packages from Arcade-Services include some native libraries that
30+
our current symbol uploader can't handle. Below is a workaround until
31+
we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
32+
-->
33+
<PackagesToPublishToSymbolServer Remove="$(BlobBasePath)\Microsoft.DotNet.Darc.*" />
34+
<PackagesToPublishToSymbolServer Remove="$(BlobBasePath)\Microsoft.DotNet.Maestro.Tasks.*" />
35+
</ItemGroup>
36+
37+
<PropertyGroup>
38+
<DotNetSymbolExpirationInDays Condition="'$(DotNetSymbolExpirationInDays)' == ''">3650</DotNetSymbolExpirationInDays>
39+
<PublishToSymbolServer>true</PublishToSymbolServer>
40+
<PublishToSymbolServer Condition="'@(FilesToPublishToSymbolServer)' == '' and '@(PackagesToPublishToSymbolServer)' == ''">false</PublishToSymbolServer>
41+
</PropertyGroup>
42+
43+
<Message
44+
Importance="High"
45+
Text="No symbol package(s) were found to publish."
46+
Condition="$(PublishToSymbolServer) == false" />
47+
48+
<!-- Symbol Uploader: MSDL -->
49+
<Message Importance="High" Text="Publishing symbol packages to MSDL ..." Condition="$(PublishToSymbolServer)" />
50+
<PublishSymbols PackagesToPublish="@(PackagesToPublishToSymbolServer)"
51+
FilesToPublish="@(FilesToPublishToSymbolServer)"
52+
PersonalAccessToken="$(DotNetSymbolServerTokenMsdl)"
53+
SymbolServerPath="https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection"
54+
ExpirationInDays="$(DotNetSymbolExpirationInDays)"
55+
VerboseLogging="true"
56+
DryRun="false"
57+
ConvertPortablePdbsToWindowsPdbs="false"
58+
PdbConversionTreatAsWarning=""
59+
Condition="$(PublishToSymbolServer)"/>
60+
61+
<!--
62+
Symbol Uploader: SymWeb
63+
Watson, VS insertion testings and the typical internal dev usage require SymWeb.
64+
Currently we need to call the task twice (https://github.com/dotnet/core-eng/issues/3489).
65+
-->
66+
<Message Importance="High" Text="Publishing symbol packages to SymWeb ..." Condition="$(PublishToSymbolServer)" />
67+
<PublishSymbols PackagesToPublish="@(PackagesToPublishToSymbolServer)"
68+
FilesToPublish="@(FilesToPublishToSymbolServer)"
69+
PersonalAccessToken="$(DotNetSymbolServerTokenSymWeb)"
70+
SymbolServerPath="https://microsoft.artifacts.visualstudio.com/DefaultCollection"
71+
ExpirationInDays="$(DotNetSymbolExpirationInDays)"
72+
VerboseLogging="true"
73+
DryRun="false"
74+
ConvertPortablePdbsToWindowsPdbs="false"
75+
PdbConversionTreatAsWarning=""
76+
Condition="$(PublishToSymbolServer)"/>
77+
</Target>
78+
79+
<ItemGroup>
80+
<PackageReference Include="Microsoft.SymbolUploader.Build.Task" Version="$(SymbolUploaderVersion)" />
81+
</ItemGroup>
82+
</Project>

‎eng/common/README.md

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Don't touch this folder
2+
3+
uuuuuuuuuuuuuuuuuuuu
4+
u" uuuuuuuuuuuuuuuuuu "u
5+
u" u$$$$$$$$$$$$$$$$$$$$u "u
6+
u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u
7+
u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
8+
u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
9+
u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
10+
$ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
11+
$ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
12+
$ $$$" ... "$... ...$" ... "$$$ ... "$$$ $
13+
$ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $
14+
$ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $
15+
$ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $
16+
$ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $
17+
$ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
18+
"u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
19+
"u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
20+
"u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
21+
"u "$$$$$$$$$$$$$$$$$$$$$$$$" u"
22+
"u "$$$$$$$$$$$$$$$$$$$$" u"
23+
"u """""""""""""""""" u"
24+
""""""""""""""""""""
25+
26+
!!! Changes made in this directory are subject to being overwritten by automation !!!
27+
28+
The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first.

0 commit comments

Comments
 (0)