save and publish log from package publishing step (#6141)

save and publish log from package publishing step

Author: brettfo

.gitignore

@@ -1,10 +1,9 @@
 # output location
 artifacts/
 
-/tests/scripts/current
+packages/
 
-# auto-generated during the build
-global.json
+/tests/scripts/current
 
 # Patches that may have been generated by scripts.
 # (These aren't generally useful to commit directly; if anything, they should be applied.)

.vsts-signed.yaml

@@ -1,10 +1,17 @@
+variables:
+- name: PB_PublishBlobFeedUrl
+  value: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+- group: DotNet-Blob-Feed
+- name: PB_PublishBlobFeedKey
+  value: $(dotnetfeed-storage-access-key-1)
+
 jobs:
 - job: Full_Signed
   pool:
     name: VSEng-MicroBuildVS2017
   timeoutInMinutes: 300
   variables:
-    MSBuildConfiguration: 'Release'
+    BuildConfiguration: 'Release'
   steps:
   # Install Signing Plugin
   - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1
@@ -29,15 +36,44 @@ jobs:
     displayName: Publish nightly package to MyGet
     inputs:
       scriptName: 'setup\publish-assets.ps1'
-      arguments: '-binariesPath artifacts\bin -configuration $(MSBuildConfiguration) -branchName $(Build.SourceBranch) -apiKey $(FSharp.MyGetApiKey)'
+      arguments: '-binariesPath artifacts\bin -configuration $(BuildConfiguration) -branchName $(Build.SourceBranch) -apiKey $(FSharp.MyGetApiKey)'
     condition: and(succeeded(), contains(variables['PB_PublishType'], 'myget'))
 
-  # Publish packages to Azure Blob Storage
+  # Package publish
+  - task: CmdLine@1
+    displayName: Restore package publishing
+    inputs:
+      filename: '.nuget\NuGet.exe'
+      arguments: 'restore packages.config -PackagesDirectory packages -Source https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json'
+    condition: and(succeeded(), contains(variables['PB_PublishType'], 'blob'))
   - task: MSBuild@1
     displayName: Publish packages to Azure Blob Storage
     inputs:
       solution: PublishToBlob.proj
-      msbuildArguments: '/t:Build /p:ExpectedFeedUrl=$(PB_PublishBlobFeedUrl) /p:AccountKey=$(PB_PublishBlobFeedKey) /p:ManifestBranch=$(SourceBranch) /p:ManifestCommit=$(SourceVersion) /p:ManifestBuildId=$(OfficialBuildId)'
+      msbuildArguments: '/t:Build /p:Configuration=$(BuildConfiguration) /p:ExpectedFeedUrl=$(PB_PublishBlobFeedUrl) /p:AccountKey=$(PB_PublishBlobFeedKey) /p:ManifestRepouri=$(Build.Repository.Uri) /p:ManifestBranch=$(Build.SourceBranch) /p:ManifestCommit=$(Build.SourceVersion) /p:ManifestBuildId=$(Build.BuildNumber) /bl:$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)/pub/publish.binlog'
+    condition: and(succeeded(), contains(variables['PB_PublishType'], 'blob'))
+  - task: PublishBuildArtifacts@1
+    displayName: Publish publishing bin log
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)/pub'
+      ArtifactName: 'Publish_bin_log'
+      publishLocation: Container
+    continueOnError: true
+    condition: and(succeeded(), contains(variables['PB_PublishType'], 'blob'))
+  - task: CopyFiles@2
+    displayName: Gather Asset Manifests
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)/AssetManifest'
+      TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
+    continueOnError: true
+    condition: and(succeeded(), contains(variables['PB_PublishType'], 'blob'))
+  - task: PublishBuildArtifacts@1
+    displayName: Push Asset Manifests
+    inputs:
+      PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
+      PublishLocation: Container
+      ArtifactName: AssetManifests
+    continueOnError: true
     condition: and(succeeded(), contains(variables['PB_PublishType'], 'blob'))
 
   # Create static drop
@@ -69,7 +105,7 @@ jobs:
   - task: ms-vseng.MicroBuildTasks.4305a8de-ba66-4d8b-b2d1-0dc4ecbbf5e8.MicroBuildUploadVstsDropFolder@1
     displayName: Upload VSTS Drop
     inputs:
-      DropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(MSBuildConfiguration)\Insertion'
+      DropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(BuildConfiguration)\Insertion'
     condition: and(succeeded(), contains(variables['PB_PublishType'], 'vsts'))
 
   # Execute cleanup tasks
@@ -94,3 +130,11 @@ jobs:
       sourcePath: '$(DropRoot)\$(Build.DefinitionName)\$(Build.SourceBranchName)\$(Build.BuildNumber)\SymStore'
       usePat: false
     condition: and(succeeded(), contains(variables['PB_PublishType'], 'symweb'))
+
+- template: /eng/common/templates/job/publish-build-assets.yml
+  parameters:
+    dependsOn:
+    - Full_Signed
+    pool:
+      vmImage: vs2017-win2016
+    enablePublishBuildArtifacts: true

Makefile

@@ -8,10 +8,7 @@ all: proto restore build test
 tools:
 	$(CURDIR)/scripts/dotnet-install.sh --version $(DotNetVersion) --install-dir "$(DotNetToolPath)"
 
-global.json: tools
-	echo { \"sdk\": { \"version\": \"$(DotNetVersion)\" } }>global.json
-
-proto: global.json
+proto: tools
 	$(DotNetExe) build-server shutdown
 	$(DotNetExe) restore src/buildtools/buildtools.proj
 	$(DotNetExe) restore src/fsharp/FSharp.Build/FSharp.Build.fsproj
@@ -20,7 +17,7 @@ proto: global.json
 	$(DotNetExe) build src/fsharp/FSharp.Build/FSharp.Build.fsproj -f netstandard2.0 -c Proto
 	$(DotNetExe) build src/fsharp/fsc/fsc.fsproj -f netcoreapp2.1 -c Proto
 
-restore: global.json
+restore:
 	$(DotNetExe) restore src/fsharp/FSharp.Core/FSharp.Core.fsproj
 	$(DotNetExe) restore src/fsharp/FSharp.Build/FSharp.Build.fsproj
 	$(DotNetExe) restore src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj

NuGet.Config

@@ -3,4 +3,13 @@
   <solution>
     <add key="disableSourceControlIntegration" value="true" />
   </solution>
+  <!-- Only specify feed for Arcade SDK (see https://github.com/Microsoft/msbuild/issues/2982) -->
+  <packageSources>
+    <clear />
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
 </configuration>

PublishToBlob.proj

@@ -10,26 +10,33 @@
   <PropertyGroup>
     <FeedTasksPackage>Microsoft.DotNet.Build.Tasks.Feed</FeedTasksPackage>
     <!-- This version should be kept in sync with `packages.config` -->
-    <FeedTasksPackageVersion>2.1.0-prerelease-02419-02</FeedTasksPackageVersion>
+    <FeedTasksPackageVersion>2.2.0-beta.19066.1</FeedTasksPackageVersion>
   </PropertyGroup>
 
   <Import Project="$(MSBuildThisFileDirectory)packages\$(FeedTasksPackage).$(FeedTasksPackageVersion)\build\$(FeedTasksPackage).targets" />
 
   <ItemGroup>
-    <!-- this should pick up both *.nupkg and *.symbols.nupkg -->
-    <NuGetPackages Include="$(MSBuildThisFileDirectory)\artifacts\packages\$(Configuration)\Microsoft.FSharp.Compiler.*.nupkg" />
+    <!-- the string '-rtm-' is important, because that's the package with a per-build unique version number -->
+    <NuGetPackages Include="$(MSBuildThisFileDirectory)artifacts\packages\$(Configuration)\Microsoft.FSharp.Compiler.*-rtm-*.nupkg" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <ArtifactsLogDir>$(MSBuildThisFileDirectory)artifacts\log\$(Configuration)\</ArtifactsLogDir>
+    <Platform Condition="'$(Platform)' == ''">AnyCPU</Platform>
+    <PlatformName Condition="'$(PlatformName)' == ''">$(Platform)</PlatformName>
+    <AssetManifestFilePath>$(ArtifactsLogDir)AssetManifest\$(OS)-$(PlatformName).xml</AssetManifestFilePath>
+  </PropertyGroup>
+
   <Target Name="Build">
     <PushToBlobFeed ExpectedFeedUrl="$(ExpectedFeedUrl)"
                     AccountKey="$(AccountKey)"
                     ItemsToPush="@(NuGetPackages)"
-                    Overwrite="$(PublishOverwrite)"
+                    ManifestBuildData="Location=$(ExpectedFeedUrl)"
+                    ManifestRepoUri="$(ManifestRepoUri)"
                     ManifestBranch="$(ManifestBranch)"
                     ManifestBuildId="$(ManifestBuildId)"
                     ManifestCommit="$(ManifestCommit)"
-                    ManifestName="fsharp"
-                    SkipCreateManifest="false" />
+                    AssetManifestPath="$(AssetManifestFilePath)" />
   </Target>
 
 </Project>

build.cmd

@@ -613,10 +613,6 @@ if not exist %_ngenexe% echo Error: Could not find ngen.exe. && goto :failure
 
 echo ---------------- Done with prepare, starting package restore ----------------
 
-:: create a global.json
-set /p DOTNET_TOOLS_VERSION=<"%~dp0DotnetCLIToolsVersion.txt"
-echo { "sdk": { "version": "%DOTNET_TOOLS_VERSION%" } }>global.json
-
 :: Restore the Tools directory
 call "%~dp0init-tools.cmd"
 set _dotnetexe=%~dp0artifacts\toolset\dotnet\dotnet.exe

eng/common/CIBuild.cmd

@@ -0,0 +1,2 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"

eng/common/PublishBuildAssets.cmd

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0sdk-task.ps1""" -msbuildEngine dotnet -restore -projects PublishBuildAssets.proj -ci %*"
+exit /b %ErrorLevel%

eng/common/PublishToPackageFeed.proj

@@ -0,0 +1,37 @@
+<!--
+  This MSBuild file is intended to be used as the body of the default 
+  publishing release pipeline. The release pipeline will use this file
+  to invoke the PushToStaticFeed task that will read the build asset
+  manifest and publish the assets described in the manifest to
+  informed target feeds.
+-->
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+  </PropertyGroup>
+
+  <Import Project="$(MSBuildThisFileDirectory)MicrosoftDotNetBuildTasksFeedVersion.props" />
+  <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
+
+  <Target Name="PublishToFeed">
+    <Error Condition="'$(TargetStaticFeed)' == ''" Text="TargetStaticFeed: Target feed for publishing assets wasn't provided." />
+    <Error Condition="'$(AccountKeyToStaticFeed)' == ''" Text="AccountKeyToStaticFeed: Account key for target feed wasn't provided." />
+    <Error Condition="'$(FullPathAssetManifest)' == ''" Text="Full path to asset manifest wasn't provided." />
+    <Error Condition="'$(FullPathBlobBasePath)' == '' AND '$(FullPathPackageBasePath)' == ''" Text="A valid full path to BlobBasePath of PackageBasePath is required." />
+
+    <PushArtifactsInManifestToFeed
+      ExpectedFeedUrl="$(TargetStaticFeed)"
+      AccountKey="$(AccountKeyToStaticFeed)"
+      Overwrite="$(OverrideAssetsWithSameName)"
+      PassIfExistingItemIdentical="$(PassIfExistingItemIdentical)"
+      MaxClients="$(MaxParallelUploads)"
+      UploadTimeoutInMinutes="$(MaxUploadTimeoutInMinutes)"
+      AssetManifestPath="$(FullPathAssetManifest)"
+      BlobAssetsBasePath="$(FullPathBlobBasePath)"
+      PackageAssetsBasePath="$(FullPathPackageBasePath)" />
+  </Target>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.Build.Tasks.Feed" Version="$(MicrosoftDotNetBuildTasksFeedVersion)" />
+  </ItemGroup>
+</Project>

eng/common/README.md

@@ -0,0 +1,28 @@
+# Don't touch this folder
+
+                uuuuuuuuuuuuuuuuuuuu
+              u" uuuuuuuuuuuuuuuuuu "u
+            u" u$$$$$$$$$$$$$$$$$$$$u "u
+          u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u
+        u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+      u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+    u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+    $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+    $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+    $ $$$" ... "$...  ...$" ... "$$$  ... "$$$ $
+    $ $$$u `"$$$$$$$  $$$  $$$$$  $$  $$$  $$$ $
+    $ $$$$$$uu "$$$$  $$$  $$$$$  $$  """ u$$$ $
+    $ $$$""$$$  $$$$  $$$u "$$$" u$$  $$$$$$$$ $
+    $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $
+    $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+    "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+      "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+        "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+          "u "$$$$$$$$$$$$$$$$$$$$$$$$" u"
+            "u "$$$$$$$$$$$$$$$$$$$$" u"
+              "u """""""""""""""""" u"
+                """"""""""""""""""""
+
+!!! Changes made in this directory are subject to being overwritten by automation !!!
+
+The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first.