Improve components infrastructure (#12145)

* Remove generated Web.JS javascript files files

* Build NodeJS project referenced by managed projects by default

* Build Web.JS before other CI jobs

- We can't build Web.JS on Linux Musl for example

* Make BuildAll not override explicit BuildJava/NodeJS/Native/Managed settings

* Capture CodeCheck logs for debugging

* Skip NodeJS projects when generation ref assemblies

* Do not build dependent projects if NodeJS is not built

* Make build fail immediately if BuildNodeJS is true but NodeJS is not installed

* Do not build nodejs in installer builds

Author: John Luo

.azure/pipelines/ci.yml

@@ -29,7 +29,31 @@ variables:
 - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
   - name: _BuildArgs
     value: ''
+
 jobs:
+# Build Web.JS
+- template: jobs/default-build.yml
+  parameters:
+    codeSign: true
+    jobName: WebJS_build
+    jobDisplayName: "Build: Web.JS"
+    agentOs: Linux
+    steps:
+    - script: ./build.sh
+          --ci
+          --projects $(Build.SourcesDirectory)/src/Components/Web.JS/Microsoft.AspNetCore.Components.Web.JS.npmproj
+          -bl:artifacts/log/build.linux-x64.binlog
+          $(_BuildArgs)
+      displayName: Run build.sh
+    - publish: src/Components/Web.JS/dist/
+      artifact: WebJS_Javascript
+    installJdk: false
+    artifacts:
+    - name: WebJS_Logs
+      path: artifacts/log/
+      publishOnError: true
+
+# Code check
 - template: jobs/default-build.yml
   parameters:
     jobName: Code_check
@@ -38,10 +62,16 @@ jobs:
     steps:
     - powershell: ./eng/scripts/CodeCheck.ps1 -ci
       displayName: Run eng/scripts/CodeCheck.ps1
+    artifacts:
+    - name: Code_Check_Logs
+      path: artifacts/log/
+      publishOnError: true
 
 # Build Windows (x64/x86)
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     codeSign: true
     jobName: Windows_build
     jobDisplayName: "Build: Windows x64/x86"
@@ -53,6 +83,10 @@ jobs:
     - script: "echo ##vso[build.addbuildtag]release-candidate"
       condition: and(ne(variables['Build.Reason'], 'PullRequest'), in(variables['DotNetFinalVersionKind'], 'release', 'prerelease'))
       displayName: 'Set CI tags'
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     # !!! NOTE !!! Some of these steps have disabled code signing.
     # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even
     # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing.
@@ -130,10 +164,17 @@ jobs:
 # Build Windows ARM
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     codeSign: true
     jobName: Windows_arm_build
     jobDisplayName: "Build: Windows ARM"
     agentOs: Windows
+    beforeBuild:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     buildArgs:
       -arch arm
       -sign
@@ -159,9 +200,16 @@ jobs:
 # Build MacOS
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     jobName: MacOs_x64_build
     jobDisplayName: "Build: macOS"
     agentOs: macOs
+    beforeBuild:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     buildArgs:
       --pack
       --all
@@ -171,6 +219,7 @@ jobs:
       -bl:artifacts/log/build.macos.binlog
       $(_BuildArgs)
     installNodeJs: false
+    installJdk: false
     artifacts:
     - name: MacOS_x64_Packages
       path: artifacts/packages/
@@ -188,11 +237,16 @@ jobs:
 # Build Linux x64
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     jobName: Linux_x64_build
     jobDisplayName: "Build: Linux x64"
     agentOs: Linux
-    installNodeJs: false
     steps:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     - script: ./build.sh
           --ci
           --arch x64
@@ -211,6 +265,7 @@ jobs:
           --arch x64 \
           --build-installers \
           --no-build-deps \
+          --no-build-nodejs \
           -p:OnlyPackPlatformSpecificPackages=true \
           -p:BuildRuntimeArchive=false \
           -p:LinuxInstallerType=deb \
@@ -224,12 +279,15 @@ jobs:
           --arch x64 \
           --build-installers \
           --no-build-deps \
+          --no-build-nodejs \
           -p:OnlyPackPlatformSpecificPackages=true \
           -p:BuildRuntimeArchive=false \
           -p:LinuxInstallerType=rpm \
           -bl:artifacts/log/build.rpm.binlog \
           $(_BuildArgs)
       displayName: Build RPM installers
+    installNodeJs: false
+    installJdk: false
     artifacts:
     - name: Linux_x64_Packages
       path: artifacts/packages/
@@ -247,9 +305,16 @@ jobs:
 # Build Linux ARM
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     jobName: Linux_arm_build
     jobDisplayName: "Build: Linux ARM"
     agentOs: Linux
+    beforeBuild:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     buildArgs:
       --arch arm
       --pack
@@ -260,6 +325,7 @@ jobs:
       -bl:artifacts/log/build.linux-arm.binlog
       $(_BuildArgs)
     installNodeJs: false
+    installJdk: false
     artifacts:
     - name: Linux_arm_Packages
       path: artifacts/packages/
@@ -277,9 +343,16 @@ jobs:
 # Build Linux ARM64
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     jobName: Linux_arm64_build
     jobDisplayName: "Build: Linux ARM64"
     agentOs: Linux
+    beforeBuild:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     buildArgs:
       --arch arm64
       --all
@@ -290,6 +363,7 @@ jobs:
       -bl:artifacts/log/build.arm64.binlog
       $(_BuildArgs)
     installNodeJs: false
+    installJdk: false
     artifacts:
     - name: Linux_arm64_Packages
       path: artifacts/packages/
@@ -307,9 +381,16 @@ jobs:
 # Build Linux Musl x64
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     jobName: Linux_musl_x64_build
     jobDisplayName: "Build: Linux Musl x64"
     agentOs: Linux
+    beforeBuild:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     buildScript: ./dockerbuild.sh alpine
     buildArgs:
       --ci
@@ -323,6 +404,7 @@ jobs:
       -bl:artifacts/log/build.musl.binlog
       $(_BuildArgs)
     installNodeJs: false
+    installJdk: false
     artifacts:
     - name: Linux_musl_x64_Packages
       path: artifacts/packages/
@@ -337,12 +419,19 @@ jobs:
   parameters:
     inputName: Linux_musl_x64
 
-# Build Linux Musl arm64
+# Build Linux Musl ARM64
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     jobName: Linux_musl_arm64_build
     jobDisplayName: "Build: Linux Musl ARM64"
     agentOs: Linux
+    beforeBuild:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     buildScript: ./dockerbuild.sh ubuntu-alpine37
     buildArgs:
       --ci
@@ -356,6 +445,7 @@ jobs:
       -bl:artifacts/log/build.musl.binlog
       $(_BuildArgs)
     installNodeJs: false
+    installJdk: false
     artifacts:
     - name: Linux_musl_arm64_Packages
       path: artifacts/packages/
@@ -373,6 +463,8 @@ jobs:
 # Test jobs
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     condition: ne(variables['SkipTests'], 'true')
     jobName: Windows_Test
     jobDisplayName: "Test: Windows Server 2016 x64"
@@ -382,6 +474,10 @@ jobs:
     beforeBuild:
     - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"
       displayName: Setup IISExpress test certificates and schema
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     afterBuild:
     - powershell: "& ./build.ps1 -CI -NoBuild -Test /p:RunFlakyTests=true"
       displayName: Run Flaky Tests
@@ -396,12 +492,18 @@ jobs:
 
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     condition: ne(variables['SkipTests'], 'true')
     jobName: Windows_Templates_Test
     jobDisplayName: "Test: Templates - Windows Server 2016 x64"
     agentOs: Windows
     isTestingJob: true
     steps:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     - script: ./build.cmd -ci -all -pack
       displayName: Build Repo
     - script: ./src/ProjectTemplates/build.cmd -ci -pack -NoRestore -NoBuilddeps "/p:RunTemplateTests=true /bl:artifacts/log/template.pack.binlog"
@@ -418,6 +520,8 @@ jobs:
 
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     condition: ne(variables['SkipTests'], 'true')
     jobName: MacOs_Test
     jobDisplayName: "Test: macOS 10.13"
@@ -427,6 +531,10 @@ jobs:
     beforeBuild:
     - bash: "./eng/scripts/install-nginx-mac.sh"
       displayName: Installing Nginx
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     afterBuild:
     - bash: ./build.sh --ci --pack --no-build --no-restore --no-build-deps "/bl:artifacts/log/packages.pack.binlog"
       displayName: Pack Packages (for Template tests)
@@ -445,6 +553,8 @@ jobs:
 
 - template: jobs/default-build.yml
   parameters:
+    dependsOn:
+    - WebJS_build
     condition: ne(variables['SkipTests'], 'true')
     jobName: Linux_Test
     jobDisplayName: "Test: Ubuntu 16.04 x64"
@@ -456,6 +566,10 @@ jobs:
       displayName: Installing Nginx
     - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p"
       displayName: Increase inotify limit
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: WebJS_Javascript
+        path: $(Build.SourcesDirectory)/src/Components/Web.JS/dist/
     afterBuild:
     - bash: ./build.sh --ci --pack --no-build --no-restore --no-build-deps "/bl:artifacts/log/packages.pack.binlog"
       displayName: Pack Packages (for Template tests)
@@ -489,6 +603,10 @@ jobs:
       chmod +x $HOME/bin/jq
       echo "##vso[task.prependpath]$HOME/bin"
     displayName: Install jq
+  - task: NodeTool@0
+    displayName: Install Node 10.x
+    inputs:
+      versionSpec: 10.x
   - task: UseDotNet@2
     displayName: 'Use .NET Core sdk'
     inputs:

.azure/pipelines/helix-test.yml

@@ -36,11 +36,10 @@ jobs:
     steps:
     - script: ./restore.sh -ci
       displayName: Restore
-    - script: ./build.sh -ci --arch arm64 -test --no-build-nodejs -projects $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true -bl
+    - script: ./build.sh -ci --arch arm64 -test -projects $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true -bl
       displayName: Run build.sh helix arm64 target
       env:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
-    installNodeJs: false
     artifacts:
     - name: Helix_arm64_logs
       path: artifacts/logs/

build.ps1

@@ -183,14 +183,18 @@ elseif ($Projects) {
 }
 # When adding new sub-group build flags, add them to this check.
 elseif((-not $BuildNative) -and (-not $BuildManaged) -and (-not $BuildNodeJS) -and (-not $BuildInstallers) -and (-not $BuildJava)) {
-    Write-Warning "No default group of projects was specified, so building the 'managed' subsets of projects. Run ``build.cmd -help`` for more details."
+    Write-Warning "No default group of projects was specified, so building the 'managed' and its dependent subsets of projects. Run ``build.cmd -help`` for more details."
 
     # This goal of this is to pick a sensible default for `build.cmd` with zero arguments.
     # Now that we support subfolder invokations of build.cmd, we will be pushing to have build.cmd build everything (-all) by default
 
     $BuildManaged = $true
 }
 
+if ($BuildManaged -and ($NoBuildNodeJS)) {
+    Write-Warning "Some managed projects that depend on NodeJS projects will be skipped since building NodeJS is disabled."
+}
+
 if ($BuildInstallers) { $MSBuildArguments += "/p:BuildInstallers=true" }
 if ($BuildManaged) { $MSBuildArguments += "/p:BuildManaged=true" }
 if ($BuildNative) { $MSBuildArguments += "/p:BuildNative=true" }

build.sh

@@ -213,14 +213,18 @@ elif [ ! -z "$build_projects" ]; then
 elif [ -z "$build_managed" ] && [ -z "$build_nodejs" ] && [ -z "$build_java" ] && [ -z "$build_native" ] && [ -z "$build_installers" ]; then
     # This goal of this is to pick a sensible default for `build.sh` with zero arguments.
     # We believe the most common thing our contributors will work on is C#, so if no other build group was picked, build the C# projects.
-    __warn "No default group of projects was specified, so building the 'managed' subset of projects. Run ``build.sh --help`` for more details."
+    __warn "No default group of projects was specified, so building the 'managed' and its dependent subset of projects. Run ``build.sh --help`` for more details."
     build_managed=true
 fi
 
 if [ "$build_deps" = false ]; then
     msbuild_args[${#msbuild_args[*]}]="-p:BuildProjectReferences=false"
 fi
 
+if [ "$build_nodejs" = false ] && [ "$build_managed" = true ]; then
+    __warn "Some managed projects that depend on NodeJS projects will be skipped since building NodeJS is disabled."
+fi
+
 # Only set these MSBuild properties if they were explicitly set by build parameters.
 [ ! -z "$build_java" ] && msbuild_args[${#msbuild_args[*]}]="-p:BuildJava=$build_java"
 [ ! -z "$build_native" ] && msbuild_args[${#msbuild_args[*]}]="-p:BuildNative=$build_native"