# This is a combination of 18 commits.

# This is the 1st commit message:

Remove CoreFX runtest dependency

This change will start using build-test.sh generatelayoutonly to build
the coreoverlay directory for use with runtest.sh.

# This is the commit message #2:

Missing ./

# This is the commit message #3:

Fix untar location

# This is the commit message #4:

Use portable builds

# This is the commit message #5:

Get ci green

# This is the commit message #6:

Small changes

# This is the commit message #7:

Small changes

# This is the commit message #8:

Small changes

# This is the commit message #9:

Add generateonly builds.sh

# This is the commit message #10:

Ensure correct test location

# This is the commit message #11:

netci change

# This is the commit message #12:

netci change

# This is the commit message #13:

Working on setting up build-test

# This is the commit message #14:

Remove generatelayoutonly

# This is the commit message #15:

Undo some of the changes to use generatelayoutonly

# This is the commit message #16:

Correctly remove build-test.sh invocation

# This is the commit message #17:

Fix build-test generatelayoutonly

# This is the commit message #18:

Fix a few netci issues

Author: jashook

build-test.sh

@@ -6,7 +6,7 @@ initHostDistroRid()
     if [ "$__HostOS" == "Linux" ]; then
         if [ -e /etc/os-release ]; then
             source /etc/os-release
-            if [[ $ID == "alpine" ]]; then
+            if [[ $ID == "alpine" || $ID == "rhel" ]]; then
                 # remove the last version digit
                 VERSION_ID=${VERSION_ID%.*}
             fi
@@ -18,13 +18,25 @@ initHostDistroRid()
             fi
         fi
     fi
+
+    if [ "$__HostOS" == "OSX" ]; then
+        __PortableBuild=1
+    fi
+
+    # Portable builds target the base RID
+    if [ "$__PortableBuild" == 1 ]; then
+        if [ "$__BuildOS" == "OSX" ]; then
+            export __HostDistroRid="osx-$__BuildArch"
+        fi
+    fi
+
     if [ "$__HostOS" == "FreeBSD" ]; then
         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
     fi
 
     if [ "$__HostDistroRid" == "" ]; then
-        echo "WARNING: Cannot determine runtime id for current distro."
+        echo "WARNING: Can not determine runtime id for current distro."
     fi
 }
 
@@ -33,8 +45,13 @@ initTargetDistroRid()
     if [ $__CrossBuild == 1 ]; then
         if [ "$__BuildOS" == "Linux" ]; then
             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
-                echo "WARNING: Can not determine runtime id for current distro."
-                export __DistroRid=""
+                if [ -e $ROOTFS_DIR/android_platform ]; then
+                    source $ROOTFS_DIR/android_platform
+                    export __DistroRid="$RID"
+                else
+                    echo "WARNING: Can not determine runtime id for current distro."
+                    export __DistroRid=""
+                fi
             else
                 source $ROOTFS_DIR/etc/os-release
                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
@@ -52,16 +69,21 @@ initTargetDistroRid()
     if [ "$__PortableBuild" == 1 ]; then
         if [ "$__BuildOS" == "Linux" ]; then
             export __DistroRid="linux-$__BuildArch"
+            export __RuntimeId="linux-$__BuildArch"
         elif [ "$__BuildOS" == "OSX" ]; then
             export __DistroRid="osx-$__BuildArch"
+            export __RuntimeId="osx-$__BuildArch"
+        elif [ "$__BuildOS" == "FreeBSD" ]; then
+            export __DistroRid="freebsd-$__BuildArch"
+            export __RuntimeId="freebsd-$__BuildArch"
         fi
     fi
 
-   if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
+    if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
      export __DistroRid="ubuntu.14.04-$__BuildArch"
-   fi
+    fi
 
-   echo "__DistroRid: " $__DistroRid
+    echo "__DistroRid: " $__DistroRid
 }
 
 isMSBuildOnNETCoreSupported()
@@ -89,7 +111,7 @@ isMSBuildOnNETCoreSupported()
     fi
 }
 
-build_Tests()
+generate_layout()
 {
     __TestDir=$__ProjectDir/tests
     __ProjectFilesDir=$__TestDir
@@ -108,10 +130,10 @@ build_Tests()
         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
     fi
 
-	echo "__BuildOS: ${__BuildOS}"
-	echo "__BuildArch: ${__BuildArch}"
-	echo "__BuildType: ${__BuildType}"
-	echo "__TestIntermediateDir: ${__TestIntermediateDir}"
+    echo "__BuildOS: ${__BuildOS}"
+    echo "__BuildArch: ${__BuildArch}"
+    echo "__BuildType: ${__BuildType}"
+    echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 
     if [ ! -f "$__TestBinDir" ]; then
         echo "Creating TestBinDir: ${__TestBinDir}"
@@ -132,27 +154,96 @@ build_Tests()
 
     build_Tests_internal "Restore_Product" "${__ProjectDir}/tests/build.proj" " -BatchRestorePackages" "Restore product binaries (build tests)"
 
-    build_Tests_internal "Tests_GenerateRuntimeLayout" "${__ProjectDir}/tests/runtest.proj" "-BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct" "Restore product binaries (run tests)"
-
     if [ -n "$__UpdateInvalidPackagesArg" ]; then
         __up=-updateinvalidpackageversion
     fi
 
+    echo "${__MsgPrefix}Creating test overlay..."
+
+    if [ -z "$XuintTestBinBase" ]; then
+      XuintTestBinBase=$__TestWorkingDir
+    fi
+
+    export CORE_ROOT=$XuintTestBinBase/Tests/Core_Root
+
+    if [ ! -f "${CORE_ROOT}" ]; then
+      mkdir -p $CORE_ROOT
+    else
+      rm -rf $CORE_ROOT/*
+    fi
+
+    build_Tests_internal "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "-testOverlay" "Creating test overlay"
+
+    if [ $__ZipTests -ne 0 ]; then
+        echo "${__MsgPrefix}ZIP tests packages..."
+        build_Tests_internal "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" " " "Prep test binaries for Helix publishing"
+    fi
+
+    # Make sure to copy over the pulled down packages
+    cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
+
     # Work hardcoded path around
     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets" ]; then
         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Core.targets" "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets"
     fi
     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.targets" ]; then
         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Targets" "${__BuildToolsDir}/Microsoft.CSharp.targets"
     fi
+}
+
+build_Tests()
+{
+    __TestDir=$__ProjectDir/tests
+    __ProjectFilesDir=$__TestDir
+    __TestBinDir=$__TestWorkingDir
+
+    if [ $__RebuildTests -ne 0 ]; then
+        if [ -d "${__TestBinDir}" ]; then
+            echo "Removing tests build dir: ${__TestBinDir}"
+            rm -rf $__TestBinDir
+        fi
+    fi
+
+    __CMakeBinDir="${__TestBinDir}"
+
+    if [ -z "$__TestIntermediateDir" ]; then
+        __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
+    fi
+
+    echo "__BuildOS: ${__BuildOS}"
+    echo "__BuildArch: ${__BuildArch}"
+    echo "__BuildType: ${__BuildType}"
+    echo "__TestIntermediateDir: ${__TestIntermediateDir}"
+
+    if [ ! -f "$__TestBinDir" ]; then
+        echo "Creating TestBinDir: ${__TestBinDir}"
+        mkdir -p $__TestBinDir
+    fi
+    if [ ! -f "$__LogsDir" ]; then
+        echo "Creating LogsDir: ${__LogsDir}"
+        mkdir -p $__LogsDir
+    fi
+
+    __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
+
+    # =========================================================================================
+    # ===
+    # === Restore product binaries from packages
+    # ===
+    # =========================================================================================
+
+    if [ -n "$__BuildAgainstPackagesArg" ]; then
+        build_Tests_internal "Restore_Product" "${__ProjectDir}/tests/build.proj" " -BatchRestorePackages" "Restore product binaries (build tests)"
+        build_Tests_internal "Tests_GenerateRuntimeLayout" "${__ProjectDir}/tests/runtest.proj" "-BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct" "Restore product binaries (run tests)"
+    fi
 
     echo "Starting the Managed Tests Build..."
 
     __ManagedTestBuiltMarker=${__TestBinDir}/managed_test_build
 
     if [ ! -f $__ManagedTestBuiltMarker ]; then
 
-	    build_Tests_internal "Tests_Managed" "$__ProjectDir/tests/build.proj" "$__up" "Managed tests build (build tests)"
+        build_Tests_internal "Tests_Managed" "$__ProjectDir/tests/build.proj" "$__up" "Managed tests build (build tests)"
 
         if [ $? -ne 0 ]; then
             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
@@ -188,6 +279,10 @@ build_Tests()
         fi
     fi
 
+    if [ -n "$__UpdateInvalidPackagesArg" ]; then
+        __up=-updateinvalidpackageversion
+    fi
+
     echo "${__MsgPrefix}Creating test overlay..."
 
     if [ -z "$XuintTestBinBase" ]; then
@@ -202,25 +297,34 @@ build_Tests()
       rm -rf $CORE_ROOT/*
     fi
 
-    cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
-
-    build_Tests_internal "Tests_Overlay_Managed" "$__ProjectDir/tests/runtest.proj" "-testOverlay" "Creating test overlay"
+    build_Tests_internal "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "-testOverlay" "Creating test overlay"
 
     if [ $__ZipTests -ne 0 ]; then
         echo "${__MsgPrefix}ZIP tests packages..."
         build_Tests_internal "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" " " "Prep test binaries for Helix publishing"
     fi
+
+    # Make sure to copy over the pulled down packages
+    cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
+
+    # Work hardcoded path around
+    if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets" ]; then
+        ln -s "${__BuildToolsDir}/Microsoft.CSharp.Core.targets" "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets"
+    fi
+    if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.targets" ]; then
+        ln -s "${__BuildToolsDir}/Microsoft.CSharp.Targets" "${__BuildToolsDir}/Microsoft.CSharp.targets"
+    fi
 }
 
 build_Tests_internal()
 {
-	subDirectoryName=$1
-	projectName=$2
-	extraBuildParameters=$3
-	stepName="$4"
+    subDirectoryName=$1
+    projectName=$2
+    extraBuildParameters=$3
+    stepName="$4"
 
-	# Set up directories and file names
-	__BuildLogRootName=$subDirectoryName
+    # Set up directories and file names
+    __BuildLogRootName=$subDirectoryName
     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
@@ -262,8 +366,11 @@ usage()
     echo "crosscomponent - optional argument to build cross-architecture component,"
     echo "               - will use CAC_ROOTFS_DIR environment variable if set."
     echo "portableLinux - build for Portable Linux Distribution"
+    echo "portablebuild - Use portable build."
     echo "verbose - optional argument to enable verbose build output."
     echo "rebuild - if tests have already been built - rebuild them"
+    echo "generatelayoutonly - only pull down dependencies and build coreroot"
+    echo "buildagainstpackages - pull down and build using packages."
     echo "runtests - run tests after building them"
     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
@@ -376,6 +483,7 @@ __ClangMajorVersion=0
 __ClangMinorVersion=0
 __NuGetPath="$__PackagesDir/NuGet.exe"
 __HostDistroRid=""
+__BuildAgainstPackagesArg=
 __DistroRid=""
 __cmakeargs=""
 __PortableLinux=0
@@ -385,9 +493,9 @@ __NativeTestIntermediatesDir=
 __RunTests=0
 __RebuildTests=0
 __BuildTestWrappers=0
+__GenarateLayoutOnly=
 CORE_ROOT=
 
-
 while :; do
     if [ $# -le 0 ]; then
         break
@@ -440,6 +548,10 @@ while :; do
             __CrossBuild=1
             ;;
 
+        portableBuild)
+            __PortableBuild=1
+            ;;
+
         portablelinux)
             if [ "$__BuildOS" == "Linux" ]; then
                 __PortableLinux=1
@@ -494,6 +606,14 @@ while :; do
             __ZipTests=1
             ;;
 
+        generatelayoutonly)
+            __GenarateLayoutOnly=1
+            ;;
+
+        buildagainstpackages)
+            __BuildAgainstPackagesArg=1
+            ;;
+
         bindir)
             if [ -n "$2" ]; then
                 __RootBinDir="$2"
@@ -527,7 +647,7 @@ __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
 # Configure environment if we are doing a verbose build
 if [ $__VerboseBuild == 1 ]; then
     export VERBOSE=1
-	__RunArgs="$__RunArgs -verbose"
+    __RunArgs="$__RunArgs -verbose"
 fi
 
 # Set default clang version
@@ -588,8 +708,9 @@ export __CMakeBinDir="$__BinDir"
 
 if [ ! -d "$__BinDir" ] || [ ! -d "$__BinDir/bin" ]; then
 
-    echo "Has not been found built CoreCLR instance"
-    echo "Please build it before tests using './build.sh $__BuildArch $__BuildType'"
+    echo "Cannot find build directory for the Coreclr Product."
+    echo "Please make sure CoreCLR is built before building tests."
+    echo "Example use: './build.sh $__BuildArch $__BuildType'"
     exit 1
 fi
 
@@ -611,7 +732,11 @@ __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClr
 
 echo "Building Tests..."
 
-build_Tests
+if [ -z "__GenerateLayoutOnly" ]; then
+    build_Tests
+else
+    generate_layout
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed to build tests"