env/windows: automate buildlet image creation

johnsonj · bradfitz · commit 3c22c38761a2 · 2017-04-27T23:11:12.000Z
Generate a Windows image that will start a buildlet on boot and have the dependencies needed for building/testing go/cgo. provisioning: - `sysprep.ps1`: disables unneeded features (eg UAC) and downloads dependencies (stage0, gcc) - `startup.ps1`: sets up a user account for unattended login. this can't be done in the sysprep stage because the `local machine` does not yet exist to create accounts under. helpers: - `build.bash`: builds a single image, creates a vm from the image and verifies it with `test_buildlet.bash` - `make.bash`: builds a set of images - `connect.bash`: helper to RDP into a machine for troubleshooting - `test_buildlet.bash`: validation script to exercise a buildlet Updates golang/go#17513 Change-Id: I4812ed1fc9862ae0aa44b712ea270fd52d0c505f Reviewed-on: https://go-review.googlesource.com/41142 Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
diff --git a/env/windows/.gitignore b/env/windows/.gitignore
@@ -0,0 +1,3 @@
+instance.txt
+.envrc
+out/*
diff --git a/env/windows/README.md b/env/windows/README.md
@@ -0,0 +1,47 @@
+# Windows buildlet images
+
+Windows images are built by creating and configuring VMs in GCP then capturing the image to the GCP Project.
+
+The provisioning happens in two stages:
+  - [sysprep.ps1](./sysprep.ps1): Downloads and unpacks dependencies, disabled unneeded Windows features (eg UAC)
+  - [startup.ps1](./startup.ps1): Creates and configures user for unattended login to launch the buildlet
+
+## Prerequisite: Setup a firewall rule
+Allow traffic to instances tagged `allow-dev-access` on tcp:80, tcp:3389
+
+```bash
+# restrict this down to your local network
+source_range=0.0.0.0/0
+
+gcloud compute firewall-rules create --allow=tcp:80,tcp:3389 --target-tags allow-dev-access --source-ranges $source_range allow-dev-access
+```
+
+## Examples/Tools
+
+### Build and test a single base image
+Builds a buildlet from the BASE_IMAGE and sets it up with  and  An image is captured and then a new VM is created from that image and validated with [test_buildlet.bash](./test_buildlet.bash).
+
+```bash
+export PROJECT_ID=YOUR_GCP_PROJECT
+export BASE_IMAGE=windows-server-2016-dc-core-v20170214
+export IMAGE_PROJECT=windows-cloud
+
+./build.bash
+```
+
+### Build all targets
+```bash
+./make.bash
+```
+
+### Build/test golang
+```bash
+instance_name=golang-buildlet-test
+external_ip=$(gcloud compute instances describe golang-buildlet-test --project=${PROJECT_ID} --zone=${ZONE} --format="value(networkInterfaces[0].accessConfigs[0].natIP)")
+./test_buildlet.bash $external_ip
+```
+
+### Troubleshoot via RDP
+```bash
+./connect.bash <instance_name>
+```
diff --git a/env/windows/build.bash b/env/windows/build.bash
@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# Copyright 2017 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -eu
+
+ZONE="us-central1-f"
+BUILDER_PREFIX="${1-golang}"
+IMAGE_NAME="${1-${BASE_IMAGE}}"
+INSTANCE_NAME="${BUILDER_PREFIX}-buildlet"
+TEST_INSTANCE_NAME="${BUILDER_PREFIX}-buildlet-test"
+MACHINE_TYPE="n1-standard-4"
+BUILDLET_IMAGE="windows-amd64-${IMAGE_NAME}"
+IMAGE_PROJECT=$IMAGE_PROJECT
+BASE_IMAGE=$BASE_IMAGE
+
+function wait_for_buildlet() {
+  external_ip=$1
+  seconds=5
+
+  echo "Waiting for buildlet at ${external_ip} to become responsive"
+  until curl "http://${external_ip}" 2>/dev/null; do
+    echo "retrying ${external_ip} in ${seconds} seconds"
+    sleep "${seconds}"
+  done
+}
+
+#
+# 0. Cleanup images/instances from prior runs
+#
+echo "Destroying existing instances (if exists)"
+yes "Y" | gcloud compute instances delete "$INSTANCE_NAME" --project="$PROJECT_ID" --zone="$ZONE" || true
+yes "Y" | gcloud compute instances delete "$TEST_INSTANCE_NAME" --project="$PROJECT_ID" --zone="$ZONE" || true
+echo "Destroying existing image (if exists)"
+yes "Y" | gcloud compute images delete "$BUILDLET_IMAGE" --project="$PROJECT_ID" || true
+
+
+#
+# 1. Create base instance
+# 
+echo "Creating target instance"
+gcloud compute instances create --machine-type="$MACHINE_TYPE" "$INSTANCE_NAME" \
+        --image "$BASE_IMAGE" --image-project "$IMAGE_PROJECT" \
+        --project="$PROJECT_ID" --zone="$ZONE" \
+        --metadata="buildlet-binary-url=https://storage.googleapis.com/go-builder-data/buildlet.windows-amd64" \
+        --metadata-from-file=sysprep-specialize-script-ps1=sysprep.ps1,windows-startup-script-ps1=startup.ps1 --tags=allow-dev-access 
+
+echo ""
+echo "Fetch logs with:"
+echo ""
+echo gcloud compute instances get-serial-port-output "$INSTANCE_NAME" --zone="$ZONE" --project="$PROJECT_ID" 
+echo ""
+external_ip=$(gcloud compute instances describe "$INSTANCE_NAME" --project="$PROJECT_ID" --zone="$ZONE" --format="value(networkInterfaces[0].accessConfigs[0].natIP)")
+
+wait_for_buildlet "$external_ip"
+
+#
+# 2. Image base instance
+#
+
+echo "Shutting down instance"
+gcloud compute instances stop "$INSTANCE_NAME" \
+        --project="$PROJECT_ID" --zone="$ZONE"
+
+echo "Capturing image"
+gcloud compute images create "$BUILDLET_IMAGE" --source-disk "$INSTANCE_NAME" --source-disk-zone "$ZONE" --project="$PROJECT_ID"
+
+echo "Removing base machine"
+yes "Y" | gcloud compute instances delete "$INSTANCE_NAME" --project="$PROJECT_ID" --zone="$ZONE" || true
+
+#
+# 3. Verify image is valid
+#
+
+echo "Creating new machine with image"
+gcloud compute instances create --machine-type="$MACHINE_TYPE" --image "$BUILDLET_IMAGE" "$TEST_INSTANCE_NAME" \
+       --project="$PROJECT_ID" --metadata="buildlet-binary-url=https://storage.googleapis.com/go-builder-data/buildlet.windows-amd64" \
+       --tags=allow-dev-access --zone="$ZONE"
+
+test_image_ip=$(gcloud compute instances describe "$TEST_INSTANCE_NAME" --project="$PROJECT_ID" --zone="$ZONE" --format="value(networkInterfaces[0].accessConfigs[0].natIP)")
+wait_for_buildlet "$test_image_ip"
+
+echo "Performing test build"
+./test_buildlet.bash "$test_image_ip"
+
+echo "Removing test instance"
+yes "Y" | gcloud compute instances delete "$TEST_INSTANCE_NAME" --project="$PROJECT_ID" --zone="$ZONE" || true
+
+echo "Success! A new buildlet can be created with the following command"
+echo "gcloud compute instances create --machine-type='$MACHINE_TYPE' '$INSTANCE_NAME' \
+--metadata='buildlet-binary-url=https://storage.googleapis.com/go-builder-data/buildlet.windows-amd64' \
+--image '$BUILDLET_IMAGE' --image-project '$PROJECT_ID'"
diff --git a/env/windows/connect.bash b/env/windows/connect.bash
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright 2017 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -eu
+
+ZONE=us-central1-f
+INSTANCE_NAME="${1:-golang-buildlet}"
+
+# Set, fetch credentials
+yes "Y" | gcloud compute reset-windows-password "${INSTANCE_NAME}" --user wingopher --project="${PROJECT_ID}" --zone="${ZONE}" > instance.txt
+
+echo ""
+echo "Instance credentials: "
+echo ""
+cat instance.txt
+
+echo ""
+echo "Connecting to instance: "
+echo ""
+
+username="$(grep username instance.txt | cut -d ':' -f 2 | xargs echo -n)"
+password="$(grep password instance.txt | sed 's/password:\s*//' | xargs echo -n)"
+hostname="$(grep ip_address instance.txt | cut -d ':' -f 2 | xargs echo -n)"
+
+echo xfreerdp -u "${username}" -p "'${password}'" -n "${hostname}" --ignore-certificate "${hostname}"
+xfreerdp -u "${username}" -p "${password}" -n "${hostname}" --ignore-certificate "${hostname}"
diff --git a/env/windows/make.bash b/env/windows/make.bash
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright 2017 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -e -u
+
+declare -A public_images
+
+public_images=( 
+         ['server-2016-v2']='windows-server-2016-dc-core-v20170214'
+         ['server-2008r2-v2']='windows-server-2008-r2-dc-v20170214' 
+         ['server-2012r2-v2']='windows-server-2012-r2-dc-core-v20170214'
+       )
+
+mkdir -p out
+
+for image in "${!public_images[@]}"; do
+    prefix=$image
+    base_image=${public_images[$image]}
+
+    BASE_IMAGE="$base_image" IMAGE_PROJECT='windows-cloud' ./build.bash "$prefix" |& tee "out/${base_image}.txt" &
+done
+
+
+wait
diff --git a/env/windows/old/README b/env/windows/old/README
diff --git a/env/windows/startup.ps1 b/env/windows/startup.ps1
@@ -0,0 +1,35 @@
+# Copyright 2017 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+Set-StrictMode -Version Latest
+
+function Test-RegistryKeyExists($path, $name)
+{
+    $key = Get-Item -LiteralPath $path -ErrorAction SilentlyContinue
+    ($key -and $null -ne $key.GetValue($name, $null)) -ne $false
+}
+
+$builder_dir = "C:\golang"
+$bootstrap_exe_path = "$builder_dir\bootstrap.exe"
+
+# Create a buildlet user
+$buildlet_user = "buildlet"
+$buildlet_password = "bUi-dL3ttt"
+net user $buildlet_user $buildlet_password /ADD
+net localgroup administrators $buildlet_user /ADD
+
+# Run the bootstrap program on login
+$bootstrap_cmd = "cmd /k ""cd $builder_dir && $bootstrap_exe_path"""
+New-ItemProperty -Path "HKLM:\Software\Microsoft\Windows\CurrentVersion\Run" -Name "Buildlet" -PropertyType ExpandString -Value $bootstrap_cmd -Force
+
+# Setup autologon and reboot
+$RegPath = "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon"
+if ((Test-RegistryKeyExists $RegPath "DefaultUsername") -eq $false) {
+  Remove-ItemProperty -Path 'HKLM:SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon' -Name 'AutoLogonCount' -Force
+  Set-ItemProperty $RegPath "AutoAdminLogon" -Value "1" -type String 
+  Set-ItemProperty $RegPath "DefaultUsername" -Value "$buildlet_user" -type String 
+  Set-ItemProperty $RegPath "DefaultPassword" -Value "$buildlet_password" -type String
+  Set-ItemProperty $RegPath "LogonCount" -Value "99999999" -type String
+  shutdown /r /t 0
+}
diff --git a/env/windows/sysprep.ps1 b/env/windows/sysprep.ps1
@@ -0,0 +1,68 @@
+# Copyright 2017 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+Set-StrictMode -Version Latest
+
+# Helpers
+function Get-FileFromUrl(
+	[string] $URL,
+	[string] $Output)
+{
+    Add-Type -AssemblyName "System.Net.Http"
+
+    $client = New-Object System.Net.Http.HttpClient
+    $request = New-Object System.Net.Http.HttpRequestMessage -ArgumentList @([System.Net.Http.HttpMethod]::Get, $URL)
+    $responseMsg = $client.SendAsync($request)
+    $responseMsg.Wait()
+
+    if (!$responseMsg.IsCanceled)
+    {
+	$response = $responseMsg.Result
+	if ($response.IsSuccessStatusCode)
+	{
+	    $downloadedFileStream = [System.IO.File]::Create($Output)
+	    $copyStreamOp = $response.Content.CopyToAsync($downloadedFileStream)
+	    $copyStreamOp.Wait()
+	    $downloadedFileStream.Close()
+	    if ($copyStreamOp.Exception -ne $null)
+	    {
+		throw $copyStreamOp.Exception
+	    }
+	}
+    }
+}
+
+# Disable automatic updates, windows firewall, error reporting, and UAC
+# 
+# - They'll just interrupt the builds later. 
+# - We don't care about security since this isn't going to be Internet-facing. 
+# - No ports will be accessible once the image is built.
+# - We can be trusted to run as a real Administrator
+New-ItemProperty "HKLM:\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate\AU" -Name NoAutoUpdate -Value 1 -Force | Out-Null
+new-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting" -Name Disabled -Value 1 -Force | Out-Null
+new-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting" -Name DontShowUI -Value 1 -Force | Out-Null
+netsh advfirewall set allprofiles state off
+netsh firewall set opmode mode=disable profile=ALL
+New-ItemProperty -Path HKLM:Software\Microsoft\Windows\CurrentVersion\policies\system -Name EnableLUA -PropertyType DWord -Value 0 -Force | Out-Null
+
+# Download buildlet
+$url = "https://storage.googleapis.com/go-builder-data/buildlet-stage0.windows-amd64.untar"
+$builder_dir = "C:\golang"
+$bootstrap_exe_path = "$builder_dir\bootstrap.exe"
+mkdir $builder_dir
+Get-FileFromUrl -URL $url -Output $bootstrap_exe_path
+
+# Download and unpack dependencies
+$dep_dir = "C:\godep"
+$gcc32_tar = "$dep_dir\gcc32.tar.gz"
+$gcc64_tar = "$dep_dir\gcc64.tar.gz"
+mkdir $dep_dir
+Get-FileFromUrl -URL "https://storage.googleapis.com/godev/gcc5-1-tdm32.tar.gz" -Output "$gcc32_tar"
+Get-FileFromUrl -URL "https://storage.googleapis.com/godev/gcc5-1-tdm64.tar.gz" -Output "$gcc64_tar"
+
+# Extract GCC
+$extract32_args=@("--untar-file=$gcc32_tar", "--untar-dest-dir=$dep_dir")
+& $bootstrap_exe_path $extract32_args 
+$extract64_args=@("--untar-file=$gcc64_tar", "--untar-dest-dir=$dep_dir")
+& $bootstrap_exe_path $extract64_args 
diff --git a/env/windows/test_buildlet.bash b/env/windows/test_buildlet.bash
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Copyright 2017 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -ue
+
+hostname="$1"
+BUILDLET="windows-amd64-gce@${hostname}"
+
+echo "Pushing go1.4, go1.8 to buildlet"
+gomote puttar -url https://storage.googleapis.com/golang/go1.8.src.tar.gz "$BUILDLET"
+gomote put14 "$BUILDLET"
+
+echo "Building go (32-bit)"
+gomote run -e GOARCH=386 -e GOHOSTARCH=386 -path 'C:/godep/gcc32/bin,$WORKDIR/go/bin,$PATH' -e 'GOROOT=c:\workdir\go' "$BUILDLET" go/src/make.bat
+
+# Go1.8 has failing tests on some windows versions.
+# Push a new release when avaliable or update this to use master.
+#echo "Running tests for go (32-bit)"
+#gomote run -e GOARCH=386 -e GOHOSTARCH=386 -path 'C:/godep/gcc32/bin,$WORKDIR/go/bin,$PATH' -e 'GOROOT=C:\workdir\go' "$BUILDLET" go/bin/go.exe tool dist test -v --no-rebuild
+
+echo "Building go (64-bit)"
+gomote run -path '$PATH,C:/godep/gcc64/bin,$WORKDIR/go/bin,$PATH' -e 'GOROOT=c:\workdir\go' "$BUILDLET" go/src/make.bat
+
+#echo "Running tests for go (64-bit)"
+#gomote run -path 'C:/godep/gcc64/bin,$WORKDIR/go/bin,$PATH' -e 'GOROOT=C:\workdir\go' "$BUILDLET" go/bin/go.exe tool dist test -v --no-rebuild
