Merge branch 'master' of https://github.com/taskcluster/generic-worker into bug1534223

Author: petemoore

.taskcluster.yml

@@ -120,6 +120,11 @@ tasks:
       features:
         taskclusterProxy: true
       maxRunTime: 3600
+      env:
+        # Python 2 is needed for TestDesktopResizeAndMovePointer
+        # Powershell is needed for TestMounts
+        # System32 is needed for e.g. cmd.exe, icacls.exe, ...
+        PATH: C:\Windows\System32;C:\Windows\System32\WindowsPowerShell\v1.0;C:\mozilla-build\python
       command:
         - set CGO_ENABLED=0
         - set GOPATH=%CD%\gopath1.10.8

README.md

@@ -26,6 +26,12 @@ If you prefer not to use a prepackaged binary, or want to have the latest unrele
 * Run `go get github.com/taskcluster/livelog`
 * Run `go get github.com/taskcluster/taskcluster-proxy`
 
+Run `go get -tags nativeEngine github.com/taskcluster/generic-worker` (all platforms) and/or `go get -tags dockerEngine github.com/taskcluster/generic-worker` (linux only). This should also build binaries for your platform.
+
+Run `./build.sh` to check go version, generate code, build binaries, compile (but not run) tests, perform linting, and ensure there are no ineffective assignments in go code.
+
+`./build.sh` takes optional arguments, `-a` to build all platforms, and `-t` to run tests. By default tests are not run and only the current platform is built.
+
 All being well, the binaries will be built under `${GOPATH}/bin`.
 
 # Acquire taskcluster credentials for running tests
@@ -86,7 +92,7 @@ Once you have been granted the above scope:
 To see a full description of all the config options available to you, run `generic-worker --help`:
 
 ```
-generic-worker 14.1.0
+generic-worker 14.1.1
 
 generic-worker is a taskcluster worker that can run on any platform that supports go (golang).
 See http://taskcluster.github.io/generic-worker/ for more details. Essentially, the worker is
@@ -340,7 +346,7 @@ and reports back results to the queue.
           taskclusterProxyPort              Port number for taskcluster-proxy HTTP requests.
                                             [default: 80]
           tasksDir                          The location where task directories should be
-                                            created on the worker. [default: /home]
+                                            created on the worker. [default: /Users]
           workerGroup                       Typically this would be an aws region - an
                                             identifier to uniquely identify which pool of
                                             workers this worker logically belongs to.
@@ -441,7 +447,7 @@ go test -v ./...
 Run the `release.sh` script like so:
 
 ```
-$ ./release.sh 14.1.0
+$ ./release.sh 14.1.1
 ```
 
 This will perform some checks, tag the repo, push the tag to github, which will then trigger travis-ci to run tests, and publish the new release.
@@ -452,9 +458,17 @@ See [worker_types README.md](https://github.com/taskcluster/generic-worker/blob/
 
 # Release notes
 
+In v14.1.1 since v14.1.0
+========================
+
+* [Bug 1548829 - generic-worker: log header should mention provisionerId](https://bugzil.la/1548829)
+* [Bug 1551164 - [mounts] Not able to rename dir caches/*** as tasks/task_***/***: rename caches/*** tasks/task_***/***: file exists](https://bugzil.la/1551164)
+
 In v14.1.0 since v14.0.2
 ========================
 
+__Please don't use release 14.1.0 on macOS/linux due to [bug 1551164](https://bugzil.la/1551164) which will be fixed in v14.1.1.__
+
 * [Bug 1436195 - Add support for live-logs via websocktunnel](https://bugzil.la/1436195) (note that existing livelog support via stateless DNS still works)
 
   To enable websocktunnel, include configuration options `wstAudience` and `wstServerURL` to correspond to the websocktunnel server the worker should use.

main.go

@@ -56,7 +56,7 @@ var (
 	logName = "public/logs/live_backing.log"
 	logPath = filepath.Join("generic-worker", "live_backing.log")
 
-	version  = "14.1.0"
+	version  = "14.1.1"
 	revision = "" // this is set during build with `-ldflags "-X main.revision=$(git rev-parse HEAD)"`
 )
 
@@ -431,11 +431,6 @@ func RunWorker() (exitCode ExitCode) {
 		return INTERNAL_ERROR
 	}
 
-	reboot := PrepareTaskEnvironment()
-	if reboot {
-		return REBOOT_REQUIRED
-	}
-
 	// number of tasks resolved since worker first ran
 	// stored in a json file, since we may reboot between tasks etc
 	tasksResolved := ReadTasksResolvedFile()
@@ -446,11 +441,7 @@ func RunWorker() (exitCode ExitCode) {
 			panic(err)
 		}
 	}(&tasksResolved)
-	err = purgeOldTasks()
-	// errors are not fatal
-	if err != nil {
-		log.Printf("WARNING: failed to remove old task directories/users: %v", err)
-	}
+
 	// Queue is the object we will use for accessing queue api
 	queue = config.Queue()
 	provisioner = config.AWSProvisioner()
@@ -476,6 +467,17 @@ func RunWorker() (exitCode ExitCode) {
 	signal.Notify(sigInterrupt, os.Interrupt)
 	for {
 
+		reboot := PrepareTaskEnvironment()
+		if reboot {
+			return REBOOT_REQUIRED
+		}
+
+		err = purgeOldTasks()
+		// errors are not fatal
+		if err != nil {
+			log.Printf("WARNING: failed to remove old task directories/users: %v", err)
+		}
+
 		// See https://bugzil.la/1298010 - routinely check if this worker type is
 		// outdated, and shut down if a new deployment is required.
 		// Round(0) forces wall time calculation instead of monotonic time in case machine slept etc

mounts_broken_on_docker_test.go

@@ -5,6 +5,7 @@ package main
 import (
 	"encoding/json"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -115,7 +116,12 @@ func TestMounts(t *testing.T) {
 	payload := GenericWorkerPayload{
 		Mounts: toMountArray(t, &mounts),
 		// since this checks that SHA values of files as the task user, is also ensures they are readable by task user
-		Command:    checkSHASums(),
+		Command: checkSHASums(),
+		// Don't assume powershell is in the default system PATH, but rather
+		// require that powershell is in the PATH of the test process.
+		Env: map[string]string{
+			"PATH": os.Getenv("PATH"),
+		},
 		MaxRunTime: 180,
 	}
 

mozilla-try-scripts/gecko-try.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -exv
+#!/bin/bash -e
 
 ######
 # This script allows you to test a new generic-worker Windows release on
@@ -34,7 +34,7 @@ if [ -z "${NEW_GW_VERSION}" ] || [ -z "${NEW_TP_VERSION}" ]; then
 fi
 
 echo "Checking system dependencies..."
-for command in curl jq grep sleep file git hg cat sed rm mktemp python go patch basename tr; do
+for command in curl jq grep sleep file git hg cat sed rm mktemp python2 go patch basename tr; do
   if ! which "${command}" >/dev/null; then
     echo -e "  \xE2\x9D\x8C ${command}"
     echo "${0} requires ${command} to be installed and available in your PATH - please fix and rerun" >&2
@@ -75,7 +75,7 @@ function add_github {
   "${THIS_SCRIPT_DIR}/lib/tooltool.py" add --visibility internal "${LOCAL_FILE}"
   # Bug 1460178 - sanity check binary downloads of generic-worker before publishing to tooltool...
   if ! file "${LOCAL_FILE}" | grep -F "${FILETYPE}" | grep -F 'for MS Windows'; then
-    echo "Downloaded file doesn't appear to be '${FILETYPE}':" >&2
+    echo "Downloaded file '${LOCAL_FILE}' (from dir '$(pwd)') doesn't appear to be '${FILETYPE}':" >&2
     file "${LOCAL_FILE}" >&2
     exit 69
   fi
@@ -102,7 +102,7 @@ cat manifest.tt
 
 git clone [email protected]:mozilla-releng/OpenCloudConfig.git
 cd OpenCloudConfig/userdata/Manifest
-for MANIFEST in *-b.json *-cu.json *-beta.json; do
+for MANIFEST in gecko-t-win10-64-gpu-b gecko-t-win7-32-gpu-b gecko-t-win10-64-cu gecko-t-win7-32-cu gecko-1-b-win2012-beta gecko-t-win10-64-beta gecko-t-win7-32-beta; do
   cat "${MANIFEST}" > "${MANIFEST}.bak"
   cat "${MANIFEST}.bak" \
     | sed "s_\\(generic-worker/releases/download/v\\)[^/]*\\(/generic-worker-nativeEngine-windows-\\)_\\1${NEW_GW_VERSION}\\2_" | sed "s_\\(\"generic-worker \\)[^ ]*\\(.*\\)\$_\\1${NEW_GW_VERSION}\\2_" \
@@ -111,7 +111,7 @@ for MANIFEST in *-b.json *-cu.json *-beta.json; do
   cat "${MANIFEST}" > "${MANIFEST}.bak"
   THIS_ARCH="$(cat "${MANIFEST}" | sed -n 's/.*\/generic-worker-nativeEngine-windows-\(.*\)\.exe.*/\1/p' | sort -u)"
   if [ "${THIS_ARCH}" != "386" ] && [ "${THIS_ARCH}" != "amd64" ]; then
-    echo "NOOOOOOO - cannot recognise ARCH '${THIS_ARCH}'" >&2
+    echo "NOOOOOOO - cannot recognise ARCH '${THIS_ARCH}' of generic-worker binary in manifest '${MANIFEST}' (from dir '$(pwd)')" >&2
     exit 67
   fi
   updateSHA512 "generic-worker-nativeEngine-windows-${THIS_ARCH}.exe" "GenericWorkerDownload"
@@ -130,7 +130,7 @@ This change does _not_ affect any production workers. Commit made with:
 
     ${0} $(echo ${@})
 
-See https://github.com/taskcluster/generic-worker/blob/$THIS_REV/$THIS_FILE -m \"${DEPLOY}\""
+See https://github.com/taskcluster/generic-worker/blob/$THIS_REV/$THIS_FILE" -m "${DEPLOY}"
 OCC_COMMIT="$(git rev-parse HEAD)"
 git push
 

mozilla-try-scripts/lib/tooltool.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # tooltool is a lookaside cache implemented in Python
 # Copyright (C) 2011 John H. Ford <[email protected]>

plat_all-unix-style_test.go

@@ -2,7 +2,13 @@
 
 package main
 
-import "testing"
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+)
 
 func TestSeparateTaskUsersNotSupported(t *testing.T) {
 	defer setup(t)()
@@ -12,3 +18,39 @@ func TestSeparateTaskUsersNotSupported(t *testing.T) {
 		t.Fatalf("Got exit code %v but was expecting exit code %v", exitCode, INVALID_CONFIG)
 	}
 }
+
+func TestNewTaskDirectoryForEachTask(t *testing.T) {
+	defer setup(t)()
+	config.NumberOfTasksToRun = 3
+	payload := GenericWorkerPayload{
+		Command:    returnExitCode(0),
+		MaxRunTime: 10,
+	}
+	td := testTask(t)
+	for i := uint(0); i < config.NumberOfTasksToRun; i++ {
+		_ = scheduleTask(t, td, payload)
+	}
+
+	execute(t, TASKS_COMPLETE)
+
+	// scan task directories, to make sure there are three unique backing log files,
+	// implying that each task ran in its own directory
+
+	var backingLogsFound uint = 0
+	err := filepath.Walk(config.TasksDir, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+		if info.Name() != "live_backing.log" {
+			return fmt.Errorf("Discovered file %q but was expecting %q", info.Name(), "live_backing.log")
+		}
+		backingLogsFound++
+		return nil
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+	if backingLogsFound != config.NumberOfTasksToRun {
+		log.Fatalf("Expected to find %v backing logs, but found %v", config.NumberOfTasksToRun, backingLogsFound)
+	}
+}

plat_windows_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"os"
 	"testing"
 )
 
@@ -101,6 +102,11 @@ func TestDesktopResizeAndMovePointer(t *testing.T) {
 	payload := GenericWorkerPayload{
 		Command:    commands,
 		MaxRunTime: 90,
+		// Don't assume python 2 is in the default system PATH, but rather
+		// require that python 2 is in the PATH of the test process.
+		Env: map[string]string{
+			"PATH": os.Getenv("PATH"),
+		},
 	}
 	td := testTask(t)
 

worker_types/deepspeech-win-gpu-b/userdata

@@ -84,7 +84,7 @@ Expand-ZIPFile -File "C:\nssm-2.24.zip" -Destination "C:\" -Url "http://www.nssm
 
 # download generic-worker
 md C:\generic-worker
-$client.DownloadFile("https://github.com/taskcluster/generic-worker/releases/download/v14.1.0/generic-worker-nativeEngine-windows-amd64.exe", "C:\generic-worker\generic-worker.exe")
+$client.DownloadFile("https://github.com/taskcluster/generic-worker/releases/download/v14.1.1/generic-worker-nativeEngine-windows-amd64.exe", "C:\generic-worker\generic-worker.exe")
 
 # download livelog
 $client.DownloadFile("https://github.com/taskcluster/livelog/releases/download/v1.1.0/livelog-windows-amd64.exe", "C:\generic-worker\livelog.exe")

worker_types/deepspeech-win-gpu/userdata

@@ -130,7 +130,7 @@ Expand-ZIPFile -File "C:\nssm-2.24.zip" -Destination "C:\" -Url "http://www.nssm
 
 # download generic-worker
 md C:\generic-worker
-$client.DownloadFile("https://github.com/taskcluster/generic-worker/releases/download/v14.1.0/generic-worker-nativeEngine-windows-amd64.exe", "C:\generic-worker\generic-worker.exe")
+$client.DownloadFile("https://github.com/taskcluster/generic-worker/releases/download/v14.1.1/generic-worker-nativeEngine-windows-amd64.exe", "C:\generic-worker\generic-worker.exe")
 
 # download livelog
 $client.DownloadFile("https://github.com/taskcluster/livelog/releases/download/v1.1.0/livelog-windows-amd64.exe", "C:\generic-worker\livelog.exe")