Minor testing improvements

README.md

@@ -75,14 +75,37 @@ go build arduino.cc/arduino-builder
 In order to run the tests, type:
 
 ```
-go test -timeout 60m -v ./src/arduino.cc/builder/test/...
+go test arduino.cc/builder/test
+```
+
+This runs all tests, showing any failures and a summary at the end.
+Add the -v option to show each test as it is being ran. Currently,
+arduino-builder itself also generates copious output, even for
+non-failing testcases and without -v, and testing does not stop at the
+first failure, so you probably want to redirect test output so you can
+scroll back to find any failures.
+
+To run a single test, use the -run option, which accepts a regular
+expression (see also go help testflag).
+
+```
+go test arduino.cc/builder/test -run 'TestBuilderEmptySketch'
+go test arduino.cc/builder/test -run 'TestPrototypesAdder.*'
 ```
 
 In jenkins, use
 ```
-go test -timeout 60m -v ./src/arduino.cc/builder/test/... | bin/go-junit-report > report.xml
+go test -v arduino.cc/builder/test | bin/go-junit-report > report.xml
 ```
 
+The first time you run the tests, some needed files (toolchains and
+source files) will be downloaded, which needs about 1GB of space (at the
+time of writing). If you have a slow connection, this download might
+exceed the default 10 minute timeout for a single test. If you run into
+this, add `-timeout 60m` or similar to the commandline to extend the
+timeout. If you are running on slower system (like a rasbperry pi),
+increasing the timeout might be needed as well.
+
 ### License and Copyright
 
 `arduino-builder` is licensed under General Public License version 2, as published by the Free Software Foundation. See [LICENSE.txt](LICENSE.txt).

src/arduino.cc/builder/test/.gitignore

@@ -0,0 +1 @@
+downloaded_*

src/arduino.cc/builder/test/helper_tools_downloader.go

@@ -139,6 +139,8 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) {
 }
 
 func patchFiles(t *testing.T) {
+	err := utils.EnsureFolderExists(PATCHES_FOLDER)
+	NoError(t, err)
 	files, err := ioutil.ReadDir(PATCHES_FOLDER)
 	NoError(t, err)
 

src/arduino.cc/builder/test/tools_loader_test.go

@@ -64,7 +64,7 @@ func TestLoadTools(t *testing.T) {
 	NoError(t, err)
 
 	tools := ctx.Tools
-	require.Equal(t, 7, len(tools))
+	require.Equal(t, 6, len(tools))
 
 	sort.Sort(ByToolIDAndVersion(tools))
 
@@ -88,6 +88,10 @@ func TestLoadTools(t *testing.T) {
 	require.Equal(t, "bossac", tools[idx].Name)
 	require.Equal(t, "1.6.1-arduino", tools[idx].Version)
 	require.Equal(t, Abs(t, "./downloaded_tools/bossac/1.6.1-arduino"), tools[idx].Folder)
+	idx++
+	require.Equal(t, "ctags", tools[idx].Name)
+	require.Equal(t, "5.8-arduino10", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_tools/ctags/5.8-arduino10"), tools[idx].Folder)
 }
 
 func TestLoadToolsWithBoardManagerFolderStructure(t *testing.T) {
@@ -106,14 +110,18 @@ func TestLoadToolsWithBoardManagerFolderStructure(t *testing.T) {
 
 	sort.Sort(ByToolIDAndVersion(tools))
 
-	require.Equal(t, "CMSIS", tools[0].Name)
-	require.Equal(t, "arm-none-eabi-gcc", tools[1].Name)
-	require.Equal(t, "4.8.3-2014q1", tools[1].Version)
-	require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/RFduino/tools/arm-none-eabi-gcc/4.8.3-2014q1"), tools[1].Folder)
-
-	require.Equal(t, "openocd", tools[2].Name)
-	require.Equal(t, "0.9.0-arduino", tools[2].Version)
-	require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/openocd/0.9.0-arduino"), tools[2].Folder)
+	idx := 0
+	require.Equal(t, "CMSIS", tools[idx].Name)
+	require.Equal(t, "4.0.0-atmel", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/CMSIS/4.0.0-atmel"), tools[idx].Folder)
+	idx++
+	require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name)
+	require.Equal(t, "4.8.3-2014q1", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/RFduino/tools/arm-none-eabi-gcc/4.8.3-2014q1"), tools[idx].Folder)
+	idx++
+	require.Equal(t, "openocd", tools[idx].Name)
+	require.Equal(t, "0.9.0-arduino", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/openocd/0.9.0-arduino"), tools[idx].Folder)
 }
 
 func TestLoadLotsOfTools(t *testing.T) {
@@ -128,13 +136,40 @@ func TestLoadLotsOfTools(t *testing.T) {
 	NoError(t, err)
 
 	tools := ctx.Tools
-	require.Equal(t, 9, len(tools))
+	require.Equal(t, 8, len(tools))
 
-	require.Equal(t, "arm-none-eabi-gcc", tools[0].Name)
-	require.Equal(t, "4.8.3-2014q1", tools[0].Version)
+	sort.Sort(ByToolIDAndVersion(tools))
 
-	require.Equal(t, "CMSIS", tools[7].Name)
-	require.Equal(t, "openocd", tools[8].Name)
-	require.Equal(t, "0.9.0-arduino", tools[8].Version)
-	require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/openocd/0.9.0-arduino"), tools[8].Folder)
+	idx := 0
+	require.Equal(t, "CMSIS", tools[idx].Name)
+	require.Equal(t, "4.0.0-atmel", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/CMSIS/4.0.0-atmel"), tools[idx].Folder)
+	idx++
+	require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name)
+	require.Equal(t, "4.8.3-2014q1", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_tools/arm-none-eabi-gcc/4.8.3-2014q1"), tools[idx].Folder)
+	idx++
+	require.Equal(t, "avr-gcc", tools[idx].Name)
+	require.Equal(t, "4.8.1-arduino5", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_tools/avr-gcc/4.8.1-arduino5"), tools[idx].Folder)
+	idx++
+	require.Equal(t, "avrdude", tools[idx].Name)
+	require.Equal(t, "6.0.1-arduino5", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_tools/avrdude/6.0.1-arduino5"), tools[idx].Folder)
+	idx++
+	require.Equal(t, "bossac", tools[idx].Name)
+	require.Equal(t, "1.5-arduino", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_tools/bossac/1.5-arduino"), tools[idx].Folder)
+	idx++
+	require.Equal(t, "bossac", tools[idx].Name)
+	require.Equal(t, "1.6.1-arduino", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_tools/bossac/1.6.1-arduino"), tools[idx].Folder)
+	idx++
+	require.Equal(t, "ctags", tools[idx].Name)
+	require.Equal(t, "5.8-arduino10", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_tools/ctags/5.8-arduino10"), tools[idx].Folder)
+	idx++
+	require.Equal(t, "openocd", tools[idx].Name)
+	require.Equal(t, "0.9.0-arduino", tools[idx].Version)
+	require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/openocd/0.9.0-arduino"), tools[idx].Folder)
 }