Merge branch 'master' into feat/purge-build-cache

Author: Luca Bianconi

arduino/libraries/librariesmanager/install.go

@@ -64,6 +64,7 @@ func (lm *LibrariesManager) InstallPrerequisiteCheck(name string, version *semve
 		return nil, err
 	}
 
+	lm.RescanLibraries()
 	libs := lm.FindByReference(&librariesindex.Reference{Name: name}, installLocation)
 
 	if len(libs) > 1 {

arduino/libraries/librariesmanager/librariesmanager.go

@@ -132,6 +132,7 @@ func (lm *LibrariesManager) AddPlatformReleaseLibrariesDir(plaftormRelease *core
 
 // RescanLibraries reload all installed libraries in the system.
 func (lm *LibrariesManager) RescanLibraries() []*status.Status {
+	lm.clearLibraries()
 	statuses := []*status.Status{}
 	for _, dir := range lm.LibrariesDir {
 		if errs := lm.LoadLibrariesFromDir(dir); len(errs) > 0 {
@@ -217,3 +218,9 @@ func (lm *LibrariesManager) FindByReference(libRef *librariesindex.Reference, in
 	}
 	return alternatives.FilterByVersionAndInstallLocation(libRef.Version, installLocation)
 }
+
+func (lm *LibrariesManager) clearLibraries() {
+	for k := range lm.Libraries {
+		delete(lm.Libraries, k)
+	}
+}

arduino/libraries/librariesmanager/librariesmanager_test.go

@@ -0,0 +1,32 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+package librariesmanager
+
+import (
+	"testing"
+
+	"github.com/arduino/arduino-cli/arduino/libraries"
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_RescanLibrariesCallClear(t *testing.T) {
+	baseDir := paths.New(t.TempDir())
+	lm := NewLibraryManager(baseDir.Join("index_dir"), baseDir.Join("downloads_dir"))
+	lm.Libraries["testLibA"] = libraries.List{}
+	lm.Libraries["testLibB"] = libraries.List{}
+	lm.RescanLibraries()
+	require.Len(t, lm.Libraries, 0)
+}

arduino/sketch/sketch.go

@@ -32,8 +32,7 @@ import (
 type Sketch struct {
 	Name             string
 	MainFile         *paths.Path
-	FullPath         *paths.Path // FullPath is the path to the Sketch folder
-	BuildPath        *paths.Path
+	FullPath         *paths.Path    // FullPath is the path to the Sketch folder
 	OtherSketchFiles paths.PathList // Sketch files that end in .ino other than main file
 	AdditionalFiles  paths.PathList
 	RootFolderFiles  paths.PathList // All files that are in the Sketch root
@@ -81,7 +80,6 @@ func New(path *paths.Path) (*Sketch, error) {
 		Name:             path.Base(),
 		MainFile:         mainFile,
 		FullPath:         path,
-		BuildPath:        GenBuildPath(path),
 		OtherSketchFiles: paths.PathList{},
 		AdditionalFiles:  paths.PathList{},
 		RootFolderFiles:  paths.PathList{},
@@ -293,19 +291,15 @@ func CheckForPdeFiles(sketch *paths.Path) []*paths.Path {
 	return files
 }
 
-// GenBuildPath generates a suitable name for the build folder.
-// The sketchPath, if not nil, is also used to furhter differentiate build paths.
-func GenBuildPath(sketchPath *paths.Path) *paths.Path {
-	path := ""
-	if sketchPath != nil {
-		path = sketchPath.String()
-	}
-	md5SumBytes := md5.Sum([]byte(path))
-	md5Sum := strings.ToUpper(hex.EncodeToString(md5SumBytes[:]))
-
-	return getSketchesCacheDir().Join(md5Sum)
+// DefaultBuildPath generates the default build directory for a given sketch.
+// The build path is in a temporary directory and is unique for each sketch.
+func (s *Sketch) DefaultBuildPath() *paths.Path {
+	return paths.TempDir().Join("arduino", "sketches", s.Hash())
 }
 
-func getSketchesCacheDir() *paths.Path {
-	return paths.TempDir().Join("arduino", "sketches").Canonical()
+// Hash generate a unique hash for the given sketch.
+func (s *Sketch) Hash() string {
+	path := s.FullPath.String()
+	md5SumBytes := md5.Sum([]byte(path))
+	return strings.ToUpper(hex.EncodeToString(md5SumBytes[:]))
 }

arduino/sketch/sketch_test.go

@@ -287,10 +287,8 @@ func TestNewSketchFolderSymlink(t *testing.T) {
 
 func TestGenBuildPath(t *testing.T) {
 	want := paths.TempDir().Join("arduino", "sketches", "ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
-	assert.True(t, GenBuildPath(paths.New("foo")).EquivalentTo(want))
-
-	want = paths.TempDir().Join("arduino", "sketches", "D41D8CD98F00B204E9800998ECF8427E")
-	assert.True(t, GenBuildPath(nil).EquivalentTo(want))
+	assert.True(t, (&Sketch{FullPath: paths.New("foo")}).DefaultBuildPath().EquivalentTo(want))
+	assert.Equal(t, "ACBD18DB4CC2F85CEDEF654FCCC4A4D8", (&Sketch{FullPath: paths.New("foo")}).Hash())
 }
 
 func TestCheckForPdeFiles(t *testing.T) {

commands/compile/compile.go

@@ -118,7 +118,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	}
 	builderCtx.UseCachedLibrariesResolution = req.GetSkipLibrariesDiscovery()
 	builderCtx.FQBN = fqbn
-	builderCtx.SketchLocation = sk.FullPath
+	builderCtx.Sketch = sk
 	builderCtx.ProgressCB = progressCB
 
 	// FIXME: This will be redundant when arduino-builder will be part of the cli
@@ -130,7 +130,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	builderCtx.OtherLibrariesDirs.Add(configuration.LibrariesDir(configuration.Settings))
 	builderCtx.LibraryDirs = paths.NewPathList(req.Library...)
 	if req.GetBuildPath() == "" {
-		builderCtx.BuildPath = sk.BuildPath
+		builderCtx.BuildPath = sk.DefaultBuildPath()
 	} else {
 		builderCtx.BuildPath = paths.New(req.GetBuildPath()).Canonical()
 	}
@@ -150,7 +150,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 
 	// Optimize for debug
 	builderCtx.OptimizeForDebug = req.GetOptimizeForDebug()
-	builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores")
 
 	builderCtx.Jobs = int(req.GetJobs())
 
@@ -160,12 +159,17 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	builderCtx.CustomBuildProperties = append(req.GetBuildProperties(), "build.warn_data_percentage=75")
 	builderCtx.CustomBuildProperties = append(req.GetBuildProperties(), securityKeysOverride...)
 
-	if req.GetBuildCachePath() != "" {
-		builderCtx.BuildCachePath = paths.New(req.GetBuildCachePath())
-		err = builderCtx.BuildCachePath.MkdirAll()
+	if req.GetBuildCachePath() == "" {
+		builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores")
+	} else {
+		buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs()
 		if err != nil {
 			return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err}
 		}
+		if err := buildCachePath.MkdirAll(); err != nil {
+			return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err}
+		}
+		builderCtx.CoreBuildCachePath = buildCachePath.Join("core")
 	}
 
 	builderCtx.BuiltInLibrariesDirs = configuration.IDEBuiltinLibrariesDir(configuration.Settings)

commands/daemon/daemon.go

@@ -343,7 +343,6 @@ func (s *ArduinoCoreServerImpl) LibraryDownload(req *rpc.LibraryDownloadRequest,
 	resp, err := lib.LibraryDownload(
 		stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryDownloadResponse{Progress: p}) },
-		"",
 	)
 	if err != nil {
 		return convertErrorToRPCStatus(err)
@@ -357,7 +356,6 @@ func (s *ArduinoCoreServerImpl) LibraryInstall(req *rpc.LibraryInstallRequest, s
 		stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryInstallResponse{Progress: p}) },
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryInstallResponse{TaskProgress: p}) },
-		"",
 	)
 	if err != nil {
 		return convertErrorToRPCStatus(err)

commands/debug/debug_info.go

@@ -115,9 +115,11 @@ func getDebugProperties(req *debug.DebugConfigRequest, pme *packagemanager.Explo
 		}
 	}
 
-	importPath := sk.BuildPath
+	var importPath *paths.Path
 	if importDir := req.GetImportDir(); importDir != "" {
 		importPath = paths.New(importDir)
+	} else {
+		importPath = sk.DefaultBuildPath()
 	}
 	if !importPath.Exist() {
 		return nil, &arduino.NotFoundError{Message: tr("Compiled sketch not found in %s", importPath)}

commands/lib/download.go

@@ -32,8 +32,7 @@ var tr = i18n.Tr
 
 // LibraryDownload executes the download of the library.
 // A DownloadProgressCB callback function must be passed to monitor download progress.
-// queryParameter is passed for analysis purposes.
-func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downloadCB rpc.DownloadProgressCB, queryParameter string) (*rpc.LibraryDownloadResponse, error) {
+func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downloadCB rpc.DownloadProgressCB) (*rpc.LibraryDownloadResponse, error) {
 	logrus.Info("Executing `arduino-cli lib download`")
 
 	lm := commands.GetLibraryManager(req)
@@ -48,7 +47,7 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downl
 		return nil, err
 	}
 
-	if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}, queryParameter); err != nil {
+	if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}, "download"); err != nil {
 		return nil, err
 	}
 

commands/lib/install.go

@@ -31,8 +31,7 @@ import (
 )
 
 // LibraryInstall resolves the library dependencies, then downloads and installs the libraries into the install location.
-// queryParameter is passed for analysis purposes and forwarded to the called functions. It is set to "depends" when a dependency is installed
-func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB, queryParameter string) error {
+func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error {
 	lm := commands.GetLibraryManager(req)
 	if lm == nil {
 		return &arduino.InvalidInstanceError{}
@@ -98,21 +97,19 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa
 
 	for libRelease, installTask := range libReleasesToInstall {
 		// Checks if libRelease is the requested library and not a dependency
+		downloadReason := "depends"
 		if libRelease.GetName() == req.Name {
-			if err := downloadLibrary(lm, libRelease, downloadCB, taskCB, queryParameter); err != nil {
-				return err
-			}
-			if err := installLibrary(lm, libRelease, installTask, taskCB); err != nil {
-				return err
-			}
-		} else {
-			if err := downloadLibrary(lm, libRelease, downloadCB, taskCB, "depends"); err != nil {
-				return err
-			}
-			if err := installLibrary(lm, libRelease, installTask, taskCB); err != nil {
-				return err
+			downloadReason = "install"
+			if installTask.ReplacedLib != nil {
+				downloadReason = "upgrade"
 			}
 		}
+		if err := downloadLibrary(lm, libRelease, downloadCB, taskCB, downloadReason); err != nil {
+			return err
+		}
+		if err := installLibrary(lm, libRelease, installTask, taskCB); err != nil {
+			return err
+		}
 	}
 
 	if err := commands.Init(&rpc.InitRequest{Instance: req.Instance}, nil); err != nil {