diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index e7b0d56fe41..4a227e76154 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -38,35 +38,30 @@ import ( var tr = i18n.Tr -func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { - files, err := utils.ReadDirFiltered(sourcePath, utils.FilterFiles()) +// DirContentIsOlderThan returns true if the content of the given directory is +// older than target file. If extensions are given, only the files with these +// extensions are tested. +func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...string) (bool, error) { + targetStat, err := target.Stat() if err != nil { - return nil, errors.WithStack(err) - } - var sources []string - for _, file := range files { - sources = append(sources, filepath.Join(sourcePath, file.Name())) + return false, err } + targetModTime := targetStat.ModTime() - if recurse { - folders, err := utils.ReadDirFiltered(sourcePath, utils.FilterDirs) + files, err := utils.FindFilesInFolder(dir, true, extensions...) + if err != nil { + return false, err + } + for _, file := range files { + file, err := file.Stat() if err != nil { - return nil, errors.WithStack(err) + return false, err } - - for _, folder := range folders { - if !utils.IsSCCSOrHiddenFile(folder) { - // Skip SCCS directories as they do not influence the build and can be very large - otherSources, err := findAllFilesInFolder(filepath.Join(sourcePath, folder.Name()), recurse) - if err != nil { - return nil, errors.WithStack(err) - } - sources = append(sources, otherSources...) - } + if file.ModTime().After(targetModTime) { + return false, nil } } - - return sources, nil + return true, nil } func CompileFiles(ctx *types.Context, sourcePath *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { @@ -331,54 +326,6 @@ func removeEndingBackSlash(s string) string { return strings.TrimSuffix(s, "\\") } -func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths.Path) bool { - - targetFileStat, err := targetFile.Stat() - if err == nil { - files, err := findAllFilesInFolder(corePath.String(), true) - if err != nil { - return true - } - for _, file := range files { - fileStat, err := os.Stat(file) - if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) { - return true - } - } - if targetCorePath != nil && !strings.EqualFold(corePath.String(), targetCorePath.String()) { - return CoreOrReferencedCoreHasChanged(targetCorePath, nil, targetFile) - } - return false - } - return true -} - -func TXTBuildRulesHaveChanged(corePath, targetCorePath, targetFile *paths.Path) bool { - - targetFileStat, err := targetFile.Stat() - if err == nil { - files, err := findAllFilesInFolder(corePath.String(), true) - if err != nil { - return true - } - for _, file := range files { - // report changes only for .txt files - if filepath.Ext(file) != ".txt" { - continue - } - fileStat, err := os.Stat(file) - if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) { - return true - } - } - if targetCorePath != nil && !corePath.EqualsTo(targetCorePath) { - return TXTBuildRulesHaveChanged(targetCorePath, nil, targetFile) - } - return false - } - return true -} - func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map) (*paths.Path, error) { archiveFilePath := buildPath.JoinPath(archiveFile) diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 4c78e18ce18..5d849b927ed 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -427,7 +427,7 @@ func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.Uniqu for k := range globals.SourceFilesValidExtensions { sourceFileExtensions = append(sourceFileExtensions, k) } - filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions) + filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions...) if err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 9cc38434286..acbce09e2cb 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -17,6 +17,8 @@ package builder import ( "fmt" + "io" + "os" "path/filepath" "regexp" "strings" @@ -39,6 +41,130 @@ type ExportProjectCMake struct { var lineMatcher = regexp.MustCompile(`^#line\s\d+\s"`) func (s *ExportProjectCMake) Run(ctx *types.Context) error { + // copies the contents of the file named src to the file named + // by dst. The file will be created if it does not already exist. If the + // destination file exists, all it's contents will be replaced by the contents + // of the source file. The file mode will be copied from the source and + // the copied data is synced/flushed to stable storage. + // TODO: Replace with call to go-paths-helper... + copyFile := func(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return + } + + err = out.Sync() + if err != nil { + return + } + + si, err := os.Stat(src) + if err != nil { + return + } + err = os.Chmod(dst, si.Mode()) + if err != nil { + return + } + + return + } + + // recursively copies a directory tree, attempting to preserve permissions. + // Source directory must exist, destination directory must *not* exist. + // Symlinks are ignored and skipped. + // TODO: Replace with call to go-paths-helper... + var copyDir func(src string, dst string, extensions []string) (err error) + copyDir = func(src string, dst string, extensions []string) (err error) { + isAcceptedExtension := func(ext string) bool { + ext = strings.ToLower(ext) + for _, valid := range extensions { + if ext == valid { + return true + } + } + return false + } + + src = filepath.Clean(src) + dst = filepath.Clean(dst) + + si, err := os.Stat(src) + if err != nil { + return err + } + if !si.IsDir() { + return fmt.Errorf(tr("source is not a directory")) + } + + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return + } + if err == nil { + return fmt.Errorf(tr("destination already exists")) + } + + err = os.MkdirAll(dst, si.Mode()) + if err != nil { + return + } + + entries, err := os.ReadDir(src) + if err != nil { + return + } + + for _, dirEntry := range entries { + entry, scopeErr := dirEntry.Info() + if scopeErr != nil { + return + } + + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + err = copyDir(srcPath, dstPath, extensions) + if err != nil { + return + } + } else { + // Skip symlinks. + if entry.Mode()&os.ModeSymlink != 0 { + continue + } + + if !isAcceptedExtension(filepath.Ext(srcPath)) { + continue + } + + err = copyFile(srcPath, dstPath) + if err != nil { + return + } + } + } + + return + } + var validExportExtensions = []string{".a", ".properties"} for ext := range globals.SourceFilesValidExtensions { validExportExtensions = append(validExportExtensions, ext) @@ -76,7 +202,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { // Copy used libraries in the correct folder libDir := libBaseFolder.Join(library.DirName) mcu := ctx.BuildProperties.Get(constants.BUILD_PROPERTIES_BUILD_MCU) - utils.CopyDir(library.InstallDir.String(), libDir.String(), validExportExtensions) + copyDir(library.InstallDir.String(), libDir.String(), validExportExtensions) // Read cmake options if available isStaticLib := true @@ -96,7 +222,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { } // Remove stray folders contining incompatible or not needed libraries archives - files, _ := utils.FindFilesInFolder(libDir.Join("src"), true, validStaticLibExtensions) + files, _ := utils.FindFilesInFolder(libDir.Join("src"), true, validStaticLibExtensions...) for _, file := range files { staticLibDir := file.Parent() if !isStaticLib || !strings.Contains(staticLibDir.String(), mcu) { @@ -106,11 +232,11 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { } // Copy core + variant in use + preprocessed sketch in the correct folders - err := utils.CopyDir(ctx.BuildProperties.Get("build.core.path"), coreFolder.String(), validExportExtensions) + err := copyDir(ctx.BuildProperties.Get("build.core.path"), coreFolder.String(), validExportExtensions) if err != nil { fmt.Println(err) } - err = utils.CopyDir(ctx.BuildProperties.Get("build.variant.path"), coreFolder.Join("variant").String(), validExportExtensions) + err = copyDir(ctx.BuildProperties.Get("build.variant.path"), coreFolder.Join("variant").String(), validExportExtensions) if err != nil { fmt.Println(err) } @@ -119,13 +245,13 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { return err } - err = utils.CopyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions) + err = copyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions) if err != nil { fmt.Println(err) } // remove "#line 1 ..." from exported c_make folder sketch - sketchFiles, _ := utils.FindFilesInFolder(cmakeFolder.Join("sketch"), false, validExportExtensions) + sketchFiles, _ := utils.FindFilesInFolder(cmakeFolder.Join("sketch"), false, validExportExtensions...) for _, file := range sketchFiles { input, err := file.ReadFile() @@ -159,11 +285,11 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { extractCompileFlags(ctx, "recipe.cpp.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories) // Extract folders with .h in them for adding in include list - headerFiles, _ := utils.FindFilesInFolder(cmakeFolder, true, validHeaderExtensions) + headerFiles, _ := utils.FindFilesInFolder(cmakeFolder, true, validHeaderExtensions...) foldersContainingHeaders := findUniqueFoldersRelative(headerFiles.AsStrings(), cmakeFolder.String()) // Extract folders with .a in them for adding in static libs paths list - staticLibs, _ := utils.FindFilesInFolder(cmakeFolder, true, validStaticLibExtensions) + staticLibs, _ := utils.FindFilesInFolder(cmakeFolder, true, validStaticLibExtensions...) // Generate the CMakeLists global file @@ -232,25 +358,34 @@ func canExportCmakeProject(ctx *types.Context) bool { } func extractCompileFlags(ctx *types.Context, recipe string, defines, dynamicLibs, linkerflags, linkDirectories *[]string) { + appendIfNotPresent := func(target []string, elements ...string) []string { + for _, element := range elements { + if !slices.Contains(target, element) { + target = append(target, element) + } + } + return target + } + command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) for _, arg := range command.Args { if strings.HasPrefix(arg, "-D") { - *defines = utils.AppendIfNotPresent(*defines, arg) + *defines = appendIfNotPresent(*defines, arg) continue } if strings.HasPrefix(arg, "-l") { - *dynamicLibs = utils.AppendIfNotPresent(*dynamicLibs, arg[2:]) + *dynamicLibs = appendIfNotPresent(*dynamicLibs, arg[2:]) continue } if strings.HasPrefix(arg, "-L") { - *linkDirectories = utils.AppendIfNotPresent(*linkDirectories, arg[2:]) + *linkDirectories = appendIfNotPresent(*linkDirectories, arg[2:]) continue } if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "-I") && !strings.HasPrefix(arg, "-o") { // HACK : from linkerflags remove MMD (no cache is produced) if !strings.HasPrefix(arg, "-MMD") { - *linkerflags = utils.AppendIfNotPresent(*linkerflags, arg) + *linkerflags = appendIfNotPresent(*linkerflags, arg) } } } diff --git a/legacy/builder/gohasissues/go_has_issues.go b/legacy/builder/gohasissues/go_has_issues.go deleted file mode 100644 index 316308de429..00000000000 --- a/legacy/builder/gohasissues/go_has_issues.go +++ /dev/null @@ -1,119 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package gohasissues - -import ( - "io/fs" - "os" - "path/filepath" - "sort" -) - -func Walk(root string, walkFn filepath.WalkFunc) error { - info, err := os.Stat(root) - if err != nil { - return walkFn(root, nil, err) - } - return walk(root, info, walkFn) -} - -func walk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error { - err := walkFn(path, info, nil) - if err != nil { - if info.IsDir() && err == filepath.SkipDir { - return nil - } - return err - } - - if !info.IsDir() { - return nil - } - - names, err := readDirNames(path) - if err != nil { - return walkFn(path, info, err) - } - - for _, name := range names { - filename := filepath.Join(path, name) - fileInfo, err := os.Stat(filename) - if err != nil { - if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { - return err - } - } else { - err = walk(filename, fileInfo, walkFn) - if err != nil { - if !fileInfo.IsDir() || err != filepath.SkipDir { - return err - } - } - } - } - return nil -} - -// readDirNames reads the directory named by dirname and returns -// a sorted list of directory entries. -func readDirNames(dirname string) ([]string, error) { - f, err := os.Open(dirname) - if err != nil { - return nil, err - } - names, err := f.Readdirnames(-1) - f.Close() - if err != nil { - return nil, err - } - sort.Strings(names) - return names, nil -} - -func ReadDir(dirname string) ([]os.FileInfo, error) { - entries, err := os.ReadDir(dirname) - if err != nil { - return nil, err - } - - infos := make([]fs.FileInfo, 0, len(entries)) - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - return nil, err - } - - info, err = resolveSymlink(dirname, info) - if err != nil { - // unresolvable symlinks should be skipped silently - continue - } - infos = append(infos, info) - } - - return infos, nil -} - -func resolveSymlink(parentFolder string, info os.FileInfo) (os.FileInfo, error) { - if !isSymlink(info) { - return info, nil - } - return os.Stat(filepath.Join(parentFolder, info.Name())) -} - -func isSymlink(info os.FileInfo) bool { - return info.Mode()&os.ModeSymlink != 0 -} diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index e60af767b42..5c94573f349 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -16,6 +16,8 @@ package phases import ( + "crypto/md5" + "encoding/hex" "fmt" "os" "strings" @@ -90,26 +92,33 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path } } - // Recreate the archive if ANY of the core files (including platform.txt) has changed - realCoreFolder := coreFolder.Parent().Parent() - var targetArchivedCore *paths.Path - var buildCacheErr error if buildCachePath != nil { + realCoreFolder := coreFolder.Parent().Parent() archivedCoreName := GetCachedCoreArchiveDirName( buildProperties.Get("build.fqbn"), buildProperties.Get("compiler.optimization_flags"), realCoreFolder) targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a") - _, buildCacheErr = buildcache.New(buildCachePath).GetOrCreate(archivedCoreName) - if errors.Is(buildCacheErr, buildcache.CreateDirErr) { + if _, err := buildcache.New(buildCachePath).GetOrCreate(archivedCoreName); errors.Is(err, buildcache.CreateDirErr) { return nil, nil, fmt.Errorf(tr("creating core cache folder: %s", err)) } - canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase && - !ctx.Clean && - !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) + var canUseArchivedCore bool + if ctx.OnlyUpdateCompilationDatabase || ctx.Clean { + canUseArchivedCore = false + } else if isOlder, err := builder_utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder { + // Recreate the archive if ANY of the core files (including platform.txt) has changed + canUseArchivedCore = false + } else if targetCoreFolder == nil || realCoreFolder.EquivalentTo(targetCoreFolder) { + canUseArchivedCore = true + } else if isOlder, err := builder_utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder { + // Recreate the archive if ANY of the build core files (including platform.txt) has changed + canUseArchivedCore = false + } else { + canUseArchivedCore = true + } if canUseArchivedCore { // use archived core @@ -157,11 +166,16 @@ func GetCachedCoreArchiveDirName(fqbn string, optimizationFlags string, coreFold if absCoreFolder, err := coreFolder.Abs(); err == nil { coreFolder = absCoreFolder } // silently continue if absolute path can't be detected - hash := utils.MD5Sum([]byte(coreFolder.String() + optimizationFlags)) + + md5Sum := func(data []byte) string { + md5sumBytes := md5.Sum(data) + return hex.EncodeToString(md5sumBytes[:]) + } + hash := md5Sum([]byte(coreFolder.String() + optimizationFlags)) realName := fqbnToUnderscore + "_" + hash if len(realName) > 100 { - // avoid really long names, simply hash the name - realName = utils.MD5Sum([]byte(fqbnToUnderscore + "_" + hash)) + // avoid really long names, simply hash the name again + realName = md5Sum([]byte(realName)) } return realName } diff --git a/legacy/builder/test/helper_tools_downloader.go b/legacy/builder/test/helper_tools_downloader.go index 133122c4ab6..c32042c0c15 100644 --- a/legacy/builder/test/helper_tools_downloader.go +++ b/legacy/builder/test/helper_tools_downloader.go @@ -22,13 +22,11 @@ import ( "net/http" "os" "os/exec" - "path/filepath" "runtime" "strings" "testing" "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/gohasissues" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -455,32 +453,17 @@ func downloadAndUnpackCore(core Core, url string, targetPath *paths.Path) error packagerPath := targetPath.Join(core.Maintainer) corePath := targetPath.Join(core.Maintainer, core.Arch) + if err := packagerPath.MkdirAll(); err != nil { + return errors.WithStack(err) + } if corePath.Exist() { if err := corePath.RemoveAll(); err != nil { return errors.WithStack(err) } } - - if len(files) == 1 && files[0].IsDir() { - if err := packagerPath.MkdirAll(); err != nil { - return errors.WithStack(err) - } - err = copyRecursive(unpackFolder.Join(files[0].Name()), targetPath.Join(core.Maintainer, core.Arch)) - if err != nil { - return errors.WithStack(err) - } - } else { - if err := targetPath.Join(core.Maintainer, core.Arch).MkdirAll(); err != nil { - return errors.WithStack(err) - } - for _, file := range files { - err = copyRecursive(unpackFolder.Join(file.Name()), targetPath.Join(core.Maintainer, core.Arch, file.Name())) - if err != nil { - return errors.WithStack(err) - } - } + if err := unpackFolder.Join(files[0].Base()).CopyDirTo(corePath); err != nil { + return errors.WithStack(err) } - return nil } @@ -505,27 +488,12 @@ func downloadAndUnpackBoardManagerCore(core Core, url string, targetPath *paths. return errors.WithStack(err) } } - - if len(files) == 1 && files[0].IsDir() { - if err := corePath.MkdirAll(); err != nil { - return errors.WithStack(err) - } - err = copyRecursive(unpackFolder.Join(files[0].Name()), corePath.Join(core.Version)) - if err != nil { - return errors.WithStack(err) - } - } else { - if err := corePath.Join(core.Version).MkdirAll(); err != nil { - return errors.WithStack(err) - } - for _, file := range files { - err = copyRecursive(unpackFolder.Join(file.Name()), corePath.Join(core.Version, file.Name())) - if err != nil { - return errors.WithStack(err) - } - } + if err := corePath.MkdirAll(); err != nil { + return errors.WithStack(err) + } + if err := unpackFolder.Join(files[0].Base()).CopyDirTo(corePath.Join(core.Version)); err != nil { + return errors.WithStack(err) } - return nil } @@ -544,26 +512,12 @@ func downloadAndUnpackBoardsManagerTool(tool Tool, url string, targetPath *paths } defer unpackFolder.RemoveAll() - if len(files) == 1 && files[0].IsDir() { - if err := targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name).MkdirAll(); err != nil { - return errors.WithStack(err) - } - err = copyRecursive(unpackFolder.Join(files[0].Name()), targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version)) - if err != nil { - return errors.WithStack(err) - } - } else { - if err := targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version).MkdirAll(); err != nil { - return errors.WithStack(err) - } - for _, file := range files { - err = copyRecursive(unpackFolder.Join(file.Name()), targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version, file.Name())) - if err != nil { - return errors.WithStack(err) - } - } + if err := targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name).MkdirAll(); err != nil { + return errors.WithStack(err) + } + if err := unpackFolder.Join(files[0].Base()).CopyDirTo(targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version)); err != nil { + return errors.WithStack(err) } - return nil } @@ -590,31 +544,22 @@ func downloadAndUnpackTool(tool Tool, url string, targetPath *paths.Path, delete } } } - + if err := toolPath.MkdirAll(); err != nil { + return errors.WithStack(err) + } + destDir := toolPath.Join(tool.Version) if len(files) == 1 && files[0].IsDir() { - if err := toolPath.MkdirAll(); err != nil { - return errors.WithStack(err) - } - err = copyRecursive(unpackFolder.Join(files[0].Name()), toolPath.Join(tool.Version)) - if err != nil { + if err := unpackFolder.Join(files[0].Base()).CopyDirTo(destDir); err != nil { return errors.WithStack(err) } } else { - if err := toolPath.Join(tool.Version).MkdirAll(); err != nil { - return errors.WithStack(err) - } - for _, file := range files { - err = copyRecursive(unpackFolder.Join(file.Name()), toolPath.Join(tool.Version, file.Name())) - if err != nil { - return errors.WithStack(err) - } - } + unpackFolder.CopyDirTo(destDir) } return nil } -func downloadAndUnpack(url string) (*paths.Path, []os.FileInfo, error) { +func downloadAndUnpack(url string) (*paths.Path, paths.PathList, error) { fmt.Fprintln(os.Stderr, "Downloading "+url) unpackFolder, err := paths.MkTempDir("", "arduino-builder-tool") @@ -651,7 +596,7 @@ func downloadAndUnpack(url string) (*paths.Path, []os.FileInfo, error) { archiveFilePath.Remove() - files, err := gohasissues.ReadDir(unpackFolder.String()) + files, err := unpackFolder.ReadDir() if err != nil { return nil, nil, errors.WithStack(err) } @@ -773,65 +718,9 @@ func downloadAndUnpackLibrary(library Library, url string, targetPath *paths.Pat } } - err = copyRecursive(unpackFolder.Join(files[0].Name()), libPath) - if err != nil { + if err := unpackFolder.Join(files[0].Base()).CopyDirTo(libPath); err != nil { return errors.WithStack(err) } return nil } - -func copyRecursive(from, to *paths.Path) error { - copyFunc := func(currentPath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - rel, err := filepath.Rel(from.String(), currentPath) - if err != nil { - return errors.WithStack(err) - } - targetPath := filepath.Join(to.String(), rel) - if info.IsDir() { - err := os.MkdirAll(targetPath, info.Mode()) - if err != nil { - return errors.WithStack(err) - } - } else if info.Mode().IsRegular() { - fromFile, err := os.Open(currentPath) - if err != nil { - return errors.WithStack(err) - } - defer fromFile.Close() - targetFile, err := os.Create(targetPath) - if err != nil { - return errors.WithStack(err) - } - defer targetFile.Close() - _, err = io.Copy(targetFile, fromFile) - if err != nil { - return errors.WithStack(err) - } - err = os.Chmod(targetPath, info.Mode()) - if err != nil { - return errors.WithStack(err) - } - } else if info.Mode()&os.ModeSymlink == os.ModeSymlink { - linkedFile, err := os.Readlink(currentPath) - if err != nil { - return errors.WithStack(err) - } - fromFile := filepath.Join(filepath.Dir(targetPath), linkedFile) - err = os.Symlink(fromFile, targetPath) - if err != nil { - return errors.WithStack(err) - } - } else { - return errors.Errorf("unable to copy file " + currentPath) - } - - return nil - } - err := gohasissues.Walk(from.String(), copyFunc) - return errors.WithStack(err) -} diff --git a/legacy/builder/test/wipeout_build_path_if_build_options_changed_test.go b/legacy/builder/test/wipeout_build_path_if_build_options_changed_test.go index 2e28b3eff07..e46a66a7dbb 100644 --- a/legacy/builder/test/wipeout_build_path_if_build_options_changed_test.go +++ b/legacy/builder/test/wipeout_build_path_if_build_options_changed_test.go @@ -19,7 +19,6 @@ import ( "testing" "github.com/arduino/arduino-cli/legacy/builder" - "github.com/arduino/arduino-cli/legacy/builder/gohasissues" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/stretchr/testify/require" ) @@ -48,7 +47,7 @@ func TestWipeoutBuildPathIfBuildOptionsChanged(t *testing.T) { NoError(t, err) require.True(t, exist) - files, err := gohasissues.ReadDir(buildPath.String()) + files, err := buildPath.ReadDir() NoError(t, err) require.Equal(t, 0, len(files)) @@ -80,7 +79,7 @@ func TestWipeoutBuildPathIfBuildOptionsChangedNoPreviousBuildOptions(t *testing. NoError(t, err) require.True(t, exist) - files, err := gohasissues.ReadDir(buildPath.String()) + files, err := buildPath.ReadDir() NoError(t, err) require.Equal(t, 1, len(files)) diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 3efc8b73d6d..102b007883f 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -17,74 +17,20 @@ package utils import ( "bytes" - "crypto/md5" - "encoding/hex" - "fmt" - "io" "os" "os/exec" - "path/filepath" "strings" "unicode" - "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/gohasissues" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/pkg/errors" - "golang.org/x/exp/slices" "golang.org/x/text/runes" "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" ) -type filterFiles func([]os.FileInfo) []os.FileInfo - -var tr = i18n.Tr - -func ReadDirFiltered(folder string, fn filterFiles) ([]os.FileInfo, error) { - files, err := gohasissues.ReadDir(folder) - if err != nil { - return nil, errors.WithStack(err) - } - return fn(files), nil -} - -func FilterDirs(files []os.FileInfo) []os.FileInfo { - var filtered []os.FileInfo - for _, info := range files { - if info.IsDir() { - filtered = append(filtered, info) - } - } - return filtered -} - -func FilterFilesWithExtensions(extensions ...string) filterFiles { - return func(files []os.FileInfo) []os.FileInfo { - var filtered []os.FileInfo - for _, file := range files { - if !file.IsDir() && slices.Contains(extensions, filepath.Ext(file.Name())) { - filtered = append(filtered, file) - } - } - return filtered - } -} - -func FilterFiles() filterFiles { - return func(files []os.FileInfo) []os.FileInfo { - var filtered []os.FileInfo - for _, file := range files { - if !file.IsDir() { - filtered = append(filtered, file) - } - } - return filtered - } -} - var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} // FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name @@ -106,20 +52,6 @@ func FilterReadableFiles(file *paths.Path) bool { return true } -func IsSCCSOrHiddenFile(file os.FileInfo) bool { - return IsSCCSFile(file) || IsHiddenFile(file) -} - -func IsHiddenFile(file os.FileInfo) bool { - name := filepath.Base(file.Name()) - return name[0] == '.' -} - -func IsSCCSFile(file os.FileInfo) bool { - name := filepath.Base(file.Name()) - return SOURCE_CONTROL_FOLDERS[name] -} - func WrapWithHyphenI(value string) string { return "\"-I" + value + "\"" } @@ -194,29 +126,19 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) return outbytes, errbytes, errors.WithStack(err) } -func AbsolutizePaths(files []string) ([]string, error) { - for idx, file := range files { - if file == "" { - continue - } - absFile, err := filepath.Abs(file) - if err != nil { - return nil, errors.WithStack(err) - } - files[idx] = absFile - } - - return files, nil -} - -func FindFilesInFolder(dir *paths.Path, recurse bool, extensions []string) (paths.PathList, error) { +func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { fileFilter := paths.AndFilter( - paths.FilterSuffixes(extensions...), FilterOutHiddenFiles, FilterOutSCCS, paths.FilterOutDirectories(), FilterReadableFiles, ) + if len(extensions) > 0 { + fileFilter = paths.AndFilter( + paths.FilterSuffixes(extensions...), + fileFilter, + ) + } if recurse { dirFilter := paths.AndFilter( FilterOutHiddenFiles, @@ -227,20 +149,6 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions []string) (path return dir.ReadDir(fileFilter) } -func AppendIfNotPresent(target []string, elements ...string) []string { - for _, element := range elements { - if !slices.Contains(target, element) { - target = append(target, element) - } - } - return target -} - -func MD5Sum(data []byte) string { - md5sumBytes := md5.Sum(data) - return hex.EncodeToString(md5sumBytes[:]) -} - type loggerAction struct { onlyIfVerbose bool warn bool @@ -269,124 +177,3 @@ func NormalizeUTF8(buf []byte) []byte { result, _, _ := transform.Bytes(t, buf) return result } - -// CopyFile copies the contents of the file named src to the file named -// by dst. The file will be created if it does not already exist. If the -// destination file exists, all it's contents will be replaced by the contents -// of the source file. The file mode will be copied from the source and -// the copied data is synced/flushed to stable storage. -func CopyFile(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - - out, err := os.Create(dst) - if err != nil { - return - } - defer func() { - if e := out.Close(); e != nil { - err = e - } - }() - - _, err = io.Copy(out, in) - if err != nil { - return - } - - err = out.Sync() - if err != nil { - return - } - - si, err := os.Stat(src) - if err != nil { - return - } - err = os.Chmod(dst, si.Mode()) - if err != nil { - return - } - - return -} - -// CopyDir recursively copies a directory tree, attempting to preserve permissions. -// Source directory must exist, destination directory must *not* exist. -// Symlinks are ignored and skipped. -func CopyDir(src string, dst string, extensions []string) (err error) { - isAcceptedExtension := func(ext string) bool { - ext = strings.ToLower(ext) - for _, valid := range extensions { - if ext == valid { - return true - } - } - return false - } - - src = filepath.Clean(src) - dst = filepath.Clean(dst) - - si, err := os.Stat(src) - if err != nil { - return err - } - if !si.IsDir() { - return fmt.Errorf(tr("source is not a directory")) - } - - _, err = os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - return - } - if err == nil { - return fmt.Errorf(tr("destination already exists")) - } - - err = os.MkdirAll(dst, si.Mode()) - if err != nil { - return - } - - entries, err := os.ReadDir(src) - if err != nil { - return - } - - for _, dirEntry := range entries { - entry, scopeErr := dirEntry.Info() - if scopeErr != nil { - return - } - - srcPath := filepath.Join(src, entry.Name()) - dstPath := filepath.Join(dst, entry.Name()) - - if entry.IsDir() { - err = CopyDir(srcPath, dstPath, extensions) - if err != nil { - return - } - } else { - // Skip symlinks. - if entry.Mode()&os.ModeSymlink != 0 { - continue - } - - if !isAcceptedExtension(filepath.Ext(srcPath)) { - continue - } - - err = CopyFile(srcPath, dstPath) - if err != nil { - return - } - } - } - - return -} diff --git a/legacy/builder/wipeout_build_path_if_build_options_changed.go b/legacy/builder/wipeout_build_path_if_build_options_changed.go index 44e5207a619..2a6873de97b 100644 --- a/legacy/builder/wipeout_build_path_if_build_options_changed.go +++ b/legacy/builder/wipeout_build_path_if_build_options_changed.go @@ -66,9 +66,11 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { coreFolder := buildProperties.GetPath("build.core.path") realCoreFolder := coreFolder.Parent().Parent() jsonPath := ctx.BuildPath.Join(constants.BUILD_OPTIONS_FILE) - coreHasChanged := builder_utils.TXTBuildRulesHaveChanged(realCoreFolder, targetCoreFolder, jsonPath) - - if !coreHasChanged { + coreUnchanged, _ := builder_utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt") + if coreUnchanged && targetCoreFolder != nil && !realCoreFolder.EqualsTo(targetCoreFolder) { + coreUnchanged, _ = builder_utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt") + } + if coreUnchanged { return nil } }