From b3dd26f46051b7cfb35f0142bda0fe258c87fb61 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 16 Mar 2017 13:22:28 +0100 Subject: [PATCH 1/3] Allow zip sketches with embedded libraries --- container_setup.go | 2 +- libraries_loader.go | 4 +++ resolve_library.go | 14 +++++++++ sketch_loader.go | 29 +++++++++++++++++++ types/context.go | 3 ++ utils/utils.go | 70 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 1 deletion(-) diff --git a/container_setup.go b/container_setup.go index 93ee6d73..d8406676 100644 --- a/container_setup.go +++ b/container_setup.go @@ -46,8 +46,8 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) &ToolsLoader{}, &TargetBoardResolver{}, &AddBuildBoardPropertyIfMissing{}, - &LibrariesLoader{}, &SketchLoader{}, + &LibrariesLoader{}, &SetupBuildProperties{}, &LoadVIDPIDSpecificProperties{}, &SetCustomBuildProperties{}, diff --git a/libraries_loader.go b/libraries_loader.go index 1730bf44..6c8aca8d 100644 --- a/libraries_loader.go +++ b/libraries_loader.go @@ -63,6 +63,10 @@ func (s *LibrariesLoader) Run(ctx *types.Context) error { sortedLibrariesFolders = appendPathToLibrariesFolders(sortedLibrariesFolders, filepath.Join(platform.Folder, constants.FOLDER_LIBRARIES)) + if ctx.SketchZipped { + sortedLibrariesFolders = appendPathToLibrariesFolders(sortedLibrariesFolders, filepath.Join(filepath.Dir(ctx.SketchLocation), constants.FOLDER_LIBRARIES)) + } + librariesFolders := ctx.OtherLibrariesFolders librariesFolders, err = utils.AbsolutizePaths(librariesFolders) if err != nil { diff --git a/resolve_library.go b/resolve_library.go index dd96ab75..c69fd845 100644 --- a/resolve_library.go +++ b/resolve_library.go @@ -84,6 +84,11 @@ func ResolveLibrary(ctx *types.Context, header string) *types.Library { library = useAlreadyImportedLibraryWithSameNameIfExists(library, importedLibraries) + if ctx.SketchZipped { + // select embedded library, in any case + library = libraryContainedInSketchFolder(libraries, filepath.Dir(ctx.SketchLocation)) + } + libraryResolutionResults[header] = types.LibraryResolutionResult{ Library: library, NotUsedLibraries: filterOutLibraryFrom(libraries, library), @@ -119,6 +124,15 @@ func useAlreadyImportedLibraryWithSameNameIfExists(library *types.Library, impor return library } +func libraryContainedInSketchFolder(libraries []*types.Library, sketchFolder string) *types.Library { + for _, lib := range libraries { + if strings.Contains(lib.Folder, sketchFolder) { + return lib + } + } + return libraries[0] +} + func filterOutLibraryFrom(libraries []*types.Library, libraryToRemove *types.Library) []*types.Library { filteredOutLibraries := []*types.Library{} for _, lib := range libraries { diff --git a/sketch_loader.go b/sketch_loader.go index f8ab8caa..bf3b50a8 100644 --- a/sketch_loader.go +++ b/sketch_loader.go @@ -30,7 +30,9 @@ package builder import ( + "bufio" "io/ioutil" + "net/http" "os" "path/filepath" "sort" @@ -63,6 +65,17 @@ func (s *SketchLoader) Run(ctx *types.Context) error { sketchLocation = filepath.Join(sketchLocation, mainSketchStat.Name()+".ino") } + if mimeType(sketchLocation) == "application/zip" { + dir, _ := ioutil.TempDir("", "arduino_sketch_zip_temp") + sketchLocation, err = utils.ExtractZip(sketchLocation, dir) + if err != nil { + return nil + } + mainSketchFileName := filepath.Base(sketchLocation) + ".ino" + sketchLocation = filepath.Join(sketchLocation, mainSketchFileName) + ctx.SketchZipped = true + } + ctx.SketchLocation = sketchLocation allSketchFilePaths, err := collectAllSketchFiles(filepath.Dir(sketchLocation)) @@ -87,6 +100,22 @@ func (s *SketchLoader) Run(ctx *types.Context) error { return nil } +func mimeType(fileName string) string { + fs, err := os.Open(fileName) + + if err != nil { + return "" + } + + defer fs.Close() + + reader := bufio.NewReader(fs) + + buf := make([]byte, 512) + reader.Read(buf) + return http.DetectContentType(buf) +} + func collectAllSketchFiles(from string) ([]string, error) { filePaths := []string{} // Source files in the root are compiled, non-recursively. This diff --git a/types/context.go b/types/context.go index bc9491dd..debf2192 100644 --- a/types/context.go +++ b/types/context.go @@ -79,6 +79,9 @@ type Context struct { Verbose bool DebugPreprocessor bool + // inoz handling + SketchZipped bool + // Contents of a custom build properties file (line by line) CustomBuildProperties []string diff --git a/utils/utils.go b/utils/utils.go index 318b3984..e17cc183 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -30,8 +30,10 @@ package utils import ( + "archive/zip" "crypto/md5" "encoding/hex" + "io" "io/ioutil" "os" "os/exec" @@ -488,3 +490,71 @@ func ParseCppString(line string) (string, string, bool) { i += width } } + +func ExtractZip(filePath string, location string) (string, error) { + r, err := zip.OpenReader(filePath) + if err != nil { + return location, err + } + + var dirList []string + + for _, f := range r.File { + dirList = append(dirList, f.Name) + } + + basedir := findBaseDir(dirList) + + for _, f := range r.File { + fullname := filepath.Join(location, strings.Replace(f.Name, "", "", -1)) + if f.FileInfo().IsDir() { + os.MkdirAll(fullname, f.FileInfo().Mode().Perm()) + } else { + os.MkdirAll(filepath.Dir(fullname), 0755) + perms := f.FileInfo().Mode().Perm() + out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms) + if err != nil { + return location, err + } + rc, err := f.Open() + if err != nil { + return location, err + } + _, err = io.CopyN(out, rc, f.FileInfo().Size()) + if err != nil { + return location, err + } + rc.Close() + out.Close() + + mtime := f.FileInfo().ModTime() + err = os.Chtimes(fullname, mtime, mtime) + if err != nil { + return location, err + } + } + } + return filepath.Join(location, basedir), nil +} + +func findBaseDir(dirList []string) string { + baseDir := "" + // https://github.com/backdrop-ops/contrib/issues/55#issuecomment-73814500 + dontdiff := []string{"pax_global_header"} + for index := range dirList { + if SliceContains(dontdiff, dirList[index]) { + continue + } + candidateBaseDir := dirList[index] + for i := index; i < len(dirList); i++ { + if !strings.Contains(dirList[i], candidateBaseDir) { + return baseDir + } + } + // avoid setting the candidate if it is the last file + if dirList[len(dirList)-1] != candidateBaseDir { + baseDir = candidateBaseDir + } + } + return baseDir +} From e3a30966b139a876cbc3fcdeb6b111ef1f691786 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 16 Mar 2017 18:48:37 +0100 Subject: [PATCH 2/3] Fix permission problems while unzipping --- sketch_loader.go | 2 +- utils/utils.go | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sketch_loader.go b/sketch_loader.go index bf3b50a8..f2b92355 100644 --- a/sketch_loader.go +++ b/sketch_loader.go @@ -69,7 +69,7 @@ func (s *SketchLoader) Run(ctx *types.Context) error { dir, _ := ioutil.TempDir("", "arduino_sketch_zip_temp") sketchLocation, err = utils.ExtractZip(sketchLocation, dir) if err != nil { - return nil + panic(err) } mainSketchFileName := filepath.Base(sketchLocation) + ".ino" sketchLocation = filepath.Join(sketchLocation, mainSketchFileName) diff --git a/utils/utils.go b/utils/utils.go index e17cc183..13960d4e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -508,9 +508,12 @@ func ExtractZip(filePath string, location string) (string, error) { for _, f := range r.File { fullname := filepath.Join(location, strings.Replace(f.Name, "", "", -1)) if f.FileInfo().IsDir() { - os.MkdirAll(fullname, f.FileInfo().Mode().Perm()) + os.MkdirAll(fullname, 0755) } else { - os.MkdirAll(filepath.Dir(fullname), 0755) + _, err := os.Stat(filepath.Dir(fullname)) + if err != nil { + os.MkdirAll(filepath.Dir(fullname), 0755) + } perms := f.FileInfo().Mode().Perm() out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms) if err != nil { From 920572964becaf638ff93dba94343d4d67e204be Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 17 Mar 2017 09:29:58 +0100 Subject: [PATCH 3/3] rethink zip extraction --- sketch_loader.go | 2 +- utils/utils.go | 27 ++++++++++----------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/sketch_loader.go b/sketch_loader.go index f2b92355..ca6858f9 100644 --- a/sketch_loader.go +++ b/sketch_loader.go @@ -85,7 +85,7 @@ func (s *SketchLoader) Run(ctx *types.Context) error { logger := ctx.GetLogger() - if !utils.SliceContains(allSketchFilePaths, sketchLocation) { + if !utils.SliceContains(allSketchFilePaths, sketchLocation) && !ctx.SketchZipped { return i18n.ErrorfWithLogger(logger, constants.MSG_CANT_FIND_SKETCH_IN_PATH, sketchLocation, filepath.Dir(sketchLocation)) } diff --git a/utils/utils.go b/utils/utils.go index 13960d4e..67f575b9 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -499,19 +499,15 @@ func ExtractZip(filePath string, location string) (string, error) { var dirList []string - for _, f := range r.File { - dirList = append(dirList, f.Name) - } - - basedir := findBaseDir(dirList) - for _, f := range r.File { fullname := filepath.Join(location, strings.Replace(f.Name, "", "", -1)) if f.FileInfo().IsDir() { + dirList = append(dirList, fullname) os.MkdirAll(fullname, 0755) } else { _, err := os.Stat(filepath.Dir(fullname)) if err != nil { + dirList = append(dirList, filepath.Dir(fullname)) os.MkdirAll(filepath.Dir(fullname), 0755) } perms := f.FileInfo().Mode().Perm() @@ -537,26 +533,23 @@ func ExtractZip(filePath string, location string) (string, error) { } } } + basedir := filepath.Base(findBaseDir(dirList)) return filepath.Join(location, basedir), nil } func findBaseDir(dirList []string) string { baseDir := "" + minLen := 256 // https://github.com/backdrop-ops/contrib/issues/55#issuecomment-73814500 dontdiff := []string{"pax_global_header"} - for index := range dirList { - if SliceContains(dontdiff, dirList[index]) { + for _, dir := range dirList { + if SliceContains(dontdiff, dir) { continue } - candidateBaseDir := dirList[index] - for i := index; i < len(dirList); i++ { - if !strings.Contains(dirList[i], candidateBaseDir) { - return baseDir - } - } - // avoid setting the candidate if it is the last file - if dirList[len(dirList)-1] != candidateBaseDir { - baseDir = candidateBaseDir + //get the shortest string + if len(dir) < minLen { + baseDir = dir + minLen = len(dir) } } return baseDir