Fix Vendoring Issues with Globs and Symlinks #984
+479
−6
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,255 @@ | ||
| package exec | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/cloudposse/atmos/pkg/schema" | ||
| u "github.com/cloudposse/atmos/pkg/utils" | ||
| cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required. | ||
| ) | ||
|
|
||
| // copyFile copies a single file from src to dst while preserving file permissions. | ||
| func copyFile(atmosConfig schema.AtmosConfiguration, src, dst string) error { | ||
| sourceFile, err := os.Open(src) | ||
| if err != nil { | ||
| return fmt.Errorf("opening source file %q: %w", src, err) | ||
| } | ||
| defer sourceFile.Close() | ||
|
|
||
| if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { | ||
| return fmt.Errorf("creating destination directory for %q: %w", dst, err) | ||
| } | ||
|
|
||
| destinationFile, err := os.Create(dst) | ||
| if err != nil { | ||
| return fmt.Errorf("creating destination file %q: %w", dst, err) | ||
| } | ||
| defer destinationFile.Close() | ||
|
|
||
| if _, err := io.Copy(destinationFile, sourceFile); err != nil { | ||
| return fmt.Errorf("copying content from %q to %q: %w", src, dst, err) | ||
| } | ||
|
|
||
| info, err := os.Stat(src) | ||
| if err != nil { | ||
| return fmt.Errorf("getting file info for %q: %w", src, err) | ||
| } | ||
| if err := os.Chmod(dst, info.Mode()); err != nil { | ||
| return fmt.Errorf("setting permissions on %q: %w", dst, err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // skipFunc determines whether to skip a file/directory based on its relative path to baseDir. | ||
| // If an error occurs during matching for an exclusion or inclusion pattern, it logs the error and proceeds. | ||
| func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { | ||
| if info.Name() == ".git" { | ||
| return true, nil | ||
| } | ||
| relPath, err := filepath.Rel(baseDir, srcPath) | ||
| if err != nil { | ||
| l.LogDebug("Error computing relative path", 'srcPath', srcPath, 'err', err) | ||
|
Check failure on line 55 in internal/exec/copy_glob.go
|
||
| return true, nil // treat error as a signal to skip | ||
| } | ||
| relPath = filepath.ToSlash(relPath) | ||
|
|
||
| // Process exclusion patterns. | ||
| for _, pattern := range excluded { | ||
| matched, err := u.PathMatch(pattern, relPath) | ||
| if err != nil { | ||
| u.LogTrace(fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", pattern, relPath, err)) | ||
| continue | ||
| } else if matched { | ||
| u.LogTrace(fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, pattern)) | ||
| return true, nil | ||
| } | ||
| } | ||
|
|
||
| // Process inclusion patterns (only for non-directory files). | ||
| if len(included) > 0 && !info.IsDir() { | ||
| matchedAny := false | ||
| for _, pattern := range included { | ||
| matched, err := u.PathMatch(pattern, relPath) | ||
| if err != nil { | ||
| u.LogTrace(fmt.Sprintf("Error matching inclusion pattern %q with %q: %v", pattern, relPath, err)) | ||
| continue | ||
| } else if matched { | ||
| u.LogTrace(fmt.Sprintf("Including %q because it matches inclusion pattern %q", relPath, pattern)) | ||
| matchedAny = true | ||
| break | ||
| } | ||
| } | ||
| if !matchedAny { | ||
| u.LogTrace(fmt.Sprintf("Excluding %q because it does not match any inclusion pattern", relPath)) | ||
| return true, nil | ||
| } | ||
| } | ||
| return false, nil | ||
| } | ||
|
|
||
| // copyDirRecursive recursively copies srcDir to dstDir using skipFunc filtering. | ||
| func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, baseDir string, excluded, included []string) error { | ||
| entries, err := os.ReadDir(srcDir) | ||
| if err != nil { | ||
| return fmt.Errorf("reading directory %q: %w", srcDir, err) | ||
| } | ||
| for _, entry := range entries { | ||
| srcPath := filepath.Join(srcDir, entry.Name()) | ||
| dstPath := filepath.Join(dstDir, entry.Name()) | ||
|
|
||
| info, err := entry.Info() | ||
| if err != nil { | ||
| return fmt.Errorf("getting info for %q: %w", srcPath, err) | ||
| } | ||
|
|
||
| skip, err := skipFunc(atmosConfig, info, srcPath, baseDir, excluded, included) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if skip { | ||
| continue | ||
| } | ||
|
|
||
| // Skip symlinks. | ||
| if info.Mode()&os.ModeSymlink != 0 { | ||
| u.LogTrace(fmt.Sprintf("Skipping symlink: %q", srcPath)) | ||
| continue | ||
| } | ||
|
|
||
| if info.IsDir() { | ||
| if err := os.MkdirAll(dstPath, info.Mode()); err != nil { | ||
| return fmt.Errorf("creating directory %q: %w", dstPath, err) | ||
| } | ||
| if err := copyDirRecursive(atmosConfig, srcPath, dstPath, baseDir, excluded, included); err != nil { | ||
| return err | ||
| } | ||
| } else { | ||
| if err := copyFile(atmosConfig, srcPath, dstPath); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. | ||
| // If no matches are found, it logs a trace and returns an empty slice. | ||
| // When the pattern ends with "/*", it retries with a recursive "/**" variant. | ||
| func getMatchesForPattern(atmosConfig schema.AtmosConfiguration, sourceDir, pattern string) ([]string, error) { | ||
| fullPattern := filepath.Join(sourceDir, pattern) | ||
| matches, err := u.GetGlobMatches(fullPattern) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err) | ||
| } | ||
| if len(matches) == 0 { | ||
| if strings.HasSuffix(pattern, "/*") { | ||
| recursivePattern := strings.TrimSuffix(pattern, "/*") + "/**" | ||
| fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) | ||
| matches, err = u.GetGlobMatches(fullRecursivePattern) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) | ||
| } | ||
| if len(matches) == 0 { | ||
| u.LogTrace(fmt.Sprintf("No matches found for recursive pattern %q - target directory will be empty", fullRecursivePattern)) | ||
| return []string{}, nil | ||
| } | ||
| return matches, nil | ||
| } | ||
| u.LogTrace(fmt.Sprintf("No matches found for pattern %q - target directory will be empty", fullPattern)) | ||
| return []string{}, nil | ||
| } | ||
| return matches, nil | ||
| } | ||
|
|
||
| // copyToTargetWithPatterns copies the contents from sourceDir to targetPath, | ||
| // applying inclusion and exclusion patterns from the vendor source configuration. | ||
| // If sourceIsLocalFile is true and targetPath lacks an extension, the sanitized URI is appended. | ||
| // If no included paths are defined, all files (except those matching excluded paths) are copied. | ||
| // In the special case where neither inclusion nor exclusion patterns are defined, | ||
| // the optimized cp library (github.com/otiai10/copy) is used. | ||
| func copyToTargetWithPatterns( | ||
| atmosConfig schema.AtmosConfiguration, | ||
| sourceDir, targetPath string, | ||
| s *schema.AtmosVendorSource, | ||
| sourceIsLocalFile bool, | ||
| uri string, | ||
| ) error { | ||
| if sourceIsLocalFile && filepath.Ext(targetPath) == "" { | ||
| targetPath = filepath.Join(targetPath, SanitizeFileName(uri)) | ||
| } | ||
| u.LogTrace(fmt.Sprintf("Copying from %q to %q", sourceDir, targetPath)) | ||
| if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { | ||
| return fmt.Errorf("creating target directory %q: %w", targetPath, err) | ||
| } | ||
|
|
||
| // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library for fast copying. | ||
| if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { | ||
| u.LogTrace("No inclusion or exclusion patterns defined; using cp library for fast copy") | ||
| return cp.Copy(sourceDir, targetPath) | ||
| } | ||
|
|
||
| // If inclusion patterns are provided, use them to determine which files to copy. | ||
| if len(s.IncludedPaths) > 0 { | ||
| filesToCopy := make(map[string]struct{}) | ||
| for _, pattern := range s.IncludedPaths { | ||
| matches, err := getMatchesForPattern(atmosConfig, sourceDir, pattern) | ||
| if err != nil { | ||
| u.LogTrace(fmt.Sprintf("Warning: error getting matches for pattern %q: %v", pattern, err)) | ||
| continue | ||
| } | ||
| for _, match := range matches { | ||
| filesToCopy[match] = struct{}{} | ||
| } | ||
| } | ||
| if len(filesToCopy) == 0 { | ||
| u.LogTrace("No files matched the inclusion patterns - target directory will be empty") | ||
Listener430 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return nil | ||
| } | ||
| for file := range filesToCopy { | ||
| relPath, err := filepath.Rel(sourceDir, file) | ||
| if err != nil { | ||
| return fmt.Errorf("computing relative path for %q: %w", file, err) | ||
| } | ||
| relPath = filepath.ToSlash(relPath) | ||
| skip := false | ||
| for _, ex := range s.ExcludedPaths { | ||
| matched, err := u.PathMatch(ex, relPath) | ||
| if err != nil { | ||
| u.LogTrace(fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", ex, relPath, err)) | ||
| continue | ||
| } else if matched { | ||
| u.LogTrace(fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, ex)) | ||
| skip = true | ||
| break | ||
| } | ||
| } | ||
| if skip { | ||
| continue | ||
| } | ||
| dstPath := filepath.Join(targetPath, relPath) | ||
| info, err := os.Stat(file) | ||
| if err != nil { | ||
| return fmt.Errorf("stating file %q: %w", file, err) | ||
| } | ||
| if info.IsDir() { | ||
| if err := copyDirRecursive(atmosConfig, file, dstPath, file, s.ExcludedPaths, nil); err != nil { | ||
| return err | ||
| } | ||
| } else { | ||
| if err := copyFile(atmosConfig, file, dstPath); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
| } else { | ||
| // No inclusion patterns defined; copy everything except those matching excluded items. | ||
| if err := copyDirRecursive(atmosConfig, sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { | ||
| return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
💡 Verification agent
🧩 Analysis chain
Handle potential race condition in directory creation.
The
MkdirAllfollowed by operations on the directory could be subject to a race condition if multiple processes are involved.Consider adding file locking or checking for concurrent access issues. Let's check if there are any existing synchronization mechanisms in the codebase:
🏁 Script executed:
Length of output: 28258
Handle race condition in directory creation in internal/exec/copy_glob.go
After reviewing the codebase, it’s clear that although other parts of the repository (for example, in pkg/config/cache.go) leverage file locks (using gofrs/flock) or internal mutexes to coordinate concurrent file operations, the directory creation in copy_glob.go is done solely via os.MkdirAll without any explicit locking mechanism. This could lead to potential race conditions in environments where multiple processes or goroutines attempt to create or operate on the same directory concurrently.
• The os.MkdirAll call is inherently idempotent, but subsequent operations on the newly created directory might be affected by concurrent access.
• Unlike other modules that incorporate file locking, no synchronization guards the MkdirAll in copy_glob.go.
Consider introducing a locking mechanism (such as using a file-based lock or an internal mutex) around directory creation to ensure safe concurrent access.