Fix Vendoring Issues with Globs and Symlinks

internal/exec/copy_glob.go

@@ -0,0 +1,255 @@
+package exec
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	l "github.com/charmbracelet/log"
+	"github.com/cloudposse/atmos/pkg/schema"
+	u "github.com/cloudposse/atmos/pkg/utils"
+	cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required.
+)
+
+// copyFile copies a single file from src to dst while preserving file permissions.
+func copyFile(src, dst string) error {
+	sourceFile, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("opening source file %q: %w", src, err)
+	}
+	defer sourceFile.Close()
+
+	if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
+		return fmt.Errorf("creating destination directory for %q: %w", dst, err)
+	}
+
+	destinationFile, err := os.Create(dst)
+	if err != nil {
+		return fmt.Errorf("creating destination file %q: %w", dst, err)
+	}
+	defer destinationFile.Close()
+
+	if _, err := io.Copy(destinationFile, sourceFile); err != nil {
+		return fmt.Errorf("copying content from %q to %q: %w", src, dst, err)
+	}
+
+	info, err := os.Stat(src)
+	if err != nil {
+		return fmt.Errorf("getting file info for %q: %w", src, err)
+	}
+	if err := os.Chmod(dst, info.Mode()); err != nil {
+		return fmt.Errorf("setting permissions on %q: %w", dst, err)
+	}
+	return nil
+}
+
+// shouldSkipEntry determines whether to skip a file/directory based on its relative path to baseDir.
+// If an error occurs during matching for an exclusion or inclusion pattern, it logs the error and proceeds.
+func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) {
+	if info.Name() == ".git" {
+		return true, nil
+	}
+	relPath, err := filepath.Rel(baseDir, srcPath)
+	if err != nil {
+		l.Debug("Error computing relative path", "srcPath", srcPath, "error", err)
+		return true, nil // treat error as a signal to skip
+	}
+	relPath = filepath.ToSlash(relPath)
+
+	// Process exclusion patterns.
+	for _, pattern := range excluded {
+		matched, err := u.PathMatch(pattern, relPath)
+		if err != nil {
+			l.Debug("Error matching exclusion pattern", "pattern", pattern, "path", relPath, "error", err)
+			continue
+		} else if matched {
+			l.Debug("Excluding path due to exclusion pattern", "path", relPath, "pattern", pattern)
+			return true, nil
+		}
+	}
+
+	// Process inclusion patterns (only for non-directory files).
+	if len(included) > 0 && !info.IsDir() {
+		matchedAny := false
+		for _, pattern := range included {
+			matched, err := u.PathMatch(pattern, relPath)
+			if err != nil {
+				l.Debug("Error matching inclusion pattern", "pattern", pattern, "path", relPath, "error", err)
+				continue
+			} else if matched {
+				l.Debug("Including path due to inclusion pattern", "path", relPath, "pattern", pattern)
+				matchedAny = true
+				break
+			}
+		}
+		if !matchedAny {
+			l.Debug("Excluding path because it does not match any inclusion pattern", "path", relPath)
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering.
+func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []string) error {
+	entries, err := os.ReadDir(srcDir)
+	if err != nil {
+		return fmt.Errorf("reading directory %q: %w", srcDir, err)
+	}
+	for _, entry := range entries {
+		srcPath := filepath.Join(srcDir, entry.Name())
+		dstPath := filepath.Join(dstDir, entry.Name())
+
+		info, err := entry.Info()
+		if err != nil {
+			return fmt.Errorf("getting info for %q: %w", srcPath, err)
+		}
+
+		skip, err := shouldSkipEntry(info, srcPath, baseDir, excluded, included)
+		if err != nil {
+			return err
+		}
+		if skip {
+			continue
+		}
+
+		// Skip symlinks.
+		if info.Mode()&os.ModeSymlink != 0 {
+			l.Debug("Skipping symlink", "path", srcPath)
+			continue
+		}
+
+		if info.IsDir() {
+			if err := os.MkdirAll(dstPath, info.Mode()); err != nil {
+				return fmt.Errorf("creating directory %q: %w", dstPath, err)
+			}
+			if err := copyDirRecursive(srcPath, dstPath, baseDir, excluded, included); err != nil {
+				return err
+			}
+		} else {
+			if err := copyFile(srcPath, dstPath); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// getMatchesForPattern returns files/directories matching a pattern relative to sourceDir.
+// If no matches are found, it logs a debug message and returns an empty slice.
+// When the pattern ends with "/*", it retries with a recursive "/**" variant.
+func getMatchesForPattern(sourceDir, pattern string) ([]string, error) {
+	fullPattern := filepath.Join(sourceDir, pattern)
+	matches, err := u.GetGlobMatches(fullPattern)
+	if err != nil {
+		return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err)
+	}
+	if len(matches) == 0 {
+		if strings.HasSuffix(pattern, "/*") {
+			recursivePattern := strings.TrimSuffix(pattern, "/*") + "/**"
+			fullRecursivePattern := filepath.Join(sourceDir, recursivePattern)
+			matches, err = u.GetGlobMatches(fullRecursivePattern)
+			if err != nil {
+				return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err)
+			}
+			if len(matches) == 0 {
+				l.Debug("No matches found for recursive pattern; target directory will be empty", "pattern", fullRecursivePattern)
+				return []string{}, nil
+			}
+			return matches, nil
+		}
+		l.Debug("No matches found for pattern; target directory will be empty", "pattern", fullPattern)
+		return []string{}, nil
+	}
+	return matches, nil
+}
+
+// copyToTargetWithPatterns copies the contents from sourceDir to targetPath,
+// applying inclusion and exclusion patterns from the vendor source configuration.
+// If sourceIsLocalFile is true and targetPath lacks an extension, the sanitized URI is appended.
+// If no included paths are defined, all files (except those matching excluded paths) are copied.
+// In the special case where neither inclusion nor exclusion patterns are defined,
+// the optimized cp library (github.com/otiai10/copy) is used.
+func copyToTargetWithPatterns(
+	sourceDir, targetPath string,
+	s *schema.AtmosVendorSource,
+	sourceIsLocalFile bool,
+	uri string,
+) error {
+	if sourceIsLocalFile && filepath.Ext(targetPath) == "" {
+		targetPath = filepath.Join(targetPath, SanitizeFileName(uri))
+	}
+	l.Debug("Copying files", "source", sourceDir, "target", targetPath)
+	if err := os.MkdirAll(targetPath, os.ModePerm); err != nil {
+		return fmt.Errorf("creating target directory %q: %w", targetPath, err)
+	}
+
+	// Optimization: if no inclusion and no exclusion patterns are defined, use the cp library for fast copying.
+	if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 {
+		l.Debug("No inclusion or exclusion patterns defined; using cp library for fast copy")
+		return cp.Copy(sourceDir, targetPath)
+	}
+
+	// If inclusion patterns are provided, use them to determine which files to copy.
+	if len(s.IncludedPaths) > 0 {
+		filesToCopy := make(map[string]struct{})
+		for _, pattern := range s.IncludedPaths {
+			matches, err := getMatchesForPattern(sourceDir, pattern)
+			if err != nil {
+				l.Debug("Warning: error getting matches for pattern", "pattern", pattern, "error", err)
+				continue
+			}
+			for _, match := range matches {
+				filesToCopy[match] = struct{}{}
+			}
+		}
+		if len(filesToCopy) == 0 {
+			l.Debug("No files matched the inclusion patterns - target directory will be empty")
+			return nil
+		}
+		for file := range filesToCopy {
+			relPath, err := filepath.Rel(sourceDir, file)
+			if err != nil {
+				return fmt.Errorf("computing relative path for %q: %w", file, err)
+			}
+			relPath = filepath.ToSlash(relPath)
+			skip := false
+			for _, ex := range s.ExcludedPaths {
+				matched, err := u.PathMatch(ex, relPath)
+				if err != nil {
+					l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err)
+					continue
+				} else if matched {
+					l.Debug("Excluding file due to exclusion pattern", "file", relPath, "pattern", ex)
+					skip = true
+					break
+				}
+			}
+			if skip {
+				continue
+			}
+			dstPath := filepath.Join(targetPath, relPath)
+			info, err := os.Stat(file)
+			if err != nil {
+				return fmt.Errorf("stating file %q: %w", file, err)
+			}
+			if info.IsDir() {
+				if err := copyDirRecursive(file, dstPath, file, s.ExcludedPaths, nil); err != nil {
+					return err
+				}
+			} else {
+				if err := copyFile(file, dstPath); err != nil {
+					return err
+				}
+			}
+		}
+	} else {
+		// No inclusion patterns defined; copy everything except those matching excluded items.
+		if err := copyDirRecursive(sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil {
+			return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err)
+		}
+	}
+	return nil
+}

internal/exec/go_getter_utils.go

@@ -65,6 +65,7 @@ func IsValidScheme(scheme string) bool {
 // do a git-based clone with a token.
 type CustomGitHubDetector struct {
 	AtmosConfig schema.AtmosConfiguration
+	source      string
 }
 
 // Detect implements the getter.Detector interface for go-getter v1.
@@ -94,6 +95,14 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) {
 		return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path)
 	}
 
+	if !strings.Contains(d.source, "//") {
+		// means user typed something like "github.com/org/repo.git" with NO subdir
+		if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 {
+			u.LogDebug("Detected top-level repo with no subdir: appending '//.'\n")
+			parsedURL.Path = parsedURL.Path + "//."
+		}
+	}
+
 	atmosGitHubToken := os.Getenv("ATMOS_GITHUB_TOKEN")
 	gitHubToken := os.Getenv("GITHUB_TOKEN")
 
@@ -127,17 +136,28 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) {
 		}
 	}
 
+	// Set "depth=1" for a shallow clone if not specified.  
+        // In Go-Getter, "depth" controls how many revisions are cloned:  
+        // - `depth=1` fetches only the latest commit (faster, less bandwidth).  
+        // - `depth=` (empty) performs a full clone (default Git behavior).  
+        // - `depth=N` clones the last N revisions. 
+	q := parsedURL.Query()
+	if _, exists := q["depth"]; !exists {
+		q.Set("depth", "1")
+	}
+	parsedURL.RawQuery = q.Encode()
+
 	finalURL := "git::" + parsedURL.String()
 
 	return finalURL, true, nil
 }
 
 // RegisterCustomDetectors prepends the custom detector so it runs before
 // the built-in ones. Any code that calls go-getter should invoke this.
-func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) {
+func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration, source string) {
 	getter.Detectors = append(
 		[]getter.Detector{
-			&CustomGitHubDetector{AtmosConfig: atmosConfig},
+			&CustomGitHubDetector{AtmosConfig: atmosConfig, source: source},
 		},
 		getter.Detectors...,
 	)
@@ -154,24 +174,70 @@ func GoGetterGet(
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	// Register custom detectors
-	RegisterCustomDetectors(atmosConfig)
+	// Register custom detectors, passing the original `src` to the CustomGitHubDetector.
+	// go-getter typically strips subdirectories before calling the detector, so the
+	// unaltered source is needed to identify whether a top-level repository or a
+	// subdirectory was specified (e.g., for appending "//." only when no subdir is present).
+	RegisterCustomDetectors(atmosConfig, src)
 
 	client := &getter.Client{
 		Ctx: ctx,
 		Src: src,
 		// Destination where the files will be stored. This will create the directory if it doesn't exist
 		Dst:  dest,
 		Mode: clientMode,
-	}
+		Getters: map[string]getter.Getter{
+			// Overriding 'git'
+			"git":   &CustomGitGetter{},
+			"file":  &getter.FileGetter{},
+			"hg":    &getter.HgGetter{},
+			"http":  &getter.HttpGetter{},
+			"https": &getter.HttpGetter{},
+			// "s3": &getter.S3Getter{}, // add as needed
+			// "gcs": &getter.GCSGetter{},
 
+		},
+	}
 	if err := client.Get(); err != nil {
 		return err
 	}
 
 	return nil
 }
 
+// CustomGitGetter is a custom getter for git (git::) that removes symlinks
+type CustomGitGetter struct {
+	getter.GitGetter
+}
+
+// Implements the custom getter logic removing symlinks
+func (c *CustomGitGetter) Get(dst string, url *url.URL) error {
+	// Normal clone
+	if err := c.GitGetter.Get(dst, url); err != nil {
+		return err
+	}
+	// Remove symlinks
+	return removeSymlinks(dst)
+}
+
+// removeSymlinks walks the directory and removes any symlinks
+// it encounters.
+func removeSymlinks(root string) error {
+	return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.Mode()&os.ModeSymlink != 0 {
+			// Symlinks are removed for the entire repo, regardless if there are any subfolders specified
+			// Thus logging is disabled
+			// u.LogWarning(fmt.Sprintf("Removing symlink: %s", path))
+			// It's a symlink, remove it
+			return os.Remove(path)
+		}
+		return nil
+	})
+}
+
 // DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type
 func DownloadDetectFormatAndParseFile(atmosConfig schema.AtmosConfiguration, file string) (any, error) {
 	tempDir := os.TempDir()

internal/exec/vendor_model.go

@@ -307,7 +307,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos
 			}
 
 		}
-		if err := copyToTarget(atmosConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil {
+		if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil {
 			return installedPkgMsg{
 				err:  fmt.Errorf("failed to copy package: %w", err),
 				name: p.name,