diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go new file mode 100644 index 000000000..105fa50bd --- /dev/null +++ b/internal/exec/copy_glob.go @@ -0,0 +1,376 @@ +package exec + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + l "github.com/charmbracelet/log" + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" + cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required. +) + +// copyFile copies a single file from src to dst while preserving file permissions. +func copyFile(src, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("opening source file %q: %w", src, err) + } + defer sourceFile.Close() + + if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { + return fmt.Errorf("creating destination directory for %q: %w", dst, err) + } + + destinationFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("creating destination file %q: %w", dst, err) + } + defer destinationFile.Close() + + if _, err := io.Copy(destinationFile, sourceFile); err != nil { + return fmt.Errorf("copying content from %q to %q: %w", src, dst, err) + } + + info, err := os.Stat(src) + if err != nil { + return fmt.Errorf("getting file info for %q: %w", src, err) + } + if err := os.Chmod(dst, info.Mode()); err != nil { + return fmt.Errorf("setting permissions on %q: %w", dst, err) + } + return nil +} + +// shouldSkipEntry determines whether to skip a file/directory based on its relative path to baseDir. +// If an error occurs during matching for an exclusion or inclusion pattern, it logs the error and proceeds. +func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { + if info.Name() == ".git" { + return true, nil + } + relPath, err := filepath.Rel(baseDir, srcPath) + if err != nil { + l.Debug("Error computing relative path", "srcPath", srcPath, "error", err) + return true, nil // treat error as a signal to skip + } + // Ensure uniform path separator. + relPath = filepath.ToSlash(relPath) + + // Process exclusion patterns. + // For directories, check with and without a trailing slash. + for _, pattern := range excluded { + // First check the plain relative path. + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + l.Debug("Error matching exclusion pattern", "pattern", pattern, "path", relPath, "error", err) + continue + } + if matched { + l.Debug("Excluding path due to exclusion pattern (plain match)", "path", relPath, "pattern", pattern) + return true, nil + } + // If it is a directory, also try matching with a trailing slash. + if info.IsDir() { + matched, err = u.PathMatch(pattern, relPath+"/") + if err != nil { + l.Debug("Error matching exclusion pattern with trailing slash", "pattern", pattern, "path", relPath+"/", "error", err) + continue + } + if matched { + l.Debug("Excluding directory due to exclusion pattern (with trailing slash)", "path", relPath+"/", "pattern", pattern) + return true, nil + } + } + } + + // Process inclusion patterns (only for non-directory files). + // (Directories are generally picked up by the inclusion branch in copyToTargetWithPatterns.) + if len(included) > 0 && !info.IsDir() { + matchedAny := false + for _, pattern := range included { + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + l.Debug("Error matching inclusion pattern", "pattern", pattern, "path", relPath, "error", err) + continue + } + if matched { + l.Debug("Including path due to inclusion pattern", "path", relPath, "pattern", pattern) + matchedAny = true + break + } + } + if !matchedAny { + l.Debug("Excluding path because it does not match any inclusion pattern", "path", relPath) + return true, nil + } + } + return false, nil +} + +// copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering. +// This function is used in cases where the entire sourceDir is the base for relative paths. +func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", srcDir, err) + } + for _, entry := range entries { + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + // Check if this entry should be skipped. + skip, err := shouldSkipEntry(info, srcPath, baseDir, excluded, included) + if err != nil { + return err + } + if skip { + l.Debug("Skipping entry", "srcPath", srcPath) + continue + } + + // Skip symlinks. + if info.Mode()&os.ModeSymlink != 0 { + l.Debug("Skipping symlink", "path", srcPath) + continue + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + if err := copyDirRecursive(srcPath, dstPath, baseDir, excluded, included); err != nil { + return err + } + } else { + if err := copyFile(srcPath, dstPath); err != nil { + return err + } + } + } + return nil +} + +// copyDirRecursiveWithPrefix recursively copies srcDir to dstDir while preserving the global relative path. +// Instead of using the local srcDir as the base for computing relative paths, this function uses the original +// source directory (globalBase) and an accumulated prefix that represents the relative path from globalBase. +func copyDirRecursiveWithPrefix(srcDir, dstDir, globalBase, prefix string, excluded []string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", srcDir, err) + } + for _, entry := range entries { + // Compute the full relative path from the original source. + fullRelPath := filepath.ToSlash(filepath.Join(prefix, entry.Name())) + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + // Skip .git directories. + if entry.Name() == ".git" { + l.Debug("Skipping .git directory", "path", fullRelPath) + continue + } + + // Check exclusion patterns using the full relative path. + skip := false + for _, pattern := range excluded { + // Check plain match. + matched, err := u.PathMatch(pattern, fullRelPath) + if err != nil { + l.Debug("Error matching exclusion pattern in prefix function", "pattern", pattern, "path", fullRelPath, "error", err) + continue + } + if matched { + l.Debug("Excluding (prefix) due to exclusion pattern (plain match)", "path", fullRelPath, "pattern", pattern) + skip = true + break + } + // For directories, also try with a trailing slash. + if info.IsDir() { + matched, err = u.PathMatch(pattern, fullRelPath+"/") + if err != nil { + l.Debug("Error matching exclusion pattern with trailing slash in prefix function", "pattern", pattern, "path", fullRelPath+"/", "error", err) + continue + } + if matched { + l.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", "path", fullRelPath+"/", "pattern", pattern) + skip = true + break + } + } + } + if skip { + continue + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + // Recurse with updated prefix. + if err := copyDirRecursiveWithPrefix(srcPath, dstPath, globalBase, fullRelPath, excluded); err != nil { + return err + } + } else { + if err := copyFile(srcPath, dstPath); err != nil { + return err + } + } + } + return nil +} + +// getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. +// If no matches are found, it logs a debug message and returns an empty slice. +// When the pattern ends with "/*", it retries with a recursive "/**" variant. +func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { + fullPattern := filepath.Join(sourceDir, pattern) + matches, err := u.GetGlobMatches(fullPattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err) + } + if len(matches) == 0 { + if strings.HasSuffix(pattern, "/*") { + recursivePattern := strings.TrimSuffix(pattern, "/*") + "/**" + fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) + matches, err = u.GetGlobMatches(fullRecursivePattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) + } + if len(matches) == 0 { + l.Debug("No matches found for recursive pattern; target directory will be empty", "pattern", fullRecursivePattern) + return []string{}, nil + } + return matches, nil + } + l.Debug("No matches found for pattern; target directory will be empty", "pattern", fullPattern) + return []string{}, nil + } + return matches, nil +} + +// copyToTargetWithPatterns copies the contents from sourceDir to targetPath, +// applying inclusion and exclusion patterns from the vendor source configuration. +// If sourceIsLocalFile is true and targetPath lacks an extension, the sanitized URI is appended. +// If no included paths are defined, all files (except those matching excluded paths) are copied. +// In the special case where neither inclusion nor exclusion patterns are defined, +// the optimized cp library (github.com/otiai10/copy) is used. +func copyToTargetWithPatterns( + sourceDir, targetPath string, + s *schema.AtmosVendorSource, + sourceIsLocalFile bool, + uri string, +) error { + if sourceIsLocalFile && filepath.Ext(targetPath) == "" { + targetPath = filepath.Join(targetPath, SanitizeFileName(uri)) + } + l.Debug("Copying files", "source", sourceDir, "target", targetPath) + if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { + return fmt.Errorf("creating target directory %q: %w", targetPath, err) + } + + // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library for fast copying. + if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { + l.Debug("No inclusion or exclusion patterns defined; using cp library for fast copy") + return cp.Copy(sourceDir, targetPath) + } + + // If inclusion patterns are provided, use them to determine which files to copy. + if len(s.IncludedPaths) > 0 { + filesToCopy := make(map[string]struct{}) + for _, pattern := range s.IncludedPaths { + matches, err := getMatchesForPattern(sourceDir, pattern) + if err != nil { + l.Debug("Warning: error getting matches for pattern", "pattern", pattern, "error", err) + continue + } + for _, match := range matches { + filesToCopy[match] = struct{}{} + } + } + if len(filesToCopy) == 0 { + l.Debug("No files matched the inclusion patterns - target directory will be empty") + return nil + } + for file := range filesToCopy { + // Retrieve file information early so that we can adjust exclusion checks if this is a directory. + info, err := os.Stat(file) + if err != nil { + return fmt.Errorf("stating file %q: %w", file, err) + } + relPath, err := filepath.Rel(sourceDir, file) + if err != nil { + return fmt.Errorf("computing relative path for %q: %w", file, err) + } + relPath = filepath.ToSlash(relPath) + skip := false + // For directories, check both the plain relative path and with a trailing slash. + for _, ex := range s.ExcludedPaths { + if info.IsDir() { + matched, err := u.PathMatch(ex, relPath) + if err != nil { + l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) + } else if matched { + l.Debug("Excluding directory due to exclusion pattern (plain match)", "directory", relPath, "pattern", ex) + skip = true + break + } + // Also try matching with a trailing slash. + matched, err = u.PathMatch(ex, relPath+"/") + if err != nil { + l.Debug("Error matching exclusion pattern with trailing slash", "pattern", ex, "path", relPath+"/", "error", err) + } else if matched { + l.Debug("Excluding directory due to exclusion pattern (with trailing slash)", "directory", relPath+"/", "pattern", ex) + skip = true + break + } + } else { + // For files, just check the plain relative path. + matched, err := u.PathMatch(ex, relPath) + if err != nil { + l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) + } else if matched { + l.Debug("Excluding file due to exclusion pattern", "file", relPath, "pattern", ex) + skip = true + break + } + } + } + if skip { + continue + } + + // Build the destination path. + dstPath := filepath.Join(targetPath, relPath) + if info.IsDir() { + // Instead of resetting the base for relative paths, + // use the new recursive function that preserves the global relative path. + if err := copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, s.ExcludedPaths); err != nil { + return err + } + } else { + if err := copyFile(file, dstPath); err != nil { + return err + } + } + } + } else { + // No inclusion patterns defined; copy everything except those matching excluded items. + if err := copyDirRecursive(sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { + return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) + } + } + return nil +} diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 8a99a5f6c..f76699f58 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -65,6 +65,7 @@ func IsValidScheme(scheme string) bool { // do a git-based clone with a token. type CustomGitHubDetector struct { AtmosConfig schema.AtmosConfiguration + source string } // Detect implements the getter.Detector interface for go-getter v1. @@ -94,6 +95,14 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path) } + if !strings.Contains(d.source, "//") { + // means user typed something like "github.com/org/repo.git" with NO subdir + if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 { + u.LogDebug("Detected top-level repo with no subdir: appending '//.'\n") + parsedURL.Path = parsedURL.Path + "//." + } + } + atmosGitHubToken := os.Getenv("ATMOS_GITHUB_TOKEN") gitHubToken := os.Getenv("GITHUB_TOKEN") @@ -127,6 +136,17 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } + // Set "depth=1" for a shallow clone if not specified. + // In Go-Getter, "depth" controls how many revisions are cloned: + // - `depth=1` fetches only the latest commit (faster, less bandwidth). + // - `depth=` (empty) performs a full clone (default Git behavior). + // - `depth=N` clones the last N revisions. + q := parsedURL.Query() + if _, exists := q["depth"]; !exists { + q.Set("depth", "1") + } + parsedURL.RawQuery = q.Encode() + finalURL := "git::" + parsedURL.String() return finalURL, true, nil @@ -134,10 +154,10 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { // RegisterCustomDetectors prepends the custom detector so it runs before // the built-in ones. Any code that calls go-getter should invoke this. -func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) { +func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration, source string) { getter.Detectors = append( []getter.Detector{ - &CustomGitHubDetector{AtmosConfig: atmosConfig}, + &CustomGitHubDetector{AtmosConfig: atmosConfig, source: source}, }, getter.Detectors..., ) @@ -154,8 +174,11 @@ func GoGetterGet( ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - // Register custom detectors - RegisterCustomDetectors(atmosConfig) + // Register custom detectors, passing the original `src` to the CustomGitHubDetector. + // go-getter typically strips subdirectories before calling the detector, so the + // unaltered source is needed to identify whether a top-level repository or a + // subdirectory was specified (e.g., for appending "//." only when no subdir is present). + RegisterCustomDetectors(atmosConfig, src) client := &getter.Client{ Ctx: ctx, @@ -163,8 +186,18 @@ func GoGetterGet( // Destination where the files will be stored. This will create the directory if it doesn't exist Dst: dest, Mode: clientMode, - } + Getters: map[string]getter.Getter{ + // Overriding 'git' + "git": &CustomGitGetter{}, + "file": &getter.FileGetter{}, + "hg": &getter.HgGetter{}, + "http": &getter.HttpGetter{}, + "https": &getter.HttpGetter{}, + // "s3": &getter.S3Getter{}, // add as needed + // "gcs": &getter.GCSGetter{}, + }, + } if err := client.Get(); err != nil { return err } @@ -172,6 +205,39 @@ func GoGetterGet( return nil } +// CustomGitGetter is a custom getter for git (git::) that removes symlinks +type CustomGitGetter struct { + getter.GitGetter +} + +// Implements the custom getter logic removing symlinks +func (c *CustomGitGetter) Get(dst string, url *url.URL) error { + // Normal clone + if err := c.GitGetter.Get(dst, url); err != nil { + return err + } + // Remove symlinks + return removeSymlinks(dst) +} + +// removeSymlinks walks the directory and removes any symlinks +// it encounters. +func removeSymlinks(root string) error { + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + // Symlinks are removed for the entire repo, regardless if there are any subfolders specified + // Thus logging is disabled + // u.LogWarning(fmt.Sprintf("Removing symlink: %s", path)) + // It's a symlink, remove it + return os.Remove(path) + } + return nil + }) +} + // DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type func DownloadDetectFormatAndParseFile(atmosConfig schema.AtmosConfiguration, file string) (any, error) { tempDir := os.TempDir() diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 2fb300f49..c12b0045b 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -307,7 +307,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } } - if err := copyToTarget(atmosConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { + if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ err: fmt.Errorf("failed to copy package: %w", err), name: p.name, diff --git a/tests/fixtures/scenarios/vendor/vendor.yaml b/tests/fixtures/scenarios/vendor/vendor.yaml index 82cf62016..ba3c016f5 100644 --- a/tests/fixtures/scenarios/vendor/vendor.yaml +++ b/tests/fixtures/scenarios/vendor/vendor.yaml @@ -51,3 +51,24 @@ spec: - "**/*.tftmpl" - "**/modules/**" excluded_paths: [] + + - component: "test globs" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/{demo-library,demo-stacks}/**/*.{tf,md}" + excluded_paths: + - "**/demo-library/**/*.{tfvars,tf}" + targets: + - "components/library/" + tags: + - demo + + - component: "test globs without double stars upfront" + source: "github.com/cloudposse/atmos.git//examples/demo-library?ref={{.Version}}" + included_paths: + - "/weather/*.md" + version: "main" + targets: + - "components/library/" + tags: + - demo diff --git a/tests/test-cases/demo-stacks.yaml b/tests/test-cases/demo-stacks.yaml index 3fc5c07a3..367742db9 100644 --- a/tests/test-cases/demo-stacks.yaml +++ b/tests/test-cases/demo-stacks.yaml @@ -183,4 +183,14 @@ tests: - "./components/terraform/vpc-src/outputs.tf" - "./components/terraform/vpc-src/variables.tf" - "./components/terraform/vpc-src/versions.tf" + - "./components/library/examples/demo-library/github/stargazers/README.md" + - "./components/library/examples/demo-library/ipinfo/README.md" + - "./components/library/examples/demo-library/weather/README.md" + - "./components/library/examples/demo-library/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/main.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/outputs.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" + - "./components/library/weather/README.md" exit_code: 0