github token vendoring

Author: Listener430

internal/exec/vendor_utils.go

@@ -265,6 +265,8 @@ func ExecuteAtmosVendorInternal(
 		return fmt.Errorf("either 'spec.sources' or 'spec.imports' (or both) must be defined in the vendor config file '%s'", vendorConfigFileName)
 	}
 
+	var tempDir string
+
 	// Process imports and return all sources from all the imports and from `vendor.yaml`
 	sources, _, err := processVendorImports(
 		atmosConfig,
@@ -314,6 +316,16 @@ func ExecuteAtmosVendorInternal(
 		)
 	}
 
+	tempDir, err = os.MkdirTemp("", "atmos_vendor_")
+	if err != nil {
+		return fmt.Errorf("failed to create temp directory: %w", err)
+	}
+	defer func() {
+		if err := os.RemoveAll(tempDir); err != nil {
+			u.LogWarning(atmosConfig, fmt.Sprintf("failed to clean up temp directory %s: %v", tempDir, err))
+		}
+	}()
+
 	// Allow having duplicate targets in different sources.
 	// This can be used to vendor mixins (from local and remote sources) and write them to the same targets.
 	// TODO: consider adding a flag to `atmos vendor pull` to specify if duplicate targets are allowed or not.
@@ -356,7 +368,24 @@ func ExecuteAtmosVendorInternal(
 			return err
 		}
 
-		useOciScheme, useLocalFileSystem, sourceIsLocalFile := determineSourceType(&uri, vendorConfigFilePath)
+		useOciScheme, useLocalFileSystem, sourceIsLocalFile, isGitHubSource := determineSourceType(&uri, vendorConfigFilePath)
+
+		// Handle GitHub source
+		if isGitHubSource {
+			u.LogInfo(atmosConfig, fmt.Sprintf("Fetching GitHub source: %s", uri))
+			fileContents, err := u.DownloadFileFromGitHub(uri)
+			if err != nil {
+				return fmt.Errorf("failed to download GitHub file: %w", err)
+			}
+			// Save the downloaded file to the existing tempDir
+			tempGitHubFile := filepath.Join(tempDir, filepath.Base(uri))
+			err = os.WriteFile(tempGitHubFile, fileContents, os.ModePerm)
+			if err != nil {
+				return fmt.Errorf("failed to save GitHub file to temp location: %w", err)
+			}
+			// Update the URI to point to the saved file in the temp directory
+			uri = tempGitHubFile
+		}
 
 		// Determine package type
 		var pType pkgType
@@ -494,23 +523,32 @@ func shouldSkipSource(s *schema.AtmosVendorSource, component string, tags []stri
 	return (component != "" && s.Component != component) || (len(tags) > 0 && len(lo.Intersect(tags, s.Tags)) == 0)
 }
 
-func determineSourceType(uri *string, vendorConfigFilePath string) (bool, bool, bool) {
-	// Determine if the URI is an OCI scheme, a local file, or remote
+func determineSourceType(uri *string, vendorConfigFilePath string) (bool, bool, bool, bool) {
+	// Determine if the URI is an OCI scheme, a local file, a remote GitHub source, or a generic remote
 	useOciScheme := strings.HasPrefix(*uri, "oci://")
 	if useOciScheme {
 		*uri = strings.TrimPrefix(*uri, "oci://")
 	}
 
 	useLocalFileSystem := false
 	sourceIsLocalFile := false
+	isGitHubSource := false
+
 	if !useOciScheme {
-		if absPath, err := u.JoinAbsolutePathWithPath(vendorConfigFilePath, *uri); err == nil {
-			uri = &absPath
-			useLocalFileSystem = true
-			sourceIsLocalFile = u.FileExists(*uri)
+		if strings.Contains(*uri, "github.com") {
+			// Check if the URL is a GitHub source
+			isGitHubSource = true
+		} else {
+			// Handle local file system sources
+			if absPath, err := u.JoinAbsolutePathWithPath(vendorConfigFilePath, *uri); err == nil {
+				uri = &absPath
+				useLocalFileSystem = true
+				sourceIsLocalFile = u.FileExists(*uri)
+			}
 		}
 	}
-	return useOciScheme, useLocalFileSystem, sourceIsLocalFile
+
+	return useOciScheme, useLocalFileSystem, sourceIsLocalFile, isGitHubSource
 }
 
 func copyToTarget(atmosConfig schema.AtmosConfiguration, tempDir, targetPath string, s *schema.AtmosVendorSource, sourceIsLocalFile bool, uri string) error {

pkg/utils/file_utils.go

@@ -242,3 +242,27 @@ func GetFileNameFromURL(rawURL string) (string, error) {
 	}
 	return fileName, nil
 }
+
+// ParseGitHubURL parses a GitHub URL and returns the owner, repo, file path and branch
+func ParseGitHubURL(rawURL string) (owner, repo, filePath, branch string, err error) {
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return "", "", "", "", fmt.Errorf("invalid URL: %w", err)
+	}
+
+	if !strings.Contains(parsedURL.Host, "github.com") {
+		return "", "", "", "", fmt.Errorf("URL is not a GitHub URL")
+	}
+
+	pathParts := strings.Split(strings.Trim(parsedURL.Path, "/"), "/")
+	if len(pathParts) < 4 || pathParts[2] != "blob" {
+		return "", "", "", "", fmt.Errorf("URL format not supported. Expected: /owner/repo/blob/branch/filepath")
+	}
+
+	owner = pathParts[0]
+	repo = pathParts[1]
+	branch = pathParts[3]
+	filePath = strings.Join(pathParts[4:], "/")
+
+	return owner, repo, filePath, branch, nil
+}

pkg/utils/github_utils.go

@@ -2,6 +2,10 @@ package utils
 
 import (
 	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
 	"time"
 
 	"github.com/google/go-github/v59/github"
@@ -28,3 +32,38 @@ func GetLatestGitHubRepoRelease(owner string, repo string) (string, error) {
 
 	return "", nil
 }
+
+// ParseGitHubURL parses a GitHub URL and returns the owner, repo, file path and branch
+func DownloadFileFromGitHub(rawURL string) ([]byte, error) {
+	owner, repo, filePath, branch, err := ParseGitHubURL(rawURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse GitHub URL: %w", err)
+	}
+
+	githubToken := os.Getenv("GITHUB_TOKEN")
+	if githubToken == "" {
+		return nil, fmt.Errorf("GITHUB_TOKEN is not set")
+	}
+
+	apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s?ref=%s", owner, repo, filePath, branch)
+	req, err := http.NewRequest("GET", apiURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("Authorization", "Bearer "+githubToken)
+	req.Header.Set("Accept", "application/vnd.github.v3.raw")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to perform request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to download file: %s", resp.Status)
+	}
+
+	return io.ReadAll(resp.Body)
+}

pkg/vender/vendor_config_test.go

@@ -142,3 +142,54 @@ spec:
 		assert.Nil(t, err)
 	})
 }
+
+// New separate test function that directly calls ExecuteAtmosVendorInternal
+// after setting a GitHub token and reading a vendor.yaml referencing a GitHub source.
+func TestExecuteAtmosVendorInternalWithToken(t *testing.T) {
+	testDir := t.TempDir()
+
+	// Set the GitHub token environment variable
+	os.Setenv("GITHUB_TOKEN", "my-test-token")
+	defer os.Unsetenv("GITHUB_TOKEN")
+
+	// Create a vendor.yaml file referencing a GitHub source
+	vendorYaml := `
+apiVersion: atmos/v1
+kind: AtmosVendorConfig
+metadata:
+  name: test-vendor-config-with-token
+spec:
+  sources:
+    - component: github-component
+      source: https://github.com/example/repo/blob/main/test-file.yaml
+      included_paths:
+        - "**/*.yaml"
+      targets:
+        - vendor/github-component
+`
+	vendorYamlPath := filepath.Join(testDir, "vendor.yaml")
+	err := os.WriteFile(vendorYamlPath, []byte(vendorYaml), 0644)
+	assert.Nil(t, err)
+
+	// Initialize CLI configuration for this test
+	atmosConfig := schema.AtmosConfiguration{
+		BasePath: testDir,
+		Components: schema.Components{
+			Terraform: schema.Terraform{
+				BasePath: "components/terraform",
+			},
+		},
+	}
+	atmosConfig.Logs.Level = "Trace"
+
+	// Read the vendor config
+	vendorConfig, exists, foundVendorConfigFile, err := e.ReadAndProcessVendorConfigFile(atmosConfig, vendorYamlPath, true)
+	assert.Nil(t, err)
+	assert.True(t, exists)
+	assert.NotEmpty(t, foundVendorConfigFile)
+
+	// Now call ExecuteAtmosVendorInternal directly
+	// We use dryRun to avoid downloading and writing files
+	err = e.ExecuteAtmosVendorInternal(atmosConfig, foundVendorConfigFile, vendorConfig.Spec, "github-component", []string{}, true)
+	assert.Nil(t, err, "ExecuteAtmosVendorInternal should run without errors using the token in dry-run mode")
+}