find setup files on separate command

Author: David Cavazos

.github/cloud-samples-tools/cmd/main.go

@@ -30,7 +30,7 @@ var usage = `usage: tools <command> ...
 
 commands:
   affected path/to/config.jsonc path/to/diffs.txt
-  run-all path/to/config.jsonc path/to/script.sh
+  setups path/to/config.jsonc path/to/affected-paths.txt
 `
 
 // Entry point to validate command line arguments.
@@ -54,15 +54,17 @@ func main() {
 			log.Fatalln("❌ no diffs file specified\n", usage)
 		}
 
-		affectedCmd(configFile, diffsFile)
+		outputFile := flag.Arg(3)
+
+		affectedCmd(configFile, diffsFile, outputFile)
 
 	default:
 		log.Fatalln("❌ unknown command: ", command, "\n", usage)
 	}
 }
 
 // affected command entry point to validate inputs.
-func affectedCmd(configFile string, diffsFile string) {
+func affectedCmd(configFile string, diffsFile string, outputFile string) {
 	config, err := c.LoadConfig(configFile)
 	if err != nil {
 		log.Fatalln("❌ error loading the config file: ", configFile, "\n", err)
@@ -76,23 +78,32 @@ func affectedCmd(configFile string, diffsFile string) {
 	diffs := strings.Split(strings.TrimSpace(string(diffsBytes)), "\n")
 
 	// Log to stderr since GitHub Actions expects the output on stdout.
-	packages, err := config.Affected(os.Stderr, diffs)
+	paths, err := config.Affected(os.Stderr, diffs)
 	if err != nil {
 		log.Fatalln("❌ error finding the affected packages.\n", err)
 	}
-	if len(packages) > 256 {
+	if len(paths) > 256 {
 		log.Fatalln(
 			"❌ Error: GitHub Actions only supports up to 256 packages, got ",
-			len(packages),
+			len(paths),
 			" packages, for more details see:\n",
 			"https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow",
 		)
 	}
 
-	packagesJson, err := json.Marshal(packages)
+	if outputFile != "" {
+		file, err := os.Create(outputFile)
+		if err != nil {
+			log.Fatalln("❌ eror creating output file.\n", err)
+		}
+		for _, path := range paths {
+			fmt.Fprintf(file, "%v\n", path)
+		}
+	}
+
+	output, err := json.Marshal(paths)
 	if err != nil {
 		log.Fatalln("❌ error marshaling packages to JSON.\n", err)
 	}
-
-	fmt.Println(string(packagesJson))
+	fmt.Println(string(output))
 }

.github/cloud-samples-tools/pkg/config/config.go

@@ -32,11 +32,9 @@ type Config struct {
 	// Filename to look for the root of a package.
 	PackageFile []string `json:"package-file"`
 
-	// Setup file, must be located in the same directory as the package file.
-	Setup struct {
-		FileName string         `json:"filename"`
-		Defaults map[string]any `json:"defaults"`
-	} `json:"setup"`
+	// CI setup file, must be located in the same directory as the package file.
+	CISetupFileName string   `json:"ci-setup-filename"`
+	CISetupDefaults *CISetup `json:"ci-setup-defaults"`
 
 	// Pattern to match filenames or directories.
 	Match []string `json:"match"`
@@ -48,13 +46,7 @@ type Config struct {
 	ExcludePackages []string `json:"exclude-packages"`
 }
 
-type Package struct {
-	// Package directory path.
-	Path string `json:"path"`
-
-	// Package setup configurations.
-	Setup *map[string]any `json:"setup"`
-}
+type CISetup = map[string]any
 
 // Saves the config to the given file.
 func (c *Config) Save(file *os.File) error {
@@ -73,7 +65,7 @@ func (c *Config) Save(file *os.File) error {
 func LoadConfig(path string) (*Config, error) {
 	// Read the config file.
 	var config Config
-	err := ReadJsonc(path, &config)
+	err := readJsonc(path, &config)
 	if err != nil {
 		return nil, err
 	}
@@ -85,6 +77,9 @@ func LoadConfig(path string) (*Config, error) {
 	if config.Match == nil {
 		config.Match = []string{"*"}
 	}
+	if config.CISetupFileName == "" {
+		config.CISetupFileName = "ci-setup.json"
+	}
 
 	return &config, nil
 }
@@ -111,15 +106,14 @@ func (c *Config) Matches(path string) bool {
 // IsPackageDir returns true if the path is a package directory.
 func (c *Config) IsPackageDir(dir string) bool {
 	for _, filename := range c.PackageFile {
-		packageFile := filepath.Join(dir, filename)
-		if fileExists(packageFile) {
+		if fileExists(filepath.Join(dir, filename)) {
 			return true
 		}
 	}
 	return false
 }
 
-// FindPackage returns the package name for the given path.
+// FindPackage returns the most specific package path for the given filename.
 func (c *Config) FindPackage(path string) string {
 	dir := filepath.Dir(path)
 	if dir == "." || c.IsPackageDir(dir) {
@@ -128,9 +122,9 @@ func (c *Config) FindPackage(path string) string {
 	return c.FindPackage(dir)
 }
 
-// FindAllPackages finds all the packages in the given root directory.
+// FindAllPackages finds all the package paths in the given root directory.
 func (c *Config) FindAllPackages(root string) ([]string, error) {
-	var packages []string
+	var paths []string
 	err := fs.WalkDir(os.DirFS(root), ".",
 		func(path string, d os.DirEntry, err error) error {
 			if err != nil {
@@ -143,45 +137,15 @@ func (c *Config) FindAllPackages(root string) ([]string, error) {
 				return nil
 			}
 			if d.IsDir() && c.Matches(path) && c.IsPackageDir(path) {
-				packages = append(packages, path)
+				paths = append(paths, path)
 				return nil
 			}
 			return nil
 		})
 	if err != nil {
 		return []string{}, err
 	}
-	return packages, nil
-}
-
-// Affected returns the packages that have been affected from diffs.
-// If there are diffs on at leat one global file affecting all packages,
-// then this returns all packages matched by the config.
-func (c *Config) Affected(log io.Writer, diffs []string) ([]Package, error) {
-	changed := c.Changed(log, diffs)
-	if slices.Contains(changed, ".") {
-		allPackages, err := c.FindAllPackages(".")
-		if err != nil {
-			return nil, err
-		}
-		changed = allPackages
-	}
-
-	packages := make([]Package, 0, len(changed))
-	for _, path := range changed {
-		pkg := Package{Path: path, Setup: &c.Setup.Defaults}
-		if c.Setup.FileName != "" {
-			setup_file := filepath.Join(path, c.Setup.FileName)
-			if fileExists(setup_file) {
-				err := ReadJsonc(setup_file, pkg.Setup)
-				if err != nil {
-					return nil, err
-				}
-			}
-		}
-		packages = append(packages, pkg)
-	}
-	return packages, nil
+	return paths, nil
 }
 
 // Changed returns the packages that have changed.
@@ -193,17 +157,54 @@ func (c *Config) Changed(log io.Writer, diffs []string) []string {
 		if !c.Matches(diff) {
 			continue
 		}
-		pkg := c.FindPackage(diff)
-		changedUnique[pkg] = true
+		path := c.FindPackage(diff)
+		if path == "." {
+			fmt.Fprintf(log, "ℹ️ Global file changed: %q\n", diff)
+		}
+		changedUnique[path] = true
 	}
 
 	changed := make([]string, 0, len(changedUnique))
-	for pkg := range changedUnique {
-		if slices.Contains(c.ExcludePackages, pkg) {
-			fmt.Fprintf(log, "ℹ️ Excluded package %q, skipping.\n", pkg)
+	for path := range changedUnique {
+		if slices.Contains(c.ExcludePackages, path) {
+			fmt.Fprintf(log, "ℹ️ Excluded package %q, skipping.\n", path)
 			continue
 		}
-		changed = append(changed, pkg)
+		changed = append(changed, path)
 	}
 	return changed
 }
+
+// Affected returns the packages that have been affected from diffs.
+// If there are diffs on at leat one global file affecting all packages,
+// then this returns all packages matched by the config.
+func (c *Config) Affected(log io.Writer, diffs []string) ([]string, error) {
+	paths := c.Changed(log, diffs)
+	if slices.Contains(paths, ".") {
+		fmt.Fprintf(log, "One or more global files were affected, all packages marked as affected.")
+		allPackages, err := c.FindAllPackages(".")
+		if err != nil {
+			return nil, err
+		}
+		paths = allPackages
+	}
+	return paths, nil
+}
+
+func (c *Config) FindSetupFiles(paths []string) (*map[string]CISetup, error) {
+	setups := make(map[string]CISetup, len(paths))
+	for _, path := range paths {
+		setup := *c.CISetupDefaults
+		setupFile := filepath.Join(path, c.CISetupFileName)
+		if fileExists(setupFile) {
+			// This mutates `setup` so there's no need to reassign it.
+			// It keeps the default values if they're not in the JSON file.
+			err := readJsonc(setupFile, &setup)
+			if err != nil {
+				return nil, err
+			}
+		}
+		setups[path] = setup
+	}
+	return &setups, nil
+}

.github/cloud-samples-tools/pkg/config/utils.go

@@ -26,7 +26,7 @@ import (
 var multiLineCommentsRegex = regexp.MustCompile(`(?s)\s*/\*.*?\*/`)
 var singleLineCommentsRegex = regexp.MustCompile(`\s*//.*\s*`)
 
-func ReadJsonc[a any](path string, ref *a) error {
+func readJsonc[a any](path string, ref *a) error {
 	// Read the JSONC file.
 	sourceJsonc, err := os.ReadFile(path)
 	if err != nil {

.github/config/nodejs-dev.jsonc

@@ -16,12 +16,9 @@
 
 {
   "package-file": [ "package.json" ],
-  "setup": {
-    "filename": "setup.json",
-    "defaults": {
-      "nodejs_version": 20,
-      "secrets": null
-    }
+  "ci-setup-defaults": {
+    "node-version": 20,
+    "secrets": null
   },
   "ignore": [
     ".eslintignore",

.github/config/nodejs-prod.jsonc

@@ -16,7 +16,6 @@
 
 {
   "package-file": [ "package.json" ],
-  "setup-file": "setup.json",
   "ignore": [
     ".eslintignore",
     ".eslintrc.json",

.github/workflows/ci-dev.yaml

@@ -31,7 +31,8 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 2
     outputs:
-      nodejs: ${{ steps.nodejs.outputs.packages }}
+      nodejs-paths: ${{ steps.nodejs.outputs.paths }}
+      nodejs-setups: ${{ steps.nodejs.outputs.setups }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -45,37 +46,40 @@ jobs:
         run: git --no-pager diff --name-only HEAD origin/main | tee diffs.txt
       - name: Find Node.js affected packages
         id: nodejs
-        run: echo "packages=$(./tools affected .github/config/nodejs-dev.jsonc diffs.txt)" | tee -a $GITHUB_OUTPUT
+        run: |
+          echo "paths=$(./tools affected .github/config/nodejs-dev.jsonc diffs.txt paths.txt)" >> $GITHUB_OUTPUT
+          cat paths.txt
+          echo "setups=$(./tools setups .github/config/nodejs-dev.jsonc paths.txt)" >> $GITHUB_OUTPUT
 
   nodejs-test:
-    name: Node.js ${{ matrix.package.setup.nodejs_version }} test
+    name: Node.js test
     needs: affected
     runs-on: ubuntu-latest
     permissions:
       id-token: write # needed for google-github-actions/auth
     strategy:
       fail-fast: false
       matrix:
-        package: ${{ fromJson(github.event_name == 'pull_request' && needs.affected.outputs.nodejs || '[]') }}
+        path: ${{ fromJson(github.event_name == 'pull_request' && needs.affected.outputs.nodejs-paths || '[]') }}
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - uses: actions/setup-node@v4
         with:
-          node-version: ${{ matrix.package.setup.nodejs_version }}
+          node-version: ${{ needs.affected.outputs.nodejs-setups[matrix.path].node-version }}
       - uses: google-github-actions/auth@v2
         with:
           project_id: long-door-651
           workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider
           service_account: [email protected]
           access_token_lifetime: 600s # 10 minutes
-      - if: matrix.package.secrets != '[]'
+      - if: needs.affected.outputs.nodejs-setups[matrix.path].secrets != null
         uses: google-github-actions/get-secretmanager-secrets@v2
         with:
-          secrets: ${{ join(matrix.package.secrets) }}
+          secrets: ${{ join(needs.affected.outputs.nodejs-setups[matrix.path].secrets) }}
           export_to_environment: true
-      - name: Test ${{ matrix.package.path }}
-        run: make test dir=${{ matrix.package.path }}
+      - name: 🛠️ Test ${{ matrix.path }}
+        run: make test dir=${{ matrix.path }}
         env:
           GOOGLE_SAMPLES_PROJECT: long-door-651

generative-ai/snippets/setup.json generative-ai/snippets/ci-setup.jsongenerative-ai/snippets/setup.json renamed to generative-ai/snippets/ci-setup.json

@@ -1,5 +1,5 @@
 {
-  "nodejs_version": 18,
+  "node-version": 18,
   "secrets": [
     "CAIP_PROJECT_ID:nodejs-docs-samples-tests/nodejs-docs-samples-ai-platform-caip-project-id",
     "LOCATION:nodejs-docs-samples-tests/nodejs-docs-samples-ai-platform-location",