diff --git a/.github/cloud-samples-tools/cmd/main.go b/.github/cloud-samples-tools/cmd/main.go index 556b7140a0..944e176d3c 100644 --- a/.github/cloud-samples-tools/cmd/main.go +++ b/.github/cloud-samples-tools/cmd/main.go @@ -29,8 +29,8 @@ import ( var usage = `usage: tools ... commands: - affected path/to/config.jsonc path/to/diffs.txt - run-all path/to/config.jsonc path/to/script.sh + affected path/to/config.jsonc diffs.txt paths.txt + setup-files path/to/config.jsonc paths.txt ` // Entry point to validate command line arguments. @@ -48,21 +48,34 @@ func main() { if configFile == "" { log.Fatalln("❌ no config file specified\n", usage) } - diffsFile := flag.Arg(2) if diffsFile == "" { log.Fatalln("❌ no diffs file specified\n", usage) } + pathsFile := flag.Arg(3) + if pathsFile == "" { + log.Fatalln("❌ no paths file specified\n", usage) + } + affectedCmd(configFile, diffsFile, pathsFile) - affectedCmd(configFile, diffsFile) + case "setup-files": + configFile := flag.Arg(1) + if configFile == "" { + log.Fatalln("❌ no config file specified\n", usage) + } + pathsFile := flag.Arg(2) + if pathsFile == "" { + log.Fatalln("❌ no paths file specified\n", usage) + } + setupFilesCmd(configFile, pathsFile) default: log.Fatalln("❌ unknown command: ", command, "\n", usage) } } -// affected command entry point to validate inputs. -func affectedCmd(configFile string, diffsFile string) { +// affectedCmd command entry point to validate inputs. +func affectedCmd(configFile string, diffsFile string, pathsFile string) { config, err := c.LoadConfig(configFile) if err != nil { log.Fatalln("❌ error loading the config file: ", configFile, "\n", err) @@ -76,23 +89,58 @@ func affectedCmd(configFile string, diffsFile string) { diffs := strings.Split(strings.TrimSpace(string(diffsBytes)), "\n") // Log to stderr since GitHub Actions expects the output on stdout. - packages, err := config.Affected(os.Stderr, diffs) + paths, err := config.Affected(os.Stderr, diffs) if err != nil { log.Fatalln("❌ error finding the affected packages.\n", err) } - if len(packages) > 256 { + if len(paths) > 256 { log.Fatalln( "❌ Error: GitHub Actions only supports up to 256 packages, got ", - len(packages), + len(paths), " packages, for more details see:\n", "https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow", ) } - packagesJson, err := json.Marshal(packages) + if pathsFile != "" { + file, err := os.Create(pathsFile) + if err != nil { + log.Fatalln("❌ eror creating output file.\n", err) + } + for _, path := range paths { + fmt.Fprintf(file, "%v\n", path) + } + } + + output, err := json.Marshal(paths) + if err != nil { + log.Fatalln("❌ error marshaling paths to JSON.\n", err) + } + fmt.Println(string(output)) +} + +// setupFilesCmd command entry point to validate inputs. +func setupFilesCmd(configFile string, pathsFile string) { + config, err := c.LoadConfig(configFile) if err != nil { - log.Fatalln("❌ error marshaling packages to JSON.\n", err) + log.Fatalln("❌ error loading the config file: ", configFile, "\n", err) } - fmt.Println(string(packagesJson)) + pathsBytes, err := os.ReadFile(pathsFile) + if err != nil { + log.Fatalln("❌ error getting the diffs: ", pathsFile, "\n", err) + } + // Trim whitespace to remove extra newline from diff output. + paths := strings.Split(strings.TrimSpace(string(pathsBytes)), "\n") + + setups, err := config.FindSetupFiles(paths) + if err != nil { + log.Fatalln("❌ error finding setup files.\n", err) + } + + output, err := json.Marshal(setups) + if err != nil { + log.Fatalln("❌ error marshaling setups to JSON.\n", err) + } + fmt.Println(string(output)) } diff --git a/.github/cloud-samples-tools/pkg/config/config.go b/.github/cloud-samples-tools/pkg/config/config.go index c8e7a22936..447eca1b07 100644 --- a/.github/cloud-samples-tools/pkg/config/config.go +++ b/.github/cloud-samples-tools/pkg/config/config.go @@ -24,7 +24,6 @@ import ( "io/fs" "os" "path/filepath" - "regexp" "slices" "strings" ) @@ -33,6 +32,12 @@ type Config struct { // Filename to look for the root of a package. PackageFile []string `json:"package-file"` + // CI setup file, must be located in the same directory as the package file. + CISetupFileName string `json:"ci-setup-filename"` + + // CI setup defaults, used when no setup file or field is not sepcified in file. + CISetupDefaults CISetup `json:"ci-setup-defaults"` + // Pattern to match filenames or directories. Match []string `json:"match"` @@ -43,8 +48,7 @@ type Config struct { ExcludePackages []string `json:"exclude-packages"` } -var multiLineCommentsRegex = regexp.MustCompile(`(?s)\s*/\*.*?\*/`) -var singleLineCommentsRegex = regexp.MustCompile(`\s*//.*\s*`) +type CISetup = map[string]any // Saves the config to the given file. func (c *Config) Save(file *os.File) error { @@ -61,30 +65,22 @@ func (c *Config) Save(file *os.File) error { // LoadConfig loads the config from the given path. func LoadConfig(path string) (*Config, error) { - // Read the JSONC file. - sourceJsonc, err := os.ReadFile(path) - if err != nil { - return nil, err + // Set the config default values. + config := Config{ + Match: []string{"*"}, } - // Strip the comments and load the JSON. - sourceJson := multiLineCommentsRegex.ReplaceAll(sourceJsonc, []byte{}) - sourceJson = singleLineCommentsRegex.ReplaceAll(sourceJson, []byte{}) - - var config Config - err = json.Unmarshal(sourceJson, &config) + // This mutates `config` so there's no need to reassign it. + // It keeps the default values if they're not in the JSON file. + err := readJsonc(path, &config) if err != nil { return nil, err } - // Set default values if they are not set. + // Validate for required values. if config.PackageFile == nil { return nil, errors.New("package-file is required") } - if config.Match == nil { - config.Match = []string{"*"} - } - return &config, nil } @@ -110,15 +106,14 @@ func (c *Config) Matches(path string) bool { // IsPackageDir returns true if the path is a package directory. func (c *Config) IsPackageDir(dir string) bool { for _, filename := range c.PackageFile { - packageFile := filepath.Join(dir, filename) - if fileExists(packageFile) { + if fileExists(filepath.Join(dir, filename)) { return true } } return false } -// FindPackage returns the package name for the given path. +// FindPackage returns the most specific package path for the given filename. func (c *Config) FindPackage(path string) string { dir := filepath.Dir(path) if dir == "." || c.IsPackageDir(dir) { @@ -127,9 +122,9 @@ func (c *Config) FindPackage(path string) string { return c.FindPackage(dir) } -// FindAllPackages finds all the packages in the given root directory. +// FindAllPackages finds all the package paths in the given root directory. func (c *Config) FindAllPackages(root string) ([]string, error) { - var packages []string + var paths []string err := fs.WalkDir(os.DirFS(root), ".", func(path string, d os.DirEntry, err error) error { if err != nil { @@ -142,7 +137,7 @@ func (c *Config) FindAllPackages(root string) ([]string, error) { return nil } if d.IsDir() && c.Matches(path) && c.IsPackageDir(path) { - packages = append(packages, path) + paths = append(paths, path) return nil } return nil @@ -150,18 +145,7 @@ func (c *Config) FindAllPackages(root string) ([]string, error) { if err != nil { return []string{}, err } - return packages, nil -} - -// Affected returns the packages that have been affected from diffs. -// If there are diffs on at leat one global file affecting all packages, -// then this returns all packages matched by the config. -func (c *Config) Affected(log io.Writer, diffs []string) ([]string, error) { - changed := c.Changed(log, diffs) - if slices.Contains(changed, ".") { - return c.FindAllPackages(".") - } - return changed, nil + return paths, nil } // Changed returns the packages that have changed. @@ -173,17 +157,57 @@ func (c *Config) Changed(log io.Writer, diffs []string) []string { if !c.Matches(diff) { continue } - pkg := c.FindPackage(diff) - changedUnique[pkg] = true + path := c.FindPackage(diff) + if path == "." { + fmt.Fprintf(log, "ℹ️ Global file changed: %q\n", diff) + } + changedUnique[path] = true } changed := make([]string, 0, len(changedUnique)) - for pkg := range changedUnique { - if slices.Contains(c.ExcludePackages, pkg) { - fmt.Fprintf(log, "ℹ️ Excluded package %q, skipping.\n", pkg) + for path := range changedUnique { + if slices.Contains(c.ExcludePackages, path) { + fmt.Fprintf(log, "ℹ️ Excluded package %q, skipping.\n", path) continue } - changed = append(changed, pkg) + changed = append(changed, path) } return changed } + +// Affected returns the packages that have been affected from diffs. +// If there are diffs on at leat one global file affecting all packages, +// then this returns all packages matched by the config. +func (c *Config) Affected(log io.Writer, diffs []string) ([]string, error) { + paths := c.Changed(log, diffs) + if slices.Contains(paths, ".") { + fmt.Fprintf(log, "One or more global files were affected, all packages marked as affected.\n") + allPackages, err := c.FindAllPackages(".") + if err != nil { + return nil, err + } + paths = allPackages + } + return paths, nil +} + +func (c *Config) FindSetupFiles(paths []string) (*map[string]CISetup, error) { + setups := make(map[string]CISetup, len(paths)) + for _, path := range paths { + setup := make(CISetup, len(c.CISetupDefaults)) + for k, v := range c.CISetupDefaults { + setup[k] = v + } + setupFile := filepath.Join(path, c.CISetupFileName) + if c.CISetupFileName != "" && fileExists(setupFile) { + // This mutates `setup` so there's no need to reassign it. + // It keeps the default values if they're not in the JSON file. + err := readJsonc(setupFile, &setup) + if err != nil { + return nil, err + } + } + setups[path] = setup + } + return &setups, nil +} diff --git a/.github/cloud-samples-tools/pkg/config/config_test.go b/.github/cloud-samples-tools/pkg/config/config_test.go index a95c5c7e8e..81b2eb75eb 100644 --- a/.github/cloud-samples-tools/pkg/config/config_test.go +++ b/.github/cloud-samples-tools/pkg/config/config_test.go @@ -44,7 +44,7 @@ func TestLoadConfig(t *testing.T) { t.Fatal("error loading config\n", err) } if !reflect.DeepEqual(test.config, got) { - t.Fatal("expected equal\n", test.config, got) + t.Fatal("expected equal\n", test.config, "\n", got) } } } @@ -78,7 +78,7 @@ func TestSaveLoadConfig(t *testing.T) { } if !reflect.DeepEqual(&config, loadedConfig) { - t.Fatal("expected equal\n", &config, loadedConfig) + t.Fatal("expected equal\n", &config, "\n", loadedConfig) } } @@ -108,7 +108,7 @@ func TestMatch(t *testing.T) { for _, test := range tests { got := c.Match(test.patterns, test.path) if got != test.expected { - t.Fatal("expected equal\n", test.expected, got) + t.Fatal("expected equal\n", test.expected, "\n", got) } } } @@ -132,7 +132,7 @@ func TestIsPackage(t *testing.T) { for _, test := range tests { got := config.IsPackageDir(test.path) if test.expected != got { - t.Fatal("expected equal\n", test.expected, got) + t.Fatal("expected equal\n", test.expected, "\n", got) } } } @@ -160,7 +160,7 @@ func TestFindPackage(t *testing.T) { for _, test := range tests { got := config.FindPackage(test.path) if test.expected != got { - t.Fatal("expected equal\n", test.expected, got) + t.Fatal("expected equal\n", test.expected, "\n", got) } } } @@ -202,7 +202,49 @@ func TestChanged(t *testing.T) { for _, test := range tests { got := config.Changed(os.Stderr, test.diffs) if !reflect.DeepEqual(test.expected, got) { - t.Fatal("expected equal\n", test.expected, got) + t.Fatal("expected equal\n", test.expected, "\n", got) } } } + +func TestFindSetupFiles(t *testing.T) { + config := c.Config{ + PackageFile: []string{"package.json"}, + CISetupFileName: "ci-setup.json", + CISetupDefaults: c.CISetup{ + "my-number": 3.14, + "my-string": "hello", + "my-array": []any{"a", "b", "c"}, + }, + } + + emptyPath := filepath.Join("testdata", "setup", "empty") + defaultsPath := filepath.Join("testdata", "setup", "defaults") + overridePath := filepath.Join("testdata", "setup", "override") + paths := []string{emptyPath, defaultsPath, overridePath} + expected := &map[string]c.CISetup{ + emptyPath: { + "my-number": 3.14, + "my-string": "hello", + "my-array": []any{"a", "b", "c"}, + }, + defaultsPath: { + "my-number": 3.14, + "my-string": "hello", + "my-array": []any{"a", "b", "c"}, + }, + overridePath: { + "my-number": 3.14, + "my-string": "custom-value", + "my-array": []any{"A", "B", "C"}, + }, + } + + got, err := config.FindSetupFiles(paths) + if err != nil { + t.Fatal("error finding setup files\n", err) + } + if !reflect.DeepEqual(expected, got) { + t.Fatal("expected equal\n", expected, "\n", got) + } +} diff --git a/.github/cloud-samples-tools/pkg/config/testdata/setup/defaults/ci-setup.json b/.github/cloud-samples-tools/pkg/config/testdata/setup/defaults/ci-setup.json new file mode 100644 index 0000000000..ffcd4415b0 --- /dev/null +++ b/.github/cloud-samples-tools/pkg/config/testdata/setup/defaults/ci-setup.json @@ -0,0 +1 @@ +{ } diff --git a/.github/cloud-samples-tools/pkg/config/testdata/setup/defaults/package.json b/.github/cloud-samples-tools/pkg/config/testdata/setup/defaults/package.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/cloud-samples-tools/pkg/config/testdata/setup/empty/package.json b/.github/cloud-samples-tools/pkg/config/testdata/setup/empty/package.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/cloud-samples-tools/pkg/config/testdata/setup/override/ci-setup.json b/.github/cloud-samples-tools/pkg/config/testdata/setup/override/ci-setup.json new file mode 100644 index 0000000000..adcf598a7d --- /dev/null +++ b/.github/cloud-samples-tools/pkg/config/testdata/setup/override/ci-setup.json @@ -0,0 +1,4 @@ +{ + "my-string": "custom-value", + "my-array": [ "A", "B", "C" ] +} diff --git a/.github/cloud-samples-tools/pkg/config/testdata/setup/override/package.json b/.github/cloud-samples-tools/pkg/config/testdata/setup/override/package.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/cloud-samples-tools/pkg/config/utils.go b/.github/cloud-samples-tools/pkg/config/utils.go index b227f9cd99..94f0944ad4 100644 --- a/.github/cloud-samples-tools/pkg/config/utils.go +++ b/.github/cloud-samples-tools/pkg/config/utils.go @@ -17,10 +17,28 @@ package config import ( + "encoding/json" "errors" "os" + "regexp" ) +var multiLine = regexp.MustCompile(`(?s)\s*/\*.*?\*/`) +var singleLine = regexp.MustCompile(`\s*//.*\s*`) + +func readJsonc[a any](path string, ref *a) error { + // Read the JSONC file. + sourceJsonc, err := os.ReadFile(path) + if err != nil { + return err + } + + // Strip the comments and load the JSON. + sourceJson := multiLine.ReplaceAll(sourceJsonc, []byte{}) + sourceJson = singleLine.ReplaceAll(sourceJson, []byte{}) + return json.Unmarshal(sourceJson, ref) +} + // fileExists returns true if the file exists. func fileExists(path string) bool { if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { diff --git a/.github/config/nodejs-dev.jsonc b/.github/config/nodejs-dev.jsonc index 45e7874ee5..18374f30e9 100644 --- a/.github/config/nodejs-dev.jsonc +++ b/.github/config/nodejs-dev.jsonc @@ -16,6 +16,17 @@ { "package-file": [ "package.json" ], + "ci-setup-filename": "ci-setup.json", + "ci-setup-defaults": { + "node-version": 20, + "timeout-minutes": 10, + "project-id": "long-door-651", + "workload-identity-provider": "projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider", + "service-account": "kokoro-system-test@long-door-651.iam.gserviceaccount.com", + "access-token-lifetime": "600s", // 10 minutes + "env": { }, + "secrets": { } + }, "ignore": [ ".eslintignore", ".eslintrc.json", @@ -26,10 +37,12 @@ ".github/auto-label.yaml", ".github/blunderbuss.yaml", ".github/cloud-samples-tools/", + ".github/config/", ".github/flakybot.yaml", ".github/header-checker-lint.yaml", ".github/snippet-bot.yml", ".github/trusted-contribution.yml", + ".github/workflows/", ".gitignore", ".kokoro/", ".prettierignore", @@ -54,7 +67,6 @@ "functions/firebase", // parent directory "functions/helloworld", // parent directory "functions/http", // parent directory - "functions/http/uploadFile", // parent directory "functions/log", // parent directory "functions/pubsub", // parent directory "memorystore/redis", // parent directory @@ -159,6 +171,7 @@ "functions/v2/tips/retry", "functions/v2/typed/googlechatbot", "functions/v2/typed/greeting", + "generative-ai/snippets", "healthcare/consent", "healthcare/datasets", "healthcare/dicom", diff --git a/.github/config/nodejs-prod.jsonc b/.github/config/nodejs-prod.jsonc index 32629dfc9b..88c00e353e 100644 --- a/.github/config/nodejs-prod.jsonc +++ b/.github/config/nodejs-prod.jsonc @@ -16,6 +16,17 @@ { "package-file": [ "package.json" ], + "ci-setup-filename": "ci-setup.json", + "ci-setup-defaults": { + "node-version": 20, + "timeout-minutes": 10, + "project-id": "long-door-651", + "workload-identity-provider": "projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider", + "service-account": "kokoro-system-test@long-door-651.iam.gserviceaccount.com", + "access-token-lifetime": "600s", // 10 minutes + "env": { }, + "secrets": { } + }, "ignore": [ ".eslintignore", ".eslintrc.json", @@ -26,10 +37,12 @@ ".github/auto-label.yaml", ".github/blunderbuss.yaml", ".github/cloud-samples-tools/", + ".github/config/", ".github/flakybot.yaml", ".github/header-checker-lint.yaml", ".github/snippet-bot.yml", ".github/trusted-contribution.yml", + ".github/workflows/", ".gitignore", ".kokoro/", ".prettierignore", @@ -52,7 +65,6 @@ "functions/firebase", // parent directory "functions/helloworld", // parent directory "functions/http", // parent directory - "functions/http/uploadFile", // parent directory "functions/log", // parent directory "functions/pubsub", // parent directory "memorystore/redis", // parent directory @@ -78,11 +90,11 @@ "eventarc/audit-storage", // (untested) Environment Variable 'SERVICE_NAME' not found "eventarc/pubsub", // (untested) Environment Variable 'SERVICE_NAME' not found "functions/billing", // Error: Request failed with status code 500 + "functions/http/uploadFile", // npm error Missing script: "test" "functions/imagemagick", // Error: A bucket name is needed to use Cloud Storage "functions/ocr/app", // Error: Bucket not provided. Make sure you have a "bucket" property in your request "functions/slack", // TypeError [ERR_INVALID_ARG_TYPE]: The "key" argument must be of type ... Received undefined "functions/v2/imagemagick", // Error: A bucket name is needed to use Cloud Storage. - "generative-ai/snippets", // [VertexAI.ClientError]: got status: 403 Forbidden. "healthcare/fhir", // Error: Cannot find module 'whatwg-url' "iam/deny", // PERMISSION_DENIED: Permission iam.googleapis.com/denypolicies.create denied on resource cloudresourcemanager.googleapis.com/projects/long-door-651 "recaptcha_enterprise/snippets", // Cannot use import statement outside a module diff --git a/.github/workflows/ci-dev.yaml b/.github/workflows/ci-dev.yaml index 5f51c82b30..19a9fdb046 100644 --- a/.github/workflows/ci-dev.yaml +++ b/.github/workflows/ci-dev.yaml @@ -31,7 +31,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 2 outputs: - nodejs: ${{ steps.nodejs.outputs.packages }} + nodejs-paths: ${{ steps.nodejs.outputs.paths }} + nodejs-setups: ${{ steps.nodejs.outputs.setups }} steps: - uses: actions/checkout@v4 with: @@ -45,34 +46,65 @@ jobs: run: git --no-pager diff --name-only HEAD origin/main | tee diffs.txt - name: Find Node.js affected packages id: nodejs - run: echo "packages=$(./tools affected .github/config/nodejs-dev.jsonc diffs.txt)" | tee -a $GITHUB_OUTPUT + run: | + echo "paths=$(./tools affected .github/config/nodejs-dev.jsonc diffs.txt paths.txt)" >> $GITHUB_OUTPUT + cat paths.txt + echo "setups=$(./tools setup-files .github/config/nodejs-dev.jsonc paths.txt)" >> $GITHUB_OUTPUT nodejs-test: - name: Node.js 20 test + name: Node.js test needs: affected runs-on: ubuntu-latest + timeout-minutes: 120 # 2 hours hard limit permissions: id-token: write # needed for google-github-actions/auth strategy: fail-fast: false matrix: - package: ${{ fromJson(github.event_name == 'pull_request' && needs.affected.outputs.nodejs || '[]') }} + path: ${{ fromJson(github.event_name == 'pull_request' && needs.affected.outputs.nodejs-paths || '[]') }} + env: + CI_SETUP: ${{ toJson(fromJson(needs.affected.outputs.nodejs-setups)[matrix.path])}} steps: + - name: CI Setup + run: echo "${{ env.CI_SETUP }}" - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: ${{ fromJson(env.CI_SETUP).node-version }} - uses: google-github-actions/auth@v2 with: - project_id: long-door-651 - workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider - service_account: kokoro-system-test@long-door-651.iam.gserviceaccount.com - access_token_lifetime: 600s # 10 minutes - - name: Test ${{ matrix.package }} + project_id: ${{ fromJson(env.CI_SETUP).project-id }} + workload_identity_provider: ${{ fromJson(env.CI_SETUP).workload-identity-provider }} + service_account: ${{ fromJson(env.CI_SETUP).service-account }} + access_token_lifetime: ${{ fromJson(env.CI_SETUP).access-token-lifetime }} + - name: Export environment variables + uses: actions/github-script@v7 + id: vars + with: + script: | + const setup = JSON.parse(process.env.CI_SETUP) + const env = { + 'GOOGLE_SAMPLES_PROJECT': setup['project-id'], + ...setup.env + } + for (const key in env) { + console.log(`${key}: ${env[key]}`) + core.exportVariable(key, env[key]) + } + return { + "env": env, + "secrets": Object.keys(setup.secrets) + .map(key => `${key}:${setup.secrets[key]}`) + .join('\n'), + } + - uses: google-github-actions/get-secretmanager-secrets@v2 + if: ${{ fromJson(steps.vars.outputs.result).secrets }} + with: + secrets: ${{ fromJson(steps.vars.outputs.result).secrets }} + export_to_environment: true + - name: 🛠️ Test ${{ matrix.path }} run: | - npm install - bash .github/scripts/nodejs-test.sh ${{ matrix.package }} - env: - GOOGLE_SAMPLES_PROJECT: long-door-651 + timeout ${{ fromJson(env.CI_SETUP).timeout-minutes }}m \ + make test dir=${{ matrix.path }} diff --git a/.github/workflows/ci-prod.yaml b/.github/workflows/ci-prod.yaml index 6e74b62937..dbf619cc03 100644 --- a/.github/workflows/ci-prod.yaml +++ b/.github/workflows/ci-prod.yaml @@ -31,7 +31,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 2 outputs: - nodejs: ${{ steps.nodejs.outputs.packages }} + nodejs-paths: ${{ steps.nodejs.outputs.paths }} + nodejs-setups: ${{ steps.nodejs.outputs.setups }} steps: - uses: actions/checkout@v4 with: @@ -45,7 +46,10 @@ jobs: run: git --no-pager diff --name-only HEAD origin/main | tee diffs.txt - name: Find Node.js affected packages id: nodejs - run: echo "packages=$(./tools affected .github/config/nodejs-prod.jsonc diffs.txt)" | tee -a $GITHUB_OUTPUT + run: | + echo "paths=$(./tools affected .github/config/nodejs-prod.jsonc diffs.txt paths.txt)" >> $GITHUB_OUTPUT + cat paths.txt + echo "setups=$(./tools setup-files .github/config/nodejs-prod.jsonc paths.txt)" >> $GITHUB_OUTPUT nodejs-lint: name: Node.js lint @@ -60,36 +64,62 @@ jobs: - run: ./.github/workflows/utils/region-tags-tests.sh nodejs-test: - name: Node.js 20 test + name: Node.js test needs: affected runs-on: ubuntu-latest + timeout-minutes: 120 # 2 hours hard limit permissions: id-token: write # needed for google-github-actions/auth strategy: fail-fast: false matrix: - package: ${{ fromJson(github.event_name == 'pull_request' && needs.affected.outputs.nodejs || '[]') }} + path: ${{ fromJson(github.event_name == 'pull_request' && needs.affected.outputs.nodejs-paths || '[]') }} env: - MOCHA_REPORTER_OUTPUT: ${{github.run_id}}_spong_log.xml + CI_SETUP: ${{ toJson(fromJson(needs.affected.outputs.nodejs-setups)[matrix.path])}} steps: + - name: CI Setup + run: echo "${{ env.CI_SETUP }}" - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: ${{ fromJson(env.CI_SETUP).node-version }} - uses: google-github-actions/auth@v2 with: - project_id: long-door-651 - workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider - service_account: kokoro-system-test@long-door-651.iam.gserviceaccount.com - access_token_lifetime: 600s # 10 minutes - - name: Test ${{ matrix.package }} + project_id: ${{ fromJson(env.CI_SETUP).project-id }} + workload_identity_provider: ${{ fromJson(env.CI_SETUP).workload-identity-provider }} + service_account: ${{ fromJson(env.CI_SETUP).service-account }} + access_token_lifetime: ${{ fromJson(env.CI_SETUP).access-token-lifetime }} + - name: Export environment variables + uses: actions/github-script@v7 + id: vars + with: + script: | + const setup = JSON.parse(process.env.CI_SETUP) + const env = { + 'GOOGLE_SAMPLES_PROJECT': setup['project-id'], + ...setup.env + } + for (const key in env) { + console.log(`${key}: ${env[key]}`) + core.exportVariable(key, env[key]) + } + return { + "env": env, + "secrets": Object.keys(setup.secrets) + .map(key => `${key}:${setup.secrets[key]}`) + .join('\n'), + } + - uses: google-github-actions/get-secretmanager-secrets@v2 + if: ${{ fromJson(steps.vars.outputs.result).secrets }} + with: + secrets: ${{ fromJson(steps.vars.outputs.result).secrets }} + export_to_environment: true + - name: 🛠️ Test ${{ matrix.path }} run: | - npm install - bash .github/scripts/nodejs-test.sh ${{ matrix.package }} - env: - GOOGLE_SAMPLES_PROJECT: long-door-651 + timeout ${{ fromJson(env.CI_SETUP).timeout-minutes }}m \ + make test dir=${{ matrix.path }} # - name: Upload test results for FlakyBot workflow # if: github.event.action == 'schedule' && always() # always() submits logs even if tests fail # uses: actions/upload-artifact@v4 diff --git a/generative-ai/snippets/ci-setup.json b/generative-ai/snippets/ci-setup.json new file mode 100644 index 0000000000..e196013a68 --- /dev/null +++ b/generative-ai/snippets/ci-setup.json @@ -0,0 +1,7 @@ +{ + "secrets": { + "CAIP_PROJECT_ID": "nodejs-docs-samples-tests/nodejs-docs-samples-ai-platform-caip-project-id", + "LOCATION": "nodejs-docs-samples-tests/nodejs-docs-samples-ai-platform-location", + "DATASTORE_ID": "nodejs-docs-samples-tests/nodejs-docs-samples-ai-platform-datastore-id" + } +}