refact: clustering operations in cobuilds

Author: chengcyber

build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json

@@ -40,6 +40,7 @@
         "cwd": "${workspaceFolder}/sandbox/repo",
         "env": {
           "RUSH_COBUILD_CONTEXT_ID": "integration-test",
+          "RUSH_COBUILD_RUNNER_ID": "runner1",
           "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED": "1",
           "REDIS_PASS": "redis123"
         }
@@ -63,6 +64,7 @@
         "cwd": "${workspaceFolder}/sandbox/repo",
         "env": {
           "RUSH_COBUILD_CONTEXT_ID": "integration-test",
+          "RUSH_COBUILD_RUNNER_ID": "runner2",
           "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED": "1",
           "REDIS_PASS": "redis123"
         }

build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json

@@ -2,50 +2,11 @@
  * Defines configuration used by core Heft.
  */
 {
-  "$schema": "https://developer.microsoft.com/json-schemas/heft/heft.schema.json",
-
-  "eventActions": [
-    {
-      /**
-       * The kind of built-in operation that should be performed.
-       * The "deleteGlobs" action deletes files or folders that match the
-       * specified glob patterns.
-       */
-      "actionKind": "deleteGlobs",
-
-      /**
-       * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json
-       * occur at the end of the stage of the Heft run.
-       */
-      "heftEvent": "clean",
-
-      /**
-       * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other
-       * configs.
-       */
-      "actionId": "defaultClean",
-
-      /**
-       * Glob patterns to be deleted. The paths are resolved relative to the project folder.
-       */
-      "globsToDelete": ["dist", "lib", "temp"]
-    }
-  ],
+  "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json",
 
   /**
-   * The list of Heft plugins to be loaded.
+   * Optionally specifies another JSON config file that this file extends from. This provides a way for standard
+   * settings to be shared across multiple projects.
    */
-  "heftPlugins": [
-    // {
-    //  /**
-    //   * The path to the plugin package.
-    //   */
-    //  "plugin": "path/to/my-plugin",
-    //
-    //  /**
-    //   * An optional object that provides additional settings that may be defined by the plugin.
-    //   */
-    //  // "options": { }
-    // }
-  ]
+  "extends": "@rushstack/heft-node-rig/profiles/default/config/heft.json"
 }

build-tests/rush-redis-cobuild-plugin-integration-test/package.json

@@ -13,6 +13,7 @@
     "@microsoft/rush-lib": "workspace:*",
     "@rushstack/eslint-config": "workspace:*",
     "@rushstack/heft": "workspace:*",
+    "@rushstack/heft-node-rig": "workspace:*",
     "@rushstack/node-core-library": "workspace:*",
     "@rushstack/rush-redis-cobuild-plugin": "workspace:*",
     "@types/http-proxy": "~1.17.8",

build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts

@@ -20,15 +20,21 @@ async function main(): Promise<void> {
   const lockProvider: RedisCobuildLockProvider = new RedisCobuildLockProvider(options, rushSession as any);
   await lockProvider.connectAsync();
   const context: ICobuildContext = {
-    contextId: 'test-context-id',
-    version: 1,
-    cacheId: 'test-cache-id'
+    contextId: 'context_id',
+    cacheId: 'cache_id',
+    lockKey: 'lock_key',
+    lockExpireTimeInSeconds: 30,
+    completedStateKey: 'completed_state_key',
+    clusterId: 'cluster_id',
+    runnerId: 'runner_id',
+    packageName: 'package_name',
+    phaseName: 'phase_name'
   };
   await lockProvider.acquireLockAsync(context);
   await lockProvider.renewLockAsync(context);
   await lockProvider.setCompletedStateAsync(context, {
     status: OperationStatus.Success,
-    cacheId: 'test-cache-id'
+    cacheId: 'cache_id'
   });
   const completedState = await lockProvider.getCompletedStateAsync(context);
   console.log('Completed state: ', completedState);

common/config/rush/pnpm-lock.yaml

@@ -1,5 +1,5 @@
 // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush.
 {
-  "pnpmShrinkwrapHash": "343a65be71bc82e4e59e741b0c4d29b4eaf5c561",
+  "pnpmShrinkwrapHash": "abf9008e956d819415ba592c72fb2f5750e33e27",
   "preferredVersionsHash": "1926a5b12ac8f4ab41e76503a0d1d0dccc9c0e06"
 }

common/config/rush/repo-state.json

@@ -102,8 +102,6 @@ export class CobuildConfiguration {
     get cobuildLockProvider(): ICobuildLockProvider;
     readonly cobuildRunnerId: string;
     // (undocumented)
-    get contextId(): string | undefined;
-    // (undocumented)
     createLockProviderAsync(terminal: ITerminal): Promise<void>;
     // (undocumented)
     destroyLockProviderAsync(): Promise<void>;
@@ -283,8 +281,14 @@ export interface ICobuildCompletedState {
 // @beta (undocumented)
 export interface ICobuildContext {
     cacheId: string;
+    clusterId: string;
+    completedStateKey: string;
     contextId: string;
-    version: number;
+    lockExpireTimeInSeconds: number;
+    lockKey: string;
+    packageName: string;
+    phaseName: string;
+    runnerId: string;
 }
 
 // @beta (undocumented)
@@ -297,18 +301,12 @@ export interface ICobuildJson {
 
 // @beta (undocumented)
 export interface ICobuildLockProvider {
-    // (undocumented)
-    acquireLockAsync(context: ICobuildContext): Promise<boolean>;
-    // (undocumented)
+    acquireLockAsync(context: Readonly<ICobuildContext>): Promise<boolean>;
     connectAsync(): Promise<void>;
-    // (undocumented)
     disconnectAsync(): Promise<void>;
-    // (undocumented)
-    getCompletedStateAsync(context: ICobuildContext): Promise<ICobuildCompletedState | undefined>;
-    // (undocumented)
-    renewLockAsync(context: ICobuildContext): Promise<void>;
-    // (undocumented)
-    setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise<void>;
+    getCompletedStateAsync(context: Readonly<ICobuildContext>): Promise<ICobuildCompletedState | undefined>;
+    renewLockAsync(context: Readonly<ICobuildContext>): Promise<void>;
+    setCompletedStateAsync(context: Readonly<ICobuildContext>, state: ICobuildCompletedState): Promise<void>;
 }
 
 // @public
@@ -1061,7 +1059,6 @@ export class RushConstants {
     static readonly bypassPolicyFlagLongName: '--bypass-policy';
     static readonly changeFilesFolderName: string;
     static readonly cobuildFilename: string;
-    static readonly cobuildLockVersion: number;
     static readonly commandLineFilename: string;
     static readonly commonFolderName: string;
     static readonly commonVersionsFilename: string;

common/reviews/api/rush-lib.api.md

@@ -6,13 +6,13 @@
 
 /// <reference types="node" />
 
-import type { ICobuildCompletedState } from '@rushstack/rush-sdk';
-import type { ICobuildContext } from '@rushstack/rush-sdk';
-import type { ICobuildLockProvider } from '@rushstack/rush-sdk';
+import { ICobuildCompletedState } from '@rushstack/rush-sdk';
+import { ICobuildContext } from '@rushstack/rush-sdk';
+import { ICobuildLockProvider } from '@rushstack/rush-sdk';
 import type { IRushPlugin } from '@rushstack/rush-sdk';
 import type { RedisClientOptions } from '@redis/client';
 import type { RushConfiguration } from '@rushstack/rush-sdk';
-import type { RushSession } from '@rushstack/rush-sdk';
+import { RushSession } from '@rushstack/rush-sdk';
 
 // @beta
 export interface IRedisCobuildLockProviderOptions extends RedisClientOptions {
@@ -22,7 +22,6 @@ export interface IRedisCobuildLockProviderOptions extends RedisClientOptions {
 // @beta (undocumented)
 export class RedisCobuildLockProvider implements ICobuildLockProvider {
     constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession);
-    // (undocumented)
     acquireLockAsync(context: ICobuildContext): Promise<boolean>;
     // (undocumented)
     connectAsync(): Promise<void>;
@@ -32,8 +31,6 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider {
     static expandOptionsWithEnvironmentVariables(options: IRedisCobuildLockProviderOptions, environment?: NodeJS.ProcessEnv): IRedisCobuildLockProviderOptions;
     // (undocumented)
     getCompletedStateAsync(context: ICobuildContext): Promise<ICobuildCompletedState | undefined>;
-    getCompletedStateKey(context: ICobuildContext): string;
-    getLockKey(context: ICobuildContext): string;
     // (undocumented)
     renewLockAsync(context: ICobuildContext): Promise<void>;
     // (undocumented)

common/reviews/api/rush-redis-cobuild-plugin.api.md

@@ -133,10 +133,6 @@ export class CobuildConfiguration {
     });
   }
 
-  public get contextId(): string | undefined {
-    return this.cobuildContextId;
-  }
-
   public async createLockProviderAsync(terminal: ITerminal): Promise<void> {
     if (this.cobuildEnabled) {
       terminal.writeLine(`Running cobuild (runner ${this.cobuildContextId}/${this.cobuildRunnerId})`);

libraries/rush-lib/src/api/CobuildConfiguration.ts

@@ -156,7 +156,10 @@ export const EnvironmentVariableNames = {
   RUSH_COBUILD_ENABLED: 'RUSH_COBUILD_ENABLED',
 
   /**
-   * Setting this environment variable opt into running with cobuilds.
+   * Setting this environment variable opt into running with cobuilds. The context id should be the same across
+   * multiple VMs, but changed when it is a new round of cobuilds.
+   *
+   * e.g. `Build.BuildNumber` in Azure DevOps Pipeline.
    *
    * @remarks
    * If there is no cobuild configured, then this environment variable is ignored.