ui, sql: Record KV CPU Time in statement and transaction statistics

alyshanjahani-crl · alyshanjahani-crl · commit 77852ac54535 · 2025-11-26T15:48:43.000-05:00
This commit starts recording the newly added kvCPUTime field from topLevelQueryStats and exposes it in the UI. Fixes: https://cockroachlabs.atlassian.net/browse/CRDB-57265 Release note (ui change): KV CPU Time is now recorded to statement_statistics and transaction_statistics and is displayed in the SQL Activity page.
diff --git a/pkg/sql/appstatspb/app_stats.go b/pkg/sql/appstatspb/app_stats.go
@@ -131,6 +131,7 @@ func (t *TransactionStatistics) Add(other *TransactionStatistics) {
 	t.RowsRead.Add(other.RowsRead, t.Count, other.Count)
 	t.BytesRead.Add(other.BytesRead, t.Count, other.Count)
 	t.RowsWritten.Add(other.RowsWritten, t.Count, other.Count)
+	t.KVCPUTime.Add(other.KVCPUTime, t.Count, other.Count)
 
 	t.ExecStats.Add(other.ExecStats)
 
@@ -181,6 +182,7 @@ func (s *StatementStatistics) Add(other *StatementStatistics) {
 	s.BytesRead.Add(other.BytesRead, s.Count, other.Count)
 	s.RowsRead.Add(other.RowsRead, s.Count, other.Count)
 	s.RowsWritten.Add(other.RowsWritten, s.Count, other.Count)
+	s.KVCPUTime.Add(other.KVCPUTime, s.Count, other.Count)
 	s.Nodes = util.CombineUnique(s.Nodes, other.Nodes)
 	s.KVNodeIDs = util.CombineUnique(s.KVNodeIDs, other.KVNodeIDs)
 	s.Regions = util.CombineUnique(s.Regions, other.Regions)
diff --git a/pkg/sql/appstatspb/app_stats.proto b/pkg/sql/appstatspb/app_stats.proto
@@ -142,6 +142,11 @@ message StatementStatistics {
   // system.statement_hints table.
   optional int64 stmt_hints_count = 37 [(gogoproto.nullable) = false];
 
+  // kv_cpu_time is the CPU time in nanoseconds spent by KV. This is
+  // representative of the "synchronous" portion of work performed by KV
+  // (ex/ not replication related work).
+  optional NumericStat kv_cpu_time = 38 [(gogoproto.nullable) = false, (gogoproto.customname) = "KVCPUTime"];
+
   // Note: be sure to update `sql/app_stats.go` when adding/removing fields here!
 
   reserved 13, 14, 17, 18, 19, 20;
@@ -188,6 +193,11 @@ message TransactionStatistics {
 
   // RowsWritten collects the number of rows written to disk.
   optional NumericStat rows_written = 10 [(gogoproto.nullable) = false];
+  
+  // kv_cpu_time is the CPU time in nanoseconds spent by KV. This is
+  // representative of the "synchronous" portion of work performed by KV
+  // (ex/ not replication related work).
+  optional NumericStat kv_cpu_time = 12 [(gogoproto.nullable) = false, (gogoproto.customname) = "KVCPUTime"];
 }
 
 
diff --git a/pkg/sql/conn_executor.go b/pkg/sql/conn_executor.go
@@ -1757,15 +1757,14 @@ type connExecutor struct {
 		// client to send statements while holding the transaction open.
 		idleLatency time.Duration
 
-		// rowsRead and bytesRead are separate from QueryLevelStats because they are
-		// accumulated independently since they are always collected, as opposed to
-		// QueryLevelStats which are sampled.
+		// rowsRead, bytesRead, kvCPUTime, and rowsWritten are separate from accumulatedStats
+		// since they are always collected as opposed to QueryLevelStats which are sampled.
 		rowsRead  int64
 		bytesRead int64
-
 		// rowsWritten tracks the number of rows written (modified) by all
 		// statements in this txn so far.
 		rowsWritten int64
+		kvCPUTime   time.Duration
 
 		// rowsWrittenLogged and rowsReadLogged indicates whether we have
 		// already logged an event about reaching written/read rows setting,
diff --git a/pkg/sql/conn_executor_exec.go b/pkg/sql/conn_executor_exec.go
@@ -2951,6 +2951,7 @@ func (ex *connExecutor) dispatchToExecutionEngine(
 	ex.extraTxnState.rowsRead += stats.rowsRead
 	ex.extraTxnState.bytesRead += stats.bytesRead
 	ex.extraTxnState.rowsWritten += stats.rowsWritten
+	ex.extraTxnState.kvCPUTime += stats.kvCPUTime
 
 	if ppInfo := getPausablePortalInfo(planner); ppInfo != nil && !ppInfo.dispatchToExecutionEngine.cleanup.isComplete {
 		// We need to ensure that we're using the planner bound to the first-time
@@ -4199,6 +4200,7 @@ func (ex *connExecutor) onTxnRestart(ctx context.Context) {
 		ex.extraTxnState.rowsRead = 0
 		ex.extraTxnState.bytesRead = 0
 		ex.extraTxnState.rowsWritten = 0
+		ex.extraTxnState.kvCPUTime = 0
 
 		if ex.server.cfg.TestingKnobs.BeforeRestart != nil {
 			ex.server.cfg.TestingKnobs.BeforeRestart(ctx, ex.state.mu.autoRetryReason)
@@ -4230,6 +4232,7 @@ func (ex *connExecutor) recordTransactionStart(txnID uuid.UUID) {
 	ex.extraTxnState.accumulatedStats = execstats.QueryLevelStats{}
 	ex.extraTxnState.idleLatency = 0
 	ex.extraTxnState.rowsRead = 0
+	ex.extraTxnState.kvCPUTime = 0
 	ex.extraTxnState.bytesRead = 0
 	ex.extraTxnState.rowsWritten = 0
 	ex.extraTxnState.rowsWrittenLogged = false
@@ -4342,6 +4345,7 @@ func (ex *connExecutor) recordTransactionFinish(
 		RowsRead:                ex.extraTxnState.rowsRead,
 		RowsWritten:             ex.extraTxnState.rowsWritten,
 		BytesRead:               ex.extraTxnState.bytesRead,
+		KVCPUTime:               ex.extraTxnState.kvCPUTime,
 		Priority:                ex.state.mu.priority,
 		// TODO(107318): add isolation level
 		// TODO(107318): add qos
diff --git a/pkg/sql/executor_statement_metrics.go b/pkg/sql/executor_statement_metrics.go
@@ -208,7 +208,7 @@ func (ex *connExecutor) recordStatementSummary(
 			).
 			PlanGist(planner.instrumentation.planGist.String(), planner.instrumentation.planGist.Hash()).
 			LatencyRecorder(ex.statsCollector).
-			QueryLevelStats(stats.bytesRead, stats.rowsRead, stats.rowsWritten).
+			QueryLevelStats(stats.bytesRead, stats.rowsRead, stats.rowsWritten, stats.kvCPUTime.Nanoseconds()).
 			ExecStats(queryLevelStats).
 			// TODO(mgartner): Use a slice of struct{uint64, uint64} instead of
 			// converting to strings.
diff --git a/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_encoding_test.go b/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_encoding_test.go
@@ -97,6 +97,10 @@ func TestSQLStatsJsonEncoding(t *testing.T) {
            "mean": {{.Float}},
            "sqDiff": {{.Float}}
          },
+         "kvCPUTime": {
+           "mean": {{.Float}},
+           "sqDiff": {{.Float}}
+         },
          "nodes": [{{joinInts .IntArray}}],
          "kvNodeIds": [{{joinInt32s .Int32Array}}],
          "regions": [{{joinStrings .StringArray}}],
@@ -470,6 +474,10 @@ func TestSQLStatsJsonEncoding(t *testing.T) {
     "rowsWritten": {
       "mean": {{.Float}},
       "sqDiff": {{.Float}}
+    },
+    "kvCPUTime": {
+      "mean": {{.Float}},
+      "sqDiff": {{.Float}}
     }
   },
   "execution_statistics": {
diff --git a/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_impl.go b/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_impl.go
@@ -306,6 +306,7 @@ func (t *innerTxnStats) jsonFields() jsonFields {
 		{"bytesRead", (*numericStats)(&t.BytesRead)},
 		{"rowsRead", (*numericStats)(&t.RowsRead)},
 		{"rowsWritten", (*numericStats)(&t.RowsWritten)},
+		{"kvCPUTime", (*numericStats)(&t.KVCPUTime)},
 	}
 }
 
@@ -335,6 +336,7 @@ func (s *innerStmtStats) jsonFields() jsonFields {
 		{"bytesRead", (*numericStats)(&s.BytesRead)},
 		{"rowsRead", (*numericStats)(&s.RowsRead)},
 		{"rowsWritten", (*numericStats)(&s.RowsWritten)},
+		{"kvCPUTime", (*numericStats)(&s.KVCPUTime)},
 		{"nodes", (*int64Array)(&s.Nodes)},
 		{"kvNodeIds", (*int32Array)(&s.KVNodeIDs)},
 		{"regions", (*stringArray)(&s.Regions)},
diff --git a/pkg/sql/sqlstats/ssmemstorage/ss_mem_writer.go b/pkg/sql/sqlstats/ssmemstorage/ss_mem_writer.go
@@ -100,6 +100,7 @@ func (s *Container) RecordStatement(ctx context.Context, value *sqlstats.Recorde
 	stats.mu.data.BytesRead.Record(stats.mu.data.Count, float64(value.BytesRead))
 	stats.mu.data.RowsRead.Record(stats.mu.data.Count, float64(value.RowsRead))
 	stats.mu.data.RowsWritten.Record(stats.mu.data.Count, float64(value.RowsWritten))
+	stats.mu.data.KVCPUTime.Record(stats.mu.data.Count, float64(value.KVCPUTime))
 	stats.mu.data.LastExecTimestamp = s.getTimeNow()
 	stats.mu.data.Nodes = util.CombineUnique(stats.mu.data.Nodes, value.Nodes)
 	stats.mu.data.KVNodeIDs = util.CombineUnique(stats.mu.data.KVNodeIDs, value.KVNodeIDs)
@@ -239,6 +240,7 @@ func (s *Container) RecordTransaction(ctx context.Context, value *sqlstats.Recor
 	stats.mu.data.RowsRead.Record(stats.mu.data.Count, float64(value.RowsRead))
 	stats.mu.data.RowsWritten.Record(stats.mu.data.Count, float64(value.RowsWritten))
 	stats.mu.data.BytesRead.Record(stats.mu.data.Count, float64(value.BytesRead))
+	stats.mu.data.KVCPUTime.Record(stats.mu.data.Count, float64(value.KVCPUTime.Nanoseconds()))
 
 	if value.CollectedExecStats {
 		stats.mu.data.ExecStats.Count++
diff --git a/pkg/sql/sqlstats/ssmemstorage/ss_mem_writer_test.go b/pkg/sql/sqlstats/ssmemstorage/ss_mem_writer_test.go
@@ -117,6 +117,7 @@ func TestContainer_Add(t *testing.T) {
 			BytesRead:          100,
 			RowsRead:           20,
 			RowsWritten:        5,
+			KVCPUTime:          30,
 			Failed:             true,
 			Generic:            true,
 			AppliedStmtHints:   true,
@@ -137,6 +138,7 @@ func TestContainer_Add(t *testing.T) {
 			RowsRead:       20,
 			RowsWritten:    5,
 			BytesRead:      100,
+			KVCPUTime:      30 * time.Nanosecond,
 		}
 		require.NoError(t, src.RecordTransaction(ctx, txnStats))
 
@@ -160,6 +162,7 @@ func TestContainer_Add(t *testing.T) {
 			BytesRead:          60,
 			RowsRead:           70,
 			RowsWritten:        80,
+			KVCPUTime:          90,
 			Failed:             true,
 			Generic:            true,
 			AppliedStmtHints:   true,
@@ -177,6 +180,7 @@ func TestContainer_Add(t *testing.T) {
 			RowsRead:       20,
 			RowsWritten:    5,
 			BytesRead:      100,
+			KVCPUTime:      90 * time.Nanosecond,
 		}
 		require.NoError(t, dest.RecordStatement(ctx, reducedStmtStats))
 		require.NoError(t, dest.RecordTransaction(ctx, reducedTxnStats))
@@ -221,6 +225,7 @@ func verifyStmtStatsMultiple(
 	require.InEpsilon(t, float64(stmtStats.BytesRead), destStmtStats.mu.data.BytesRead.Mean, epsilon)
 	require.InEpsilon(t, float64(stmtStats.RowsRead), destStmtStats.mu.data.RowsRead.Mean, epsilon)
 	require.InEpsilon(t, float64(stmtStats.RowsWritten), destStmtStats.mu.data.RowsWritten.Mean, epsilon)
+	require.InEpsilon(t, float64(stmtStats.KVCPUTime), destStmtStats.mu.data.KVCPUTime.Mean, epsilon)
 }
 
 // verifyStmtStatsReduced verifies that statement statistics have been properly
@@ -241,6 +246,7 @@ func verifyStmtStatsReduced(
 	require.InEpsilon(t, float64(stmtStats.BytesRead)/cnt, destStmtStats.mu.data.BytesRead.Mean, epsilon)
 	require.InEpsilon(t, float64(stmtStats.RowsRead)/cnt, destStmtStats.mu.data.RowsRead.Mean, epsilon)
 	require.InEpsilon(t, float64(stmtStats.RowsWritten)/cnt, destStmtStats.mu.data.RowsWritten.Mean, epsilon)
+	require.InEpsilon(t, float64(stmtStats.KVCPUTime)/cnt, destStmtStats.mu.data.KVCPUTime.Mean, epsilon)
 }
 
 // verifyTxnStatsMultiple verifies that transaction statistics have been recorded
@@ -259,6 +265,7 @@ func verifyTxnStatsMultiple(
 	require.InEpsilon(t, float64(txnStats.RowsRead), destTxnStats.mu.data.RowsRead.Mean, epsilon)
 	require.InEpsilon(t, float64(txnStats.RowsWritten), destTxnStats.mu.data.RowsWritten.Mean, epsilon)
 	require.InEpsilon(t, float64(txnStats.BytesRead), destTxnStats.mu.data.BytesRead.Mean, epsilon)
+	require.InEpsilon(t, float64(txnStats.KVCPUTime.Nanoseconds()), destTxnStats.mu.data.KVCPUTime.Mean, epsilon)
 }
 
 // verifyTxnStatsReduced verifies that transaction statistics have been properly
@@ -277,6 +284,7 @@ func verifyTxnStatsReduced(
 	require.InEpsilon(t, float64(txnStats.RowsRead)/cnt, destTxnStats.mu.data.RowsRead.Mean, epsilon)
 	require.InEpsilon(t, float64(txnStats.RowsWritten)/cnt, destTxnStats.mu.data.RowsWritten.Mean, epsilon)
 	require.InEpsilon(t, float64(txnStats.BytesRead)/cnt, destTxnStats.mu.data.BytesRead.Mean, epsilon)
+	require.InEpsilon(t, float64(txnStats.KVCPUTime.Nanoseconds())/cnt, destTxnStats.mu.data.KVCPUTime.Mean, epsilon)
 }
 
 func testMonitor(
diff --git a/pkg/sql/sqlstats/ssprovider.go b/pkg/sql/sqlstats/ssprovider.go
@@ -85,6 +85,7 @@ type RecordedStmtStats struct {
 	BytesRead                int64
 	RowsRead                 int64
 	RowsWritten              int64
+	KVCPUTime                int64
 	Nodes                    []int64
 	KVNodeIDs                []int32
 	StatementType            tree.StatementType
@@ -123,6 +124,7 @@ type RecordedTxnStats struct {
 	RowsRead                int64
 	RowsWritten             int64
 	BytesRead               int64
+	KVCPUTime               time.Duration
 	Priority                roachpb.UserPriority
 	TxnErr                  error
 	Application             string
@@ -233,14 +235,15 @@ func (b *RecordedStatementStatsBuilder) LatencyRecorder(
 }
 
 func (b *RecordedStatementStatsBuilder) QueryLevelStats(
-	bytesRead int64, rowsRead int64, rowsWritten int64,
+	bytesRead int64, rowsRead int64, rowsWritten int64, kvCPUTime int64,
 ) *RecordedStatementStatsBuilder {
 	if b == nil {
 		return b
 	}
 	b.stmtStats.BytesRead = bytesRead
 	b.stmtStats.RowsRead = rowsRead
 	b.stmtStats.RowsWritten = rowsWritten
+	b.stmtStats.KVCPUTime = kvCPUTime
 	return b
 }
 
diff --git a/pkg/ui/workspaces/cluster-ui/src/api/statementsApi.ts b/pkg/ui/workspaces/cluster-ui/src/api/statementsApi.ts
@@ -170,6 +170,7 @@ type Statistics = {
   runLat: NumericStat;
   svcLat: NumericStat;
   regions: string[];
+  kvCPUTime?: NumericStat;
 };
 
 type ExecStats = {
@@ -253,6 +254,7 @@ export function convertStatementRawFormatToAggregatedStatistics(
       service_lat: s.statistics.statistics.svcLat,
       sql_type: s.metadata.stmtType,
       regions: s.statistics.statistics.regions,
+      kv_cpu_time: s.statistics.statistics.kvCPUTime,
     },
   };
 }
diff --git a/pkg/ui/workspaces/cluster-ui/src/barCharts/barCharts.tsx b/pkg/ui/workspaces/cluster-ui/src/barCharts/barCharts.tsx
@@ -52,6 +52,13 @@ const cpuBars = [
   ),
 ];
 
+const kvCPUTimeBars = [
+  bar(
+    "kv-cpu-time",
+    (d: StatementStatistics) => d.stats.kv_cpu_time?.mean,
+  ),
+];
+
 const maxMemUsageBars = [
   bar(
     "max-mem-usage",
@@ -87,6 +94,9 @@ const contentionStdDev = bar(cx("contention-dev"), (d: StatementStatistics) =>
 const cpuStdDev = bar(cx("cpu-dev"), (d: StatementStatistics) =>
   stdDevLong(d.stats.exec_stats.cpu_sql_nanos, d.stats.exec_stats.count),
 );
+const kvCPUTimeStdDev = bar(cx("kv-cpu-time-dev"), (d: StatementStatistics) =>
+  stdDevLong(d.stats.kv_cpu_time, d.stats.count),
+);
 const maxMemUsageStdDev = bar(
   cx("max-mem-usage-dev"),
   (d: StatementStatistics) =>
@@ -123,6 +133,12 @@ export const cpuBarChart = barChartFactory(
   v => Duration(v),
   cpuStdDev,
 );
+export const kvCPUTimeBarChart = barChartFactory(
+  "grey",
+  kvCPUTimeBars,
+  v => Duration(v),
+  kvCPUTimeStdDev,
+);
 export const maxMemUsageBarChart = barChartFactory(
   "grey",
   maxMemUsageBars,
diff --git a/pkg/ui/workspaces/cluster-ui/src/statementDetails/statementDetails.tsx b/pkg/ui/workspaces/cluster-ui/src/statementDetails/statementDetails.tsx
@@ -837,6 +837,13 @@ export class StatementDetails extends React.Component<
                     Duration,
                   )}
                 />
+                <SummaryCardItem
+                  label="KV CPU Time"
+                  value={formatNumberForDisplay(
+                    stats?.kv_cpu_time?.mean,
+                    Duration,
+                  )}
+                />
                 <SummaryCardItem
                   label="Client Wait Time"
                   value={formatNumberForDisplay(stats?.idle_lat.mean, duration)}
diff --git a/pkg/ui/workspaces/cluster-ui/src/statementsTable/statementsTable.tsx b/pkg/ui/workspaces/cluster-ui/src/statementsTable/statementsTable.tsx
@@ -13,6 +13,7 @@ import {
   latencyBarChart,
   contentionBarChart,
   cpuBarChart,
+  kvCPUTimeBarChart,
   maxMemUsageBarChart,
   networkBytesBarChart,
   retryBarChart,
@@ -152,6 +153,7 @@ export function makeStatementsColumns(
     sampledExecStatsBarChartOptions,
   );
   const cpuBar = cpuBarChart(statements, sampledExecStatsBarChartOptions);
+  const kvCPUTimeBar = kvCPUTimeBarChart(statements, sampledExecStatsBarChartOptions);
   const maxMemUsageBar = maxMemUsageBarChart(
     statements,
     sampledExecStatsBarChartOptions,
@@ -231,6 +233,13 @@ export function makeStatementsColumns(
       sort: (stmt: AggregateStatistics) =>
         FixLong(Number(stmt.stats.exec_stats.cpu_sql_nanos?.mean)),
     },
+    {
+      name: "kvCPUTime",
+      title: statisticsTableTitles.kvCPUTime(statType),
+      cell: kvCPUTimeBar,
+      sort: (stmt: AggregateStatistics) =>
+        FixLong(Number(stmt.stats.kv_cpu_time?.mean)),
+    },
     {
       name: "latencyMin",
       title: statisticsTableTitles.latencyMin(statType),
diff --git a/pkg/ui/workspaces/cluster-ui/src/statsTableUtil/statsTableUtil.tsx b/pkg/ui/workspaces/cluster-ui/src/statsTableUtil/statsTableUtil.tsx
@@ -27,6 +27,7 @@ export const statisticsColumnLabels = {
   clientAddress: "Client IP Address",
   contention: "Contention Time",
   cpu: "SQL CPU Time",
+  kvCPUTime: "KV CPU Time",
   database: "Database",
   diagnostics: "Diagnostics",
   executionCount: "Execution Count",
@@ -684,6 +685,27 @@ export const statisticsTableTitles: StatisticTableTitleType = {
       </Tooltip>
     );
   },
+  kvCPUTime: (_: StatisticType) => {
+    return (
+      <Tooltip
+        placement="bottom"
+        style="tableTitle"
+        content={
+          <>
+            <p>
+              Average KV CPU time spent executing within the specified time
+              interval. This can be thought of as KV work that is on the critical
+              path of serving the query. It does not include any asynchronous replication
+              related work. The gray bar indicates mean KV CPU time.
+              The blue bar indicates one standard deviation from the mean.
+            </p>
+          </>
+        }
+      >
+        {getLabel("kvCPUTime")}
+      </Tooltip>
+    );
+  },
   maxMemUsage: (statType: StatisticType) => {
     let contentModifier = "";
     let fingerprintModifier = "";
diff --git a/pkg/ui/workspaces/cluster-ui/src/transactionDetails/transactionDetailsUtils.tsx b/pkg/ui/workspaces/cluster-ui/src/transactionDetails/transactionDetailsUtils.tsx
@@ -158,6 +158,7 @@ function addTransactionStats(
       countB,
     ),
     exec_stats: addExecStats(a.exec_stats, b.exec_stats),
+    kv_cpu_time: aggregateNumericStats(a.kv_cpu_time, b.kv_cpu_time, countA, countB),
   };
 }
 
diff --git a/pkg/ui/workspaces/cluster-ui/src/transactionsTable/transactionsBarCharts.ts b/pkg/ui/workspaces/cluster-ui/src/transactionsTable/transactionsBarCharts.ts
diff --git a/pkg/ui/workspaces/cluster-ui/src/transactionsTable/transactionsTable.tsx b/pkg/ui/workspaces/cluster-ui/src/transactionsTable/transactionsTable.tsx
diff --git a/pkg/ui/workspaces/cluster-ui/src/util/appStats/appStats.ts b/pkg/ui/workspaces/cluster-ui/src/util/appStats/appStats.ts

Original file line number	Diff line number	Diff line change
`@@ -306,6 +306,7 @@ func (t *innerTxnStats) jsonFields() jsonFields {`
`306`	`306`	`{"bytesRead", (*numericStats)(&t.BytesRead)},`
`307`	`307`	`{"rowsRead", (*numericStats)(&t.RowsRead)},`
`308`	`308`	`{"rowsWritten", (*numericStats)(&t.RowsWritten)},`
	`309`	`+ {"kvCPUTime", (*numericStats)(&t.KVCPUTime)},`
`309`	`310`	`}`
`310`	`311`	`}`
`311`	`312`
`@@ -335,6 +336,7 @@ func (s *innerStmtStats) jsonFields() jsonFields {`
`335`	`336`	`{"bytesRead", (*numericStats)(&s.BytesRead)},`
`336`	`337`	`{"rowsRead", (*numericStats)(&s.RowsRead)},`
`337`	`338`	`{"rowsWritten", (*numericStats)(&s.RowsWritten)},`
	`339`	`+ {"kvCPUTime", (*numericStats)(&s.KVCPUTime)},`
`338`	`340`	`{"nodes", (*int64Array)(&s.Nodes)},`
`339`	`341`	`{"kvNodeIds", (*int32Array)(&s.KVNodeIDs)},`
`340`	`342`	`{"regions", (*stringArray)(&s.Regions)},`