Merge #140607

140607: logictest: raise TxnLivenessThreshold for multitenant configs r=rafiss a=rafiss

Logic tests are flaky due to overload when running in multitenant mode. This patch increases the threshold for transaction heartbeat timeouts, which will make it less likely for foreground operations to be aborted by background jobs like the span config reconciler or the job registry loop to reclaim jobs from dead sessions.

fixes #140597
fixes #140494
fixes #140578
fixes #140577
fixes #140572
fixes #140574
fixes #139890
fixes #140565
fixes #140542
fixes #140543
fixes #140545
fixes #140544
fixes #140546
fixes #140560
fixes #140552
fixes #140541
fixes #140553
fixes #139990
fixes #139776
fixes #140072
fixes #139761
fixes #140161
fixes #140063
fixes #140062
fixes #140011


Release note: None

Co-authored-by: Rafi Shamim <[email protected]>

Author: craig[bot]

pkg/sql/logictest/BUILD.bazel

@@ -62,6 +62,7 @@ go_library(
         "//pkg/kv/kvclient/rangefeed",
         "//pkg/kv/kvserver",
         "//pkg/kv/kvserver/kvserverbase",
+        "//pkg/kv/kvserver/txnwait",
         "//pkg/multitenant/tenantcapabilities",
         "//pkg/security/username",
         "//pkg/server",

pkg/sql/logictest/logic.go

@@ -41,6 +41,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait"
 	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
 	"github.com/cockroachdb/cockroach/pkg/server"
@@ -1708,6 +1709,21 @@ func (t *logicTest) newCluster(
 					t.Fatal(err)
 				}
 			}
+			if _, err := conn.Exec(
+				"RESET CLUSTER SETTING kv.closed_timestamp.target_duration",
+			); err != nil {
+				t.Fatal(err)
+			}
+			if _, err := conn.Exec(
+				"RESET CLUSTER SETTING kv.closed_timestamp.side_transport_interval",
+			); err != nil {
+				t.Fatal(err)
+			}
+			if _, err := conn.Exec(
+				"RESET CLUSTER SETTING kv.rangefeed.closed_timestamp_refresh_interval",
+			); err != nil {
+				t.Fatal(err)
+			}
 		}
 
 		capabilities := toa.capabilities
@@ -4373,6 +4389,18 @@ func RunLogicTest(
 	if *printErrorSummary {
 		defer lt.printErrorSummary()
 	}
+	if config.UseSecondaryTenant == logictestbase.Always {
+		// Under multitenant configs running in EngFlow, we have seen that logic
+		// tests can be flaky due to an overload condition where schema change
+		// transactions do not heartbeat quickly enough. This allows background jobs
+		// such as the spanconfig reconciler or the job registry "remove claims from
+		// dead sessions" loop.
+		// See https://github.com/cockroachdb/cockroach/pull/140400#issuecomment-2634346278
+		// and https://github.com/cockroachdb/cockroach/issues/140494#issuecomment-2640208187
+		// for a detailed analysis of this issue.
+		cleanup := txnwait.TestingOverrideTxnLivenessThreshold(30 * time.Second)
+		defer cleanup()
+	}
 	// Each test needs a copy because of Parallel
 	serverArgsCopy := serverArgs
 	serverArgsCopy.ForceProductionValues = serverArgs.ForceProductionValues || nonMetamorphicBatchSizes