roachtest: create secure clusters by default

In order to run on secure clusters, many tests were
changed to explicity specify a pgurl or certs directory
to authenticate with. Currently most tests authenticate
with the root user, but in the future we want to use
a non root user when possible.

This change also fixes roachtests to use
the new defaultHTTPClient helper to disable cert
verification and automatically retrieve and use
sessionID for auth.

Release note: None

Author: DarrylWong

pkg/cmd/roachtest/cluster.go

@@ -1428,7 +1428,7 @@ func (c *clusterImpl) HealthStatus(
 		return nil, errors.WithDetail(err, "Unable to get admin UI address(es)")
 	}
 	getStatus := func(ctx context.Context, node int) *HealthStatusResult {
-		url := fmt.Sprintf(`http://%s/health?ready=1`, adminAddrs[node-1])
+		url := fmt.Sprintf(`https://%s/health?ready=1`, adminAddrs[node-1])
 		resp, err := httputil.Get(ctx, url)
 		if err != nil {
 			return newHealthStatusResult(node, 0, nil, err)

pkg/cmd/roachtest/clusterstats/BUILD.bazel

@@ -15,7 +15,6 @@ go_library(
     deps = [
         "//pkg/cmd/roachtest/cluster",
         "//pkg/cmd/roachtest/option",
-        "//pkg/cmd/roachtest/roachtestutil",
         "//pkg/cmd/roachtest/test",
         "//pkg/roachprod/logger",
         "//pkg/roachprod/prometheus",

pkg/cmd/roachtest/roachtestutil/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
         "commandbuilder.go",
         "disk_usage.go",
         "health_checker.go",
+        "httpclient.go",
         "jaeger.go",
         "utils.go",
         "validation_check.go",
@@ -25,9 +26,10 @@ go_library(
         "//pkg/util/httputil",
         "//pkg/util/humanizeutil",
         "//pkg/util/log",
+        "//pkg/util/protoutil",
+        "//pkg/util/syncutil",
         "//pkg/util/timeutil",
         "@com_github_cockroachdb_errors//:errors",
-        "@com_github_pkg_errors//:errors",
     ],
 )
 

pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go

@@ -141,9 +141,7 @@ var (
 	// defaultClusterSettings is the set of cluster settings always
 	// passed to `clusterupgrade.StartWithSettings` when (re)starting
 	// nodes in a cluster.
-	defaultClusterSettings = []install.ClusterSettingOption{
-		install.SecureOption(true),
-	}
+	defaultClusterSettings = []install.ClusterSettingOption{}
 
 	// minSupportedARM64Version is the minimum version for which there
 	// is a published ARM64 build. If we are running a mixedversion test

pkg/cmd/roachtest/roachtestutil/utils.go

@@ -30,9 +30,13 @@ func SystemInterfaceSystemdUnitName() string {
 // DefaultPGUrl is a wrapper over roachprod.PgUrl that calls it with the arguments
 // that *almost* all roachtests want: single tenant and only a single node.
 func DefaultPGUrl(
-	ctx context.Context, c cluster.Cluster, l *logger.Logger, node option.NodeListOption,
+	ctx context.Context,
+	c cluster.Cluster,
+	l *logger.Logger,
+	node option.NodeListOption,
+	auth install.PGAuthMode,
 ) (string, error) {
-	opts := roachprod.PGURLOptions{Secure: c.IsSecure()}
+	opts := roachprod.PGURLOptions{Auth: auth, Secure: c.IsSecure()}
 	pgurl, err := roachprod.PgURL(ctx, l, c.MakeNodes(node), "certs", opts)
 	if err != nil {
 		return "", err

pkg/cmd/roachtest/tests/activerecord.go

@@ -51,7 +51,8 @@ func registerActiveRecord(r registry.Registry) {
 		t.Status("setting up cockroach")
 		startOpts := option.DefaultStartOptsInMemory()
 		startOpts.RoachprodOpts.SQLPort = config.DefaultSQLPort
-		c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.All())
+		// Activerecord uses root user with ssl disabled.
+		c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(install.SecureOption(false)), c.All())
 
 		version, err := fetchCockroachVersion(ctx, t.L(), c, node[0])
 		if err != nil {

pkg/cmd/roachtest/tests/admission_control_elastic_io.go

@@ -65,17 +65,16 @@ func registerElasticIO(r registry.Registry) {
 				WithGrafanaDashboardJSON(grafana.ChangefeedAdmissionControlGrafana)
 			err := c.StartGrafana(ctx, t.L(), promCfg)
 			require.NoError(t, err)
-			promClient, err := clusterstats.SetupCollectorPromClient(ctx, c, t.L(), promCfg)
-			require.NoError(t, err)
-			statCollector := clusterstats.NewStatsCollector(ctx, promClient)
-
 			c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(workAndPromNode))
 			startOpts := option.DefaultStartOptsNoBackups()
 			roachtestutil.SetDefaultAdminUIPort(c, &startOpts.RoachprodOpts)
 			startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs,
 				"--vmodule=io_load_listener=2")
 			settings := install.MakeClusterSettings()
 			c.Start(ctx, t.L(), startOpts, settings, c.Range(1, crdbNodes))
+			promClient, err := clusterstats.SetupCollectorPromClient(ctx, c, t.L(), promCfg)
+			require.NoError(t, err)
+			statCollector := clusterstats.NewStatsCollector(ctx, promClient)
 			setAdmissionControl(ctx, t, c, true)
 			duration := 30 * time.Minute
 			t.Status("running workload")
@@ -85,7 +84,7 @@ func registerElasticIO(r registry.Registry) {
 				url := fmt.Sprintf(" {pgurl:1-%d}", crdbNodes)
 				cmd := "./workload run kv --init --histograms=perf/stats.json --concurrency=512 " +
 					"--splits=1000 --read-percent=0 --min-block-bytes=65536 --max-block-bytes=65536 " +
-					"--txn-qos=background --tolerate-errors" + dur + url
+					"--txn-qos=background --tolerate-errors --secure" + dur + url
 				c.Run(ctx, option.WithNodes(c.Node(workAndPromNode)), cmd)
 				return nil
 			})

pkg/cmd/roachtest/tests/admission_control_intent_resolution.go

@@ -64,17 +64,18 @@ func registerIntentResolutionOverload(r registry.Registry) {
 				WithGrafanaDashboardJSON(grafana.ChangefeedAdmissionControlGrafana)
 			err := c.StartGrafana(ctx, t.L(), promCfg)
 			require.NoError(t, err)
-			promClient, err := clusterstats.SetupCollectorPromClient(ctx, c, t.L(), promCfg)
-			require.NoError(t, err)
-			statCollector := clusterstats.NewStatsCollector(ctx, promClient)
 
 			startOpts := option.DefaultStartOptsNoBackups()
 			startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs,
 				"--vmodule=io_load_listener=2")
-			roachtestutil.SetDefaultSQLPort(c, &startOpts.RoachprodOpts)
 			roachtestutil.SetDefaultAdminUIPort(c, &startOpts.RoachprodOpts)
 			settings := install.MakeClusterSettings()
 			c.Start(ctx, t.L(), startOpts, settings, c.Range(1, crdbNodes))
+
+			promClient, err := clusterstats.SetupCollectorPromClient(ctx, c, t.L(), promCfg)
+			require.NoError(t, err)
+			statCollector := clusterstats.NewStatsCollector(ctx, promClient)
+
 			setAdmissionControl(ctx, t, c, true)
 			t.Status("running txn")
 			m := c.NewMonitor(ctx, c.Range(1, crdbNodes))

pkg/cmd/roachtest/tests/admission_control_multitenant_fairness.go

@@ -145,7 +145,7 @@ func runMultiTenantFairness(
 	t.L().Printf("starting cockroach securely (<%s)", time.Minute)
 	c.Start(ctx, t.L(),
 		option.DefaultStartOptsNoBackups(),
-		install.MakeClusterSettings(install.SecureOption(true)),
+		install.MakeClusterSettings(),
 		crdbNode,
 	)
 

pkg/cmd/roachtest/tests/admission_control_tpcc_overload.go

@@ -124,7 +124,7 @@ func verifyNodeLiveness(
 	if err := retry.WithMaxAttempts(ctx, retry.Options{
 		MaxBackoff: 500 * time.Millisecond,
 	}, 60, func() (err error) {
-		response, err = getMetrics(ctx, adminURLs[0], now.Add(-runDuration), now, []tsQuery{
+		response, err = getMetrics(ctx, c, t, adminURLs[0], now.Add(-runDuration), now, []tsQuery{
 			{
 				name:      "cr.node.liveness.heartbeatfailures",
 				queryType: total,