Skip to content

Commit 71b3a9e

Browse files
utkuozdemirutkuozdemir
committed
test: fix flaky unit tests on overloaded CI runners
Two unit tests failed on a heavily loaded CI runner. The certificate refresh tick test allowed only a fraction of a second for each tick to arrive, which is too tight on a starved machine. It now uses the same generous deadline as the rest of the suite. Many tests also generated a fresh secrets bundle for every mocked cluster and every generated machine config, and the RSA service account key generation inside costs around a second of CPU each time. Tests now share generated bundles through a small helper that generates one bundle per Talos version per test process. This makes the heaviest test packages several times faster. Finally, the secret rotation test deadlines become one generous shared constant. The race detector multiplies the CPU cost, and on a starved runner exactly the subtests with the tightest deadlines ran out of time while their siblings with more generous ones passed. Signed-off-by: Utku Ozdemir <utku.ozdemir@siderolabs.com>
1 parent dbff805 commit 71b3a9e

14 files changed

Lines changed: 167 additions & 40 deletions

File tree

‎internal/backend/grpc/configs_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"github.com/siderolabs/go-api-signature/pkg/serviceaccount"
2424
clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config"
2525
machineryconfig "github.com/siderolabs/talos/pkg/machinery/config"
26-
talossecrets "github.com/siderolabs/talos/pkg/machinery/config/generate/secrets"
2726
"github.com/stretchr/testify/require"
2827
"go.uber.org/zap"
2928
"go.uber.org/zap/zaptest"
@@ -48,6 +47,7 @@ import (
4847
"github.com/siderolabs/omni/internal/pkg/auth"
4948
"github.com/siderolabs/omni/internal/pkg/config"
5049
"github.com/siderolabs/omni/internal/pkg/ctxstore"
50+
"github.com/siderolabs/omni/internal/pkg/testsecrets"
5151
)
5252

5353
//go:embed testdata/admin-kubeconfig.yaml
@@ -167,7 +167,7 @@ func TestGenerateConfigs(t *testing.T) {
167167

168168
secrets := omni.NewClusterSecrets(clusterName)
169169

170-
bundle, err := talossecrets.NewBundle(talossecrets.NewFixedClock(time.Now()), machineryconfig.TalosVersion1_7)
170+
bundle, err := testsecrets.Bundle(machineryconfig.TalosVersion1_7)
171171
require.NoError(t, err)
172172

173173
var data []byte

‎internal/backend/runtime/omni/controllers/helpers/helpers_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424

2525
"github.com/siderolabs/omni/client/pkg/omni/resources/omni"
2626
"github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/helpers"
27+
"github.com/siderolabs/omni/internal/pkg/testsecrets"
2728
)
2829

2930
func TestUpdateInputsVersions(t *testing.T) {
@@ -102,7 +103,7 @@ func TestGetTalosClient(t *testing.T) {
102103

103104
talosConfig := omni.NewTalosConfig(cluster.Metadata().ID())
104105

105-
bundle, err := secrets.NewBundle(secrets.NewFixedClock(time.Now()), config.TalosVersion1_10)
106+
bundle, err := testsecrets.Bundle(config.TalosVersion1_10)
106107
require.NoError(t, err)
107108

108109
talosConfig.TypedSpec().Value.Ca = base64.StdEncoding.EncodeToString(bundle.Certs.OS.Crt)

‎internal/backend/runtime/omni/controllers/omni/cert_refresh_tick_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func (suite *CertRefreshTickSuite) TestReconcile() {
3838
suite.Require().Equal(state.Created, ev.Type)
3939

4040
ticks++
41-
case <-time.After(500 * time.Millisecond):
41+
case <-suite.ctx.Done():
4242
suite.Require().FailNow("timeout waiting for tick")
4343
}
4444

‎internal/backend/runtime/omni/controllers/omni/internal/kubernetes/kubernetes_test.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
"github.com/siderolabs/omni/client/pkg/image"
2323
"github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/omni/internal/kubernetes"
24+
"github.com/siderolabs/omni/internal/pkg/testsecrets"
2425
)
2526

2627
func TestComponentLess(t *testing.T) {
@@ -70,7 +71,11 @@ func TestComponentPatch(t *testing.T) {
7071
components[i], components[j] = components[j], components[i]
7172
})
7273

73-
in, err := generate.NewInput("component-patch-test", "https://127.0.0.1/", "1.34.0", generate.WithVersionContract(vc))
74+
secretsBundle, err := testsecrets.Bundle(vc)
75+
require.NoError(t, err)
76+
77+
in, err := generate.NewInput("component-patch-test", "https://127.0.0.1/", "1.34.0",
78+
generate.WithVersionContract(vc), generate.WithSecretsBundle(secretsBundle))
7479
require.NoError(t, err)
7580

7681
cfg, err := in.Config(machine.TypeControlPlane)

‎internal/backend/runtime/omni/controllers/omni/internal/kubernetes/patch_test.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717

1818
"github.com/siderolabs/omni/client/pkg/image"
1919
"github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/omni/internal/kubernetes"
20+
"github.com/siderolabs/omni/internal/pkg/testsecrets"
2021
)
2122

2223
// TestMergePatch verifies that patches for several components accumulate into a single machine patch.
@@ -49,7 +50,11 @@ func TestMergePatch(t *testing.T) {
4950
accumulated, err = kubernetes.MergePatch(accumulated, newAPIServerPatch.Patch)
5051
require.NoError(t, err)
5152

52-
in, err := generate.NewInput("merge-patch-test", "https://127.0.0.1/", "1.34.0", generate.WithVersionContract(vc))
53+
secretsBundle, err := testsecrets.Bundle(vc)
54+
require.NoError(t, err)
55+
56+
in, err := generate.NewInput("merge-patch-test", "https://127.0.0.1/", "1.34.0",
57+
generate.WithVersionContract(vc), generate.WithSecretsBundle(secretsBundle))
5358
require.NoError(t, err)
5459

5560
cfg, err := in.Config(machine.TypeControlPlane)

‎internal/backend/runtime/omni/controllers/omni/secrets/rotation_status_test.go‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ import (
4545
"github.com/siderolabs/omni/internal/pkg/constants"
4646
)
4747

48+
// testTimeout bounds each subtest. It is generous on purpose, so that the tests survive CPU starvation on overloaded CI runners, especially when the race detector multiplies the cost.
49+
const testTimeout = 30 * time.Second
50+
4851
//nolint:maintidx
4952
func Test_TalosCARotation(t *testing.T) {
5053
t.Parallel()
@@ -53,7 +56,7 @@ func Test_TalosCARotation(t *testing.T) {
5356
t.Run("no rotation", func(t *testing.T) {
5457
t.Parallel()
5558

56-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*20)
59+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
5760
t.Cleanup(cancel)
5861

5962
testutils.WithRuntime(
@@ -113,7 +116,7 @@ func Test_TalosCARotation(t *testing.T) {
113116
t.Run("rotation", func(t *testing.T) {
114117
t.Parallel()
115118

116-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*15)
119+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
117120
t.Cleanup(cancel)
118121

119122
testutils.WithRuntime(
@@ -212,7 +215,7 @@ func Test_TalosCARotation(t *testing.T) {
212215
t.Run("cluster locked", func(t *testing.T) {
213216
t.Parallel()
214217

215-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*20)
218+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
216219
t.Cleanup(cancel)
217220

218221
testutils.WithRuntime(
@@ -278,7 +281,7 @@ func Test_TalosCARotation(t *testing.T) {
278281
t.Run("cluster unhealthy", func(t *testing.T) {
279282
t.Parallel()
280283

281-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*20)
284+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
282285
t.Cleanup(cancel)
283286

284287
testutils.WithRuntime(
@@ -348,7 +351,7 @@ func Test_TalosCARotation(t *testing.T) {
348351
t.Run("machine locked", func(t *testing.T) {
349352
t.Parallel()
350353

351-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*15)
354+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
352355
t.Cleanup(cancel)
353356

354357
testutils.WithRuntime(
@@ -422,7 +425,7 @@ func Test_TalosCARotation(t *testing.T) {
422425
t.Run("machine unhealthy", func(t *testing.T) {
423426
t.Parallel()
424427

425-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*15)
428+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
426429
t.Cleanup(cancel)
427430

428431
testutils.WithRuntime(
@@ -496,7 +499,7 @@ func Test_TalosCARotation(t *testing.T) {
496499
t.Run("rotation ongoing", func(t *testing.T) {
497500
t.Parallel()
498501

499-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*15)
502+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
500503
t.Cleanup(cancel)
501504

502505
testutils.WithRuntime(
@@ -628,7 +631,7 @@ func Test_KubernetesCARotation(t *testing.T) {
628631
t.Run("no rotation", func(t *testing.T) {
629632
t.Parallel()
630633

631-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*20)
634+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
632635
t.Cleanup(cancel)
633636

634637
testutils.WithRuntime(
@@ -688,7 +691,7 @@ func Test_KubernetesCARotation(t *testing.T) {
688691
t.Run("rotation", func(t *testing.T) {
689692
t.Parallel()
690693

691-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*15)
694+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
692695
t.Cleanup(cancel)
693696

694697
testutils.WithRuntime(
@@ -796,7 +799,7 @@ func Test_KubernetesCARotation(t *testing.T) {
796799
t.Run("cluster locked", func(t *testing.T) {
797800
t.Parallel()
798801

799-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*20)
802+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
800803
t.Cleanup(cancel)
801804

802805
testutils.WithRuntime(
@@ -871,7 +874,7 @@ func Test_KubernetesCARotation(t *testing.T) {
871874
t.Run("cluster unhealthy", func(t *testing.T) {
872875
t.Parallel()
873876

874-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*20)
877+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
875878
t.Cleanup(cancel)
876879

877880
testutils.WithRuntime(
@@ -950,7 +953,7 @@ func Test_KubernetesCARotation(t *testing.T) {
950953
t.Run("machine locked", func(t *testing.T) {
951954
t.Parallel()
952955

953-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*15)
956+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
954957
t.Cleanup(cancel)
955958

956959
testutils.WithRuntime(
@@ -1033,7 +1036,7 @@ func Test_KubernetesCARotation(t *testing.T) {
10331036
t.Run("machine unhealthy", func(t *testing.T) {
10341037
t.Parallel()
10351038

1036-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*15)
1039+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
10371040
t.Cleanup(cancel)
10381041

10391042
testutils.WithRuntime(
@@ -1116,7 +1119,7 @@ func Test_KubernetesCARotation(t *testing.T) {
11161119
t.Run("rotation ongoing", func(t *testing.T) {
11171120
t.Parallel()
11181121

1119-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*15)
1122+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
11201123
t.Cleanup(cancel)
11211124

11221125
testutils.WithRuntime(
@@ -1181,7 +1184,7 @@ func Test_KubernetesCARotation(t *testing.T) {
11811184
func Test_ConcurrentRotationRejection(t *testing.T) {
11821185
t.Parallel()
11831186

1184-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*20)
1187+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
11851188
t.Cleanup(cancel)
11861189

11871190
testutils.WithRuntime(
@@ -1230,7 +1233,7 @@ func Test_ConcurrentRotationRejection(t *testing.T) {
12301233
func Test_ComponentIsolationDuringRotation(t *testing.T) {
12311234
t.Parallel()
12321235

1233-
ctx, cancel := context.WithTimeout(t.Context(), time.Second*15)
1236+
ctx, cancel := context.WithTimeout(t.Context(), testTimeout)
12341237
t.Cleanup(cancel)
12351238

12361239
testutils.WithRuntime(

‎internal/backend/runtime/omni/controllers/omni/secrets/talosconfig_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
"github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/testutils"
3535
"github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/testutils/rmock"
3636
testoptions "github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/testutils/rmock/options"
37+
"github.com/siderolabs/omni/internal/pkg/testsecrets"
3738
)
3839

3940
//nolint:maintidx,dupl
@@ -67,7 +68,7 @@ func Test_Talosconfig(t *testing.T) {
6768

6869
require.NoError(t, st.Create(ctx, machineSet))
6970

70-
secretsBundle, err := talossecrets.NewBundle(talossecrets.NewFixedClock(time.Now()), config.TalosVersion1_11)
71+
secretsBundle, err := testsecrets.Bundle(config.TalosVersion1_11)
7172
require.NoError(t, err)
7273
data, err := json.Marshal(secretsBundle)
7374
require.NoError(t, err)
@@ -150,7 +151,7 @@ func Test_Talosconfig(t *testing.T) {
150151

151152
require.NoError(t, st.Create(ctx, machineSet))
152153

153-
secretsBundle, err := talossecrets.NewBundle(talossecrets.NewFixedClock(time.Now()), config.TalosVersion1_11)
154+
secretsBundle, err := testsecrets.Bundle(config.TalosVersion1_11)
154155
require.NoError(t, err)
155156
data, err := json.Marshal(secretsBundle)
156157
require.NoError(t, err)

‎internal/backend/runtime/omni/controllers/testutils/rmock/rmock.go‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ package rmock
99
import (
1010
"context"
1111
"encoding/base64"
12-
"encoding/json"
1312
"errors"
1413
"time"
1514

@@ -19,7 +18,6 @@ import (
1918
"github.com/cosi-project/runtime/pkg/state"
2019
"github.com/siderolabs/talos/pkg/machinery/api/machine"
2120
"github.com/siderolabs/talos/pkg/machinery/config"
22-
gensecrets "github.com/siderolabs/talos/pkg/machinery/config/generate/secrets"
2321
talosconstants "github.com/siderolabs/talos/pkg/machinery/constants"
2422
"github.com/siderolabs/talos/pkg/machinery/role"
2523

@@ -35,6 +33,7 @@ import (
3533
"github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/omni/talosupgrade"
3634
"github.com/siderolabs/omni/internal/pkg/certs"
3735
"github.com/siderolabs/omni/internal/pkg/constants"
36+
"github.com/siderolabs/omni/internal/pkg/testsecrets"
3837
)
3938

4039
const (
@@ -212,12 +211,7 @@ func init() {
212211
return err
213212
}
214213

215-
bundle, err := gensecrets.NewBundle(gensecrets.NewFixedClock(time.Now()), vc)
216-
if err != nil {
217-
return err
218-
}
219-
220-
res.TypedSpec().Value.Data, err = json.Marshal(bundle)
214+
res.TypedSpec().Value.Data, err = testsecrets.BundleData(vc)
221215

222216
return err
223217
})

‎internal/backend/runtime/omni/talosconfig_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import (
1515
"github.com/prometheus/client_golang/prometheus"
1616
clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config"
1717
"github.com/siderolabs/talos/pkg/machinery/config"
18-
talossecrets "github.com/siderolabs/talos/pkg/machinery/config/generate/secrets"
1918
"github.com/stretchr/testify/require"
2019
"go.uber.org/zap"
2120
"go.uber.org/zap/zapcore"
@@ -29,6 +28,7 @@ import (
2928
"github.com/siderolabs/omni/internal/backend/runtime/talos"
3029
"github.com/siderolabs/omni/internal/backend/services/workloadproxy"
3130
omniconfig "github.com/siderolabs/omni/internal/pkg/config"
31+
"github.com/siderolabs/omni/internal/pkg/testsecrets"
3232
)
3333

3434
func TestOperatorTalosconfig(t *testing.T) {
@@ -57,7 +57,7 @@ func TestOperatorTalosconfig(t *testing.T) {
5757

5858
secrets := omni.NewClusterSecrets("cluster1")
5959

60-
bundle, err := talossecrets.NewBundle(talossecrets.NewFixedClock(time.Now()), config.TalosVersion1_7)
60+
bundle, err := testsecrets.Bundle(config.TalosVersion1_7)
6161
require.NoError(t, err)
6262

6363
data, err := json.Marshal(bundle)

‎internal/backend/runtime/talos/clients_test.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/cosi-project/runtime/pkg/safe"
1414
"github.com/cosi-project/runtime/pkg/state"
1515
"github.com/siderolabs/talos/pkg/machinery/config/bundle"
16+
"github.com/siderolabs/talos/pkg/machinery/config/generate"
1617
"github.com/stretchr/testify/assert"
1718
"github.com/stretchr/testify/require"
1819
"go.uber.org/zap"
@@ -24,6 +25,7 @@ import (
2425
"github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/omni/secrets"
2526
"github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/testutils"
2627
"github.com/siderolabs/omni/internal/backend/runtime/talos"
28+
"github.com/siderolabs/omni/internal/pkg/testsecrets"
2729
)
2830

2931
func TestGetClientForCluster(t *testing.T) {
@@ -41,11 +43,15 @@ func TestGetClientForCluster(t *testing.T) {
4143
_, err := clientFactory.GetForCluster(ctx, clusterName)
4244
require.True(t, talos.IsClientNotReadyError(err))
4345

46+
secretsBundle, err := testsecrets.Bundle(nil)
47+
require.NoError(t, err)
48+
4449
configBundle, err := bundle.NewBundle(bundle.WithInputOptions(
4550
&bundle.InputOptions{
4651
ClusterName: clusterName,
4752
Endpoint: "https://127.0.0.1:6443",
4853
KubeVersion: "1.36.1",
54+
GenOptions: []generate.Option{generate.WithSecretsBundle(secretsBundle)},
4955
},
5056
))
5157
require.NoError(t, err)
@@ -171,11 +177,15 @@ func TestClientLifecycle(t *testing.T) {
171177

172178
// --- Setup: cluster credentials ---
173179

180+
secretsBundle, err := testsecrets.Bundle(nil)
181+
require.NoError(t, err)
182+
174183
configBundle, err := bundle.NewBundle(bundle.WithInputOptions(
175184
&bundle.InputOptions{
176185
ClusterName: clusterName,
177186
Endpoint: "https://127.0.0.1:6443",
178187
KubeVersion: "1.36.1",
188+
GenOptions: []generate.Option{generate.WithSecretsBundle(secretsBundle)},
179189
},
180190
))
181191
require.NoError(t, err)
@@ -370,11 +380,15 @@ func TestClusterClientEvictedOnClusterLeave(t *testing.T) {
370380
const clusterName = "alpha"
371381

372382
// Cluster credentials, required to build a secure cluster client.
383+
secretsBundle, err := testsecrets.Bundle(nil)
384+
require.NoError(t, err)
385+
373386
configBundle, err := bundle.NewBundle(bundle.WithInputOptions(
374387
&bundle.InputOptions{
375388
ClusterName: clusterName,
376389
Endpoint: "https://127.0.0.1:6443",
377390
KubeVersion: "1.36.1",
391+
GenOptions: []generate.Option{generate.WithSecretsBundle(secretsBundle)},
378392
},
379393
))
380394
require.NoError(t, err)

0 commit comments

Comments
 (0)