Skip to content

Commit 7a85687

Browse files
MonokaixMonokaix
authored
feat: cluster-level resource scheduling suspend and resume capabilities (#5937)
* RB suspension: API change Signed-off-by: Monokaix <[email protected]> * Feat: scheduler supports binging suspension Signed-off-by: Monokaix <[email protected]> --------- Signed-off-by: Monokaix <[email protected]>
1 parent ba1e68d commit 7a85687

File tree

10 files changed

+215
-14
lines changed

10 files changed

+215
-14
lines changed

Diff for: ‎api/openapi-spec/swagger.json

+5-1
Original file line numberDiff line numberDiff line change
@@ -20435,7 +20435,7 @@
2043520435
}
2043620436
},
2043720437
"com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.Suspension": {
20438-
"description": "Suspension defines the policy for suspending of propagation.",
20438+
"description": "Suspension defines the policy for suspending dispatching and scheduling.",
2043920439
"type": "object",
2044020440
"properties": {
2044120441
"dispatching": {
@@ -20445,6 +20445,10 @@
2044520445
"dispatchingOnClusters": {
2044620446
"description": "DispatchingOnClusters declares a list of clusters to which the dispatching should be suspended. Note: Can not co-exist with Dispatching which is used to suspend all.",
2044720447
"$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.policy.v1alpha1.SuspendClusters"
20448+
},
20449+
"scheduling": {
20450+
"description": "Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not process resource binding when the value is true and resume scheduling when it's false or nil. This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling manage resource allocation, prioritize critical workloads, etc. It is expected that third-party systems use an admission webhook to suspend scheduling at the time of ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may lead to ineffective suspension.",
20451+
"type": "boolean"
2044820452
}
2044920453
}
2045020454
},

Diff for: ‎charts/karmada/_crds/bases/work/work.karmada.io_clusterresourcebindings.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -1281,6 +1281,16 @@ spec:
12811281
type: string
12821282
type: array
12831283
type: object
1284+
scheduling:
1285+
description: |-
1286+
Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not
1287+
process resource binding when the value is true and resume scheduling when it's false or nil.
1288+
This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling
1289+
manage resource allocation, prioritize critical workloads, etc.
1290+
It is expected that third-party systems use an admission webhook to suspend scheduling at the time of
1291+
ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may
1292+
lead to ineffective suspension.
1293+
type: boolean
12841294
type: object
12851295
required:
12861296
- resource

Diff for: ‎charts/karmada/_crds/bases/work/work.karmada.io_resourcebindings.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -1281,6 +1281,16 @@ spec:
12811281
type: string
12821282
type: array
12831283
type: object
1284+
scheduling:
1285+
description: |-
1286+
Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not
1287+
process resource binding when the value is true and resume scheduling when it's false or nil.
1288+
This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling
1289+
manage resource allocation, prioritize critical workloads, etc.
1290+
It is expected that third-party systems use an admission webhook to suspend scheduling at the time of
1291+
ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may
1292+
lead to ineffective suspension.
1293+
type: boolean
12841294
type: object
12851295
required:
12861296
- resource

Diff for: ‎pkg/apis/work/v1alpha2/binding_types.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -322,9 +322,19 @@ type BindingSnapshot struct {
322322
Clusters []TargetCluster `json:"clusters,omitempty"`
323323
}
324324

325-
// Suspension defines the policy for suspending of propagation.
325+
// Suspension defines the policy for suspending dispatching and scheduling.
326326
type Suspension struct {
327327
policyv1alpha1.Suspension `json:",inline"`
328+
329+
// Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not
330+
// process resource binding when the value is true and resume scheduling when it's false or nil.
331+
// This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling
332+
// manage resource allocation, prioritize critical workloads, etc.
333+
// It is expected that third-party systems use an admission webhook to suspend scheduling at the time of
334+
// ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may
335+
// lead to ineffective suspension.
336+
// +optional
337+
Scheduling *bool `json:"scheduling,omitempty"`
328338
}
329339

330340
// ResourceBindingStatus represents the overall status of the strategy as well as the referenced resources.

Diff for: ‎pkg/apis/work/v1alpha2/zz_generated.deepcopy.go

+5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎pkg/generated/openapi/zz_generated.openapi.go

+8-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎pkg/scheduler/event_handler.go

+6
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,16 @@ func (s *Scheduler) resourceBindingEventFilter(obj interface{}) bool {
101101
if !schedulerNameFilter(s.schedulerName, t.Spec.SchedulerName) {
102102
return false
103103
}
104+
if util.IsBindingSuspendScheduling(t) {
105+
return false
106+
}
104107
case *workv1alpha2.ClusterResourceBinding:
105108
if !schedulerNameFilter(s.schedulerName, t.Spec.SchedulerName) {
106109
return false
107110
}
111+
if util.IsClusterBindingSuspendScheduling(t) {
112+
return false
113+
}
108114
}
109115

110116
return util.GetLabelValue(accessor.GetLabels(), policyv1alpha1.PropagationPolicyPermanentIDLabel) != "" ||

Diff for: ‎pkg/scheduler/event_handler_test.go

+30-11
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
corev1 "k8s.io/api/core/v1"
2525
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2626
"k8s.io/client-go/tools/cache"
27+
"k8s.io/utils/ptr"
2728

2829
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
2930
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
@@ -40,65 +41,65 @@ func TestResourceBindingEventFilter(t *testing.T) {
4041
{
4142
name: "ResourceBinding: Matching scheduler name, no labels",
4243
schedulerName: "test-scheduler",
43-
obj: createResourceBinding("test-rb", "test-scheduler", nil),
44+
obj: createResourceBinding("test-rb", "test-scheduler", nil, nil),
4445
expectedResult: false,
4546
},
4647
{
4748
name: "ResourceBinding: Non-matching scheduler name",
4849
schedulerName: "test-scheduler",
49-
obj: createResourceBinding("test-rb", "other-scheduler", nil),
50+
obj: createResourceBinding("test-rb", "other-scheduler", nil, nil),
5051
expectedResult: false,
5152
},
5253
{
5354
name: "ResourceBinding: Matching scheduler name, with PropagationPolicyPermanentIDLabel",
5455
schedulerName: "test-scheduler",
5556
obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
5657
policyv1alpha1.PropagationPolicyPermanentIDLabel: "test-id",
57-
}),
58+
}, nil),
5859
expectedResult: true,
5960
},
6061
{
6162
name: "ResourceBinding: Matching scheduler name, with ClusterPropagationPolicyPermanentIDLabel",
6263
schedulerName: "test-scheduler",
6364
obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
6465
policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: "test-id",
65-
}),
66+
}, nil),
6667
expectedResult: true,
6768
},
6869
{
6970
name: "ResourceBinding: Matching scheduler name, with BindingManagedByLabel",
7071
schedulerName: "test-scheduler",
7172
obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
7273
workv1alpha2.BindingManagedByLabel: "test-manager",
73-
}),
74+
}, nil),
7475
expectedResult: true,
7576
},
7677
{
7778
name: "ResourceBinding: Matching scheduler name, with empty PropagationPolicyPermanentIDLabel",
7879
schedulerName: "test-scheduler",
7980
obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
8081
policyv1alpha1.PropagationPolicyPermanentIDLabel: "",
81-
}),
82+
}, nil),
8283
expectedResult: false,
8384
},
8485
{
8586
name: "ClusterResourceBinding: Matching scheduler name, no labels",
8687
schedulerName: "test-scheduler",
87-
obj: createClusterResourceBinding("test-crb", "test-scheduler", nil),
88+
obj: createClusterResourceBinding("test-crb", "test-scheduler", nil, nil),
8889
expectedResult: false,
8990
},
9091
{
9192
name: "ClusterResourceBinding: Non-matching scheduler name",
9293
schedulerName: "test-scheduler",
93-
obj: createClusterResourceBinding("test-crb", "other-scheduler", nil),
94+
obj: createClusterResourceBinding("test-crb", "other-scheduler", nil, nil),
9495
expectedResult: false,
9596
},
9697
{
9798
name: "ClusterResourceBinding: Matching scheduler name, with ClusterPropagationPolicyPermanentIDLabel",
9899
schedulerName: "test-scheduler",
99100
obj: createClusterResourceBinding("test-crb", "test-scheduler", map[string]string{
100101
policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: "test-id",
101-
}),
102+
}, nil),
102103
expectedResult: true,
103104
},
104105
{
@@ -113,6 +114,22 @@ func TestResourceBindingEventFilter(t *testing.T) {
113114
obj: "not-a-valid-object",
114115
expectedResult: false,
115116
},
117+
{
118+
name: "ResourceBinding suspended",
119+
schedulerName: "test-scheduler",
120+
obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
121+
workv1alpha2.BindingManagedByLabel: "test-manager",
122+
}, &workv1alpha2.Suspension{Scheduling: ptr.To(true)}),
123+
expectedResult: false,
124+
},
125+
{
126+
name: "ClusterResourceBinding suspended",
127+
schedulerName: "test-scheduler",
128+
obj: createClusterResourceBinding("test-crb", "test-scheduler", map[string]string{
129+
policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: "test-id",
130+
}, &workv1alpha2.Suspension{Scheduling: ptr.To(true)}),
131+
expectedResult: false,
132+
},
116133
}
117134

118135
for _, tc := range testCases {
@@ -404,26 +421,28 @@ func createCluster(name string, generation int64, labels map[string]string) *clu
404421
}
405422
}
406423

407-
func createResourceBinding(name, schedulerName string, labels map[string]string) *workv1alpha2.ResourceBinding {
424+
func createResourceBinding(name, schedulerName string, labels map[string]string, suspension *workv1alpha2.Suspension) *workv1alpha2.ResourceBinding {
408425
return &workv1alpha2.ResourceBinding{
409426
ObjectMeta: metav1.ObjectMeta{
410427
Name: name,
411428
Labels: labels,
412429
},
413430
Spec: workv1alpha2.ResourceBindingSpec{
414431
SchedulerName: schedulerName,
432+
Suspension: suspension,
415433
},
416434
}
417435
}
418436

419-
func createClusterResourceBinding(name, schedulerName string, labels map[string]string) *workv1alpha2.ClusterResourceBinding {
437+
func createClusterResourceBinding(name, schedulerName string, labels map[string]string, suspension *workv1alpha2.Suspension) *workv1alpha2.ClusterResourceBinding {
420438
return &workv1alpha2.ClusterResourceBinding{
421439
ObjectMeta: metav1.ObjectMeta{
422440
Name: name,
423441
Labels: labels,
424442
},
425443
Spec: workv1alpha2.ResourceBindingSpec{
426444
SchedulerName: schedulerName,
445+
Suspension: suspension,
427446
},
428447
}
429448
}

Diff for: ‎pkg/util/binding.go

+16
Original file line numberDiff line numberDiff line change
@@ -110,3 +110,19 @@ func RescheduleRequired(rescheduleTriggeredAt, lastScheduledTime *metav1.Time) b
110110
}
111111
return rescheduleTriggeredAt.After(lastScheduledTime.Time)
112112
}
113+
114+
// IsBindingSuspendScheduling tells whether resource binding is scheduling suspended.
115+
func IsBindingSuspendScheduling(rb *workv1alpha2.ResourceBinding) bool {
116+
if rb == nil || rb.Spec.Suspension == nil || rb.Spec.Suspension.Scheduling == nil {
117+
return false
118+
}
119+
return *rb.Spec.Suspension.Scheduling
120+
}
121+
122+
// IsClusterBindingSuspendScheduling tells whether cluster resource binding is scheduling suspended.
123+
func IsClusterBindingSuspendScheduling(crb *workv1alpha2.ClusterResourceBinding) bool {
124+
if crb == nil || crb.Spec.Suspension == nil || crb.Spec.Suspension.Scheduling == nil {
125+
return false
126+
}
127+
return *crb.Spec.Suspension.Scheduling
128+
}

0 commit comments

Comments
 (0)