Add feature flag treat-pod-as-always-schedulable

The feature flag allows to declare that Pods in the system will eventually all get scheduled and Revisions should therefore not be marked unschedulable

Author: SaschaSchwarze0

config/core/configmaps/features.yaml

@@ -22,7 +22,7 @@ metadata:
     app.kubernetes.io/component: controller
     app.kubernetes.io/version: devel
   annotations:
-    knative.dev/example-checksum: "632d47dd"
+    knative.dev/example-checksum: "7b73abaa"
 data:
   _example: |-
     ################################
@@ -216,3 +216,12 @@ data:
 
     # Default queue proxy resource requests and limits to good values for most cases if set.
     queueproxy.resource-defaults: "disabled"
+
+    # treat-pod-as-always-schedulable can be used to define that Pods in the system will always be
+    # scheduled, and a Revision should not be marked unschedulable.
+    # Setting this to `enabled` makes sense if you have cluster-autoscaling set up for you cluster
+    # where unschedulable Pods trigger the addition of a new Node and are therefore a short and
+    # transient state.
+    #
+    # See https://github.com/knative/serving/issues/14862
+    treat-pod-as-always-schedulable: "disabled"

pkg/apis/config/features.go

@@ -79,6 +79,7 @@ func defaultFeaturesConfig() *Features {
 		SecurePodDefaults:                Disabled,
 		TagHeaderBasedRouting:            Disabled,
 		AutoDetectHTTP2:                  Disabled,
+		TreatPodAsAlwaysSchedulable:      Disabled,
 	}
 }
 
@@ -112,7 +113,8 @@ func NewFeaturesConfigFromMap(data map[string]string) (*Features, error) {
 		asFlag("tag-header-based-routing", &nc.TagHeaderBasedRouting),
 		asFlag("queueproxy.resource-defaults", &nc.QueueProxyResourceDefaults),
 		asFlag("queueproxy.mount-podinfo", &nc.QueueProxyMountPodInfo),
-		asFlag("autodetect-http2", &nc.AutoDetectHTTP2)); err != nil {
+		asFlag("autodetect-http2", &nc.AutoDetectHTTP2),
+		asFlag("treat-pod-as-always-schedulable", &nc.TreatPodAsAlwaysSchedulable)); err != nil {
 		return nil, err
 	}
 	return nc, nil
@@ -151,6 +153,7 @@ type Features struct {
 	SecurePodDefaults                Flag
 	TagHeaderBasedRouting            Flag
 	AutoDetectHTTP2                  Flag
+	TreatPodAsAlwaysSchedulable      Flag
 }
 
 // asFlag parses the value at key as a Flag into the target, if it exists.

pkg/apis/config/features_test.go

@@ -77,6 +77,7 @@ func TestFeaturesConfiguration(t *testing.T) {
 			SecurePodDefaults:                Enabled,
 			QueueProxyResourceDefaults:       Enabled,
 			TagHeaderBasedRouting:            Enabled,
+			TreatPodAsAlwaysSchedulable:      Enabled,
 		}),
 		data: map[string]string{
 			"multi-container":                              "Enabled",
@@ -97,6 +98,7 @@ func TestFeaturesConfiguration(t *testing.T) {
 			"secure-pod-defaults":                          "Enabled",
 			"queueproxy.resource-defaults":                 "Enabled",
 			"tag-header-based-routing":                     "Enabled",
+			"treat-pod-as-always-schedulable":              "Enabled",
 		},
 	}, {
 		name:    "multi-container Allowed",
@@ -594,6 +596,33 @@ func TestFeaturesConfiguration(t *testing.T) {
 			data: map[string]string{
 				"kubernetes.podspec-dnsconfig": "Disabled",
 			},
+		}, {
+			name:    "treat-pod-as-always-schedulable Allowed",
+			wantErr: false,
+			wantFeatures: defaultWith(&Features{
+				TreatPodAsAlwaysSchedulable: Allowed,
+			}),
+			data: map[string]string{
+				"treat-pod-as-always-schedulable": "Allowed",
+			},
+		}, {
+			name:    "treat-pod-as-always-schedulable Enabled",
+			wantErr: false,
+			wantFeatures: defaultWith(&Features{
+				TreatPodAsAlwaysSchedulable: Enabled,
+			}),
+			data: map[string]string{
+				"treat-pod-as-always-schedulable": "Enabled",
+			},
+		}, {
+			name:    "treat-pod-as-always-schedulable Disabled",
+			wantErr: false,
+			wantFeatures: defaultWith(&Features{
+				TreatPodAsAlwaysSchedulable: Disabled,
+			}),
+			data: map[string]string{
+				"treat-pod-as-always-schedulable": "Disabled",
+			},
 		}}
 
 	for _, tt := range configTests {

pkg/reconciler/revision/reconcile_resources.go

@@ -36,6 +36,7 @@ import (
 	"knative.dev/pkg/kmp"
 	"knative.dev/pkg/logging"
 	"knative.dev/pkg/logging/logkey"
+	apicfg "knative.dev/serving/pkg/apis/config"
 	v1 "knative.dev/serving/pkg/apis/serving/v1"
 	"knative.dev/serving/pkg/networking"
 	"knative.dev/serving/pkg/reconciler/revision/config"
@@ -87,10 +88,13 @@ func (c *Reconciler) reconcileDeployment(ctx context.Context, rev *v1.Revision)
 
 			// Update the revision status if pod cannot be scheduled (possibly resource constraints)
 			// If pod cannot be scheduled then we expect the container status to be empty.
-			for _, cond := range pod.Status.Conditions {
-				if cond.Type == corev1.PodScheduled && cond.Status == corev1.ConditionFalse {
-					rev.Status.MarkResourcesAvailableFalse(cond.Reason, cond.Message)
-					break
+			treatPodAsAlwaysSchedulable := config.FromContext(ctx).Features.TreatPodAsAlwaysSchedulable
+			if treatPodAsAlwaysSchedulable == apicfg.Disabled {
+				for _, cond := range pod.Status.Conditions {
+					if cond.Type == corev1.PodScheduled && cond.Status == corev1.ConditionFalse {
+						rev.Status.MarkResourcesAvailableFalse(cond.Reason, cond.Message)
+						break
+					}
 				}
 			}