Allow Multiple Cluster Pod Creation Using Distributed Petset (#17)

Signed-off-by: souravbiswassanto <[email protected]>

Author: souravbiswassanto

apis/apps/v1/constants.go

@@ -0,0 +1,24 @@
+/*
+Copyright AppsCode Inc. and Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+const (
+	ManifestWorkRoleLabel        = "open-cluster-management.io/role"
+	ManifestWorkClusterNameLabel = "open-cluster-management.io/cluster-name"
+	RolePod                      = "pod"
+	RolePVC                      = "pvc"
+)

apis/apps/v1/petset_types.go

@@ -62,6 +62,9 @@ type PetSet struct {
 
 // A PetSetSpec is the specification of a PetSet.
 type PetSetSpec struct {
+	// Distributed means manifestworks will be used to manage pods
+	Distributed bool `json:"distributed,omitempty"`
+
 	// replicas is the desired number of replicas of the given Template.
 	// These are replicas in the sense that they are instantiations of the
 	// same Template, but individual replicas also have a consistent identity.

apis/apps/v1/placementpolicy_types.go

@@ -63,6 +63,20 @@ type PlacementPolicySpec struct {
 	// If specified, the pod's scheduling constraints
 	// +optional
 	Affinity *Affinity `json:"affinity,omitempty"`
+
+	// OCM provides spec for distributed pod placements using open cluster management
+	// +optional
+	OCM *OCMSpec `json:"ocm,omitempty"`
+}
+
+type OCMSpec struct {
+	DistributionRules []DistributionRule `json:"distributionRules,omitempty"`
+	SliceName         string             `json:"sliceName,omitempty"`
+}
+
+type DistributionRule struct {
+	ClusterName string  `json:"clusterName,omitempty"`
+	Replicas    []int32 `json:"replicas,omitempty"`
 }
 
 type ZoneSpreadConstraint struct {

apis/apps/v1/zz_generated.deepcopy.go

@@ -24,12 +24,16 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/klog/v2"
+	apiworkv1 "open-cluster-management.io/api/work/v1"
 )
 
 // PetSetListerExpansion allows custom methods to be added to
 // PetSetLister.
 type PetSetListerExpansion interface {
 	GetPodPetSets(pod *v1.Pod) ([]*api.PetSet, error)
+	GetManifestWorkPetSets(mw *apiworkv1.ManifestWork) ([]*api.PetSet, error)
 }
 
 // PetSetNamespaceListerExpansion allows custom methods to be added to
@@ -77,3 +81,47 @@ func (s *petSetLister) GetPodPetSets(pod *v1.Pod) ([]*api.PetSet, error) {
 
 	return psList, nil
 }
+
+// GetManifestWorkPetSets returns a list of PetSets that potentially match a manifestwork.
+// It lists PetSets across all namespaces and matches them based on labels.
+func (s *petSetLister) GetManifestWorkPetSets(mw *apiworkv1.ManifestWork) ([]*api.PetSet, error) {
+	if len(mw.Labels) == 0 {
+		return nil, fmt.Errorf("no PetSets found for manifestwork %s because it has no labels", mw.Name)
+	}
+
+	list, err := s.List(labels.Everything())
+	if err != nil {
+		return nil, err
+	}
+
+	var psList []*api.PetSet
+	for _, ps := range list {
+		selector, err := metav1.LabelSelectorAsSelector(ps.Spec.Selector)
+		if err != nil {
+			klog.Warningf("PetSet %s/%s has an invalid selector: %v", ps.Namespace, ps.Name, err)
+			continue
+		}
+
+		if selector.Empty() || !selector.Matches(labels.Set(mw.Labels)) {
+			continue
+		}
+		psList = append(psList, ps)
+	}
+
+	if len(psList) == 0 {
+		return nil, fmt.Errorf("could not find any PetSet for manifestwork %v/%s in any namespace with labels: %v", mw.Namespace, mw.Name, mw.Labels)
+	}
+
+	if len(psList) > 1 {
+		setNames := []string{}
+		for _, s := range psList {
+			setNames = append(setNames, s.Name)
+		}
+		utilruntime.HandleError(
+			fmt.Errorf(
+				"user error: more than one PetSet is selecting manifestwork with labels: %+v. Sets: %v",
+				mw.Labels, setNames))
+	}
+
+	return psList, nil
+}

client/listers/apps/v1/petset_expansion.go

@@ -46,6 +46,10 @@ spec:
           spec:
             description: Spec defines the desired identities of pods in this set.
             properties:
+              distributed:
+                description: Distributed means manifestworks will be used to manage
+                  pods
+                type: boolean
               minReadySeconds:
                 description: |-
                   Minimum number of seconds for which a newly created pod should be ready

crds/apps.k8s.appscode.com_petsets.yaml

@@ -100,6 +100,25 @@ spec:
                 - maxSkew
                 - whenUnsatisfiable
                 type: object
+              ocm:
+                description: OCM provides spec for distributed pod placements using
+                  open cluster management
+                properties:
+                  distributionRules:
+                    items:
+                      properties:
+                        clusterName:
+                          type: string
+                        replicas:
+                          items:
+                            format: int32
+                            type: integer
+                          type: array
+                      type: object
+                    type: array
+                  sliceName:
+                    type: string
+                type: object
               zoneSpreadConstraint:
                 properties:
                   maxSkew:

crds/apps.k8s.appscode.com_placementpolicies.yaml

@@ -1,20 +1,22 @@
 module kubeops.dev/petset
 
-go 1.23.0
+go 1.23.6
 
 toolchain go1.24.0
 
 require (
 	github.com/google/cel-go v0.22.0
 	github.com/google/go-cmp v0.7.0
 	github.com/google/gofuzz v1.2.0
+	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/pflag v1.0.6
 	github.com/stretchr/testify v1.10.0
 	gomodules.xyz/logs v0.0.7
 	gomodules.xyz/pointer v0.1.0
 	gomodules.xyz/x v0.0.17
 	k8s.io/api v0.32.2
+	k8s.io/apiextensions-apiserver v0.32.2
 	k8s.io/apimachinery v0.32.2
 	k8s.io/apiserver v0.32.2
 	k8s.io/client-go v0.32.2
@@ -25,6 +27,7 @@ require (
 	k8s.io/utils v0.0.0-20241210054802-24370beab758
 	kmodules.xyz/client-go v0.32.1
 	kmodules.xyz/go-containerregistry v0.0.14
+	open-cluster-management.io/api v1.0.0
 	sigs.k8s.io/controller-runtime v0.20.3
 	sigs.k8s.io/yaml v1.4.0
 )
@@ -112,7 +115,6 @@ require (
 	github.com/onsi/gomega v1.36.2 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.20.5 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
@@ -154,6 +156,7 @@ require (
 	gomodules.xyz/clock v0.0.0-20200817085942-06523dba733f // indirect
 	gomodules.xyz/flags v0.1.3 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect
+	gomodules.xyz/mergo v0.3.13 // indirect
 	gomodules.xyz/sets v0.2.1 // indirect
 	gomodules.xyz/wait v0.2.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
@@ -162,9 +165,7 @@ require (
 	google.golang.org/protobuf v1.36.3 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.32.2 // indirect
 	k8s.io/cloud-provider v0.32.2 // indirect
 	k8s.io/component-helpers v0.32.2 // indirect
 	k8s.io/controller-manager v0.32.2 // indirect

go.mod

@@ -257,6 +257,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
+github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -643,6 +645,8 @@ gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0
 gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
 gomodules.xyz/logs v0.0.7 h1:dkhpdQuzj+pOS3S7VaOq+JV7BVU7f68/k3uDYufhPow=
 gomodules.xyz/logs v0.0.7/go.mod h1:IEIZbRl9zua2jb35NU4KoqxUEDPmKvem3PhfRHqQI54=
+gomodules.xyz/mergo v0.3.13 h1:q6cL/MMXZH/MrR2+yjSihFFq6UifXqjwaqI48B6cMEM=
+gomodules.xyz/mergo v0.3.13/go.mod h1:F/2rKC7j0URTnHUKDiTiLcGdLMhdv8jK2Za3cRTUVmc=
 gomodules.xyz/pointer v0.1.0 h1:sG2UKrYVSo6E3r4itAjXfPfe4fuXMi0KdyTHpR3vGCg=
 gomodules.xyz/pointer v0.1.0/go.mod h1:sPLsC0+yLTRecUiC5yVlyvXhZ6LAGojNCRWNNqoplvo=
 gomodules.xyz/sets v0.2.0/go.mod h1:jKgNp01/iDs+svOWXaPk5cKP3VXy0mWUoTF/ore+aMc=
@@ -706,6 +710,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk=
@@ -745,6 +750,8 @@ kmodules.xyz/client-go v0.32.1 h1:W8TWGBBaokQEqfesbdhlRFMMV0TChcRbMRQ0eTIRS88=
 kmodules.xyz/client-go v0.32.1/go.mod h1:tqe40/iDtO/RSNaDJs/a+rgnMPSOcK622HpOmOFvcds=
 kmodules.xyz/go-containerregistry v0.0.14 h1:8MgLFa74HymAJEyjH7fyQJn5u2Ok6qPPFQX8ARfcXp0=
 kmodules.xyz/go-containerregistry v0.0.14/go.mod h1:xz0iGC3noyMi5NNAzXWTH6KqfiIgFWZAomw+U2zVOXs=
+open-cluster-management.io/api v1.0.0 h1:54QllH9DTudCk6VrGt0q8CDsE3MghqJeTaTN4RHZpE0=
+open-cluster-management.io/api v1.0.0/go.mod h1:/OeqXycNBZQoe3WG6ghuWsMgsKGuMZrK8ZpsU6gWL0Y=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcpeN4baWEV2ko2Z/AsiZgEdwgcfwLgMo=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=