add controller name and kind to pod target (#146)

.github/workflows/test_and_deploy.yml

@@ -11,7 +11,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     steps:
@@ -59,7 +59,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20.x'
+          go-version: '1.21.x'
       - name: Checkout
         uses: actions/checkout@v3
       - name: Set up image tag

README.md

@@ -16,24 +16,12 @@ configurations and exports them to the different destinations.
 
 The service discovery pipeline has four jobs:
 
-1. [discovery](#Discovery) 
-
-   Dynamically discovers monitoring targets by collecting events from kubernetes API server.
-   Currently it collects POD and SERVICE events.
-
-2. [tag](#Tag) 
-
-   Dynamically add tags to discovered monitoring targets.
-   Based on the POD and SERVICE fields and using patterns on them, one or more tags are attached to the monitoring targets.
-
-3. [build](#Build) 
-
-   Dynamically creates data collection configurations for the monitored targets, using templates.
-
-4. [export](#Export) 
-
-   Dynamically exports data collection configurations to allow netdata data collection plugins to use them.
-   Data collection jobs in netdata are created and destroyed as needed.
+|           Job           | Description                                                                                                                                                                     |
+|:-----------------------:|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [discovery](#Discovery) | Dynamically discovers monitoring targets by collecting events from kubernetes API server. It collects POD and SERVICE events.                                                   |
+|       [tag](#Tag)       | Dynamically add tags to discovered monitoring targets. Based on the POD and SERVICE fields and using patterns on them, one or more tags are attached to the monitoring targets. |
+|     [build](#Build)     | Dynamically creates data collection configurations for the monitored targets, using templates.                                                                                  |
+|    [export](#Export)    | Dynamically exports data collection configurations to allow netdata data collection plugins to use them. Data collection jobs in netdata are created and destroyed as needed.   |
 
 Routing in a job and between jobs based on `tags` and `selector`.
 
@@ -85,7 +73,7 @@ k8s:
 Kubernetes discoverer retrieves targets from [Kubernetes'](https://kubernetes.io/)
 [REST API](https://kubernetes.io/docs/reference/). It always stays synchronized with the cluster state.
 
-Kubernetes discovery configuration options:
+Configuration options:
 
 ```yaml
 # Mandatory. Tags to add to all discovered targets.
@@ -116,43 +104,45 @@ and `PortProtocol` fields.
 
 Available pod target fields:
 
-| Name           | Type              | Value                                                     |
-| :------------- | :---------------- | :-------------------------------------------------------- |
-| `TUID`         | string            | `Namespace_Name_ContName_PortProtocol_Port`               |
-| `Address`      | string            | `PodIP:Port`                                              |
-| `Namespace`    | string            | _pod.metadata.namespace_                                  |
-| `Name`         | string            | _pod.metadata.name_                                       |
-| `Annotations`  | map[string]string | _pod.metadata.annotations_                                |
-| `Labels`       | map[string]string | _pod.metadata.labels_                                     |
-| `NodeName`     | string            | _pod.spec.nodeName_                                       |
-| `PodIP`        | string            | _pod.status.podIP_                                        |
-| `ContName`     | string            | _pod.spec.containers.name_                                |
-| `Image`        | string            | _pod.spec.containers.image_                               |
-| `Env`          | map[string]string | _pod.spec.containers.env_ + _pod.spec.containers.envFrom_ |
-| `Port`         | string            | _pod.spec.containers.ports.containerPort_                 |
-| `PortName`     | string            | _pod.spec.containers.ports.name_                          |
-| `PortProtocol` | string            | _pod.spec.containers.ports.protocol_                      |
+| Name             | Type              | Value                                                     |
+|:-----------------|:------------------|:----------------------------------------------------------|
+| `TUID`           | string            | `Namespace_Name_ContName_PortProtocol_Port`               |
+| `Address`        | string            | `PodIP:Port`                                              |
+| `Namespace`      | string            | _pod.metadata.namespace_                                  |
+| `Name`           | string            | _pod.metadata.name_                                       |
+| `Annotations`    | map[string]string | _pod.metadata.annotations_                                |
+| `Labels`         | map[string]string | _pod.metadata.labels_                                     |
+| `NodeName`       | string            | _pod.spec.nodeName_                                       |
+| `PodIP`          | string            | _pod.status.podIP_                                        |
+| `ControllerName` | string            | _pod.OwnerReferences.Controller.Name_                     |
+| `ControllerKind` | string            | _pod.OwnerReferences.Controller.Kind_                     |
+| `ContName`       | string            | _pod.spec.containers.name_                                |
+| `Image`          | string            | _pod.spec.containers.image_                               |
+| `Env`            | map[string]string | _pod.spec.containers.env_ + _pod.spec.containers.envFrom_ |
+| `Port`           | string            | _pod.spec.containers.ports.containerPort_                 |
+| `PortName`       | string            | _pod.spec.containers.ports.name_                          |
+| `PortProtocol`   | string            | _pod.spec.containers.ports.protocol_                      |
 
 #### Service Role
 
 The service role discovers a target for each service port for each service.
 
 Available service target fields:
 
-| Name           | Type              | Value                                                     |
-| :------------- | :---------------- | :-------------------------------------------------------- |
-| `TUID`         | string            | `Namespace_Name_PortProtocol_Port`                        |
-| `Address`      | string            | `Name.Namespace.svc:Port`                                              |
-| `Namespace`    | string            | _svc.metadata.namespace_                                  |
-| `Name`         | string            | _svc.metadata.name_                                       |
-| `Annotations`  | map[string]string | _svc.metadata.annotations_                                |
-| `Labels`       | map[string]string | _svc.metadata.labels_                                     |
-| `Port`         | string            | _pod.spec.containers.ports.containerPort_                 |
-| `PortName`     | string            | _pod.spec.containers.ports.name_                          |
-| `PortProtocol` | string            | _pod.spec.containers.ports.protocol_                      |
-| `ClusterIP`    | string            | _svc.spec.clusterIP_                                      |
-| `ExternalName` | string            | _svc.spec.externalName_                                   |
-| `Type`         | string            | _svc.spec.ports.type_                                     |
+| Name           | Type              | Value                                     |
+|:---------------|:------------------|:------------------------------------------|
+| `TUID`         | string            | `Namespace_Name_PortProtocol_Port`        |
+| `Address`      | string            | `Name.Namespace.svc:Port`                 |
+| `Namespace`    | string            | _svc.metadata.namespace_                  |
+| `Name`         | string            | _svc.metadata.name_                       |
+| `Annotations`  | map[string]string | _svc.metadata.annotations_                |
+| `Labels`       | map[string]string | _svc.metadata.labels_                     |
+| `Port`         | string            | _pod.spec.containers.ports.containerPort_ |
+| `PortName`     | string            | _pod.spec.containers.ports.name_          |
+| `PortProtocol` | string            | _pod.spec.containers.ports.protocol_      |
+| `ClusterIP`    | string            | _svc.spec.clusterIP_                      |
+| `ExternalName` | string            | _svc.spec.externalName_                   |
+| `Type`         | string            | _svc.spec.ports.type_                     |
 
 ## Tag
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/netdata/sd
 
-go 1.20
+go 1.21
 
 require (
 	github.com/Masterminds/sprig/v3 v3.2.3

go.sum

@@ -25,6 +25,7 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En
 github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
 github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -43,6 +44,7 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -62,6 +64,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -85,12 +88,15 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
+github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
 github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
 github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
@@ -175,6 +181,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
+golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pipeline/discovery/kubernetes/pod.go

@@ -34,6 +34,9 @@ type (
 		NodeName    string
 		PodIP       string
 
+		ControllerName string
+		ControllerKind string
+
 		ContName     string
 		Image        string
 		Env          map[string]interface{}
@@ -58,7 +61,7 @@ type Pod struct {
 }
 
 func NewPod(pod, cmap, secret cache.SharedInformer) *Pod {
-	queue := workqueue.NewNamed("pod")
+	queue := workqueue.NewWithConfig(workqueue.QueueConfig{Name: "pod"})
 	pod.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc:    func(obj interface{}) { enqueue(queue, obj) },
 		UpdateFunc: func(_, obj interface{}) { enqueue(queue, obj) },
@@ -151,22 +154,33 @@ func (p Pod) buildGroup(pod *apiv1.Pod) model.Group {
 }
 
 func (p Pod) buildTargets(pod *apiv1.Pod) (targets []model.Target) {
+	var name, kind string
+	for _, ref := range pod.OwnerReferences {
+		if ref.Controller != nil && *ref.Controller {
+			name = ref.Name
+			kind = ref.Kind
+			break
+		}
+	}
+
 	for _, container := range pod.Spec.Containers {
 		env := p.collectEnv(pod.Namespace, container)
 
 		if len(container.Ports) == 0 {
 			target := &PodTarget{
-				tuid:        podTUID(pod, container),
-				Address:     pod.Status.PodIP,
-				Namespace:   pod.Namespace,
-				Name:        pod.Name,
-				Annotations: toMapInterface(pod.Annotations),
-				Labels:      toMapInterface(pod.Labels),
-				NodeName:    pod.Spec.NodeName,
-				PodIP:       pod.Status.PodIP,
-				ContName:    container.Name,
-				Image:       container.Image,
-				Env:         toMapInterface(env),
+				tuid:           podTUID(pod, container),
+				Address:        pod.Status.PodIP,
+				Namespace:      pod.Namespace,
+				Name:           pod.Name,
+				Annotations:    toMapInterface(pod.Annotations),
+				Labels:         toMapInterface(pod.Labels),
+				NodeName:       pod.Spec.NodeName,
+				PodIP:          pod.Status.PodIP,
+				ControllerName: name,
+				ControllerKind: kind,
+				ContName:       container.Name,
+				Image:          container.Image,
+				Env:            toMapInterface(env),
 			}
 			hash, err := calcHash(target)
 			if err != nil {
@@ -179,20 +193,22 @@ func (p Pod) buildTargets(pod *apiv1.Pod) (targets []model.Target) {
 			for _, port := range container.Ports {
 				portNum := strconv.FormatUint(uint64(port.ContainerPort), 10)
 				target := &PodTarget{
-					tuid:         podTUIDWithPort(pod, container, port),
-					Address:      net.JoinHostPort(pod.Status.PodIP, portNum),
-					Namespace:    pod.Namespace,
-					Name:         pod.Name,
-					Annotations:  toMapInterface(pod.Annotations),
-					Labels:       toMapInterface(pod.Labels),
-					NodeName:     pod.Spec.NodeName,
-					PodIP:        pod.Status.PodIP,
-					ContName:     container.Name,
-					Image:        container.Image,
-					Env:          toMapInterface(env),
-					Port:         portNum,
-					PortName:     port.Name,
-					PortProtocol: string(port.Protocol),
+					tuid:           podTUIDWithPort(pod, container, port),
+					Address:        net.JoinHostPort(pod.Status.PodIP, portNum),
+					Namespace:      pod.Namespace,
+					Name:           pod.Name,
+					Annotations:    toMapInterface(pod.Annotations),
+					Labels:         toMapInterface(pod.Labels),
+					NodeName:       pod.Spec.NodeName,
+					PodIP:          pod.Status.PodIP,
+					ControllerName: name,
+					ControllerKind: kind,
+					ContName:       container.Name,
+					Image:          container.Image,
+					Env:            toMapInterface(env),
+					Port:           portNum,
+					PortName:       port.Name,
+					PortProtocol:   string(port.Protocol),
 				}
 				hash, err := calcHash(target)
 				if err != nil {

pipeline/discovery/kubernetes/pod_test.go

@@ -111,10 +111,10 @@ func TestPodTarget_Hash(t *testing.T) {
 				return sim
 			},
 			expectedHash: []uint64{
-				10984584459239076244,
-				10480294460002508451,
-				6511065520956605596,
-				9821662463142050012,
+				12703169414253998055,
+				13351713096133918928,
+				8241692333761256175,
+				11562466355572729519,
 			},
 		},
 	}
@@ -468,6 +468,8 @@ func mangleContainers(containers []apiv1.Container, m func(container *apiv1.Cont
 	}
 }
 
+var controllerTrue = true
+
 func newHTTPDPod() *apiv1.Pod {
 	return &apiv1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
@@ -476,6 +478,9 @@ func newHTTPDPod() *apiv1.Pod {
 			UID:         "1cebb6eb-0c1e-495b-8131-8fa3e6668dc8",
 			Annotations: map[string]string{"phase": "prod"},
 			Labels:      map[string]string{"app": "httpd", "tier": "frontend"},
+			OwnerReferences: []metav1.OwnerReference{
+				{Name: "netdata-test", Kind: "DaemonSet", Controller: &controllerTrue},
+			},
 		},
 		Spec: apiv1.PodSpec{
 			NodeName: "m01",
@@ -504,6 +509,9 @@ func newNGINXPod() *apiv1.Pod {
 			UID:         "09e883f2-d740-4c5f-970d-02cf02876522",
 			Annotations: map[string]string{"phase": "prod"},
 			Labels:      map[string]string{"app": "nginx", "tier": "frontend"},
+			OwnerReferences: []metav1.OwnerReference{
+				{Name: "netdata-test", Kind: "DaemonSet", Controller: &controllerTrue},
+			},
 		},
 		Spec: apiv1.PodSpec{
 			NodeName: "m01",
@@ -560,20 +568,22 @@ func preparePodGroup(pod *apiv1.Pod) *podGroup {
 		for _, port := range container.Ports {
 			portNum := strconv.FormatUint(uint64(port.ContainerPort), 10)
 			target := &PodTarget{
-				tuid:         podTUIDWithPort(pod, container, port),
-				Address:      net.JoinHostPort(pod.Status.PodIP, portNum),
-				Namespace:    pod.Namespace,
-				Name:         pod.Name,
-				Annotations:  toMapInterface(pod.Annotations),
-				Labels:       toMapInterface(pod.Labels),
-				NodeName:     pod.Spec.NodeName,
-				PodIP:        pod.Status.PodIP,
-				ContName:     container.Name,
-				Image:        container.Image,
-				Env:          nil,
-				Port:         portNum,
-				PortName:     port.Name,
-				PortProtocol: string(port.Protocol),
+				tuid:           podTUIDWithPort(pod, container, port),
+				Address:        net.JoinHostPort(pod.Status.PodIP, portNum),
+				Namespace:      pod.Namespace,
+				Name:           pod.Name,
+				Annotations:    toMapInterface(pod.Annotations),
+				Labels:         toMapInterface(pod.Labels),
+				NodeName:       pod.Spec.NodeName,
+				PodIP:          pod.Status.PodIP,
+				ControllerName: "netdata-test",
+				ControllerKind: "DaemonSet",
+				ContName:       container.Name,
+				Image:          container.Image,
+				Env:            nil,
+				Port:           portNum,
+				PortName:       port.Name,
+				PortProtocol:   string(port.Protocol),
 			}
 			target.hash = mustCalcHash(target)
 			target.Tags().Merge(discoveryTags)