Skip to content

Commit 74fc28a

Browse files
ihcsimihcsim
committed
Combine the mock HTTP and GRPC servers into a sample driver
Signed-off-by: Ivan Sim <[email protected]>
1 parent 56fd505 commit 74fc28a

File tree

14 files changed

+1145
-92
lines changed

14 files changed

+1145
-92
lines changed

‎Makefile

+1-2
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,7 @@ deploy-aggapi:
7373
kubectl -n $(NAMESPACE) wait --timeout=$(WAIT_TIMEOUT) --for=condition=Ready -l apiserver=true po
7474

7575
deploy-mock:
76-
kubectl apply -f yaml/mock/cbt-grpc.yaml
77-
kubectl apply -f yaml/mock/cbt-http.yaml
76+
kubectl apply -f yaml/mock/driver.yaml
7877

7978
clean:
8079
kubectl -n $(NAMESPACE) delete -R -f yaml

‎README.md

+6-6
Original file line numberDiff line numberDiff line change
@@ -115,14 +115,14 @@ The Docker images are hosted on public repositories at `quay.io/isim`.
115115
Confirm that all the pods are running:
116116

117117
```sh
118-
kubectl -n csi-cbt get po ````
118+
kubectl -n csi-cbt get po
119+
````
119120

120121
```sh
121-
NAME READY STATUS RESTARTS AGE
122-
etcd-0 1/1 Running 0 112s
123-
cbt-aggapi-77888d6579-kwtcg 1/1 Running 0 90s
124-
cbt-http-7769467bdf-vrz8g 1/1 Running 0 28s
125-
cbt-grpc-5fc84d9656-8sr7q 1/1 Running 0 28s
122+
NAME READY STATUS RESTARTS AGE
123+
etcd-0 1/1 Running 0 103m
124+
cbt-aggapi-77888d6579-cwz8n 1/1 Running 0 102m
125+
sample-driver-6c8dd6d957-xks74 2/2 Running 0 4m40s
126126
```
127127

128128
## Development

‎cmd/mock/grpc/main.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,13 @@ import (
1111
"google.golang.org/grpc"
1212
)
1313

14-
var listenAddr = flag.String("listen", ":9779", "Address of the GRPC server")
14+
var csiAddress = flag.String("csi-address", "/run/csi/socket", "Address of the CSI driver socket.")
1515

1616
func main() {
1717
flag.Parse()
1818

19-
log.Printf("listening at %s", *listenAddr)
20-
listener, err := net.Listen("tcp", *listenAddr)
19+
log.Printf("listening at %s", *csiAddress)
20+
listener, err := net.Listen("unix", *csiAddress)
2121
if err != nil {
2222
log.Fatalf("failed to listen: %v", err)
2323
}

‎cmd/mock/http/main.go

+32-23
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ import (
1515
"github.com/ihcsim/cbt-aggapi/pkg/apis/cbt/v1alpha1"
1616
"github.com/ihcsim/cbt-aggapi/pkg/generated/cbt/clientset/versioned"
1717
cbtgrpc "github.com/ihcsim/cbt-aggapi/pkg/grpc"
18+
"github.com/kubernetes-csi/csi-lib-utils/connection"
19+
"github.com/kubernetes-csi/csi-lib-utils/metrics"
1820
"google.golang.org/grpc"
19-
"google.golang.org/grpc/credentials/insecure"
2021
apierrors "k8s.io/apimachinery/pkg/api/errors"
2122
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2223
"k8s.io/apimachinery/pkg/runtime"
@@ -29,17 +30,28 @@ const defaultPort = 8080
2930

3031
var (
3132
listenAddr = flag.String("listen-addr", fmt.Sprintf(":%d", defaultPort), "Address to listen at")
32-
grpcTarget = flag.String("grpc-target", ":9779", "Address of the GRPC server")
33+
csiAddress = flag.String("csi-address", "/run/csi/socket", "Address of the CSI driver socket.")
3334
)
3435

3536
func main() {
3637
flag.Parse()
3738

38-
if _, err := registerDriver(); err != nil {
39+
var (
40+
driver = os.Getenv("CSI_DRIVER_NAME")
41+
svc = os.Getenv("SVC_NAME")
42+
namespace = os.Getenv("SVC_NAMESPACE")
43+
)
44+
svcPort, err := strconv.ParseInt(os.Getenv("SVC_PORT"), 10, 32)
45+
if err != nil {
46+
klog.Error(err)
47+
svcPort = defaultPort
48+
}
49+
50+
if _, err := registerDriver(driver, svc, namespace, svcPort); err != nil {
3951
klog.Fatal(err)
4052
}
4153

42-
server, err := newServer()
54+
server, err := newServer(driver, *csiAddress)
4355
if err != nil {
4456
klog.Fatal(err)
4557
}
@@ -53,7 +65,7 @@ func main() {
5365
}
5466
}
5567

56-
func registerDriver() (runtime.Object, error) {
68+
func registerDriver(driver, svc, namespace string, svcPort int64) (runtime.Object, error) {
5769
restConfig, err := rest.InClusterConfig()
5870
if err != nil {
5971
return nil, err
@@ -64,22 +76,16 @@ func registerDriver() (runtime.Object, error) {
6476
return nil, err
6577
}
6678

67-
svcPort, err := strconv.ParseInt(os.Getenv("SVC_PORT"), 10, 32)
68-
if err != nil {
69-
klog.Error(err)
70-
svcPort = defaultPort
71-
}
72-
7379
var (
7480
endpoint = &v1alpha1.DriverDiscovery{
7581
ObjectMeta: metav1.ObjectMeta{
76-
Name: os.Getenv("CSI_DRIVER_NAME"),
82+
Name: driver,
7783
},
7884
Spec: v1alpha1.DriverDiscoverySpec{
79-
Driver: os.Getenv("CSI_DRIVER_NAME"),
85+
Driver: driver,
8086
Service: v1alpha1.Service{
81-
Name: os.Getenv("SVC_NAME"),
82-
Namespace: os.Getenv("SVC_NAMESPACE"),
87+
Name: svc,
88+
Namespace: namespace,
8389
Path: "/",
8490
Port: svcPort,
8591
},
@@ -107,10 +113,10 @@ func registerDriver() (runtime.Object, error) {
107113
return created, err
108114
}
109115

110-
func newServer() (*http.Server, error) {
111-
serveMux, err := newServeMux(*grpcTarget)
116+
func newServer(driver, grpcTarget string) (*http.Server, error) {
117+
serveMux, err := newServeMux(driver, grpcTarget)
112118
if err != nil {
113-
klog.Fatal(err)
119+
return nil, err
114120
}
115121

116122
return &http.Server{
@@ -136,10 +142,13 @@ type serveMux struct {
136142
*http.ServeMux
137143
}
138144

139-
func newServeMux(grpcTarget string) (*serveMux, error) {
140-
klog.Infof("connecting to GRPC target at %s", grpcTarget)
141-
opts := grpc.WithTransportCredentials(insecure.NewCredentials())
142-
clientConn, err := grpc.Dial(grpcTarget, opts)
145+
func newServeMux(driverName, grpcTarget string) (*serveMux, error) {
146+
klog.Infof("connecting to CSI driver at %s", grpcTarget)
147+
metricsManager := metrics.NewCSIMetricsManager(driverName)
148+
csiConn, err := connection.Connect(
149+
grpcTarget,
150+
metricsManager,
151+
connection.OnConnectionLoss(connection.ExitOnConnectionLoss()))
143152
if err != nil {
144153
return nil, err
145154
}
@@ -148,7 +157,7 @@ func newServeMux(grpcTarget string) (*serveMux, error) {
148157
mux := http.NewServeMux()
149158
mux.HandleFunc("/", s.handle)
150159

151-
s.grpc = clientConn
160+
s.grpc = csiConn
152161
s.ServeMux = mux
153162
return s, nil
154163
}

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/ihcsim/cbt-aggapi
33
go 1.17
44

55
require (
6+
github.com/kubernetes-csi/csi-lib-utils v0.11.0
67
google.golang.org/grpc v1.47.0
78
google.golang.org/protobuf v1.27.1
89
k8s.io/apimachinery v0.23.0

‎go.sum

+8
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
148148
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
149149
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
150150
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
151+
github.com/container-storage-interface/spec v1.5.0 h1:lvKxe3uLgqQeVQcrnL2CPQKISoKjTJxojEs9cBk+HXo=
152+
github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s=
151153
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
152154
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
153155
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -539,6 +541,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
539541
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
540542
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
541543
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
544+
github.com/kubernetes-csi/csi-lib-utils v0.11.0 h1:FHWOBtAZBA/hVk7v/qaXgG9Sxv0/n06DebPFuDwumqg=
545+
github.com/kubernetes-csi/csi-lib-utils v0.11.0/go.mod h1:BmGZZB16L18+9+Lgg9YWwBKfNEHIDdgGfAyuW6p2NV0=
542546
github.com/kulti/thelper v0.4.0/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U=
543547
github.com/kunwardeep/paralleltest v1.0.2/go.mod h1:ZPqNm1fVHPllh5LPVujzbVz1JN2GhLxSfY+oqUsvG30=
544548
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
@@ -1437,11 +1441,13 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
14371441
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
14381442
honnef.co/go/tools v0.2.0/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
14391443
k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
1444+
k8s.io/api v0.22.0/go.mod h1:0AoXXqst47OI/L0oGKq9DG61dvGRPXs7X4/B7KyjBCU=
14401445
k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY=
14411446
k8s.io/api v0.23.0 h1:WrL1gb73VSC8obi8cuYETJGXEoFNEh3LU0Pt+Sokgro=
14421447
k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg=
14431448
k8s.io/apiextensions-apiserver v0.17.0/go.mod h1:XiIFUakZywkUl54fVXa7QTEHcqQz9HG55nHd1DCoHj8=
14441449
k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
1450+
k8s.io/apimachinery v0.22.0/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
14451451
k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
14461452
k8s.io/apimachinery v0.23.0 h1:mIfWRMjBuMdolAWJ3Fd+aPTMv3X9z+waiARMpvvb0HQ=
14471453
k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc=
@@ -1450,6 +1456,7 @@ k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400=
14501456
k8s.io/apiserver v0.23.0 h1:Ds/QveXWi9aJ8ISB0CJa4zBNc5njxAs5u3rmMIexqCY=
14511457
k8s.io/apiserver v0.23.0/go.mod h1:Cec35u/9zAepDPPFyT+UMrgqOCjgJ5qtfVJDxjZYmt4=
14521458
k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
1459+
k8s.io/client-go v0.22.0/go.mod h1:GUjIuXR5PiEv/RVK5OODUsm6eZk7wtSWZSaSJbpFdGg=
14531460
k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk=
14541461
k8s.io/client-go v0.23.0 h1:vcsOqyPq7XV3QmQRCBH/t9BICJM9Q1M18qahjv+rebY=
14551462
k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA=
@@ -1458,6 +1465,7 @@ k8s.io/code-generator v0.22.1/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpx
14581465
k8s.io/code-generator v0.23.0 h1:lhyd2KJVCEmpjaCpuoooGs+e3xhPwpYvupnNRidO0Ds=
14591466
k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE=
14601467
k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc=
1468+
k8s.io/component-base v0.22.0/go.mod h1:SXj6Z+V6P6GsBhHZVbWCw9hFjUdUYnJerlhhPnYCBCg=
14611469
k8s.io/component-base v0.22.1/go.mod h1:0D+Bl8rrnsPN9v0dyYvkqFfBeAd4u7n77ze+p8CMiPo=
14621470
k8s.io/component-base v0.23.0 h1:UAnyzjvVZ2ZR1lF35YwtNY6VMN94WtOnArcXBu34es8=
14631471
k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI=

‎pkg/storage/cbt-rest.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,8 @@ func (c *cbt) Connect(ctx context.Context, id string, options runtime.Object, r
123123
klog.Infof("discovered CSI driver: %s", obj.GetName())
124124

125125
// send request to the csi sidecar
126-
httpRes, err := http.Get(obj.Spec.CBTEndpoint)
126+
endpoint := fmt.Sprintf("http://%s.%s:%d", obj.Spec.Service.Name, obj.Spec.Service.Namespace, obj.Spec.Service.Port)
127+
httpRes, err := http.Get(endpoint)
127128
if err != nil {
128129
http.Error(resp, fmt.Sprintf("failed to fetch CBT entries: %s", err), http.StatusInternalServerError)
129130
return

0 commit comments

Comments
 (0)