Skip to content

Commit b18e618

Browse files
afritzlerafritzler
committed
refactor: add file watcher for TLS certificate files
- Add fsnotify dependency to go.mod - Add file watcher for certificate files in `cmd/agent/app/server.go` and `cmd/server/app/server.go` - Reload TLS config and credentials on certificate file change in `cmd/agent/app/server.go` and `cmd/server/app/server.go` - Refactor and clean up file watcher code in `cmd/agent/app/server.go` and `cmd/server/app/server.go` Signed-off-by: Andreas Fritzler <[email protected]>
1 parent 6f42ff3 commit b18e618

File tree

4 files changed

+185
-32
lines changed

4 files changed

+185
-32
lines changed

‎cmd/agent/app/server.go

Lines changed: 74 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"strconv"
2929
"time"
3030

31+
"github.com/fsnotify/fsnotify"
3132
"github.com/prometheus/client_golang/prometheus/promhttp"
3233
"github.com/spf13/cobra"
3334
"google.golang.org/grpc"
@@ -83,19 +84,82 @@ func (a *Agent) run(o *options.GrpcProxyAgentOptions) error {
8384
func (a *Agent) runProxyConnection(o *options.GrpcProxyAgentOptions, stopCh <-chan struct{}) error {
8485
var tlsConfig *tls.Config
8586
var err error
86-
if tlsConfig, err = util.GetClientTLSConfig(o.CaCert, o.AgentCert, o.AgentKey, o.ProxyServerHost, o.AlpnProtos); err != nil {
87+
88+
watcher, err := fsnotify.NewWatcher()
89+
if err != nil {
90+
klog.Fatal(err)
91+
}
92+
defer func(watcher *fsnotify.Watcher) {
93+
if err := watcher.Close(); err != nil {
94+
klog.ErrorS(err, "failed to close watcher")
95+
return
96+
}
97+
}(watcher)
98+
99+
// Watch the certificate files
100+
if err := watcher.Add(o.AgentCert); err != nil {
87101
return err
88102
}
89-
dialOptions := []grpc.DialOption{
90-
grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)),
91-
grpc.WithKeepaliveParams(keepalive.ClientParameters{
92-
Time: o.KeepaliveTime,
93-
PermitWithoutStream: true,
94-
}),
103+
if err := watcher.Add(o.AgentKey); err != nil {
104+
return err
95105
}
96-
cc := o.ClientSetConfig(dialOptions...)
97-
cs := cc.NewAgentClientSet(stopCh)
98-
cs.Serve()
106+
if err := watcher.Add(o.CaCert); err != nil {
107+
return err
108+
}
109+
110+
reload := make(chan bool)
111+
112+
// Goroutine to watch for file changes
113+
go func() {
114+
for {
115+
select {
116+
case event, ok := <-watcher.Events:
117+
if !ok {
118+
return
119+
}
120+
if event.Op&fsnotify.Write == fsnotify.Write {
121+
reload <- true
122+
}
123+
case err, ok := <-watcher.Errors:
124+
if !ok {
125+
return
126+
}
127+
klog.ErrorS(err, "failed to watch for file changes")
128+
case <-stopCh:
129+
// Handle graceful shutdown
130+
return
131+
}
132+
}
133+
}()
134+
135+
// Goroutine to handle main logic
136+
go func() {
137+
for {
138+
select {
139+
case <-reload:
140+
tlsConfig, err = util.GetClientTLSConfig(o.CaCert, o.AgentCert, o.AgentKey, o.ProxyServerHost, o.AlpnProtos)
141+
if err != nil {
142+
klog.ErrorS(err, "Failed to reload TLS config")
143+
continue
144+
}
145+
146+
dialOptions := []grpc.DialOption{
147+
grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)),
148+
grpc.WithKeepaliveParams(keepalive.ClientParameters{
149+
Time: o.KeepaliveTime,
150+
PermitWithoutStream: true,
151+
}),
152+
}
153+
cc := o.ClientSetConfig(dialOptions...)
154+
cs := cc.NewAgentClientSet(stopCh)
155+
cs.Serve()
156+
157+
case <-stopCh:
158+
// Handle server shutdown
159+
return
160+
}
161+
}
162+
}()
99163

100164
return nil
101165
}

‎cmd/server/app/server.go

Lines changed: 107 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"syscall"
3636
"time"
3737

38+
"github.com/fsnotify/fsnotify"
3839
"github.com/prometheus/client_golang/prometheus/promhttp"
3940
"github.com/spf13/cobra"
4041
"google.golang.org/grpc"
@@ -135,7 +136,7 @@ func (p *Proxy) run(o *options.ProxyRunOptions) error {
135136
}
136137

137138
klog.V(1).Infoln("Starting agent server for tunnel connections.")
138-
err = p.runAgentServer(o, server)
139+
err = p.runAgentServer(o, server, ctx.Done())
139140
if err != nil {
140141
return fmt.Errorf("failed to run the agent server: %v", err)
141142
}
@@ -353,33 +354,117 @@ func (p *Proxy) runMTLSFrontendServer(ctx context.Context, o *options.ProxyRunOp
353354
return stop, nil
354355
}
355356

356-
func (p *Proxy) runAgentServer(o *options.ProxyRunOptions, server *server.ProxyServer) error {
357+
func (p *Proxy) runAgentServer(o *options.ProxyRunOptions, server *server.ProxyServer, stopCh <-chan struct{}) error {
357358
var tlsConfig *tls.Config
358359
var err error
359-
if tlsConfig, err = p.getTLSConfig(o.ClusterCaCert, o.ClusterCert, o.ClusterKey, o.CipherSuites); err != nil {
360-
return err
360+
361+
watcher, err := fsnotify.NewWatcher()
362+
if err != nil {
363+
klog.Fatal(err)
361364
}
365+
defer func(watcher *fsnotify.Watcher) {
366+
if err := watcher.Close(); err != nil {
367+
klog.ErrorS(err, "failed to close watcher")
368+
return
369+
}
370+
}(watcher)
362371

363-
addr := net.JoinHostPort(o.AgentBindAddress, strconv.Itoa(o.AgentPort))
364-
agentServerOptions := []grpc.ServerOption{
365-
grpc.Creds(credentials.NewTLS(tlsConfig)),
366-
grpc.KeepaliveParams(keepalive.ServerParameters{Time: o.KeepaliveTime}),
367-
grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
368-
MinTime: 30 * time.Second,
369-
PermitWithoutStream: true,
370-
}),
372+
// Watch the certificate files
373+
if err := watcher.Add(o.ClusterCaCert); err != nil {
374+
return err
371375
}
372-
grpcServer := grpc.NewServer(agentServerOptions...)
373-
agent.RegisterAgentServiceServer(grpcServer, server)
374-
lis, err := net.Listen("tcp", addr)
375-
if err != nil {
376-
return fmt.Errorf("failed to listen on %s: %v", addr, err)
376+
if err := watcher.Add(o.ClusterCert); err != nil {
377+
return err
377378
}
378-
labels := runpprof.Labels(
379-
"core", "agentListener",
380-
"port", strconv.FormatUint(uint64(o.AgentPort), 10),
381-
)
382-
go runpprof.Do(context.Background(), labels, func(context.Context) { grpcServer.Serve(lis) })
379+
if err := watcher.Add(o.ClusterKey); err != nil {
380+
return err
381+
}
382+
383+
reload := make(chan bool)
384+
restartServer := make(chan bool)
385+
386+
// Goroutine to watch for file changes
387+
go func() {
388+
for {
389+
select {
390+
case event, ok := <-watcher.Events:
391+
if !ok {
392+
return
393+
}
394+
if event.Op&fsnotify.Write == fsnotify.Write {
395+
reload <- true
396+
}
397+
case err, ok := <-watcher.Errors:
398+
if !ok {
399+
return
400+
}
401+
klog.ErrorS(err, "failed to watch for file changes")
402+
case <-stopCh:
403+
// Handle graceful shutdown
404+
return
405+
}
406+
}
407+
}()
408+
409+
addr := net.JoinHostPort(o.AgentBindAddress, strconv.Itoa(o.AgentPort))
410+
411+
go func() {
412+
var grpcServer *grpc.Server
413+
var lis net.Listener
414+
415+
for {
416+
select {
417+
case <-reload:
418+
if tlsConfig, err = p.getTLSConfig(o.ClusterCaCert, o.ClusterCert, o.ClusterKey, o.CipherSuites); err != nil {
419+
klog.ErrorS(err, "Failed to reload TLS config:")
420+
}
421+
klog.V(1).Info("TLS config reloaded")
422+
restartServer <- true
423+
424+
case <-restartServer:
425+
if grpcServer != nil {
426+
grpcServer.GracefulStop()
427+
klog.V(1).Info("gRPC server stopped")
428+
}
429+
430+
agentServerOptions := []grpc.ServerOption{
431+
grpc.Creds(credentials.NewTLS(tlsConfig)),
432+
grpc.KeepaliveParams(keepalive.ServerParameters{Time: o.KeepaliveTime}),
433+
grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
434+
MinTime: 30 * time.Second,
435+
PermitWithoutStream: true,
436+
}),
437+
}
438+
grpcServer = grpc.NewServer(agentServerOptions...)
439+
agent.RegisterAgentServiceServer(grpcServer, server)
440+
441+
lis, err = net.Listen("tcp", addr)
442+
if err != nil {
443+
klog.Fatalf("failed to listen on %s: %v", addr, err)
444+
}
445+
446+
go func() {
447+
if err := grpcServer.Serve(lis); err != nil {
448+
klog.Fatalf("failed to serve: %v", err)
449+
}
450+
}()
451+
452+
klog.V(1).Info("gRPC server restarted")
453+
case <-stopCh:
454+
// Handle graceful shutdown
455+
if grpcServer != nil {
456+
grpcServer.GracefulStop()
457+
}
458+
if lis != nil {
459+
lis.Close()
460+
}
461+
return
462+
}
463+
}
464+
}()
465+
466+
// Initial restart to start the server
467+
restartServer <- true
383468

384469
return nil
385470
}

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module sigs.k8s.io/apiserver-network-proxy
33
go 1.19
44

55
require (
6+
github.com/fsnotify/fsnotify v1.6.0
67
github.com/golang/mock v1.6.0
78
github.com/google/uuid v1.1.2
89
github.com/prometheus/client_golang v1.12.1

‎go.sum

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD
107107
github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
108108
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
109109
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
110+
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
111+
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
110112
github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
111113
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
112114
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -534,6 +536,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
534536
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
535537
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
536538
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
539+
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
537540
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
538541
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
539542
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

0 commit comments

Comments
 (0)