From a256595839b1cd9f324224cd414b3d6c531d13fd Mon Sep 17 00:00:00 2001 From: Louis Royer Date: Thu, 12 Dec 2024 14:33:30 +0100 Subject: [PATCH] Wait for PFCP Entity to be ready --- go.mod | 6 +++--- go.sum | 12 ++++++------ internal/app/http.go | 26 ++++++++++++++++++++------ internal/app/setup.go | 24 ++++++++++++++---------- internal/app/upf.go | 33 +++++++++++++++++++++++++++++---- 5 files changed, 72 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 1d36f8d..2dcb94d 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/adrg/xdg v0.5.3 github.com/gin-gonic/gin v1.10.0 github.com/gofrs/uuid v4.4.0+incompatible - github.com/nextmn/go-pfcp-networking v0.0.40 + github.com/nextmn/go-pfcp-networking v0.0.41-0.20241212103129-24fe84808302 github.com/nextmn/json-api v0.0.14 github.com/nextmn/logrus-formatter v0.0.1 github.com/nextmn/rfc9433 v0.0.2 @@ -27,7 +27,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.23.0 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/goccy/go-json v0.10.4 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -40,7 +40,7 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect golang.org/x/arch v0.12.0 // indirect - golang.org/x/crypto v0.30.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.32.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect diff --git a/go.sum b/go.sum index f8eac02..b93df69 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= +github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -50,8 +50,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/nextmn/go-pfcp-networking v0.0.40 h1:Kol94dZHQ1NNRXbySwPQBPe5O7US5XqCC9D48pHz/AA= -github.com/nextmn/go-pfcp-networking v0.0.40/go.mod h1:KYoKLiltDmHL2YMU5mz2k/E1xMoz4TpmzTz6Nr5u5gA= +github.com/nextmn/go-pfcp-networking v0.0.41-0.20241212103129-24fe84808302 h1:NlmeMncMd72Qf0jTnXLPxHnXVB07tGycVn7Nrvx4YXw= +github.com/nextmn/go-pfcp-networking v0.0.41-0.20241212103129-24fe84808302/go.mod h1:KYoKLiltDmHL2YMU5mz2k/E1xMoz4TpmzTz6Nr5u5gA= github.com/nextmn/json-api v0.0.14 h1:m4uHOVcXsxkXoxbrhqemLTRG4T86eYkejjirew1nDUU= github.com/nextmn/json-api v0.0.14/go.mod h1:CQXeNPj9MDGsEExtnqJFIGjLgZAKsmOoO2fy+mep7Ak= github.com/nextmn/logrus-formatter v0.0.1 h1:Bsf78jjiEESc+rV8xE6IyKj4frDPGMwXFNrLQzm6A1E= @@ -88,8 +88,8 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGC github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/app/http.go b/internal/app/http.go index b70b76a..5ecc1f8 100644 --- a/internal/app/http.go +++ b/internal/app/http.go @@ -26,6 +26,7 @@ import ( type HttpServerEntity struct { srv *http.Server routers *RouterRegistry + closed chan struct{} } type RouterRegistry struct { @@ -53,11 +54,12 @@ func NewHttpServerEntity(httpAddr netip.AddrPort, pfcp *pfcp_networking.PFCPEnti Addr: httpAddr.String(), Handler: r, }, + closed: make(chan struct{}), } return &e } -func (e *HttpServerEntity) Start() error { +func (e *HttpServerEntity) Start(ctx context.Context) error { l, err := net.Listen("tcp", e.srv.Addr) if err != nil { return err @@ -68,14 +70,26 @@ func (e *HttpServerEntity) Start() error { logrus.WithError(err).Error("Http Server error") } }(l) + go func(ctx context.Context) { + defer close(e.closed) + select { + case <-ctx.Done(): + ctxShutdown, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := e.srv.Shutdown(ctxShutdown); err != nil { + logrus.WithError(err).Info("HTTP Server Shutdown") + } + } + }(ctx) return nil } -func (e *HttpServerEntity) Stop() { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) // context.Background() is already Done() - defer cancel() - if err := e.srv.Shutdown(ctx); err != nil { - logrus.WithError(err).Info("HTTP Server Shutdown") +func (e *HttpServerEntity) WaitShutdown(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-e.closed: + return nil } } diff --git a/internal/app/setup.go b/internal/app/setup.go index eb350d6..8e5d3b9 100644 --- a/internal/app/setup.go +++ b/internal/app/setup.go @@ -7,6 +7,7 @@ package app import ( "context" + "time" pfcp_networking "github.com/nextmn/go-pfcp-networking/pfcp" @@ -15,15 +16,16 @@ import ( type Setup struct { HTTPServer *HttpServerEntity + Upf *Upf PFCPServer *pfcp_networking.PFCPEntityUP RulesPusher *RulesPusher } func NewSetup(conf *config.CtrlConfig) Setup { - pfcp := NewPFCPNode(conf) + upf := NewUpf(conf) return Setup{ - HTTPServer: NewHttpServer(conf, pfcp), - PFCPServer: pfcp, + HTTPServer: NewHttpServer(conf, upf.pfcpentity), + Upf: upf, RulesPusher: NewRulesPusher(conf), } } @@ -32,14 +34,16 @@ func (s Setup) Run(ctx context.Context) error { if err := PFCPServerAddHooks(s.PFCPServer, s.RulesPusher); err != nil { return err } - StartPFCPServer(ctx, s.PFCPServer) - if err := s.HTTPServer.Start(); err != nil { + if err := s.Upf.Start(ctx); err != nil { return err } - select { - case <-ctx.Done(): - s.HTTPServer.Stop() - s.PFCPServer.Close() - return nil + if err := s.HTTPServer.Start(ctx); err != nil { + return err } + ctxShutdown, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + // Try to end before 1s… + s.HTTPServer.WaitShutdown(ctxShutdown) + s.Upf.WaitShutdown(ctxShutdown) + return nil } diff --git a/internal/app/upf.go b/internal/app/upf.go index 81fbdf0..54ab204 100644 --- a/internal/app/upf.go +++ b/internal/app/upf.go @@ -7,6 +7,7 @@ package app import ( "context" + "time" "github.com/nextmn/srv6-ctrl/internal/config" @@ -16,8 +17,16 @@ import ( "github.com/wmnsk/go-pfcp/message" ) -func NewPFCPNode(conf *config.CtrlConfig) *pfcp_networking.PFCPEntityUP { - return pfcp_networking.NewPFCPEntityUP(conf.PFCPAddress.String(), conf.PFCPAddress) +type Upf struct { + pfcpentity *pfcp_networking.PFCPEntityUP + closed chan struct{} +} + +func NewUpf(conf *config.CtrlConfig) *Upf { + return &Upf{ + pfcpentity: pfcp_networking.NewPFCPEntityUP(conf.PFCPAddress.String(), conf.PFCPAddress), + closed: make(chan struct{}), + } } func PFCPServerAddHooks(s *pfcp_networking.PFCPEntityUP, pusher *RulesPusher) error { if err := s.AddHandler(message.MsgTypeSessionEstablishmentRequest, func(ctx context.Context, msg pfcp_networking.ReceivedMessage) (*pfcp_networking.OutcomingMessage, error) { @@ -48,11 +57,27 @@ func NewHttpServer(conf *config.CtrlConfig, srv6Srv *pfcp_networking.PFCPEntityU return HTTPServer } -func StartPFCPServer(ctx context.Context, srv *pfcp_networking.PFCPEntityUP) { +func (upf *Upf) Start(ctx context.Context) error { go func(ctx context.Context, srv *pfcp_networking.PFCPEntityUP) { logrus.Info("Starting PFCP Server") if err := srv.ListenAndServeContext(ctx); err != nil { logrus.WithError(err).Error("PFCP Server Shutdown") + close(upf.closed) } - }(ctx, srv) + }(ctx, upf.pfcpentity) + ctxTimeout, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + if err := upf.pfcpentity.WaitReady(ctxTimeout); err != nil { + return err + } + return nil +} + +func (upf *Upf) WaitShutdown(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-upf.closed: + return nil + } }