Skip to content

Commit 97b82a8

Browse files
authored
Reduce noise in GS logs caused by gateway disconnections (#7993)
1 parent 14a2601 commit 97b82a8

8 files changed

Lines changed: 300 additions & 6 deletions

File tree

CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,13 @@ For details about compatibility between different releases, see the **Commitment
1111

1212
### Added
1313

14+
- `gs_gateways_disconnected_total` metric, counting gateway disconnections by protocol and by the error the connection was disconnected with. This makes disconnection reasons (such as gateways disappearing without a close handshake, or missing too many pongs) observable as a rate, instead of only through logs.
15+
1416
### Changed
1517

1618
- In the Semtech UDP Packet Forwarder protocol, `PUSH_ACK` and `PULL_ACK` are only sent after the gateway has connected to the Gateway Server and the gateway's `PUSH_DATA` or `PULL_DATA` respectively has been accepted.
19+
- Don't log a `Task failed` warning in GS for every task attached to a gateway connection when that connection is closed. The tasks that run for the lifetime of a gateway connection now stop without an error when the connection is closed, and the disconnection is logged once, as `Disconnected`, including the reason. This removes several duplicate warnings per gateway disconnection, which were particularly noisy for gateways on unreliable backhaul.
20+
- Websocket close errors on the LoRa Basics Station frontend (such as `websocket: close 1006 (abnormal closure): unexpected EOF`, which is what a gateway disappearing without a close handshake looks like) are now reported as the defined error `pkg/gatewayserver/io/semtechws:websocket_closed`, with the close code as an attribute and the original error as the cause. As a result, the `gs.gateway.disconnect` event for these disconnections now carries structured error details instead of a plain string; consumers that parse the event data should expect the `ErrorDetails` format.
1721

1822
### Deprecated
1923

config/messages.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5273,6 +5273,15 @@
52735273
"file": "ws.go"
52745274
}
52755275
},
5276+
"error:pkg/gatewayserver/io/semtechws:websocket_closed": {
5277+
"translations": {
5278+
"en": "websocket closed with code `{code}`"
5279+
},
5280+
"description": {
5281+
"package": "pkg/gatewayserver/io/semtechws",
5282+
"file": "ws.go"
5283+
}
5284+
},
52765285
"error:pkg/gatewayserver/io/ttigw:downlink_channel_mixed_bandwidths": {
52775286
"translations": {
52785287
"en": "downlink channel `{channel}` has mixed bandwidths `{bandwidth_low}` and `{bandwidth_high}` Hz"

pkg/gatewayserver/gatewayserver.go

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -694,12 +694,19 @@ func (gs *GatewayServer) Connect(
694694

695695
for name, handler := range gs.upstreamHandlers {
696696
connCtx := log.NewContextWithField(conn.Context(), "upstream_handler", name)
697-
handler := handler
698697
gs.StartTask(&task.Config{
699698
Context: connCtx,
700699
ID: fmt.Sprintf("%s_connect_gateway_%s", name, ids.GatewayId),
701700
Func: func(ctx context.Context) error {
702-
return handler.ConnectGateway(ctx, ids, conn)
701+
err := handler.ConnectGateway(ctx, ids, conn)
702+
if err != nil && errors.Is(err, ctx.Err()) {
703+
// Expected stop — the handler returned the error that the context was
704+
// canceled with, so the gateway is disconnected. The reason is reported
705+
// when the connection is torn down, not in this task. Any other error is
706+
// a genuine failure and is reported, even when the context is done.
707+
return nil
708+
}
709+
return err
703710
},
704711
Done: wg.Done,
705712
Restart: task.RestartOnFailure,
@@ -764,7 +771,9 @@ func (gs *GatewayServer) startDisconnectOnChangeTask(conn connectionEntry) {
764771
d := random.Jitter(gs.config.FetchGatewayInterval, gs.config.FetchGatewayJitter)
765772
select {
766773
case <-ctx.Done():
767-
return ctx.Err()
774+
// Expected stop — the context is done, gateway is disconnected. The reason
775+
// is reported when the connection is torn down, not in this task.
776+
return nil
768777
case <-time.After(d):
769778
}
770779

@@ -991,7 +1000,7 @@ func (gs *GatewayServer) handleUpstream(ctx context.Context, conn connectionEntr
9911000
defer func() {
9921001
gs.connections.Delete(unique.ID(ctx, gtw.GetIds()))
9931002
registerGatewayDisconnect(ctx, gtw.GetIds(), protocol, ctx.Err())
994-
logger.Info("Disconnected")
1003+
logger.WithError(ctx.Err()).Info("Disconnected")
9951004
}()
9961005

9971006
hosts := make([]*upstreamHost, 0, len(gs.upstreamHandlers))

pkg/gatewayserver/io/semtechws/ws.go

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,18 @@ var (
5858
errGatewayID = errors.DefineInvalidArgument("invalid_gateway_id", "invalid gateway ID `{id}`")
5959
errNoAuthProvided = errors.DefineUnauthenticated("no_auth_provided", "no auth provided `{uid}`")
6060
errMissedTooManyPongs = errors.Define("missed_too_many_pongs", "gateway missed too many pongs")
61+
errWebsocketClosed = errors.DefineAborted("websocket_closed", "websocket closed with code `{code}`")
6162
)
6263

64+
// disconnectError converts err into the error that the connection is disconnected with.
65+
func disconnectError(err error) error {
66+
var closeErr *websocket.CloseError
67+
if errors.As(err, &closeErr) {
68+
return errWebsocketClosed.WithCause(err).WithAttributes("code", closeErr.Code)
69+
}
70+
return err
71+
}
72+
6373
type srv struct {
6474
ctx context.Context
6575
server io.Server
@@ -288,7 +298,7 @@ func (s *srv) handleTraffic(w http.ResponseWriter, r *http.Request) (err error)
288298
span.RecordError(err)
289299
span.SetStatus(codes.Error, "handle traffic failed")
290300
}
291-
conn.Disconnect(err)
301+
conn.Disconnect(disconnectError(err))
292302
err = nil // Errors are sent over the websocket connection that is established by this point.
293303
}()
294304

@@ -346,7 +356,7 @@ func (s *srv) handleTraffic(w http.ResponseWriter, r *http.Request) (err error)
346356
defer ws.Close()
347357
defer func() {
348358
if err != nil {
349-
conn.Disconnect(err)
359+
conn.Disconnect(disconnectError(err))
350360
}
351361
}()
352362
for {
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// Copyright © 2026 The Things Network Foundation, The Things Industries B.V.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package semtechws
16+
17+
import (
18+
"fmt"
19+
"io"
20+
"testing"
21+
22+
"github.com/gorilla/websocket"
23+
"github.com/smarty/assertions"
24+
"go.thethings.network/lorawan-stack/v3/pkg/errors"
25+
"go.thethings.network/lorawan-stack/v3/pkg/util/test/assertions/should"
26+
)
27+
28+
func TestDisconnectError(t *testing.T) {
29+
t.Parallel()
30+
31+
for _, tc := range []struct {
32+
Name string
33+
Err error
34+
// Code is the expected close code, nil if the error is expected to pass through.
35+
Code any
36+
}{
37+
{
38+
// This is what the gateway disappearing without a close handshake looks like.
39+
Name: "AbnormalClosure",
40+
Err: &websocket.CloseError{Code: websocket.CloseAbnormalClosure, Text: io.ErrUnexpectedEOF.Error()},
41+
Code: websocket.CloseAbnormalClosure,
42+
},
43+
{
44+
Name: "GoingAway",
45+
Err: &websocket.CloseError{Code: websocket.CloseGoingAway},
46+
Code: websocket.CloseGoingAway,
47+
},
48+
{
49+
Name: "WrappedCloseError",
50+
Err: fmt.Errorf("read: %w",
51+
&websocket.CloseError{Code: websocket.CloseNoStatusReceived},
52+
),
53+
Code: websocket.CloseNoStatusReceived,
54+
},
55+
{
56+
Name: "DefinedError",
57+
Err: errMissedTooManyPongs.New(),
58+
},
59+
{
60+
Name: "OtherError",
61+
Err: io.ErrUnexpectedEOF,
62+
},
63+
} {
64+
t.Run(tc.Name, func(t *testing.T) {
65+
t.Parallel()
66+
a := assertions.New(t)
67+
68+
err := disconnectError(tc.Err)
69+
if tc.Code == nil {
70+
a.So(err, should.Equal, tc.Err)
71+
return
72+
}
73+
if !a.So(errors.IsAborted(err), should.BeTrue) {
74+
t.FailNow()
75+
}
76+
ttnErr, ok := errors.From(err)
77+
if !a.So(ok, should.BeTrue) {
78+
t.FailNow()
79+
}
80+
// The error must be classifiable, so that it can be used as a metric label.
81+
a.So(ttnErr.FullName(), should.Equal, "pkg/gatewayserver/io/semtechws:websocket_closed")
82+
a.So(ttnErr.Attributes()["code"], should.Equal, tc.Code)
83+
// The original error must be preserved for diagnostics.
84+
a.So(errors.Is(err, tc.Err), should.BeTrue)
85+
})
86+
}
87+
}

pkg/gatewayserver/observability.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,14 @@ var gsMetrics = &messageMetrics{
136136
},
137137
[]string{protocol},
138138
),
139+
gatewaysDisconnected: metrics.NewContextualCounterVec(
140+
prometheus.CounterOpts{
141+
Subsystem: subsystem,
142+
Name: "gateways_disconnected_total",
143+
Help: "Total number of gateway disconnections",
144+
},
145+
[]string{protocol, "error"},
146+
),
139147
statusReceived: metrics.NewContextualCounterVec(
140148
prometheus.CounterOpts{
141149
Subsystem: subsystem,
@@ -256,6 +264,7 @@ func init() {
256264

257265
type messageMetrics struct {
258266
gatewaysConnected *metrics.ContextualGaugeVec
267+
gatewaysDisconnected *metrics.ContextualCounterVec
259268
statusReceived *metrics.ContextualCounterVec
260269
statusForwarded *metrics.ContextualCounterVec
261270
statusDropped *metrics.ContextualCounterVec
@@ -274,6 +283,7 @@ type messageMetrics struct {
274283

275284
func (m messageMetrics) Describe(ch chan<- *prometheus.Desc) {
276285
m.gatewaysConnected.Describe(ch)
286+
m.gatewaysDisconnected.Describe(ch)
277287
m.statusReceived.Describe(ch)
278288
m.statusForwarded.Describe(ch)
279289
m.statusDropped.Describe(ch)
@@ -292,6 +302,7 @@ func (m messageMetrics) Describe(ch chan<- *prometheus.Desc) {
292302

293303
func (m messageMetrics) Collect(ch chan<- prometheus.Metric) {
294304
m.gatewaysConnected.Collect(ch)
305+
m.gatewaysDisconnected.Collect(ch)
295306
m.statusReceived.Collect(ch)
296307
m.statusForwarded.Collect(ch)
297308
m.statusDropped.Collect(ch)
@@ -320,6 +331,11 @@ func registerGatewayConnect(
320331
func registerGatewayDisconnect(ctx context.Context, ids *ttnpb.GatewayIdentifiers, protocol string, err error) {
321332
events.Publish(evtGatewayDisconnect.NewWithIdentifiersAndData(ctx, ids, err))
322333
gsMetrics.gatewaysConnected.WithLabelValues(ctx, protocol).Dec()
334+
errorLabel := unknown
335+
if ttnErr, ok := errors.From(err); ok {
336+
errorLabel = ttnErr.FullName()
337+
}
338+
gsMetrics.gatewaysDisconnected.WithLabelValues(ctx, protocol, errorLabel).Inc()
323339
}
324340

325341
func registerGatewayConnectionStats(ctx context.Context, ids *ttnpb.GatewayIdentifiers, stats *ttnpb.GatewayConnectionStats) {

0 commit comments

Comments
 (0)