From 09833aef1ba1d699a570c92691a319056f3a607f Mon Sep 17 00:00:00 2001 From: Vinicius Fortuna Date: Mon, 5 Jun 2023 15:26:37 -0400 Subject: [PATCH] refactor: split metrics collection into TCP and UDP --- cmd/outline-ss-server/main.go | 7 +- cmd/outline-ss-server/metrics.go | 228 ++++++++++++++++ .../outline-ss-server}/metrics_test.go | 37 ++- cmd/outline-ss-server/server_test.go | 3 +- internal/integration_test/integration_test.go | 15 +- service/metrics/metrics.go | 258 ------------------ service/tcp.go | 33 ++- service/tcp_test.go | 13 +- service/udp.go | 42 ++- service/udp_test.go | 15 +- 10 files changed, 335 insertions(+), 316 deletions(-) create mode 100644 cmd/outline-ss-server/metrics.go rename {service/metrics => cmd/outline-ss-server}/metrics_test.go (65%) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 04037724..013adcbd 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -30,7 +30,6 @@ import ( "github.com/Jigsaw-Code/outline-internal-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" "github.com/Jigsaw-Code/outline-ss-server/service" - "github.com/Jigsaw-Code/outline-ss-server/service/metrics" "github.com/op/go-logging" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -68,7 +67,7 @@ type ssPort struct { type SSServer struct { natTimeout time.Duration - m metrics.ShadowsocksMetrics + m *outlineMetrics replayCache service.ReplayCache ports map[int]*ssPort } @@ -179,7 +178,7 @@ func (s *SSServer) Stop() error { } // RunSSServer starts a shadowsocks server running, and returns the server or an error. -func RunSSServer(filename string, natTimeout time.Duration, sm metrics.ShadowsocksMetrics, replayHistory int) (*SSServer, error) { +func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { server := &SSServer{ natTimeout: natTimeout, m: sm, @@ -280,7 +279,7 @@ func main() { defer ip2info.Close() } - m := metrics.NewPrometheusShadowsocksMetrics(ip2info, prometheus.DefaultRegisterer) + m := newPrometheusOutlineMetrics(ip2info, prometheus.DefaultRegisterer) m.SetBuildInfo(version) _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, m, flags.replayHistory) if err != nil { diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go new file mode 100644 index 00000000..6dbe6889 --- /dev/null +++ b/cmd/outline-ss-server/metrics.go @@ -0,0 +1,228 @@ +// Copyright 2023 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "strconv" + "time" + + "github.com/Jigsaw-Code/outline-ss-server/ipinfo" + "github.com/Jigsaw-Code/outline-ss-server/service" + "github.com/Jigsaw-Code/outline-ss-server/service/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type outlineMetrics struct { + ipinfo.IPInfoMap + + buildInfo *prometheus.GaugeVec + accessKeys prometheus.Gauge + ports prometheus.Gauge + dataBytes *prometheus.CounterVec + dataBytesPerLocation *prometheus.CounterVec + timeToCipherMs *prometheus.HistogramVec + // TODO: Add time to first byte. + + tcpProbes *prometheus.HistogramVec + tcpOpenConnections *prometheus.CounterVec + tcpClosedConnections *prometheus.CounterVec + tcpConnectionDurationMs *prometheus.HistogramVec + + udpPacketsFromClientPerLocation *prometheus.CounterVec + udpAddedNatEntries prometheus.Counter + udpRemovedNatEntries prometheus.Counter +} + +var _ service.TCPMetrics = (*outlineMetrics)(nil) +var _ service.UDPMetrics = (*outlineMetrics)(nil) + +// newPrometheusOutlineMetrics constructs a metrics object that uses +// `ipCountryDB` to convert IP addresses to countries, and reports all +// metrics to Prometheus via `registerer`. `ipCountryDB` may be nil, but +// `registerer` must not be. +func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus.Registerer) *outlineMetrics { + m := &outlineMetrics{ + IPInfoMap: ip2info, + buildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "shadowsocks", + Name: "build_info", + Help: "Information on the outline-ss-server build", + }, []string{"version"}), + accessKeys: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "shadowsocks", + Name: "keys", + Help: "Count of access keys", + }), + ports: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "shadowsocks", + Name: "ports", + Help: "Count of open Shadowsocks ports", + }), + tcpProbes: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "shadowsocks", + Name: "tcp_probes", + Buckets: []float64{0, 49, 50, 51, 73, 91}, + Help: "Histogram of number of bytes from client to proxy, for detecting possible probes", + }, []string{"port", "status", "error"}), + tcpOpenConnections: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "shadowsocks", + Subsystem: "tcp", + Name: "connections_opened", + Help: "Count of open TCP connections", + }, []string{"location"}), + tcpClosedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "shadowsocks", + Subsystem: "tcp", + Name: "connections_closed", + Help: "Count of closed TCP connections", + }, []string{"location", "status", "access_key"}), + tcpConnectionDurationMs: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "shadowsocks", + Subsystem: "tcp", + Name: "connection_duration_ms", + Help: "TCP connection duration distributions.", + Buckets: []float64{ + 100, + float64(time.Second.Milliseconds()), + float64(time.Minute.Milliseconds()), + float64(time.Hour.Milliseconds()), + float64(24 * time.Hour.Milliseconds()), // Day + float64(7 * 24 * time.Hour.Milliseconds()), // Week + }, + }, []string{"status"}), + dataBytes: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "shadowsocks", + Name: "data_bytes", + Help: "Bytes transferred by the proxy, per access key", + }, []string{"dir", "proto", "access_key"}), + dataBytesPerLocation: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "shadowsocks", + Name: "data_bytes_per_location", + Help: "Bytes transferred by the proxy, per location", + }, []string{"dir", "proto", "location"}), + timeToCipherMs: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "shadowsocks", + Name: "time_to_cipher_ms", + Help: "Time needed to find the cipher", + Buckets: []float64{0.1, 1, 10, 100, 1000}, + }, []string{"proto", "found_key"}), + udpPacketsFromClientPerLocation: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "shadowsocks", + Subsystem: "udp", + Name: "packets_from_client_per_location", + Help: "Packets received from the client, per location and status", + }, []string{"location", "status"}), + udpAddedNatEntries: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "shadowsocks", + Subsystem: "udp", + Name: "nat_entries_added", + Help: "Entries added to the UDP NAT table", + }), + udpRemovedNatEntries: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "shadowsocks", + Subsystem: "udp", + Name: "nat_entries_removed", + Help: "Entries removed from the UDP NAT table", + }), + } + + // TODO: Is it possible to pass where to register the collectors? + registerer.MustRegister(m.buildInfo, m.accessKeys, m.ports, m.tcpProbes, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs, + m.dataBytes, m.dataBytesPerLocation, m.timeToCipherMs, m.udpPacketsFromClientPerLocation, m.udpAddedNatEntries, m.udpRemovedNatEntries) + return m +} + +func (m *outlineMetrics) SetBuildInfo(version string) { + m.buildInfo.WithLabelValues(version).Set(1) +} + +func (m *outlineMetrics) SetNumAccessKeys(numKeys int, ports int) { + m.accessKeys.Set(float64(numKeys)) + m.ports.Set(float64(ports)) +} + +func (m *outlineMetrics) AddOpenTCPConnection(clientInfo ipinfo.IPInfo) { + m.tcpOpenConnections.WithLabelValues(clientInfo.CountryCode.String()).Inc() +} + +// addIfNonZero helps avoid the creation of series that are always zero. +func addIfNonZero(value int64, counterVec *prometheus.CounterVec, lvs ...string) { + if value > 0 { + counterVec.WithLabelValues(lvs...).Add(float64(value)) + } +} + +func (m *outlineMetrics) AddClosedTCPConnection(clientInfo ipinfo.IPInfo, accessKey, status string, data metrics.ProxyMetrics, duration time.Duration) { + m.tcpClosedConnections.WithLabelValues(clientInfo.CountryCode.String(), status, accessKey).Inc() + m.tcpConnectionDurationMs.WithLabelValues(status).Observe(duration.Seconds() * 1000) + addIfNonZero(data.ClientProxy, m.dataBytes, "c>p", "tcp", accessKey) + addIfNonZero(data.ClientProxy, m.dataBytesPerLocation, "c>p", "tcp", clientInfo.CountryCode.String()) + addIfNonZero(data.ProxyTarget, m.dataBytes, "p>t", "tcp", accessKey) + addIfNonZero(data.ProxyTarget, m.dataBytesPerLocation, "p>t", "tcp", clientInfo.CountryCode.String()) + addIfNonZero(data.TargetProxy, m.dataBytes, "pp", "udp", accessKey) + addIfNonZero(int64(clientProxyBytes), m.dataBytesPerLocation, "c>p", "udp", clientInfo.CountryCode.String()) + addIfNonZero(int64(proxyTargetBytes), m.dataBytes, "p>t", "udp", accessKey) + addIfNonZero(int64(proxyTargetBytes), m.dataBytesPerLocation, "p>t", "udp", clientInfo.CountryCode.String()) +} + +func (m *outlineMetrics) AddUDPPacketFromTarget(clientInfo ipinfo.IPInfo, accessKey, status string, targetProxyBytes, proxyClientBytes int) { + addIfNonZero(int64(targetProxyBytes), m.dataBytes, "p 0 { - counterVec.WithLabelValues(lvs...).Add(float64(value)) - } -} - -func (m *shadowsocksMetrics) AddClosedTCPConnection(clientInfo ipinfo.IPInfo, accessKey, status string, data ProxyMetrics, duration time.Duration) { - m.tcpClosedConnections.WithLabelValues(clientInfo.CountryCode.String(), status, accessKey).Inc() - m.tcpConnectionDurationMs.WithLabelValues(status).Observe(duration.Seconds() * 1000) - addIfNonZero(data.ClientProxy, m.dataBytes, "c>p", "tcp", accessKey) - addIfNonZero(data.ClientProxy, m.dataBytesPerLocation, "c>p", "tcp", clientInfo.CountryCode.String()) - addIfNonZero(data.ProxyTarget, m.dataBytes, "p>t", "tcp", accessKey) - addIfNonZero(data.ProxyTarget, m.dataBytesPerLocation, "p>t", "tcp", clientInfo.CountryCode.String()) - addIfNonZero(data.TargetProxy, m.dataBytes, "pp", "udp", accessKey) - addIfNonZero(int64(clientProxyBytes), m.dataBytesPerLocation, "c>p", "udp", clientInfo.CountryCode.String()) - addIfNonZero(int64(proxyTargetBytes), m.dataBytes, "p>t", "udp", accessKey) - addIfNonZero(int64(proxyTargetBytes), m.dataBytesPerLocation, "p>t", "udp", clientInfo.CountryCode.String()) -} - -func (m *shadowsocksMetrics) AddUDPPacketFromTarget(clientInfo ipinfo.IPInfo, accessKey, status string, targetProxyBytes, proxyClientBytes int) { - addIfNonZero(int64(targetProxyBytes), m.dataBytes, "p