diff --git a/cmd/postgres_exporter/main.go b/cmd/postgres_exporter/main.go index c901ec722..0c1bd003e 100644 --- a/cmd/postgres_exporter/main.go +++ b/cmd/postgres_exporter/main.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -23,6 +23,7 @@ import ( "github.com/alecthomas/kingpin/v2" "github.com/prometheus-community/postgres_exporter/collector" "github.com/prometheus-community/postgres_exporter/config" + "github.com/prometheus-community/postgres_exporter/exporter" "github.com/prometheus/client_golang/prometheus" versioncollector "github.com/prometheus/client_golang/prometheus/collectors/version" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -54,20 +55,8 @@ var ( logger = promslog.NewNopLogger() ) -// Metric name parts. -const ( - // Namespace for all metrics. - namespace = "pg" - // Subsystems. - exporter = "exporter" - // The name of the exporter. - exporterName = "postgres_exporter" - // Metric label used for static string data thats handy to send to Prometheus - // e.g. version - staticLabelName = "static" - // Metric label used for server identification. - serverLabelName = "server" -) +// The name of the exporter. +const exporterName = "postgres_exporter" func main() { kingpin.Version(version.Print(exporterName)) @@ -78,7 +67,7 @@ func main() { logger = promslog.New(promslogConfig) if *onlyDumpMaps { - dumpMaps() + exporter.DumpMaps() return } @@ -87,7 +76,7 @@ func main() { logger.Warn("Error loading config", "err", err) } - dsns, err := getDataSources() + dsns, err := exporter.GetDataSources() if err != nil { logger.Error("Failed reading data sources", "err", err.Error()) os.Exit(1) @@ -108,19 +97,20 @@ func main() { logger.Warn("Constant labels on all metrics is DEPRECATED") } - opts := []ExporterOpt{ - DisableDefaultMetrics(*disableDefaultMetrics), - DisableSettingsMetrics(*disableSettingsMetrics), - AutoDiscoverDatabases(*autoDiscoverDatabases), - WithUserQueriesPath(*queriesPath), - WithConstantLabels(*constantLabelsList), - ExcludeDatabases(excludedDatabases), - IncludeDatabases(*includeDatabases), + opts := []exporter.ExporterOpt{ + exporter.DisableDefaultMetrics(*disableDefaultMetrics), + exporter.DisableSettingsMetrics(*disableSettingsMetrics), + exporter.AutoDiscoverDatabases(*autoDiscoverDatabases), + exporter.WithUserQueriesPath(*queriesPath), + exporter.WithConstantLabels(*constantLabelsList), + exporter.ExcludeDatabases(excludedDatabases), + exporter.IncludeDatabases(*includeDatabases), + exporter.WithMetricPrefix(*metricPrefix), } - exporter := NewExporter(dsns, opts...) + exporter := exporter.NewExporter(dsns, logger, opts...) defer func() { - exporter.servers.Close() + exporter.CloseServers() }() prometheus.MustRegister(versioncollector.NewCollector(exporterName)) diff --git a/cmd/postgres_exporter/probe.go b/cmd/postgres_exporter/probe.go index 2c8c7652e..66219c07d 100644 --- a/cmd/postgres_exporter/probe.go +++ b/cmd/postgres_exporter/probe.go @@ -20,6 +20,7 @@ import ( "github.com/prometheus-community/postgres_exporter/collector" "github.com/prometheus-community/postgres_exporter/config" + "github.com/prometheus-community/postgres_exporter/exporter" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -64,20 +65,21 @@ func handleProbe(logger *slog.Logger, excludeDatabases []string) http.HandlerFun registry := prometheus.NewRegistry() - opts := []ExporterOpt{ - DisableDefaultMetrics(*disableDefaultMetrics), - DisableSettingsMetrics(*disableSettingsMetrics), - AutoDiscoverDatabases(*autoDiscoverDatabases), - WithUserQueriesPath(*queriesPath), - WithConstantLabels(*constantLabelsList), - ExcludeDatabases(excludeDatabases), - IncludeDatabases(*includeDatabases), + opts := []exporter.ExporterOpt{ + exporter.DisableDefaultMetrics(*disableDefaultMetrics), + exporter.DisableSettingsMetrics(*disableSettingsMetrics), + exporter.AutoDiscoverDatabases(*autoDiscoverDatabases), + exporter.WithUserQueriesPath(*queriesPath), + exporter.WithConstantLabels(*constantLabelsList), + exporter.ExcludeDatabases(excludeDatabases), + exporter.IncludeDatabases(*includeDatabases), + exporter.WithMetricPrefix(*metricPrefix), } dsns := []string{dsn.GetConnectionString()} - exporter := NewExporter(dsns, opts...) + exporter := exporter.NewExporter(dsns, logger, opts...) defer func() { - exporter.servers.Close() + exporter.CloseServers() }() registry.MustRegister(exporter) diff --git a/cmd/postgres_exporter/datasource.go b/exporter/datasource.go similarity index 89% rename from cmd/postgres_exporter/datasource.go rename to exporter/datasource.go index cebe5072e..9ae3410fe 100644 --- a/cmd/postgres_exporter/datasource.go +++ b/exporter/datasource.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package exporter import ( "fmt" @@ -39,19 +39,19 @@ func (e *Exporter) discoverDatabaseDSNs() []string { var err error dsnURI, err = url.Parse(dsn) if err != nil { - logger.Error("Unable to parse DSN as URI", "dsn", loggableDSN(dsn), "err", err) + e.logger.Error("Unable to parse DSN as URI", "dsn", loggableDSN(dsn), "err", err) continue } } else if connstringRe.MatchString(dsn) { dsnConnstring = dsn } else { - logger.Error("Unable to parse DSN as either URI or connstring", "dsn", loggableDSN(dsn)) + e.logger.Error("Unable to parse DSN as either URI or connstring", "dsn", loggableDSN(dsn)) continue } server, err := e.servers.GetServer(dsn) if err != nil { - logger.Error("Error opening connection to database", "dsn", loggableDSN(dsn), "err", err) + e.logger.Error("Error opening connection to database", "dsn", loggableDSN(dsn), "err", err) continue } dsns[dsn] = struct{}{} @@ -61,7 +61,7 @@ func (e *Exporter) discoverDatabaseDSNs() []string { databaseNames, err := queryDatabases(server) if err != nil { - logger.Error("Error querying databases", "dsn", loggableDSN(dsn), "err", err) + e.logger.Error("Error querying databases", "dsn", loggableDSN(dsn), "err", err) continue } for _, databaseName := range databaseNames { @@ -109,7 +109,7 @@ func (e *Exporter) scrapeDSN(ch chan<- prometheus.Metric, dsn string) error { // Check if map versions need to be updated if err := e.checkMapVersions(ch, server); err != nil { - logger.Warn("Proceeding with outdated query maps, as the Postgres version could not be determined", "err", err) + e.logger.Warn("Proceeding with outdated query maps, as the Postgres version could not be determined", "err", err) } return server.Scrape(ch, e.disableSettingsMetrics) @@ -119,7 +119,7 @@ func (e *Exporter) scrapeDSN(ch chan<- prometheus.Metric, dsn string) error { // DATA_SOURCE_NAME always wins so we do not break older versions // reading secrets from files wins over secrets in environment variables // DATA_SOURCE_NAME > DATA_SOURCE_{USER|PASS}_FILE > DATA_SOURCE_{USER|PASS} -func getDataSources() ([]string, error) { +func GetDataSources() ([]string, error) { var dsn = os.Getenv("DATA_SOURCE_NAME") if len(dsn) != 0 { return strings.Split(dsn, ","), nil diff --git a/cmd/postgres_exporter/namespace.go b/exporter/namespace.go similarity index 92% rename from cmd/postgres_exporter/namespace.go rename to exporter/namespace.go index ac7a23739..5f6bd8edc 100644 --- a/cmd/postgres_exporter/namespace.go +++ b/exporter/namespace.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package exporter import ( "database/sql" @@ -129,7 +129,7 @@ func queryNamespaceMapping(server *Server, namespace string, mapping MetricMapNa nonfatalErrors = append(nonfatalErrors, errors.New(fmt.Sprintln("Missing column: ", namespace, columnName+"_sum"))) continue } - sum, ok := dbToFloat64(columnData[idx]) + sum, ok := dbToFloat64(columnData[idx], server.logger) if !ok { nonfatalErrors = append(nonfatalErrors, errors.New(fmt.Sprintln("Unexpected error parsing column: ", namespace, columnName+"_sum", columnData[idx]))) continue @@ -140,7 +140,7 @@ func queryNamespaceMapping(server *Server, namespace string, mapping MetricMapNa nonfatalErrors = append(nonfatalErrors, errors.New(fmt.Sprintln("Missing column: ", namespace, columnName+"_count"))) continue } - count, ok := dbToUint64(columnData[idx]) + count, ok := dbToUint64(columnData[idx], server.logger) if !ok { nonfatalErrors = append(nonfatalErrors, errors.New(fmt.Sprintln("Unexpected error parsing column: ", namespace, columnName+"_count", columnData[idx]))) continue @@ -152,7 +152,7 @@ func queryNamespaceMapping(server *Server, namespace string, mapping MetricMapNa labels..., ) } else { - value, ok := dbToFloat64(columnData[idx]) + value, ok := dbToFloat64(columnData[idx], server.logger) if !ok { nonfatalErrors = append(nonfatalErrors, errors.New(fmt.Sprintln("Unexpected error parsing column: ", namespace, columnName, columnData[idx]))) continue @@ -167,7 +167,7 @@ func queryNamespaceMapping(server *Server, namespace string, mapping MetricMapNa // Its not an error to fail here, since the values are // unexpected anyway. - value, ok := dbToFloat64(columnData[idx]) + value, ok := dbToFloat64(columnData[idx], server.logger) if !ok { nonfatalErrors = append(nonfatalErrors, errors.New(fmt.Sprintln("Unparseable column type - discarding: ", namespace, columnName, err))) continue @@ -189,10 +189,10 @@ func queryNamespaceMappings(ch chan<- prometheus.Metric, server *Server) map[str scrapeStart := time.Now() for namespace, mapping := range server.metricMap { - logger.Debug("Querying namespace", "namespace", namespace) + server.logger.Debug("Querying namespace", "namespace", namespace) if mapping.master && !server.master { - logger.Debug("Query skipped...") + server.logger.Debug("Query skipped...") continue } @@ -201,7 +201,7 @@ func queryNamespaceMappings(ch chan<- prometheus.Metric, server *Server) map[str serVersion, _ := semver.Parse(server.lastMapVersion.String()) runServerRange, _ := semver.ParseRange(server.runonserver) if !runServerRange(serVersion) { - logger.Debug("Query skipped for this database version", "version", server.lastMapVersion.String(), "target_version", server.runonserver) + server.logger.Debug("Query skipped for this database version", "version", server.lastMapVersion.String(), "target_version", server.runonserver) continue } } @@ -232,12 +232,12 @@ func queryNamespaceMappings(ch chan<- prometheus.Metric, server *Server) map[str // Serious error - a namespace disappeared if err != nil { namespaceErrors[namespace] = err - logger.Info("error finding namespace", "err", err) + server.logger.Info("error finding namespace", "err", err) } // Non-serious errors - likely version or parsing problems. if len(nonFatalErrors) > 0 { for _, err := range nonFatalErrors { - logger.Info("error querying namespace", "err", err) + server.logger.Info("error querying namespace", "err", err) } } diff --git a/cmd/postgres_exporter/pg_setting.go b/exporter/pg_setting.go similarity index 98% rename from cmd/postgres_exporter/pg_setting.go rename to exporter/pg_setting.go index 5b13e160f..b7adfa25f 100644 --- a/cmd/postgres_exporter/pg_setting.go +++ b/exporter/pg_setting.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package exporter import ( "fmt" @@ -31,7 +31,7 @@ var ( // Query the pg_settings view containing runtime variables func querySettings(ch chan<- prometheus.Metric, server *Server) error { - logger.Debug("Querying pg_setting view", "server", server) + server.logger.Debug("Querying pg_setting view", "server", server) // pg_settings docs: https://www.postgresql.org/docs/current/static/view-pg-settings.html // diff --git a/cmd/postgres_exporter/pg_setting_test.go b/exporter/pg_setting_test.go similarity index 99% rename from cmd/postgres_exporter/pg_setting_test.go rename to exporter/pg_setting_test.go index 9acdeaed7..7038959fd 100644 --- a/cmd/postgres_exporter/pg_setting_test.go +++ b/exporter/pg_setting_test.go @@ -13,7 +13,7 @@ //go:build !integration -package main +package exporter import ( "github.com/prometheus/client_golang/prometheus" diff --git a/cmd/postgres_exporter/postgres_exporter.go b/exporter/postgres_exporter.go similarity index 93% rename from cmd/postgres_exporter/postgres_exporter.go rename to exporter/postgres_exporter.go index a76479611..720a4667f 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/exporter/postgres_exporter.go @@ -11,13 +11,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package exporter import ( "crypto/sha256" "database/sql" "errors" "fmt" + "log/slog" "math" "os" "regexp" @@ -28,6 +29,19 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +// Metric name parts. +const ( + // Namespace for all metrics. + namespace = "pg" + // Subsystems. + exporter = "exporter" + // Metric label used for static string data thats handy to send to Prometheus + // e.g. version + staticLabelName = "static" + // Metric label used for server identification. + serverLabelName = "server" +) + // ColumnUsage should be one of several enum values which describe how a // queried row is to be converted to a Prometheus metric. type ColumnUsage int @@ -142,7 +156,7 @@ func (e *ErrorConnectToServer) Error() string { } // TODO: revisit this with the semver system -func dumpMaps() { +func DumpMaps() { // TODO: make this function part of the exporter for name, cmap := range builtinMetricMaps { query, ok := queryOverrides[name] @@ -263,13 +277,13 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ } // Turn the MetricMap column mapping into a prometheus descriptor mapping. -func makeDescMap(pgVersion semver.Version, serverLabels prometheus.Labels, metricMaps map[string]intermediateMetricMap) map[string]MetricMapNamespace { +func makeDescMap(pgVersion semver.Version, serverLabels prometheus.Labels, metricMaps map[string]intermediateMetricMap, logger *slog.Logger, metricPrefix string) map[string]MetricMapNamespace { var metricMap = make(map[string]MetricMapNamespace) for namespace, intermediateMappings := range metricMaps { thisMap := make(map[string]MetricMap) - namespace = strings.Replace(namespace, "pg", *metricPrefix, 1) + namespace = strings.Replace(namespace, "pg", metricPrefix, 1) // Get the constant labels var variableLabels []string @@ -312,7 +326,7 @@ func makeDescMap(pgVersion semver.Version, serverLabels prometheus.Labels, metri vtype: prometheus.CounterValue, desc: prometheus.NewDesc(fmt.Sprintf("%s_%s", namespace, columnName), columnMapping.description, variableLabels, serverLabels), conversion: func(in interface{}) (float64, bool) { - return dbToFloat64(in) + return dbToFloat64(in, logger) }, } case GAUGE: @@ -320,7 +334,7 @@ func makeDescMap(pgVersion semver.Version, serverLabels prometheus.Labels, metri vtype: prometheus.GaugeValue, desc: prometheus.NewDesc(fmt.Sprintf("%s_%s", namespace, columnName), columnMapping.description, variableLabels, serverLabels), conversion: func(in interface{}) (float64, bool) { - return dbToFloat64(in) + return dbToFloat64(in, logger) }, } case HISTOGRAM: @@ -329,7 +343,7 @@ func makeDescMap(pgVersion semver.Version, serverLabels prometheus.Labels, metri vtype: prometheus.UntypedValue, desc: prometheus.NewDesc(fmt.Sprintf("%s_%s", namespace, columnName), columnMapping.description, variableLabels, serverLabels), conversion: func(in interface{}) (float64, bool) { - return dbToFloat64(in) + return dbToFloat64(in, logger) }, } thisMap[columnName+"_bucket"] = MetricMap{ @@ -425,6 +439,9 @@ type Exporter struct { // servers are used to allow re-using the DB connection between scrapes. // servers contains metrics map and query overrides. servers *Servers + + logger *slog.Logger + metricPrefix string } // ExporterOpt configures Exporter. @@ -477,11 +494,18 @@ func WithUserQueriesPath(p string) ExporterOpt { // WithConstantLabels configures constant labels. func WithConstantLabels(s string) ExporterOpt { return func(e *Exporter) { - e.constantLabels = parseConstLabels(s) + e.constantLabels = parseConstLabels(s, e.logger) + } +} + +// WithMetricPrefix configures metric prefix. +func WithMetricPrefix(prefix string) ExporterOpt { + return func(e *Exporter) { + e.metricPrefix = prefix } } -func parseConstLabels(s string) prometheus.Labels { +func parseConstLabels(s string, logger *slog.Logger) prometheus.Labels { labels := make(prometheus.Labels) s = strings.TrimSpace(s) @@ -508,10 +532,11 @@ func parseConstLabels(s string) prometheus.Labels { } // NewExporter returns a new PostgreSQL exporter for the provided DSN. -func NewExporter(dsn []string, opts ...ExporterOpt) *Exporter { +func NewExporter(dsn []string, logger *slog.Logger, opts ...ExporterOpt) *Exporter { e := &Exporter{ dsn: dsn, builtinMetricMaps: builtinMetricMaps, + logger: logger, } for _, opt := range opts { @@ -519,7 +544,7 @@ func NewExporter(dsn []string, opts ...ExporterOpt) *Exporter { } e.setupInternalMetrics() - e.servers = NewServers(ServerWithLabels(e.constantLabels)) + e.servers = NewServers(ServerWithLabels(e.constantLabels), ServerWithLogger(e.logger)) return e } @@ -576,6 +601,10 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { e.userQueriesError.Collect(ch) } +func (e *Exporter) CloseServers() { + e.servers.Close() +} + func newDesc(subsystem, name, help string, labels prometheus.Labels) *prometheus.Desc { return prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, name), @@ -583,7 +612,7 @@ func newDesc(subsystem, name, help string, labels prometheus.Labels) *prometheus ) } -func checkPostgresVersion(db *sql.DB, server string) (semver.Version, string, error) { +func checkPostgresVersion(db *sql.DB, server string, logger *slog.Logger) (semver.Version, string, error) { logger.Debug("Querying PostgreSQL version", "server", server) versionRow := db.QueryRow("SELECT version();") var versionString string @@ -601,24 +630,24 @@ func checkPostgresVersion(db *sql.DB, server string) (semver.Version, string, er // Check and update the exporters query maps if the version has changed. func (e *Exporter) checkMapVersions(ch chan<- prometheus.Metric, server *Server) error { - semanticVersion, versionString, err := checkPostgresVersion(server.db, server.String()) + semanticVersion, versionString, err := checkPostgresVersion(server.db, server.String(), e.logger) if err != nil { return fmt.Errorf("Error fetching version string on %q: %v", server, err) } if !e.disableDefaultMetrics && semanticVersion.LT(lowestSupportedVersion) { - logger.Warn("PostgreSQL version is lower than our lowest supported version", "server", server, "version", semanticVersion, "lowest_supported_version", lowestSupportedVersion) + e.logger.Warn("PostgreSQL version is lower than our lowest supported version", "server", server, "version", semanticVersion, "lowest_supported_version", lowestSupportedVersion) } // Check if semantic version changed and recalculate maps if needed. if semanticVersion.NE(server.lastMapVersion) || server.metricMap == nil { - logger.Info("Semantic version changed", "server", server, "from", server.lastMapVersion, "to", semanticVersion) + e.logger.Info("Semantic version changed", "server", server, "from", server.lastMapVersion, "to", semanticVersion) server.mappingMtx.Lock() // Get Default Metrics only for master database if !e.disableDefaultMetrics && server.master { - server.metricMap = makeDescMap(semanticVersion, server.labels, e.builtinMetricMaps) - server.queryOverrides = makeQueryOverrideMap(semanticVersion, queryOverrides) + server.metricMap = makeDescMap(semanticVersion, server.labels, e.builtinMetricMaps, server.logger, e.metricPrefix) + server.queryOverrides = makeQueryOverrideMap(semanticVersion, queryOverrides, server.logger) } else { server.metricMap = make(map[string]MetricMapNamespace) server.queryOverrides = make(map[string]string) @@ -633,13 +662,13 @@ func (e *Exporter) checkMapVersions(ch chan<- prometheus.Metric, server *Server) // Calculate the hashsum of the useQueries userQueriesData, err := os.ReadFile(e.userQueriesPath) if err != nil { - logger.Error("Failed to reload user queries", "path", e.userQueriesPath, "err", err) + e.logger.Error("Failed to reload user queries", "path", e.userQueriesPath, "err", err) e.userQueriesError.WithLabelValues(e.userQueriesPath, "").Set(1) } else { hashsumStr := fmt.Sprintf("%x", sha256.Sum256(userQueriesData)) - if err := addQueries(userQueriesData, semanticVersion, server); err != nil { - logger.Error("Failed to reload user queries", "path", e.userQueriesPath, "err", err) + if err := addQueries(userQueriesData, semanticVersion, server, e.metricPrefix); err != nil { + e.logger.Error("Failed to reload user queries", "path", e.userQueriesPath, "err", err) e.userQueriesError.WithLabelValues(e.userQueriesPath, hashsumStr).Set(1) } else { // Mark user queries as successfully loaded @@ -681,7 +710,7 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric) { if err := e.scrapeDSN(ch, dsn); err != nil { errorsCount++ - logger.Error("error scraping dsn", "err", err, "dsn", loggableDSN(dsn)) + e.logger.Error("error scraping dsn", "err", err, "dsn", loggableDSN(dsn)) if _, ok := err.(*ErrorConnectToServer); ok { connectionErrorsCount++ diff --git a/cmd/postgres_exporter/postgres_exporter_integration_test.go b/exporter/postgres_exporter_integration_test.go similarity index 92% rename from cmd/postgres_exporter/postgres_exporter_integration_test.go rename to exporter/postgres_exporter_integration_test.go index 39032f333..a65dbdaab 100644 --- a/cmd/postgres_exporter/postgres_exporter_integration_test.go +++ b/exporter/postgres_exporter_integration_test.go @@ -16,7 +16,7 @@ // working. //go:build integration -package main +package exporter import ( "fmt" @@ -26,6 +26,7 @@ import ( _ "github.com/lib/pq" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/promslog" . "gopkg.in/check.v1" ) @@ -42,7 +43,7 @@ func (s *IntegrationSuite) SetUpSuite(c *C) { dsn := os.Getenv("DATA_SOURCE_NAME") c.Assert(dsn, Not(Equals), "") - exporter := NewExporter(strings.Split(dsn, ",")) + exporter := NewExporter(strings.Split(dsn, ","), promslog.NewNopLogger()) c.Assert(exporter, NotNil) // Assign the exporter to the suite s.e = exporter @@ -99,12 +100,12 @@ func (s *IntegrationSuite) TestInvalidDsnDoesntCrash(c *C) { }() // Send a bad DSN - exporter := NewExporter([]string{"invalid dsn"}) + exporter := NewExporter([]string{"invalid dsn"}, promslog.NewNopLogger()) c.Assert(exporter, NotNil) exporter.scrape(ch) // Send a DSN to a non-listening port. - exporter = NewExporter([]string{"postgresql://nothing:nothing@127.0.0.1:1/nothing"}) + exporter = NewExporter([]string{"postgresql://nothing:nothing@127.0.0.1:1/nothing"}, promslog.NewNopLogger()) c.Assert(exporter, NotNil) exporter.scrape(ch) } @@ -122,7 +123,7 @@ func (s *IntegrationSuite) TestUnknownMetricParsingDoesntCrash(c *C) { dsn := os.Getenv("DATA_SOURCE_NAME") c.Assert(dsn, Not(Equals), "") - exporter := NewExporter(strings.Split(dsn, ",")) + exporter := NewExporter(strings.Split(dsn, ","), promslog.NewNopLogger()) c.Assert(exporter, NotNil) // Convert the default maps into a list of empty maps. @@ -155,6 +156,7 @@ func (s *IntegrationSuite) TestExtendQueriesDoesntCrash(c *C) { exporter := NewExporter( strings.Split(dsn, ","), + promslog.NewNopLogger(), WithUserQueriesPath("../user_queries_test.yaml"), ) c.Assert(exporter, NotNil) @@ -168,6 +170,7 @@ func (s *IntegrationSuite) TestAutoDiscoverDatabases(c *C) { exporter := NewExporter( strings.Split(dsn, ","), + promslog.NewNopLogger(), ) c.Assert(exporter, NotNil) diff --git a/cmd/postgres_exporter/postgres_exporter_test.go b/exporter/postgres_exporter_test.go similarity index 95% rename from cmd/postgres_exporter/postgres_exporter_test.go rename to exporter/postgres_exporter_test.go index 5b2879d94..6364fc4f9 100644 --- a/cmd/postgres_exporter/postgres_exporter_test.go +++ b/exporter/postgres_exporter_test.go @@ -13,7 +13,7 @@ //go:build !integration -package main +package exporter import ( "math" @@ -24,6 +24,7 @@ import ( "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/promslog" . "gopkg.in/check.v1" ) @@ -36,7 +37,6 @@ type FunctionalSuite struct { var _ = Suite(&FunctionalSuite{}) func (s *FunctionalSuite) SetUpSuite(c *C) { - } func (s *FunctionalSuite) TestSemanticVersionColumnDiscard(c *C) { @@ -53,7 +53,7 @@ func (s *FunctionalSuite) TestSemanticVersionColumnDiscard(c *C) { { // No metrics should be eliminated - resultMap := makeDescMap(semver.MustParse("0.0.1"), prometheus.Labels{}, testMetricMap) + resultMap := makeDescMap(semver.MustParse("0.0.1"), prometheus.Labels{}, testMetricMap, promslog.NewNopLogger(), "pg") c.Check( resultMap["test_namespace"].columnMappings["metric_which_stays"].discard, Equals, @@ -74,7 +74,7 @@ func (s *FunctionalSuite) TestSemanticVersionColumnDiscard(c *C) { testMetricMap["test_namespace"].columnMappings["metric_which_discards"] = discardableMetric // Discard metric should be discarded - resultMap := makeDescMap(semver.MustParse("0.0.1"), prometheus.Labels{}, testMetricMap) + resultMap := makeDescMap(semver.MustParse("0.0.1"), prometheus.Labels{}, testMetricMap, promslog.NewNopLogger(), "pg") c.Check( resultMap["test_namespace"].columnMappings["metric_which_stays"].discard, Equals, @@ -95,7 +95,7 @@ func (s *FunctionalSuite) TestSemanticVersionColumnDiscard(c *C) { testMetricMap["test_namespace"].columnMappings["metric_which_discards"] = discardableMetric // Discard metric should be discarded - resultMap := makeDescMap(semver.MustParse("0.0.2"), prometheus.Labels{}, testMetricMap) + resultMap := makeDescMap(semver.MustParse("0.0.2"), prometheus.Labels{}, testMetricMap, promslog.NewNopLogger(), "pg") c.Check( resultMap["test_namespace"].columnMappings["metric_which_stays"].discard, Equals, @@ -125,7 +125,7 @@ func (s *FunctionalSuite) TestEnvironmentSettingWithSecretsFiles(c *C) { var expected = "postgresql://custom_username$&+,%2F%3A;=%3F%40:custom_password$&+,%2F%3A;=%3F%40@localhost:5432/?sslmode=disable" - dsn, err := getDataSources() + dsn, err := GetDataSources() if err != nil { c.Errorf("Unexpected error reading datasources") } @@ -145,7 +145,7 @@ func (s *FunctionalSuite) TestEnvironmentSettingWithDns(c *C) { c.Assert(err, IsNil) defer UnsetEnvironment(c, "DATA_SOURCE_NAME") - dsn, err := getDataSources() + dsn, err := GetDataSources() if err != nil { c.Errorf("Unexpected error reading datasources") } @@ -173,7 +173,7 @@ func (s *FunctionalSuite) TestEnvironmentSettingWithDnsAndSecrets(c *C) { c.Assert(err, IsNil) defer UnsetEnvironment(c, "DATA_SOURCE_PASS") - dsn, err := getDataSources() + dsn, err := GetDataSources() if err != nil { c.Errorf("Unexpected error reading datasources") } @@ -289,7 +289,7 @@ func (s *FunctionalSuite) TestParseConstLabels(c *C) { } for _, cs := range cases { - labels := parseConstLabels(cs.s) + labels := parseConstLabels(cs.s, promslog.NewNopLogger()) if !reflect.DeepEqual(labels, cs.labels) { c.Fatalf("labels not equal (%v -> %v)", labels, cs.labels) } @@ -319,7 +319,6 @@ func (checker *isNaNChecker) Check(params []interface{}, names []string) (result // test boolean metric type gets converted to float func (s *FunctionalSuite) TestBooleanConversionToValueAndString(c *C) { - type TestCase struct { input interface{} expectedString string @@ -388,7 +387,7 @@ func (s *FunctionalSuite) TestBooleanConversionToValueAndString(c *C) { } for _, cs := range cases { - value, ok := dbToFloat64(cs.input) + value, ok := dbToFloat64(cs.input, promslog.NewNopLogger()) if math.IsNaN(cs.expectedValue) { c.Assert(value, IsNaN) } else { @@ -396,7 +395,7 @@ func (s *FunctionalSuite) TestBooleanConversionToValueAndString(c *C) { } c.Assert(ok, Equals, cs.expectedOK) - count, ok := dbToUint64(cs.input) + count, ok := dbToUint64(cs.input, promslog.NewNopLogger()) c.Assert(count, Equals, cs.expectedCount) c.Assert(ok, Equals, cs.expectedOK) @@ -409,7 +408,7 @@ func (s *FunctionalSuite) TestBooleanConversionToValueAndString(c *C) { func (s *FunctionalSuite) TestParseUserQueries(c *C) { userQueriesData, err := os.ReadFile("./tests/user_queries_ok.yaml") if err == nil { - metricMaps, newQueryOverrides, err := parseUserQueries(userQueriesData) + metricMaps, newQueryOverrides, err := parseUserQueries(userQueriesData, promslog.NewNopLogger()) c.Assert(err, Equals, nil) c.Assert(metricMaps, NotNil) c.Assert(newQueryOverrides, NotNil) diff --git a/cmd/postgres_exporter/queries.go b/exporter/queries.go similarity index 92% rename from cmd/postgres_exporter/queries.go rename to exporter/queries.go index c0081fe02..b0f64f657 100644 --- a/cmd/postgres_exporter/queries.go +++ b/exporter/queries.go @@ -11,11 +11,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package exporter import ( "errors" "fmt" + "log/slog" "github.com/blang/semver/v4" "gopkg.in/yaml.v2" @@ -167,7 +168,7 @@ var queryOverrides = map[string][]OverrideQuery{ // Convert the query override file to the version-specific query override file // for the exporter. -func makeQueryOverrideMap(pgVersion semver.Version, queryOverrides map[string][]OverrideQuery) map[string]string { +func makeQueryOverrideMap(pgVersion semver.Version, queryOverrides map[string][]OverrideQuery, logger *slog.Logger) map[string]string { resultMap := make(map[string]string) for name, overrideDef := range queryOverrides { // Find a matching semver. We make it an error to have overlapping @@ -189,7 +190,7 @@ func makeQueryOverrideMap(pgVersion semver.Version, queryOverrides map[string][] return resultMap } -func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[string]string, error) { +func parseUserQueries(content []byte, logger *slog.Logger) (map[string]intermediateMetricMap, map[string]string, error) { var userQueries UserQueries err := yaml.Unmarshal(content, &userQueries) @@ -242,21 +243,21 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str // queries. // TODO: test code for all cu. // TODO: the YAML this supports is "non-standard" - we should move away from it. -func addQueries(content []byte, pgVersion semver.Version, server *Server) error { - metricMaps, newQueryOverrides, err := parseUserQueries(content) +func addQueries(content []byte, pgVersion semver.Version, server *Server, metricPrefix string) error { + metricMaps, newQueryOverrides, err := parseUserQueries(content, server.logger) if err != nil { return err } // Convert the loaded metric map into exporter representation - partialExporterMap := makeDescMap(pgVersion, server.labels, metricMaps) + partialExporterMap := makeDescMap(pgVersion, server.labels, metricMaps, server.logger, metricPrefix) // Merge the two maps (which are now quite flatteend) for k, v := range partialExporterMap { _, found := server.metricMap[k] if found { - logger.Debug("Overriding metric from user YAML file", "metric", k) + server.logger.Debug("Overriding metric from user YAML file", "metric", k) } else { - logger.Debug("Adding new metric from user YAML file", "metric", k) + server.logger.Debug("Adding new metric from user YAML file", "metric", k) } server.metricMap[k] = v } @@ -265,9 +266,9 @@ func addQueries(content []byte, pgVersion semver.Version, server *Server) error for k, v := range newQueryOverrides { _, found := server.queryOverrides[k] if found { - logger.Debug("Overriding query override from user YAML file", "query_override", k) + server.logger.Debug("Overriding query override from user YAML file", "query_override", k) } else { - logger.Debug("Adding new query override from user YAML file", "query_override", k) + server.logger.Debug("Adding new query override from user YAML file", "query_override", k) } server.queryOverrides[k] = v } diff --git a/cmd/postgres_exporter/server.go b/exporter/server.go similarity index 89% rename from cmd/postgres_exporter/server.go rename to exporter/server.go index a90183cd7..2aa2c6d67 100644 --- a/cmd/postgres_exporter/server.go +++ b/exporter/server.go @@ -11,16 +11,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package exporter import ( "database/sql" "fmt" + "log/slog" "sync" "time" "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/promslog" ) // Server describes a connection to Postgres. @@ -42,6 +44,7 @@ type Server struct { // Currently cached metrics metricCache map[string]cachedMetrics cacheMtx sync.Mutex + logger *slog.Logger } // ServerOpt configures a server. @@ -56,6 +59,12 @@ func ServerWithLabels(labels prometheus.Labels) ServerOpt { } } +func ServerWithLogger(logger *slog.Logger) ServerOpt { + return func(s *Server) { + s.logger = logger + } +} + // NewServer establishes a new connection using DSN. func NewServer(dsn string, opts ...ServerOpt) (*Server, error) { fingerprint, err := parseFingerprint(dsn) @@ -70,8 +79,6 @@ func NewServer(dsn string, opts ...ServerOpt) (*Server, error) { db.SetMaxOpenConns(1) db.SetMaxIdleConns(1) - logger.Info("Established new database connection", "fingerprint", fingerprint) - s := &Server{ db: db, master: false, @@ -79,12 +86,15 @@ func NewServer(dsn string, opts ...ServerOpt) (*Server, error) { serverLabelName: fingerprint, }, metricCache: make(map[string]cachedMetrics), + logger: promslog.NewNopLogger(), } for _, opt := range opts { opt(s) } + s.logger.Info("Established new database connection", "fingerprint", fingerprint) + return s, nil } @@ -97,7 +107,7 @@ func (s *Server) Close() error { func (s *Server) Ping() error { if err := s.db.Ping(); err != nil { if cerr := s.Close(); cerr != nil { - logger.Error("Error while closing non-pinging DB connection", "server", s, "err", cerr) + s.logger.Error("Error while closing non-pinging DB connection", "server", s, "err", cerr) } return err } @@ -189,7 +199,7 @@ func (s *Servers) Close() { defer s.m.Unlock() for _, server := range s.servers { if err := server.Close(); err != nil { - logger.Error("Failed to close connection", "server", server, "err", err) + server.logger.Error("Failed to close connection", "server", server, "err", err) } } } diff --git a/cmd/postgres_exporter/tests/docker-postgres-replication/Dockerfile b/exporter/tests/docker-postgres-replication/Dockerfile similarity index 100% rename from cmd/postgres_exporter/tests/docker-postgres-replication/Dockerfile rename to exporter/tests/docker-postgres-replication/Dockerfile diff --git a/cmd/postgres_exporter/tests/docker-postgres-replication/Dockerfile.p2 b/exporter/tests/docker-postgres-replication/Dockerfile.p2 similarity index 100% rename from cmd/postgres_exporter/tests/docker-postgres-replication/Dockerfile.p2 rename to exporter/tests/docker-postgres-replication/Dockerfile.p2 diff --git a/cmd/postgres_exporter/tests/docker-postgres-replication/README.md b/exporter/tests/docker-postgres-replication/README.md similarity index 100% rename from cmd/postgres_exporter/tests/docker-postgres-replication/README.md rename to exporter/tests/docker-postgres-replication/README.md diff --git a/cmd/postgres_exporter/tests/docker-postgres-replication/docker-compose.yml b/exporter/tests/docker-postgres-replication/docker-compose.yml similarity index 100% rename from cmd/postgres_exporter/tests/docker-postgres-replication/docker-compose.yml rename to exporter/tests/docker-postgres-replication/docker-compose.yml diff --git a/cmd/postgres_exporter/tests/docker-postgres-replication/docker-entrypoint.sh b/exporter/tests/docker-postgres-replication/docker-entrypoint.sh similarity index 100% rename from cmd/postgres_exporter/tests/docker-postgres-replication/docker-entrypoint.sh rename to exporter/tests/docker-postgres-replication/docker-entrypoint.sh diff --git a/cmd/postgres_exporter/tests/docker-postgres-replication/setup-replication.sh b/exporter/tests/docker-postgres-replication/setup-replication.sh similarity index 100% rename from cmd/postgres_exporter/tests/docker-postgres-replication/setup-replication.sh rename to exporter/tests/docker-postgres-replication/setup-replication.sh diff --git a/cmd/postgres_exporter/tests/test-smoke b/exporter/tests/test-smoke similarity index 100% rename from cmd/postgres_exporter/tests/test-smoke rename to exporter/tests/test-smoke diff --git a/cmd/postgres_exporter/tests/user_queries_ok.yaml b/exporter/tests/user_queries_ok.yaml similarity index 100% rename from cmd/postgres_exporter/tests/user_queries_ok.yaml rename to exporter/tests/user_queries_ok.yaml diff --git a/cmd/postgres_exporter/tests/user_queries_test.yaml b/exporter/tests/user_queries_test.yaml similarity index 100% rename from cmd/postgres_exporter/tests/user_queries_test.yaml rename to exporter/tests/user_queries_test.yaml diff --git a/cmd/postgres_exporter/tests/username_file b/exporter/tests/username_file similarity index 100% rename from cmd/postgres_exporter/tests/username_file rename to exporter/tests/username_file diff --git a/cmd/postgres_exporter/tests/userpass_file b/exporter/tests/userpass_file similarity index 100% rename from cmd/postgres_exporter/tests/userpass_file rename to exporter/tests/userpass_file diff --git a/cmd/postgres_exporter/util.go b/exporter/util.go similarity index 96% rename from cmd/postgres_exporter/util.go rename to exporter/util.go index fa692c652..895c2d07e 100644 --- a/cmd/postgres_exporter/util.go +++ b/exporter/util.go @@ -11,10 +11,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package exporter import ( "fmt" + "log/slog" "math" "net/url" "strconv" @@ -59,7 +60,7 @@ func stringToColumnUsage(s string) (ColumnUsage, error) { // Convert database.sql types to float64s for Prometheus consumption. Null types are mapped to NaN. string and []byte // types are mapped as NaN and !ok -func dbToFloat64(t interface{}) (float64, bool) { +func dbToFloat64(t interface{}, logger *slog.Logger) (float64, bool) { switch v := t.(type) { case int64: return float64(v), true @@ -97,7 +98,7 @@ func dbToFloat64(t interface{}) (float64, bool) { // Convert database.sql types to uint64 for Prometheus consumption. Null types are mapped to 0. string and []byte // types are mapped as 0 and !ok -func dbToUint64(t interface{}) (uint64, bool) { +func dbToUint64(t interface{}, logger *slog.Logger) (uint64, bool) { switch v := t.(type) { case uint64: return v, true