Merge pull request #232 from msvticket/tagselection

feat: select db instances by tags

Author: qfritz

README.md

@@ -172,24 +172,25 @@ Prometheus RDS exporter</br>
 
 Configuration could be defined in [prometheus-rds-exporter.yaml](https://github.com/qonto/prometheus-rds-exporter/blob/main/configs/prometheus-rds-exporter/prometheus-rds-exporter.yaml) or environment variables (format `PROMETHEUS_RDS_EXPORTER_<PARAMETER_NAME>`).
 
-| Parameter | Description | Default |
-| --- | --- | --- |
-| aws-assume-role-arn | AWS IAM ARN role to assume to fetch metrics | |
-| aws-assume-role-session | AWS assume role session name | prometheus-rds-exporter |
-| collect-instance-metrics | Collect AWS instances metrics (AWS Cloudwatch API) | true |
-| collect-instance-tags | Collect AWS RDS tags | true |
-| collect-instance-types | Collect AWS instance types information (AWS EC2 API) | true |
-| collect-logs-size | Collect AWS instances logs size (AWS RDS API) | true |
-| collect-maintenances | Collect AWS instances maintenances (AWS RDS API) | true |
-| collect-quotas | Collect AWS RDS quotas (AWS quotas API) | true |
-| collect-usages | Collect AWS RDS usages (AWS Cloudwatch API) | true |
-| debug | Enable debug mode | |
-| enable-otel-traces | Enable OpenTelemetry traces. See [configuration](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/) | false |
-| listen-address | Address to listen on for web interface | :9043 |
-| log-format | Log format (`text` or `json`) | json |
-| metrics-path | Path under which to expose metrics | /metrics |
-| tls-cert-path | Path to TLS certificate | |
-| tls-key-path | Path to private key for TLS |
+| Parameter                | Description                                                                                                                                                                                                                       | Default                 |
+|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|
+| aws-assume-role-arn      | AWS IAM ARN role to assume to fetch metrics                                                                                                                                                                                       |                         |
+| aws-assume-role-session  | AWS assume role session name                                                                                                                                                                                                      | prometheus-rds-exporter |
+| collect-instance-metrics | Collect AWS instances metrics (AWS Cloudwatch API)                                                                                                                                                                                | true                    |
+| collect-instance-tags    | Collect AWS RDS tags                                                                                                                                                                                                              | true                    |
+| collect-instance-types   | Collect AWS instance types information (AWS EC2 API)                                                                                                                                                                              | true                    |
+| collect-logs-size        | Collect AWS instances logs size (AWS RDS API)                                                                                                                                                                                     | true                    |
+| collect-maintenances     | Collect AWS instances maintenances (AWS RDS API)                                                                                                                                                                                  | true                    |
+| collect-quotas           | Collect AWS RDS quotas (AWS quotas API)                                                                                                                                                                                           | true                    |
+| collect-usages           | Collect AWS RDS usages (AWS Cloudwatch API)                                                                                                                                                                                       | true                    |
+| tag-selections           | Tags to select database instances with. See [configuration on TagFilters field](https://docs.aws.amazon.com/resourcegroupstagging/latest/APIReference/API_GetResources.html#resourcegrouptagging-GetResources-request-TagFilters) |                         |
+| debug                    | Enable debug mode                                                                                                                                                                                                                 |                         |
+| enable-otel-traces       | Enable OpenTelemetry traces. See [configuration](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/)                                                                                                        | false                   |
+| listen-address           | Address to listen on for web interface                                                                                                                                                                                            | :9043                   |
+| log-format               | Log format (`text` or `json`)                                                                                                                                                                                                     | json                    |
+| metrics-path             | Path under which to expose metrics                                                                                                                                                                                                | /metrics                |
+| tls-cert-path            | Path to TLS certificate                                                                                                                                                                                                           |                         |
+| tls-key-path             | Path to private key for TLS                                                                                                                                                                                                       |                         |
 
 Configuration parameters priorities:
 

cmd/root.go

@@ -7,17 +7,23 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi"
+
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	"github.com/aws/aws-sdk-go-v2/service/rds"
 	"github.com/aws/aws-sdk-go-v2/service/servicequotas"
+	"github.com/knadh/koanf/parsers/yaml"
+	"github.com/knadh/koanf/providers/env"
+	"github.com/knadh/koanf/providers/file"
+	"github.com/knadh/koanf/providers/posflag"
+	"github.com/knadh/koanf/v2"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/qonto/prometheus-rds-exporter/internal/app/exporter"
 	"github.com/qonto/prometheus-rds-exporter/internal/infra/build"
 	"github.com/qonto/prometheus-rds-exporter/internal/infra/http"
 	"github.com/qonto/prometheus-rds-exporter/internal/infra/logger"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 const (
@@ -27,25 +33,29 @@ const (
 	awsErrorExitCode      = 4
 )
 
-var cfgFile string
+var (
+	cfgFile string
+	k       = koanf.New(".")
+)
 
 type exporterConfig struct {
-	Debug                  bool   `mapstructure:"debug"`
-	LogFormat              string `mapstructure:"log-format"`
-	TLSCertPath            string `mapstructure:"tls-cert-path"`
-	TLSKeyPath             string `mapstructure:"tls-key-path"`
-	MetricPath             string `mapstructure:"metrics-path"`
-	ListenAddress          string `mapstructure:"listen-address"`
-	AWSAssumeRoleSession   string `mapstructure:"aws-assume-role-session"`
-	AWSAssumeRoleArn       string `mapstructure:"aws-assume-role-arn"`
-	CollectInstanceMetrics bool   `mapstructure:"collect-instance-metrics"`
-	CollectInstanceTags    bool   `mapstructure:"collect-instance-tags"`
-	CollectInstanceTypes   bool   `mapstructure:"collect-instance-types"`
-	CollectLogsSize        bool   `mapstructure:"collect-logs-size"`
-	CollectMaintenances    bool   `mapstructure:"collect-maintenances"`
-	CollectQuotas          bool   `mapstructure:"collect-quotas"`
-	CollectUsages          bool   `mapstructure:"collect-usages"`
-	OTELTracesEnabled      bool   `mapstructure:"enable-otel-traces"`
+	Debug                  bool                `koanf:"debug"`
+	LogFormat              string              `koanf:"log-format"`
+	TLSCertPath            string              `koanf:"tls-cert-path"`
+	TLSKeyPath             string              `koanf:"tls-key-path"`
+	MetricPath             string              `koanf:"metrics-path"`
+	ListenAddress          string              `koanf:"listen-address"`
+	AWSAssumeRoleSession   string              `koanf:"aws-assume-role-session"`
+	AWSAssumeRoleArn       string              `koanf:"aws-assume-role-arn"`
+	CollectInstanceMetrics bool                `koanf:"collect-instance-metrics"`
+	CollectInstanceTags    bool                `koanf:"collect-instance-tags"`
+	CollectInstanceTypes   bool                `koanf:"collect-instance-types"`
+	CollectLogsSize        bool                `koanf:"collect-logs-size"`
+	CollectMaintenances    bool                `koanf:"collect-maintenances"`
+	CollectQuotas          bool                `koanf:"collect-quotas"`
+	CollectUsages          bool                `koanf:"collect-usages"`
+	OTELTracesEnabled      bool                `koanf:"enable-otel-traces"`
+	TagSelections          map[string][]string `koanf:"tag-selections"`
 }
 
 func run(configuration exporterConfig) {
@@ -55,6 +65,8 @@ func run(configuration exporterConfig) {
 		panic(err)
 	}
 
+	logger.Debug(fmt.Sprintf("Config: %+v\n", configuration))
+
 	cfg, err := getAWSConfiguration(logger, configuration.AWSAssumeRoleArn, configuration.AWSAssumeRoleSession)
 	if err != nil {
 		logger.Error("can't initialize AWS configuration", "reason", err)
@@ -68,6 +80,13 @@ func run(configuration exporterConfig) {
 	}
 
 	rdsClient := rds.NewFromConfig(cfg)
+
+	var tagClient *resourcegroupstaggingapi.Client
+
+	if configuration.TagSelections != nil {
+		tagClient = resourcegroupstaggingapi.NewFromConfig(cfg)
+	}
+
 	ec2Client := ec2.NewFromConfig(cfg)
 	cloudWatchClient := cloudwatch.NewFromConfig(cfg)
 	servicequotasClient := servicequotas.NewFromConfig(cfg)
@@ -80,9 +99,10 @@ func run(configuration exporterConfig) {
 		CollectMaintenances:    configuration.CollectMaintenances,
 		CollectQuotas:          configuration.CollectQuotas,
 		CollectUsages:          configuration.CollectUsages,
+		TagSelections:          configuration.TagSelections,
 	}
 
-	collector := exporter.NewCollector(*logger, collectorConfiguration, awsAccountID, awsRegion, rdsClient, ec2Client, cloudWatchClient, servicequotasClient)
+	collector := exporter.NewCollector(*logger, collectorConfiguration, awsAccountID, awsRegion, rdsClient, ec2Client, cloudWatchClient, servicequotasClient, tagClient)
 
 	prometheus.MustRegister(collector)
 
@@ -111,10 +131,16 @@ func NewRootCommand() (*cobra.Command, error) {
 		Long: `Collect AWS RDS key metrics from AWS APIs
 	and expose them as Prometheus metrics.`,
 		Run: func(cmd *cobra.Command, args []string) {
-			var c exporterConfig
-			err := viper.Unmarshal(&c)
+			err := k.Load(posflag.Provider(cmd.Flags(), ".", k), nil)
 			if err != nil {
-				fmt.Println("ERROR: Unable to decode configuration, %w", err)
+				fmt.Printf("ERROR: Unable to interpret flags, %v\n", err)
+
+				return
+			}
+
+			var c exporterConfig
+			if err := k.Unmarshal("", &c); err != nil {
+				fmt.Printf("ERROR: Unable to decode configuration, %v\n", err)
 
 				return
 			}
@@ -142,86 +168,6 @@ func NewRootCommand() (*cobra.Command, error) {
 	cmd.Flags().BoolP("collect-quotas", "", true, "Collect AWS RDS quotas")
 	cmd.Flags().BoolP("collect-usages", "", true, "Collect AWS RDS usages")
 
-	err := viper.BindPFlag("debug", cmd.Flags().Lookup("debug"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'debug' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("log-format", cmd.Flags().Lookup("log-format"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'log-format' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("enable-otel-traces", cmd.Flags().Lookup("enable-otel-traces"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'enable-otel-traces' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("metrics-path", cmd.Flags().Lookup("metrics-path"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'metrics-path' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("tls-cert-path", cmd.Flags().Lookup("tls-cert-path"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'tls-cert-path' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("tls-key-path", cmd.Flags().Lookup("tls-key-path"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'tls-key-path' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("listen-address", cmd.Flags().Lookup("listen-address"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'listen-address' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("aws-assume-role-arn", cmd.Flags().Lookup("aws-assume-role-arn"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'aws-assume-role-arn' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("aws-assume-role-session", cmd.Flags().Lookup("aws-assume-role-session"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'aws-assume-role-session' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("collect-instance-metrics", cmd.Flags().Lookup("collect-instance-metrics"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'collect-instance-metrics' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("collect-instance-tags", cmd.Flags().Lookup("collect-instance-tags"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'collect-instance-tags' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("collect-instance-types", cmd.Flags().Lookup("collect-instance-types"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'collect-instance-types' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("collect-quotas", cmd.Flags().Lookup("collect-quotas"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'collect-quotas' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("collect-usages", cmd.Flags().Lookup("collect-usages"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'collect-usages' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("collect-logs-size", cmd.Flags().Lookup("collect-logs-size"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'collect-logs-size' parameter: %w", err)
-	}
-
-	err = viper.BindPFlag("collect-maintenances", cmd.Flags().Lookup("collect-maintenances"))
-	if err != nil {
-		return cmd, fmt.Errorf("failed to bind 'collect-maintenances' parameter: %w", err)
-	}
-
 	return cmd, nil
 }
 
@@ -243,32 +189,30 @@ func Execute() {
 func initConfig() {
 	if cfgFile != "" {
 		// Use config file from the flag.
-		viper.SetConfigFile(cfgFile)
+		err := k.Load(file.Provider(cfgFile), yaml.Parser())
+		cobra.CheckErr(err)
 	} else {
-		// Find home directory
+		// Find home directory.
 		home, err := os.UserHomeDir()
 		cobra.CheckErr(err)
 
-		// Search config in home directory or current directory with name "prometheus-rds-exporter.yaml"
-
+		// Search config in home directory or current directory with name "prometheus-rds-exporter.yaml".
 		configurationFilename := "prometheus-rds-exporter.yaml"
 		currentPathFilename := configurationFilename
 		homeFilename := filepath.Join(home, configurationFilename)
 
-		if _, err := os.Stat(homeFilename); err == nil {
-			viper.SetConfigFile(homeFilename)
+		if err := k.Load(file.Provider(homeFilename), yaml.Parser()); err == nil {
+			fmt.Printf("Using config file: %s\n", homeFilename)
 		}
 
-		if _, err := os.Stat(currentPathFilename); err == nil {
-			viper.SetConfigFile(currentPathFilename)
+		if err := k.Load(file.Provider(currentPathFilename), yaml.Parser()); err == nil {
+			fmt.Printf("Using config file: %s\n", currentPathFilename)
 		}
 	}
 
-	if err := viper.ReadInConfig(); err == nil {
-		fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
-	}
-
-	viper.SetEnvPrefix("prometheus_rds_exporter") // will be uppercased automatically
-	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
-	viper.AutomaticEnv()
+	// Set environment variables.
+	err := k.Load(env.Provider("PROMETHEUS_RDS_EXPORTER_", ".", func(s string) string {
+		return strings.ReplaceAll(strings.ToLower(strings.TrimPrefix(s, "PROMETHEUS_RDS_EXPORTER_")), "_", ".")
+	}), nil)
+	cobra.CheckErr(err)
 }

configs/aws/policy.json

@@ -1,6 +1,14 @@
 {
     "Version": "2012-10-17",
     "Statement": [
+        {
+            "Sid": "AllowSearchDBByTags",
+            "Effect": "Allow",
+            "Action": [
+                "tag:GetResources"
+            ],
+            "Resource": "*"
+        },
         {
             "Sid": "AllowInstanceAndLogDescriptions",
             "Effect": "Allow",

configs/helm/templates/configmap.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.config }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "prometheus-rds-exporter.fullname" . }}
+  labels:
+    {{- include "prometheus-rds-exporter.labels" . | nindent 4 }}
+data:
+  prometheus-rds-exporter.yaml: | {{ toYaml .Values.config | nindent 4 }}
+{{- end }}

configs/helm/templates/deployment.yaml

@@ -63,6 +63,14 @@ spec:
               port: http
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          {{- if .Values.config }}
+          args:
+          - --config
+          - /config/prometheus-rds-exporter.yaml
+          volumeMounts:
+            - mountPath: /config
+              name: config
+          {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -75,3 +83,10 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- if .Values.config }}
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "prometheus-rds-exporter.fullname" . }}
+      {{- end }}
+

configs/helm/values.yaml

@@ -11,6 +11,9 @@ image:
 # Define environment variables
 env: {}
 
+# Application configuration. If set a configmap will be created. For example values see https://github.com/qonto/prometheus-rds-exporter/blob/main/configs/prometheus-rds-exporter/prometheus-rds-exporter.yaml
+config: {}
+
 # Name of the secret containing AWS credentials
 awsCredentialsSecret: ""
 

configs/prometheus-rds-exporter/prometheus-rds-exporter.yaml

@@ -58,3 +58,8 @@
 
 # Collect AWS RDS usages (AWS Cloudwatch API)
 # collect-usages: true
+
+# Select AWS instances by tags. See https://docs.aws.amazon.com/resourcegroupstagging/latest/APIReference/API_GetResources.html#resourcegrouptagging-GetResources-request-TagFilters
+# tag-selections:
+#   Environment:
+#   - staging

docs/add-configuration-parameter.md

@@ -10,7 +10,6 @@ Steps:
 
     1. Add a new field in `exporterConfig` structure in `cmd/root.go`
     1. Add the parameter in flag `cmd.Flags()`
-    1. Add the parameter in viper `viper.BindPFlag()`
 
 1. Add tests