Add history db sqlite format support.

gpbackup functionality of the `--gpbackup.history-file` flag has been expanded. Now it can be specified as `gpbackup_history.db` (sqllite format) or `gpbackup_history.yaml` (yaml format). If the file extension does not match, an error is returned and the exporter does not start.

In the future, the yaml format will be abandoned (see how gpbackup develops).
So this check will become irrelevant.

Author: woblerr

Diff for: gpbackup_exporter.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -36,7 +37,7 @@ func main() {
 		).Default("0").Int()
 		gpbckpHistoryFilePath = kingpin.Flag(
 			"gpbackup.history-file",
-			"Path to gpbackup_history.yaml.",
+			"Path to gpbackup_history.db or gpbackup_history.yaml.",
 		).Default("").String()
 		gpbckpIncludeDB = kingpin.Flag(
 			"gpbackup.db-include",
@@ -78,6 +79,9 @@ func main() {
 		"msg", "Starting exporter",
 		"name", filepath.Base(os.Args[0]),
 		"version", version)
+	level.Info(logger).Log(
+		"mgs", "History database file path",
+		"file", *gpbckpHistoryFilePath)
 	if *collectionDepth > 0 {
 		level.Info(logger).Log(
 			"mgs", "Metrics depth collection in days",
@@ -109,6 +113,14 @@ func main() {
 		"endpoint", *webPath,
 		"config.file", *webAdditionalToolkitFlags.WebConfigFile,
 	)
+	// Check history file type.
+	// This code will become not actual after full switch to sqlite history file format
+	historyDB, err := checkHistoryFileType(*gpbckpHistoryFilePath)
+	if err != nil {
+		level.Error(logger).Log("msg", "Check gpbackup history file format failed", "err", err)
+		// Immediately exit.
+		os.Exit(1)
+	}
 	// Start exporter.
 	gpbckpexporter.StartPromEndpoint(logger)
 	// Set up exporter info metric.
@@ -121,9 +133,32 @@ func main() {
 			*gpbckpIncludeDB,
 			*gpbckpExcludeDB,
 			*collectionDepth,
+			historyDB,
 			logger,
 		)
 		// Sleep for 'collection.interval' seconds.
 		time.Sleep(time.Duration(*collectionInterval) * time.Second)
 	}
 }
+
+// Check history db file extension is:
+//   - gpbackup_history.db (sqlite, after gpbackup 1.29.0);
+//   - gpbackup_history.yaml (before gpbackup 1.29.0);
+//
+// Returns:
+//   - true, if file extension is db;
+//   - false, if file extension is yaml.
+//   - error, in other cases.
+//
+// Check only file extension, not content or name.
+func checkHistoryFileType(filePath string) (bool, error) {
+	hFileExt := filepath.Ext(filePath)
+	switch hFileExt {
+	case ".db":
+		return true, nil
+	case ".yaml":
+		return false, nil
+	default:
+		return false, errors.New("file extension is not db or yaml")
+	}
+}

Diff for: gpbackup_exporter_test.go

@@ -18,7 +18,12 @@ func TestMain(t *testing.T) {
 		t.Errorf("\nGet error during generate random int value:\n%v", err)
 	}
 	port := 50000 + int(n.Int64())
-	os.Args = []string{"gpbackup_exporter", "--web.listen-address=:" + strconv.Itoa(port)}
+	tempFile, err := os.CreateTemp("", "gpbackup_history*.db")
+	if err != nil {
+		t.Errorf("Failed to create temp file: %v", err)
+	}
+	defer os.Remove(tempFile.Name())
+	os.Args = []string{"gpbackup_exporter", "--web.listen-address=:" + strconv.Itoa(port), "--gpbackup.history-file=" + tempFile.Name()}
 	finished := make(chan struct{})
 	go func() {
 		main()

Diff for: gpbckpexporter/gpbckp_exporter.go

@@ -3,7 +3,6 @@ package gpbckpexporter
 import (
 	"net/http"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/go-kit/log"
@@ -53,7 +52,8 @@ func StartPromEndpoint(logger log.Logger) {
 }
 
 // GetGPBackupInfo get and parse gpbackup history file
-func GetGPBackupInfo(historyFile, backupType string, dbInclude, dbExclude []string, collectDepth int, logger log.Logger) {
+func GetGPBackupInfo(historyFile, backupType string, dbInclude, dbExclude []string, collectDepth int, historyDB bool, logger log.Logger) {
+	var parseHData gpbckpconfig.History
 	// To calculate the time elapsed since the last completed backup for specific database.
 	// For all databases values are calculated relative to one value.
 	currentTime := time.Now()
@@ -67,13 +67,40 @@ func GetGPBackupInfo(historyFile, backupType string, dbInclude, dbExclude []stri
 		// Check that database from the include list is not in the exclude list.
 		// If database not set - checking for entry into the exclude list will be performed later.
 		if dbNotInExclude(db, dbExclude) {
-			historyData, err := gpbckpconfig.ReadHistoryFile(historyFile)
-			if err != nil {
-				level.Error(logger).Log("msg", "Read gpbackup history file failed", "err", err)
-			}
-			parseHData, err := gpbckpconfig.ParseResult(historyData)
-			if err != nil {
-				level.Error(logger).Log("msg", "Parse YAML failed", "err", err)
+			if historyDB {
+				hDB, err := gpbckpconfig.OpenHistoryDB(historyFile)
+				if err != nil {
+					level.Error(logger).Log("msg", "Open gpbackup history db failed", "err", err)
+				}
+				defer func() {
+					closeErr := hDB.Close()
+					if closeErr != nil {
+						level.Error(logger).Log("msg", "Close gpbackup history db failed", "err", err)
+					}
+				}()
+				// Get only active and deleted backups. Failed backups are ignored.
+				backupList, err := gpbckpconfig.GetBackupNamesDB(true, false, hDB)
+				if err != nil {
+					level.Error(logger).Log("msg", "Get backups from history db failed", "err", err)
+				}
+				// Get data for selected backups.
+				for _, backupName := range backupList {
+					backupData, err := gpbckpconfig.GetBackupDataDB(backupName, hDB)
+					if err != nil {
+						level.Error(logger).Log("msg", "Get backup data from history db failed", "err", err)
+						break
+					}
+					parseHData.BackupConfigs = append(parseHData.BackupConfigs, backupData)
+				}
+			} else {
+				historyData, err := gpbckpconfig.ReadHistoryFile(historyFile)
+				if err != nil {
+					level.Error(logger).Log("msg", "Read gpbackup history file failed", "err", err)
+				}
+				parseHData, err = gpbckpconfig.ParseResult(historyData)
+				if err != nil {
+					level.Error(logger).Log("msg", "Parse YAML failed", "err", err)
+				}
 			}
 			// Reset metrics.
 			resetMetrics()
@@ -153,16 +180,3 @@ func GetGPBackupInfo(historyFile, backupType string, dbInclude, dbExclude []stri
 func GetExporterInfo(exporterVersion string, logger log.Logger) {
 	getExporterMetrics(exporterVersion, setUpMetricValue, logger)
 }
-
-func dbNotInExclude(db string, listExclude []string) bool {
-	// Check that exclude list is empty.
-	// If so, no excluding databases are set during startup.
-	if strings.Join(listExclude, "") != "" {
-		for _, val := range listExclude {
-			if val == db {
-				return false
-			}
-		}
-	}
-	return true
-}

Diff for: gpbckpexporter/gpbckp_exporter_test.go

@@ -41,6 +41,7 @@ func TestGetGPBackupInfo(t *testing.T) {
 		bckpIncl    []string
 		bckpExcl    []string
 		cDepth      int
+		hDB         bool
 	}
 	tests := []struct {
 		name     string
@@ -111,22 +112,23 @@ func TestGetGPBackupInfo(t *testing.T) {
 				[]string{""},
 				[]string{""},
 				0,
+				false,
 			},
 			"level=debug msg=\"Set up metric\" metric=gpbackup_backup_status value=0 labels=metadata-only,test,none,none,20230118162654",
 		},
 		{
 			"FailedDataReturn",
-			args{"return error", "", []string{""}, []string{""}, 0},
+			args{"return error", "", []string{""}, []string{""}, 0, false},
 			"level=error msg=\"Parse YAML failed\" err=\"yaml: unmarshal errors:\\n  line 1: cannot unmarshal !!str `return ...` into gpbckpconfig.History\"",
 		},
 		{
 			"InvalidDataReturn",
-			args{"42", "", []string{""}, []string{""}, 0},
+			args{"42", "", []string{""}, []string{""}, 0, false},
 			"level=error msg=\"Parse YAML failed\" err=\"yaml: unmarshal errors:\\n  line 1: cannot unmarshal !!int `42` into gpbckpconfig.History\"",
 		},
 		{
 			"NoDataReturn",
-			args{"", "", []string{""}, []string{""}, 0},
+			args{"", "", []string{""}, []string{""}, 0, false},
 			"level=warn msg=\"No backup data returned\"",
 		},
 		{
@@ -163,12 +165,14 @@ func TestGetGPBackupInfo(t *testing.T) {
 				"",
 				[]string{""},
 				[]string{""},
-				14},
+				14,
+				false,
+			},
 			"level=warn msg=\"No succeed backups\"",
 		},
 		{
 			"DBinIncludeAndExclude",
-			args{"", "", []string{"test"}, []string{"test"}, 0},
+			args{"", "", []string{"test"}, []string{"test"}, 0, false},
 			"level=warn msg=\"DB is specified in include and exclude lists\" DB=test",
 		},
 	}
@@ -188,6 +192,7 @@ func TestGetGPBackupInfo(t *testing.T) {
 				tt.args.bckpIncl,
 				tt.args.bckpExcl,
 				tt.args.cDepth,
+				tt.args.hDB,
 				lc,
 			)
 			if !strings.Contains(out.String(), tt.testText) {
@@ -197,36 +202,6 @@ func TestGetGPBackupInfo(t *testing.T) {
 	}
 }
 
-func TestDBNotInExclude(t *testing.T) {
-	type args struct {
-		db          string
-		listExclude []string
-	}
-	tests := []struct {
-		name string
-		args args
-		want bool
-	}{
-		{
-			"Include",
-			args{"test", []string{"test"}},
-			false,
-		},
-		{
-			"Exclude",
-			args{"test", []string{"demo"}},
-			true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := dbNotInExclude(tt.args.db, tt.args.listExclude); got != tt.want {
-				t.Errorf("\nVariables do not match:\n%v\nwant:\n%v", got, tt.want)
-			}
-		})
-	}
-}
-
 func fakeHistoryFileData(text string) (*os.File, error) {
 	tempFile, err := os.CreateTemp("", "gpbackup_history*.yaml")
 	if err != nil {

Diff for: gpbckpexporter/gpbckp_parser.go

@@ -103,3 +103,16 @@ func setUpMetric(metric *prometheus.GaugeVec, metricName string, value float64,
 		)
 	}
 }
+
+func dbNotInExclude(db string, listExclude []string) bool {
+	// Check that exclude list is empty.
+	// If so, no excluding databases are set during startup.
+	if strings.Join(listExclude, "") != "" {
+		for _, val := range listExclude {
+			if val == db {
+				return false
+			}
+		}
+	}
+	return true
+}

Diff for: gpbckpexporter/gpbckp_parser_test.go

@@ -213,3 +213,33 @@ func returnTimeTime(sTime string) time.Time {
 	}
 	return rTime
 }
+
+func TestDBNotInExclude(t *testing.T) {
+	type args struct {
+		db          string
+		listExclude []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			"Include",
+			args{"test", []string{"test"}},
+			false,
+		},
+		{
+			"Exclude",
+			args{"test", []string{"demo"}},
+			true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := dbNotInExclude(tt.args.db, tt.args.listExclude); got != tt.want {
+				t.Errorf("\nVariables do not match:\n%v\nwant:\n%v", got, tt.want)
+			}
+		})
+	}
+}