Prometheus

cmd/guaccollect/cmd/deps_dev.go

@@ -18,6 +18,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"time"
 
@@ -40,6 +41,12 @@ type depsDevOptions struct {
 	natsAddr string
 	// run as poll collector
 	poll bool
+	// query for dependencies
+	retrieveDependencies bool
+	// enable prometheus server
+	enablePrometheus bool
+	// prometheus address
+	prometheusPort int
 }
 
 var depsDevCmd = &cobra.Command{
@@ -57,32 +64,48 @@ var depsDevCmd = &cobra.Command{
 			viper.GetBool("csub-tls-skip-verify"),
 			viper.GetBool("use-csub"),
 			viper.GetBool("service-poll"),
-			args)
+			viper.GetBool("retrieve-dependencies"),
+			args,
+			viper.GetBool("enable-prometheus"),
+			viper.GetInt("prometheus-addr"),
+		)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
 			_ = cmd.Help()
 			os.Exit(1)
 		}
-
 		// Register collector
-		depsDevCollector, err := deps_dev.NewDepsCollector(ctx, opts.dataSource, opts.poll, 30*time.Second)
+		depsDevCollector, err := deps_dev.NewDepsCollector(ctx, opts.dataSource, opts.poll, opts.retrieveDependencies, 30*time.Second)
 		if err != nil {
 			logger.Errorf("unable to register oci collector: %v", err)
 		}
 		err = collector.RegisterDocumentCollector(depsDevCollector, deps_dev.DepsCollector)
 		if err != nil {
 			logger.Errorf("unable to register oci collector: %v", err)
 		}
+		if opts.enablePrometheus {
+			go func() {
+				http.Handle("/metrics", depsDevCollector.Metrics.MetricsHandler())
+				logger.Infof("Prometheus server is listening on: %d", opts.prometheusPort)
+				if err := http.ListenAndServe(fmt.Sprintf(":%d", opts.prometheusPort), nil); err != nil {
+					logger.Fatalf("Error starting HTTP server: %v", err)
+				}
+			}()
+		}
 
 		initializeNATsandCollector(ctx, opts.natsAddr)
 	},
 }
 
-func validateDepsDevFlags(natsAddr string, csubAddr string, csubTls bool, csubTlsSkipVerify bool, useCsub bool, poll bool, args []string) (depsDevOptions, error) {
+func validateDepsDevFlags(natsAddr string, csubAddr string, csubTls bool, csubTlsSkipVerify bool, useCsub bool, poll bool, retrieveDependencies bool, args []string,
+	enablePrometheus bool, prometheusPort int,
+) (depsDevOptions, error) {
 	var opts depsDevOptions
 	opts.natsAddr = natsAddr
 	opts.poll = poll
-
+	opts.retrieveDependencies = retrieveDependencies
+	opts.enablePrometheus = enablePrometheus
+	opts.prometheusPort = prometheusPort
 	if useCsub {
 		csubOpts, err := client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify)
 		if err != nil {
@@ -104,7 +127,6 @@ func validateDepsDevFlags(natsAddr string, csubAddr string, csubTls bool, csubTl
 	sources := []datasource.Source{}
 	for _, arg := range args {
 		sources = append(sources, datasource.Source{Value: arg})
-
 	}
 
 	var err error

cmd/guaccollect/cmd/root.go

@@ -22,15 +22,14 @@ import (
 
 	"github.com/guacsec/guac/pkg/cli"
 	"github.com/guacsec/guac/pkg/version"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 func init() {
 	cobra.OnInitialize(cli.InitConfig)
 
-	set, err := cli.BuildFlags([]string{"nats-addr", "csub-addr", "use-csub", "service-poll"})
+	set, err := cli.BuildFlags([]string{"nats-addr", "csub-addr", "use-csub", "service-poll", "enable-prometheus", "prometheus-addr"})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)
 		os.Exit(1)

cmd/guacgql/cmd/ent.go

@@ -18,6 +18,8 @@
 package cmd
 
 import (
+	"context"
+
 	"github.com/guacsec/guac/pkg/assembler/backends"
 	entbackend "github.com/guacsec/guac/pkg/assembler/backends/ent/backend"
 )
@@ -29,7 +31,7 @@ func init() {
 	getOpts[ent] = getEnt
 }
 
-func getEnt() backends.BackendArgs {
+func getEnt(_ context.Context) backends.BackendArgs {
 	return &entbackend.BackendOptions{
 		DriverName:  flags.dbDriver,
 		Address:     flags.dbAddress,

cmd/guacgql/cmd/root.go

@@ -106,6 +106,7 @@ func init() {
 		"neptune-endpoint", "neptune-port", "neptune-region", "neptune-user", "neptune-realm",
 		"gql-listen-port", "gql-tls-cert-file", "gql-tls-key-file", "gql-debug", "gql-backend", "gql-trace",
 		"db-address", "db-driver", "db-debug", "db-migrate",
+		"kv-store", "kv-redis", "kv-tikv", "enable-prometheus", "prometheus-addr",
 	})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)

cmd/guacgql/cmd/server.go

@@ -35,6 +35,7 @@ import (
 	"github.com/guacsec/guac/pkg/assembler/backends/neptune"
 	"github.com/guacsec/guac/pkg/assembler/graphql/generated"
 	"github.com/guacsec/guac/pkg/assembler/graphql/resolvers"
+	"github.com/guacsec/guac/pkg/cli"
 	"github.com/guacsec/guac/pkg/logging"
 	"github.com/spf13/cobra"
 	"golang.org/x/exp/maps"
@@ -48,7 +49,7 @@ const (
 	neptunes = "neptune"
 )
 
-type optsFunc func() backends.BackendArgs
+type optsFunc func(context.Context) backends.BackendArgs
 
 var getOpts map[string]optsFunc
 
@@ -63,6 +64,7 @@ func init() {
 }
 
 func startServer(cmd *cobra.Command) {
+	var srvHandler http.Handler
 	ctx := logging.WithLogger(context.Background())
 	logger := logging.FromContext(ctx)
 
@@ -77,6 +79,18 @@ func startServer(cmd *cobra.Command) {
 		logger.Errorf("unable to initialize graphql server: %v", err)
 		os.Exit(1)
 	}
+	// Setup Prometheus metrics handler
+	metric, err := cli.SetupPrometheus(ctx, logger, "guacgql")
+
+	if err != nil {
+		logger.Fatalf("Error setting up Prometheus: %v", err)
+	}
+
+	if metric != nil {
+		srvHandler = metric.MeasureGraphQLResponseDuration(srv)
+	} else {
+		srvHandler = srv
+	}
 
 	if flags.tracegql {
 		tracer := &debug.Tracer{}
@@ -85,7 +99,7 @@ func startServer(cmd *cobra.Command) {
 
 	http.HandleFunc("/healthz", healthHandler)
 
-	http.Handle("/query", srv)
+	http.Handle("/query", srvHandler)
 	proto := "http"
 	if flags.tlsCertFile != "" && flags.tlsKeyFile != "" {
 		proto = "https"
@@ -137,13 +151,14 @@ func validateFlags() error {
 func getGraphqlServer(ctx context.Context) (*handler.Server, error) {
 	var topResolver resolvers.Resolver
 
-	backend, err := backends.Get(flags.backend, ctx, getOpts[flags.backend]())
+	backend, err := backends.Get(flags.backend, ctx, getOpts[flags.backend](ctx))
 	if err != nil {
 		return nil, fmt.Errorf("Error creating %v backend: %w", flags.backend, err)
 	}
 	topResolver = resolvers.Resolver{Backend: backend}
 
 	config := generated.Config{Resolvers: &topResolver}
+	//config.Directives.Filter = resolvers.Filter
 	srv := handler.NewDefaultServer(generated.NewExecutableSchema(config))
 
 	return srv, nil
@@ -154,15 +169,15 @@ func healthHandler(w http.ResponseWriter, r *http.Request) {
 	_, _ = fmt.Fprint(w, "Server is healthy")
 }
 
-func getArango() backends.BackendArgs {
+func getArango(_ context.Context) backends.BackendArgs {
 	return &arangodb.ArangoConfig{
 		User:   flags.arangoUser,
 		Pass:   flags.arangoPass,
 		DBAddr: flags.arangoAddr,
 	}
 }
 
-func getNeo4j() backends.BackendArgs {
+func getNeo4j(_ context.Context) backends.BackendArgs {
 	return &neo4j.Neo4jConfig{
 		User:   flags.nUser,
 		Pass:   flags.nPass,
@@ -171,11 +186,11 @@ func getNeo4j() backends.BackendArgs {
 	}
 }
 
-func getInMem() backends.BackendArgs {
+func getInMem(_ context.Context) backends.BackendArgs {
 	return nil
 }
 
-func getNeptune() backends.BackendArgs {
+func getNeptune(_ context.Context) backends.BackendArgs {
 	return &neptune.NeptuneConfig{
 		Endpoint: flags.neptuneEndpoint,
 		Port:     flags.neptunePort,

docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.9"
 services:
 
   nats:
-    networks: [frontend]
+    networks: [ frontend ]
     image: "docker.io/library/nats:2.9.17-alpine"
     command: "--config /config/nats/js.conf -m 8222"
     ports:
@@ -14,14 +14,20 @@ services:
       - ./container_files/nats:/config/nats:z
     restart: on-failure
     healthcheck:
-      test: ["CMD", "wget", "--spider", "http://localhost:8222/healthz"]
+      test:
+        [
+          "CMD",
+          "wget",
+          "--spider",
+          "http://localhost:8222/healthz"
+        ]
       interval: 10s
       timeout: 10s
       retries: 3
       start_period: 5s
 
   guac-collectsub:
-    networks: [frontend]
+    networks: [ frontend ]
     image: $GUAC_IMAGE
     command: "/opt/guac/guaccsub"
     working_dir: /guac
@@ -34,14 +40,14 @@ services:
     volumes:
       - ./container_files/guac:/guac:z
     healthcheck:
-      test: ["CMD", "wget", "--spider", "http://localhost:2782"]
+      test: [ "CMD", "wget", "--spider", "http://localhost:2782" ]
       interval: 10s
       timeout: 10s
       retries: 3
       start_period: 5s
 
   guac-ingestor:
-    networks: [frontend]
+    networks: [ frontend ]
     image: $GUAC_IMAGE
     command: "/opt/guac/guacingest"
     working_dir: /guac
@@ -55,7 +61,7 @@ services:
       - ./container_files/guac:/guac:z
 
   oci-collector:
-    networks: [frontend]
+    networks: [ frontend ]
     image: $GUAC_IMAGE
     command: "/opt/guac/guaccollect image"
     working_dir: /guac
@@ -69,7 +75,7 @@ services:
       - ./container_files/guac:/guac:z
 
   depsdev-collector:
-    networks: [frontend]
+    networks: [ frontend ]
     image: $GUAC_IMAGE
     command: "/opt/guac/guaccollect deps_dev"
     working_dir: /guac
@@ -83,9 +89,11 @@ services:
         condition: service_healthy
     volumes:
       - ./container_files/guac:/guac:z
+    ports:
+      - "9091:9091" # for prometheus metrics
 
   osv-certifier:
-    networks: [frontend]
+    networks: [ frontend ]
     image: $GUAC_IMAGE
     command: "/opt/guac/guacone certifier osv"
     working_dir: /guac