Merge branch 'main' into zenithar/doc_various_cleanup

cmd/kubehound/config.go

@@ -20,7 +20,7 @@ var (
 		Short: "Show the current configuration",
 		Long:  `[devOnly] Show the current configuration`,
 		PreRunE: func(cobraCmd *cobra.Command, args []string) error {
-			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), "", true, true)
+			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), cfgFile, true, true)
 		},
 		RunE: func(cobraCmd *cobra.Command, args []string) error {
 			// Adding datadog setup

cmd/kubehound/dumper.go

@@ -36,7 +36,7 @@ var (
 			viper.BindPFlag(config.IngestorAPIEndpoint, cobraCmd.Flags().Lookup("khaas-server")) //nolint: errcheck
 			viper.BindPFlag(config.IngestorAPIInsecure, cobraCmd.Flags().Lookup("insecure"))     //nolint: errcheck
 
-			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), "", true, true)
+			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), cfgFile, true, true)
 		},
 		RunE: func(cobraCmd *cobra.Command, args []string) error {
 			// using compress feature
@@ -62,7 +62,7 @@ var (
 				return fmt.Errorf("dump core: %w", err)
 			}
 			// Running the ingestion on KHaaS
-			if cobraCmd.Flags().Lookup("khaas-server").Value.String() != "" {
+			if khCfg.Ingestor.API.Endpoint != "" {
 				return core.CoreClientGRPCIngest(cobraCmd.Context(), khCfg.Ingestor, khCfg.Dynamic.ClusterName, khCfg.Dynamic.RunID.String())
 			}
 
@@ -77,7 +77,7 @@ var (
 		PreRunE: func(cobraCmd *cobra.Command, args []string) error {
 			viper.Set(config.CollectorFileDirectory, args[0])
 
-			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), "", true, true)
+			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), cfgFile, true, true)
 		},
 		RunE: func(cobraCmd *cobra.Command, args []string) error {
 			// Passing the Kubehound config from viper

cmd/kubehound/ingest.go

@@ -29,7 +29,7 @@ var (
 		PreRunE: func(cobraCmd *cobra.Command, args []string) error {
 			cmd.BindFlagCluster(cobraCmd)
 
-			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), "", true, true)
+			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), cfgFile, true, true)
 		},
 		RunE: func(cobraCmd *cobra.Command, args []string) error {
 			// Passing the Kubehound config from viper
@@ -56,7 +56,7 @@ var (
 				cobraCmd.MarkFlagRequired("cluster") //nolint: errcheck
 			}
 
-			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), "", false, true)
+			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), cfgFile, false, true)
 		},
 		RunE: func(cobraCmd *cobra.Command, args []string) error {
 			// Passing the Kubehound config from viper

cmd/kubehound/root.go

@@ -76,9 +76,9 @@ var (
 )
 
 func init() {
-	rootCmd.Flags().StringVarP(&cfgFile, "config", "c", cfgFile, "application config file")
+	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", cfgFile, "application config file")
 
-	rootCmd.Flags().BoolVar(&skipBackend, "skip-backend", skipBackend, "skip the auto deployment of the backend stack (janusgraph, mongodb, and UI)")
+	rootCmd.PersistentFlags().BoolVar(&skipBackend, "skip-backend", skipBackend, "skip the auto deployment of the backend stack (janusgraph, mongodb, and UI)")
 
 	cmd.InitRootCmd(rootCmd)
 }

configs/etc/kubehound-reference.yaml

@@ -62,6 +62,9 @@ janusgraph:
   # Timeout on requests to the JanusGraph DB instance
   connection_timeout: 30s
 
+  # Number of worker threads for the JanusGraph writer pool
+  writer_worker_count: 10
+
 #
 # Datadog telemetry configuration
 #
@@ -114,10 +117,10 @@ builder:
     # worker_pool_capacity: 100
 
     # # Batch size for edge inserts
-    # batch_size: 500
+    # batch_size: 250
 
     #  # Small batch size for edge inserts
-    # batch_size_small: 75
+    # batch_size_small: 50
 
     #  # Cluster impact batch size for edge inserts
     # batch_size_cluster_impact: 1

configs/etc/kubehound.yaml

@@ -37,19 +37,22 @@ janusgraph:
   # Timeout on requests to the JanusGraph DB instance
   connection_timeout: 30s
 
+  # Number of worker threads for the JanusGraph writer pool
+  writer_worker_count: 10
+
 # Graph builder configuration
 builder:
   # Vertex builder configuration
   vertex:
     # Batch size for vertex inserts
-    batch_size: 500
+    batch_size: 250
 
   # Edge builder configuration
   edge:
     worker_pool_size: 2
 
     # Batch size for edge inserts
-    batch_size: 500
+    batch_size: 250
 
     # Cluster impact batch size for edge inserts
     batch_size_cluster_impact: 10

deployments/kubehound/graph/dsl/kubehound/src/main/java/com/datadog/ase/kubehound/KubeHoundTraversalDsl.java

@@ -34,16 +34,21 @@
 import static org.apache.tinkerpop.gremlin.process.traversal.Scope.local;
 import static org.apache.tinkerpop.gremlin.structure.Column.values;
 
-
 /**
- * This KubeHound DSL is meant to be used with the Kubernetes attack graph created by the KubeHound application.
+ * This KubeHound DSL is meant to be used with the Kubernetes attack graph
+ * created by the KubeHound application.
  * <p/>
- * All DSLs should extend {@code GraphTraversal.Admin} and be suffixed with "TraversalDsl". Simply add DSL traversal
- * methods to this interface. Use Gremlin's steps to build the underlying traversal in these methods to ensure
- * compatibility with the rest of the TinkerPop stack and provider implementations.
+ * All DSLs should extend {@code GraphTraversal.Admin} and be suffixed with
+ * "TraversalDsl". Simply add DSL traversal
+ * methods to this interface. Use Gremlin's steps to build the underlying
+ * traversal in these methods to ensure
+ * compatibility with the rest of the TinkerPop stack and provider
+ * implementations.
  * <p/>
- * Arguments provided to the {@code GremlinDsl} annotation are all optional. In this case, a {@code traversalSource} is
- * specified which points to a specific implementation to use. Had that argument not been specified then a default
+ * Arguments provided to the {@code GremlinDsl} annotation are all optional. In
+ * this case, a {@code traversalSource} is
+ * specified which points to a specific implementation to use. Had that argument
+ * not been specified then a default
  * {@code TraversalSource} would have been generated.
  */
 @GremlinDsl(traversalSource = "com.datadog.ase.kubehound.KubeHoundTraversalSourceDsl")
@@ -54,148 +59,161 @@ public interface KubeHoundTraversalDsl<S, E> extends GraphTraversal.Admin<S, E>
     public static final int PATH_HOPS_MIN_DEFAULT = 6;
 
     /**
-     * From a {@code Vertex} traverse immediate edges to display the next set of possible attacks and targets.
+     * From a {@code Vertex} traverse immediate edges to display the next set of
+     * possible attacks and targets.
      *
      */
     public default GraphTraversal<S, Path> attacks() {
         return outE().inV().path();
     }
 
     /**
-     * From a {@code Vertex} filter on whether incoming vertices are critical assets.
+     * From a {@code Vertex} filter on whether incoming vertices are critical
+     * assets.
      */
-    @GremlinDsl.AnonymousMethod(returnTypeParameters = {"A", "A"}, methodTypeParameters = {"A"})
+    @GremlinDsl.AnonymousMethod(returnTypeParameters = { "A", "A" }, methodTypeParameters = { "A" })
     public default GraphTraversal<S, E> critical() {
         return has("critical", true);
     }
 
     /**
-     * From a {@code Vertex} traverse edges until {@code maxHops} is exceeded or a critical asset is reached and return all paths. 
+     * From a {@code Vertex} traverse edges until {@code maxHops} is exceeded or a
+     * critical asset is reached and return all paths.
      *
      * @param maxHops the maximum number of hops in an attack path
      */
     public default GraphTraversal<S, Path> criticalPaths(int maxHops) {
-        if (maxHops < PATH_HOPS_MIN) throw new IllegalArgumentException(String.format("maxHops must be >= %d", PATH_HOPS_MIN));
-        if (maxHops > PATH_HOPS_MAX) throw new IllegalArgumentException(String.format("maxHops must be <= %d", PATH_HOPS_MAX));
+        if (maxHops < PATH_HOPS_MIN)
+            throw new IllegalArgumentException(String.format("maxHops must be >= %d", PATH_HOPS_MIN));
+        if (maxHops > PATH_HOPS_MAX)
+            throw new IllegalArgumentException(String.format("maxHops must be <= %d", PATH_HOPS_MAX));
 
-        return repeat((
-                (KubeHoundTraversalDsl) __.outE())
+        return repeat(((KubeHoundTraversalDsl) __.outE())
                 .inV()
-                .simplePath()
-            ).until(
-                __.has("critical", true)
-                .or()
-                .loops()
-                .is(maxHops)
-            ).has("critical", true)
-            .path();
+                .simplePath()).until(
+                        __.has("critical", true)
+                                .or()
+                                .loops()
+                                .is(maxHops))
+                .has("critical", true)
+                .path();
     }
 
     /**
-     * From a {@code Vertex} traverse edges until a critical asset is reached and return all paths. 
+     * From a {@code Vertex} traverse edges until a critical asset is reached and
+     * return all paths.
      */
     public default GraphTraversal<S, Path> criticalPaths() {
         return criticalPaths(PATH_HOPS_DEFAULT);
     }
 
     /**
-     * From a {@code Vertex} traverse edges EXCLUDING labels provided in {@code exclusions} until {@code maxHops} is exceeded or 
-     * a critical asset is reached and return all paths. 
+     * From a {@code Vertex} traverse edges EXCLUDING labels provided in
+     * {@code exclusions} until {@code maxHops} is exceeded or
+     * a critical asset is reached and return all paths.
      *
-     * @param maxHops the maximum number of hops in an attack path
+     * @param maxHops    the maximum number of hops in an attack path
      * @param exclusions edge labels to exclude from paths
      */
     public default GraphTraversal<S, Path> criticalPathsFilter(int maxHops, String... exclusions) {
-        if (exclusions.length <= 0) throw new IllegalArgumentException("exclusions must be provided (otherwise use criticalPaths())");
-        if (maxHops < PATH_HOPS_MIN) throw new IllegalArgumentException(String.format("maxHops must be >= %d", PATH_HOPS_MIN));
-        if (maxHops > PATH_HOPS_MAX) throw new IllegalArgumentException(String.format("maxHops must be <= %d", PATH_HOPS_MAX));
-
-         return repeat((
-                (KubeHoundTraversalDsl) __.outE())
-                .hasLabel(P.not(P.within(exclusions)))
+        if (exclusions.length <= 0)
+            throw new IllegalArgumentException("exclusions must be provided (otherwise use criticalPaths())");
+        if (maxHops < PATH_HOPS_MIN)
+            throw new IllegalArgumentException(String.format("maxHops must be >= %d", PATH_HOPS_MIN));
+        if (maxHops > PATH_HOPS_MAX)
+            throw new IllegalArgumentException(String.format("maxHops must be <= %d", PATH_HOPS_MAX));
+
+        return repeat(((KubeHoundTraversalDsl) __.outE())
+                .has("class", P.not(P.within(exclusions)))
                 .inV()
-                .simplePath()
-            ).until(
-                __.has("critical", true)
-                .or()
-                .loops()
-                .is(maxHops)
-            ).has("critical", true)
-            .path();
+                .simplePath()).until(
+                        __.has("critical", true)
+                                .or()
+                                .loops()
+                                .is(maxHops))
+                .has("critical", true)
+                .path();
     }
 
     /**
-     * From a {@code Vertex} filter on whether incoming vertices have at least one path to a critical asset.
+     * From a {@code Vertex} filter on whether incoming vertices have at least one
+     * path to a critical asset.
      */
-    @GremlinDsl.AnonymousMethod(returnTypeParameters = {"A", "A"}, methodTypeParameters = {"A"})
+    @GremlinDsl.AnonymousMethod(returnTypeParameters = { "A", "A" }, methodTypeParameters = { "A" })
     public default GraphTraversal<S, E> hasCriticalPath() {
-        return where(__.criticalPaths().limit(1)); 
+        return where(__.criticalPaths().limit(1));
     }
 
     /**
-     * From a {@code Vertex} returns the hop count of the shortest path to a critical asset.
+     * From a {@code Vertex} returns the hop count of the shortest path to a
+     * critical asset.
      *
      */
     public default <E2 extends Comparable> GraphTraversal<S, E2> minHopsToCritical() {
         return minHopsToCritical(PATH_HOPS_MIN_DEFAULT);
     }
 
     /**
-     * From a {@code Vertex} returns the hop count of the shortest path to a critical asset.
-     *   
+     * From a {@code Vertex} returns the hop count of the shortest path to a
+     * critical asset.
+     * 
      * @param maxHops the maximum number of hops in an attack path to consider
      *
      */
     public default <E2 extends Comparable> GraphTraversal<S, E2> minHopsToCritical(int maxHops) {
-        if (maxHops < PATH_HOPS_MIN) throw new IllegalArgumentException(String.format("maxHops must be >= %d", PATH_HOPS_MIN));
-        if (maxHops > PATH_HOPS_MAX) throw new IllegalArgumentException(String.format("maxHops must be <= %d", PATH_HOPS_MAX));
-
-        return repeat((
-                (KubeHoundTraversalDsl) __.out())
-                .simplePath()
-            ).until(
-                __.has("critical", true)
-                .or()
-                .loops()
-                .is(maxHops)
-            ).has("critical", true)
-            .path()
-            .count(local)
-            .min();
+        if (maxHops < PATH_HOPS_MIN)
+            throw new IllegalArgumentException(String.format("maxHops must be >= %d", PATH_HOPS_MIN));
+        if (maxHops > PATH_HOPS_MAX)
+            throw new IllegalArgumentException(String.format("maxHops must be <= %d", PATH_HOPS_MAX));
+
+        return repeat(((KubeHoundTraversalDsl) __.out())
+                .simplePath()).until(
+                        __.has("critical", true)
+                                .or()
+                                .loops()
+                                .is(maxHops))
+                .has("critical", true)
+                .path()
+                .count(local)
+                .min();
     }
 
     /**
-     * From a {@code Vertex} returns a group count (by label) of paths to a critical asset.
+     * From a {@code Vertex} returns a group count (by label) of paths to a critical
+     * asset.
      *
      */
     public default <K> GraphTraversal<S, Map<K, Long>> criticalPathsFreq() {
-        return criticalPathsFreq(PATH_HOPS_DEFAULT);   
+        return criticalPathsFreq(PATH_HOPS_DEFAULT);
     }
 
     /**
-     * From a {@code Vertex} returns a group count (by label) of paths to a critical asset.
+     * From a {@code Vertex} returns a group count (by label) of paths to a critical
+     * asset.
      *
      * @param maxHops the maximum number of hops in an attack path
      */
     public default <K> GraphTraversal<S, Map<K, Long>> criticalPathsFreq(int maxHops) {
-        if (maxHops < PATH_HOPS_MIN) throw new IllegalArgumentException(String.format("maxHops must be >= %d", PATH_HOPS_MIN));
-        if (maxHops > PATH_HOPS_MAX) throw new IllegalArgumentException(String.format("maxHops must be <= %d", PATH_HOPS_MAX));
+        if (maxHops < PATH_HOPS_MIN)
+            throw new IllegalArgumentException(String.format("maxHops must be >= %d", PATH_HOPS_MIN));
+        if (maxHops > PATH_HOPS_MAX)
+            throw new IllegalArgumentException(String.format("maxHops must be <= %d", PATH_HOPS_MAX));
 
         return repeat(
                 (KubeHoundTraversalDsl) __.outE()
-                .inV()
-                .simplePath()
-            ).emit()
-            .until(
-                __.has("critical", true)
-                .or()
-                .loops()
-                .is(maxHops)
-            ).has("critical", true)
-            .path()
-            .by(T.label)
-            .groupCount()
-            .order(local)
-            .by(__.select(values), Order.desc);
+                        .inV()
+                        .simplePath())
+                .emit()
+                .until(
+                        __.has("critical", true)
+                                .or()
+                                .loops()
+                                .is(maxHops))
+                .has("critical", true)
+                .path()
+                .by(T.label)
+                .groupCount()
+                .order(local)
+                .by(__.select(values), Order.desc);
     }
 }