Adding local config file (#233)

* adding local config file

* removing datadog agent flags

* removing required flag as it can be handle in the config file

* typo

* inLine config path overwrite local config file

* adding comment

* cleaning the config files

cmd/kubehound/ingest.go

@@ -42,7 +42,6 @@ var (
 		Long:  `Run an ingestion on KHaaS from a bucket to build the attack path, by default it will rehydrate the latest snapshot previously dumped on a KHaaS instance from all clusters`,
 		PreRunE: func(cobraCmd *cobra.Command, args []string) error {
 			viper.BindPFlag(config.IngestorAPIEndpoint, cobraCmd.Flags().Lookup("khaas-server")) //nolint: errcheck
-			cobraCmd.MarkFlagRequired("khaas-server")                                            //nolint: errcheck
 			viper.BindPFlag(config.IngestorAPIInsecure, cobraCmd.Flags().Lookup("insecure"))     //nolint: errcheck
 
 			if !isIngestRemoteDefault() {

configs/etc/kubehound-reference.yaml

@@ -128,11 +128,14 @@ builder:
 # Ingestor configuration (for KHaaS)
 # ingestor:
 #   blob:
-#    bucket: "" # (i.e.: s3://your-bucket)
-#    region: "" # (i.e.: us-east-1)
+#     # (i.e.: s3://<your-bucket>)
+#     bucket: ""
+#     # (i.e.: us-east-1)
+#     region: "" 
 #   temp_dir: "/tmp/kubehound"
 #   archive_name: "archive.tar.gz"
-#  max_archive_size: 2147483648 # 2GB
-#   api: # GRPC endpoint for the ingestor
+#   max_archive_size: 2147483648 # 2GB
+#   # GRPC endpoint for the ingestor
+#   api: 
 #     endpoint: "127.0.0.1:9000"
-#     insecure: true
+#     insecure: true

configs/etc/kubehound.yaml

@@ -56,3 +56,18 @@ builder:
 
     # Enable for large clusters to prevent number of edges growing exponentially
     large_cluster_optimizations: true
+
+# Ingestor configuration (for KHaaS)
+ingestor:
+  blob:
+    # (i.e.: s3://<your-bucket>)
+    bucket: ""
+    # (i.e.: us-east-1)
+    region: "" 
+  temp_dir: "/tmp/kubehound"
+  archive_name: "archive.tar.gz"
+  max_archive_size: 2147483648 # 2GB
+  # GRPC endpoint for the ingestor
+  api: 
+    endpoint: "127.0.0.1:9000"
+    insecure: true

pkg/cmd/dump.go

@@ -12,21 +12,9 @@ func InitRootCmd(cmd *cobra.Command) {
 }
 
 func InitDumpCmd(cmd *cobra.Command) {
-	cmd.PersistentFlags().String("statsd", config.DefaultTelemetryStatsdUrl, "URL of the statsd endpoint")
-	viper.BindPFlag(config.TelemetryStatsdUrl, cmd.PersistentFlags().Lookup("statsd")) //nolint: errcheck
-
-	cmd.PersistentFlags().String("profiler", config.DefaultTelemetryProfilerUrl, "URL of the profiler endpoint")
-	viper.BindPFlag(config.TelemetryTracerUrl, cmd.PersistentFlags().Lookup("profiler")) //nolint: errcheck
-
 	cmd.PersistentFlags().Bool("telemetry", false, "Enable telemetry with default settings")
 	viper.BindPFlag(config.TelemetryEnabled, cmd.PersistentFlags().Lookup("telemetry")) //nolint: errcheck
 
-	cmd.PersistentFlags().Duration("period", config.DefaultProfilerPeriod, "Period specifies the interval at which to collect profiles")
-	viper.BindPFlag(config.TelemetryProfilerPeriod, cmd.PersistentFlags().Lookup("period")) //nolint: errcheck
-
-	cmd.PersistentFlags().Duration("cpu-duration", config.DefaultProfilerCPUDuration, "CPU Duration specifies the length at which to collect CPU profiles")
-	viper.BindPFlag(config.TelemetryProfilerCPUDuration, cmd.PersistentFlags().Lookup("cpu-duration")) //nolint: errcheck
-
 	cmd.PersistentFlags().Int("rate", config.DefaultK8sAPIRateLimitPerSecond, "Rate limit of requests/second to the Kubernetes API")
 	viper.BindPFlag(config.CollectorLiveRate, cmd.PersistentFlags().Lookup("rate")) //nolint: errcheck
 
@@ -51,7 +39,6 @@ func InitLocalDumpCmd(cmd *cobra.Command) {
 func InitRemoteDumpCmd(cmd *cobra.Command) {
 	cmd.Flags().String("bucket", "", "Bucket to use to push k8s resources (e.g.: s3://<your_bucket>)")
 	viper.BindPFlag(config.CollectorFileBlobBucket, cmd.Flags().Lookup("bucket")) //nolint: errcheck
-	cmd.MarkFlagRequired("bucket")                                                //nolint: errcheck
 
 	cmd.Flags().String("region", "", "Region to retrieve the configuration (only for s3) (e.g.: us-east-1)")
 	viper.BindPFlag(config.CollectorFileBlobRegion, cmd.Flags().Lookup("region")) //nolint: errcheck

pkg/config/config.go

@@ -22,6 +22,7 @@ var (
 const (
 	DefaultConfigType  = "yaml"
 	DefaultClusterName = "unknown"
+	DefaultConfigName  = "kubehound"
 
 	GlobalDebug = "debug"
 )
@@ -88,53 +89,55 @@ func NewKubehoundConfig(configPath string, inLine bool) *KubehoundConfig {
 }
 
 // SetDefaultValues loads the default value from the different modules
-func SetDefaultValues(c *viper.Viper) {
+func SetDefaultValues(v *viper.Viper) {
 	// K8s Live collector module
-	c.SetDefault(CollectorLivePageSize, DefaultK8sAPIPageSize)
-	c.SetDefault(CollectorLivePageBufferSize, DefaultK8sAPIPageBufferSize)
-	c.SetDefault(CollectorLiveRate, DefaultK8sAPIRateLimitPerSecond)
-	c.SetDefault(CollectorNonInteractive, DefaultK8sAPINonInteractive)
+	v.SetDefault(CollectorLivePageSize, DefaultK8sAPIPageSize)
+	v.SetDefault(CollectorLivePageBufferSize, DefaultK8sAPIPageBufferSize)
+	v.SetDefault(CollectorLiveRate, DefaultK8sAPIRateLimitPerSecond)
+	v.SetDefault(CollectorNonInteractive, DefaultK8sAPINonInteractive)
 
 	// File collector module
-	c.SetDefault(CollectorFileArchiveNoCompress, DefaultArchiveNoCompress)
+	v.SetDefault(CollectorFileArchiveNoCompress, DefaultArchiveNoCompress)
 
 	// Default values for storage provider
-	c.SetDefault("storage.wipe", true)
-	c.SetDefault("storage.retry", DefaultRetry)
-	c.SetDefault("storage.retry_delay", DefaultRetryDelay)
+	v.SetDefault("storage.wipe", true)
+	v.SetDefault("storage.retry", DefaultRetry)
+	v.SetDefault("storage.retry_delay", DefaultRetryDelay)
 
 	// Disable Datadog telemetry by default
-	c.SetDefault(TelemetryEnabled, false)
+	v.SetDefault(TelemetryEnabled, false)
 
 	// Default value for MongoDB
-	c.SetDefault("mongodb.url", DefaultMongoUrl)
-	c.SetDefault("mongodb.connection_timeout", DefaultConnectionTimeout)
+	v.SetDefault("mongodb.url", DefaultMongoUrl)
+	v.SetDefault("mongodb.connection_timeout", DefaultConnectionTimeout)
 
 	// Defaults values for JanusGraph
-	c.SetDefault("janusgraph.url", DefaultJanusGraphUrl)
-	c.SetDefault("janusgraph.connection_timeout", DefaultConnectionTimeout)
+	v.SetDefault("janusgraph.url", DefaultJanusGraphUrl)
+	v.SetDefault("janusgraph.connection_timeout", DefaultConnectionTimeout)
 
 	// Profiler values
-	c.SetDefault(TelemetryProfilerPeriod, DefaultProfilerPeriod)
-	c.SetDefault(TelemetryProfilerCPUDuration, DefaultProfilerCPUDuration)
+	v.SetDefault(TelemetryProfilerPeriod, DefaultProfilerPeriod)
+	v.SetDefault(TelemetryProfilerCPUDuration, DefaultProfilerCPUDuration)
 
 	// Default values for graph builder
-	c.SetDefault("builder.vertex.batch_size", DefaultVertexBatchSize)
-	c.SetDefault("builder.vertex.batch_size_small", DefaultVertexBatchSizeSmall)
-	c.SetDefault("builder.edge.worker_pool_size", DefaultEdgeWorkerPoolSize)
-	c.SetDefault("builder.edge.worker_pool_capacity", DefaultEdgeWorkerPoolCapacity)
-	c.SetDefault("builder.edge.batch_size", DefaultEdgeBatchSize)
-	c.SetDefault("builder.edge.batch_size_small", DefaultEdgeBatchSizeSmall)
-	c.SetDefault("builder.edge.batch_size_cluster_impact", DefaultEdgeBatchSizeClusterImpact)
-	c.SetDefault("builder.stop_on_error", DefaultStopOnError)
-	c.SetDefault("builder.edge.large_cluster_optimizations", DefaultLargeClusterOptimizations)
-
-	c.SetDefault(IngestorAPIEndpoint, DefaultIngestorAPIEndpoint)
-	c.SetDefault(IngestorAPIInsecure, DefaultIngestorAPIInsecure)
-	c.SetDefault(IngestorBlobBucketName, DefaultBucketName)
-	c.SetDefault(IngestorTempDir, DefaultTempDir)
-	c.SetDefault(IngestorMaxArchiveSize, DefaultMaxArchiveSize)
-	c.SetDefault(IngestorArchiveName, DefaultArchiveName)
+	v.SetDefault("builder.vertex.batch_size", DefaultVertexBatchSize)
+	v.SetDefault("builder.vertex.batch_size_small", DefaultVertexBatchSizeSmall)
+	v.SetDefault("builder.edge.worker_pool_size", DefaultEdgeWorkerPoolSize)
+	v.SetDefault("builder.edge.worker_pool_capacity", DefaultEdgeWorkerPoolCapacity)
+	v.SetDefault("builder.edge.batch_size", DefaultEdgeBatchSize)
+	v.SetDefault("builder.edge.batch_size_small", DefaultEdgeBatchSizeSmall)
+	v.SetDefault("builder.edge.batch_size_cluster_impact", DefaultEdgeBatchSizeClusterImpact)
+	v.SetDefault("builder.stop_on_error", DefaultStopOnError)
+	v.SetDefault("builder.edge.large_cluster_optimizations", DefaultLargeClusterOptimizations)
+
+	v.SetDefault(IngestorAPIEndpoint, DefaultIngestorAPIEndpoint)
+	v.SetDefault(IngestorAPIInsecure, DefaultIngestorAPIInsecure)
+	v.SetDefault(IngestorBlobBucketName, DefaultBucketName)
+	v.SetDefault(IngestorTempDir, DefaultTempDir)
+	v.SetDefault(IngestorMaxArchiveSize, DefaultMaxArchiveSize)
+	v.SetDefault(IngestorArchiveName, DefaultArchiveName)
+
+	SetLocalConfig(v)
 }
 
 // SetEnvOverrides enables environment variable overrides for the config.
@@ -173,12 +176,13 @@ func unmarshalConfig(v *viper.Viper) (*KubehoundConfig, error) {
 
 // NewConfig creates a new config instance from the provided file using viper.
 func NewConfig(v *viper.Viper, configPath string) (*KubehoundConfig, error) {
-	v.SetConfigType(DefaultConfigType)
-	v.SetConfigFile(configPath)
-
 	// Configure default values
 	SetDefaultValues(v)
 
+	// Loading inLine config path
+	v.SetConfigType(DefaultConfigType)
+	v.SetConfigFile(configPath)
+
 	// Configure environment variable override
 	SetEnvOverrides(v)
 	if err := v.ReadInConfig(); err != nil {
@@ -209,6 +213,22 @@ func NewInlineConfig(v *viper.Viper) (*KubehoundConfig, error) {
 	return kc, nil
 }
 
+// Load local config file if it exists, check for local file in current dir or in $HOME/.config/
+// Not returning any error as it is not mandatory to have a local config file
+func SetLocalConfig(v *viper.Viper) {
+	v.SetConfigName(DefaultConfigName) // name of config file (without extension)
+	v.SetConfigType(DefaultConfigType) // REQUIRED if the config file does not have the extension in the name
+	v.AddConfigPath("$HOME/.config/")  // call multiple times to add many search paths
+	v.AddConfigPath(".")               // optionally look for config in the working directory
+
+	err := v.ReadInConfig()
+	if err != nil {
+		log.I.Warnf("No local config file was found (%s.%s)", DefaultConfigName, DefaultConfigType)
+		// log.I.Debugf("Error reading config: %v", err)
+	}
+	log.I.Infof("Using %s for default config\n", viper.ConfigFileUsed())
+}
+
 // NewEmbedConfig creates a new config instance from an embedded config file using viper.
 func NewEmbedConfig(v *viper.Viper, configPath string) (*KubehoundConfig, error) {
 	v.SetConfigType(DefaultConfigType)