Skip to content

Commit 57a2384

Browse files
jt-ddjt-dd
authored
add env variable for ingestor/grpc image (#264)
* add env variable for ingestor/grpc image * change bucket_name to bucket_url * warning message for retrying connection * typo
1 parent f80397e commit 57a2384

File tree

16 files changed

+52
-31
lines changed

16 files changed

+52
-31
lines changed

‎configs/etc/kubehound-reference.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ builder:
125125
# ingestor:
126126
# blob:
127127
# # (i.e.: s3://<your-bucket>)
128-
# bucket: ""
128+
# bucket_url: ""
129129
# # (i.e.: us-east-1)
130130
# region: ""
131131
# temp_dir: "/tmp/kubehound"

‎configs/etc/kubehound.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -50,24 +50,24 @@ builder:
5050

5151
# Batch size for edge inserts
5252
batch_size: 500
53-
53+
5454
# Cluster impact batch size for edge inserts
5555
batch_size_cluster_impact: 10
56-
56+
5757
# Enable for large clusters to prevent number of edges growing exponentially
5858
large_cluster_optimizations: true
5959

6060
# Ingestor configuration (for KHaaS)
6161
ingestor:
6262
blob:
6363
# (i.e.: s3://<your-bucket>)
64-
bucket: ""
64+
bucket_url: ""
6565
# (i.e.: us-east-1)
66-
region: ""
66+
region: ""
6767
temp_dir: "/tmp/kubehound"
6868
archive_name: "archive.tar.gz"
6969
max_archive_size: 2147483648 # 2GB
7070
# GRPC endpoint for the ingestor
71-
api:
71+
api:
7272
endpoint: "127.0.0.1:9000"
7373
insecure: true

‎deployments/k8s/khaas/conf/ingestor/kubehound.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ collector:
1818

1919
# General storage configuration
2020
storage:
21-
# Whether or not to wipe all data on startup
21+
# Whether or not to wipe all data on startup
2222
wipe: false
2323

2424
# Number of connection retries before declaring an error
@@ -61,7 +61,7 @@ builder:
6161

6262
# Batch size for edge inserts
6363
batch_size: 1000
64-
64+
6565
# Cluster impact batch size for edge inserts
6666
batch_size_cluster_impact: 10
6767

@@ -70,7 +70,7 @@ builder:
7070

7171
ingestor:
7272
blob:
73-
bucket: "{{ $.Values.services.ingestor.bucket }}"
73+
bucket_url: "{{ $.Values.services.ingestor.bucket_url }}"
7474
region: "{{ $.Values.services.ingestor.region }}"
7575
temp_dir: "/tmp/kubehound"
7676
archive_name: "archive.tar.gz"

‎deployments/k8s/khaas/values.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ services:
33
ingestor:
44
image: ghcr.io/datadog/kubehound-binary
55
version: latest
6-
bucket: s3://<your_bucket>
6+
bucket_url: s3://<your_bucket>
77
region: "us-east-1"
88
resources:
99
requests:

‎docs/user-guide/khaas-101.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ If you don't want to specify the bucket every time, you can set it up in your lo
9393
ingestor:
9494
blob:
9595
# (i.e.: s3://<your-bucket>)
96-
bucket: ""
96+
bucket_url: ""
9797
# (i.e.: us-east-1)
9898
region: ""
9999
```

‎pkg/config/collector.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,6 @@ type FileArchiveConfig struct {
5050
}
5151

5252
type BlobConfig struct {
53-
Bucket string `mapstructure:"bucket"` // Bucket to use to push k8s resources (e.g.: s3://<your_bucket>)
54-
Region string `mapstructure:"region"` // Region to use for the bucket (only for s3)
53+
BucketUrl string `mapstructure:"bucket_url"` // Bucket to use to push k8s resources (e.g.: s3://<your_bucket>)
54+
Region string `mapstructure:"region"` // Region to use for the bucket (only for s3)
5555
}

‎pkg/config/config.go

+18-5
Original file line numberDiff line numberDiff line change
@@ -108,12 +108,12 @@ func SetDefaultValues(v *viper.Viper) {
108108
v.SetDefault(TelemetryEnabled, false)
109109

110110
// Default value for MongoDB
111-
v.SetDefault("mongodb.url", DefaultMongoUrl)
112-
v.SetDefault("mongodb.connection_timeout", DefaultConnectionTimeout)
111+
v.SetDefault(MongoUrl, DefaultMongoUrl)
112+
v.SetDefault(MongoConnectionTimeout, DefaultConnectionTimeout)
113113

114114
// Defaults values for JanusGraph
115-
v.SetDefault("janusgraph.url", DefaultJanusGraphUrl)
116-
v.SetDefault("janusgraph.connection_timeout", DefaultConnectionTimeout)
115+
v.SetDefault(JanusGraphUrl, DefaultJanusGraphUrl)
116+
v.SetDefault(JanusGrapTimeout, DefaultConnectionTimeout)
117117

118118
// Profiler values
119119
v.SetDefault(TelemetryProfilerPeriod, DefaultProfilerPeriod)
@@ -132,7 +132,7 @@ func SetDefaultValues(v *viper.Viper) {
132132

133133
v.SetDefault(IngestorAPIEndpoint, DefaultIngestorAPIEndpoint)
134134
v.SetDefault(IngestorAPIInsecure, DefaultIngestorAPIInsecure)
135-
v.SetDefault(IngestorBlobBucketName, DefaultBucketName)
135+
v.SetDefault(IngestorBlobBucketURL, DefaultBucketName)
136136
v.SetDefault(IngestorTempDir, DefaultTempDir)
137137
v.SetDefault(IngestorMaxArchiveSize, DefaultMaxArchiveSize)
138138
v.SetDefault(IngestorArchiveName, DefaultArchiveName)
@@ -149,6 +149,17 @@ func SetEnvOverrides(c *viper.Viper) {
149149
res = multierror.Append(res, c.BindEnv("collector.file.directory", "KH_COLLECTOR_DIR"))
150150
res = multierror.Append(res, c.BindEnv("collector.file.cluster", "KH_COLLECTOR_TARGET"))
151151

152+
res = multierror.Append(res, c.BindEnv(MongoUrl, "KH_MONGODB_URL"))
153+
res = multierror.Append(res, c.BindEnv(JanusGraphUrl, "KH_JANUSGRAPH_URL"))
154+
155+
res = multierror.Append(res, c.BindEnv(IngestorAPIEndpoint, "KH_INGESTOR_API_ENDPOINT"))
156+
res = multierror.Append(res, c.BindEnv(IngestorAPIInsecure, "KH_INGESTOR_API_INSECURE"))
157+
res = multierror.Append(res, c.BindEnv(IngestorBlobBucketURL, "KH_INGESTOR_BUCKET_URL"))
158+
res = multierror.Append(res, c.BindEnv(IngestorTempDir, "KH_INGESTOR_TEMP_DIR"))
159+
res = multierror.Append(res, c.BindEnv(IngestorMaxArchiveSize, "KH_INGESTOR_MAX_ARCHIVE_SIZE"))
160+
res = multierror.Append(res, c.BindEnv(IngestorArchiveName, "KH_INGESTOR_ARCHIVE_NAME"))
161+
res = multierror.Append(res, c.BindEnv(IngestorBlobRegion, "KH_INGESTOR_REGION"))
162+
152163
if res.ErrorOrNil() != nil {
153164
log.I.Fatalf("config environment override: %v", res.ErrorOrNil())
154165
}
@@ -234,6 +245,8 @@ func NewEmbedConfig(v *viper.Viper, configPath string) (*KubehoundConfig, error)
234245
v.SetConfigType(DefaultConfigType)
235246
SetDefaultValues(v)
236247

248+
// Configure environment variable override
249+
SetEnvOverrides(v)
237250
data, err := embedconfig.F.ReadFile(configPath)
238251
if err != nil {
239252
return nil, fmt.Errorf("reading embed config: %w", err)

‎pkg/config/config_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,8 @@ func TestMustLoadConfig(t *testing.T) {
8181
Insecure: false,
8282
},
8383
Blob: &BlobConfig{
84-
Bucket: "",
85-
Region: "",
84+
BucketUrl: "",
85+
Region: "",
8686
},
8787
TempDir: "/tmp/kubehound",
8888
ArchiveName: "archive.tar.gz",
@@ -155,8 +155,8 @@ func TestMustLoadConfig(t *testing.T) {
155155
Insecure: false,
156156
},
157157
Blob: &BlobConfig{
158-
Bucket: "",
159-
Region: "",
158+
BucketUrl: "",
159+
Region: "",
160160
},
161161
TempDir: "/tmp/kubehound",
162162
ArchiveName: "archive.tar.gz",

‎pkg/config/ingestor.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ const (
1414
IngestorTempDir = "ingestor.temp_dir"
1515
IngestorArchiveName = "ingestor.archive_name"
1616

17-
IngestorBlobBucketName = "ingestor.blob.bucket_name"
18-
IngestorBlobRegion = "ingestor.blob.region"
17+
IngestorBlobBucketURL = "ingestor.blob.bucket_url"
18+
IngestorBlobRegion = "ingestor.blob.region"
1919
)
2020

2121
type IngestorConfig struct {

‎pkg/config/janusgraph.go

+3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ import (
66

77
const (
88
DefaultJanusGraphUrl = "ws://localhost:8182/gremlin"
9+
10+
JanusGraphUrl = "janusgraph.url"
11+
JanusGrapTimeout = "janusgraph.connection_timeout"
912
)
1013

1114
// JanusGraphConfig configures JanusGraph specific parameters.

‎pkg/config/mongodb.go

+3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ import (
66

77
const (
88
DefaultMongoUrl = "mongodb://localhost:27017"
9+
10+
MongoUrl = "mongodb.url"
11+
MongoConnectionTimeout = "mongodb.connection_timeout"
912
)
1013

1114
// MongoDBConfig configures mongodb specific parameters.

‎pkg/ingestor/puller/blob/blob.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,12 @@ type BlobStore struct {
3939
var _ puller.DataPuller = (*BlobStore)(nil)
4040

4141
func NewBlobStorage(cfg *config.KubehoundConfig, blobConfig *config.BlobConfig) (*BlobStore, error) {
42-
if blobConfig.Bucket == "" {
42+
if blobConfig.BucketUrl == "" {
4343
return nil, ErrInvalidBucketName
4444
}
4545

4646
return &BlobStore{
47-
bucketName: blobConfig.Bucket,
47+
bucketName: blobConfig.BucketUrl,
4848
cfg: cfg,
4949
region: blobConfig.Region,
5050
}, nil

‎pkg/ingestor/puller/blob/blob_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ func TestNewBlobStorage(t *testing.T) {
339339
name: "empty bucket name",
340340
args: args{
341341
blobConfig: &config.BlobConfig{
342-
Bucket: "",
342+
BucketUrl: "",
343343
},
344344
cfg: &config.KubehoundConfig{
345345
Ingestor: config.IngestorConfig{
@@ -353,7 +353,7 @@ func TestNewBlobStorage(t *testing.T) {
353353
name: "valid blob storage",
354354
args: args{
355355
blobConfig: &config.BlobConfig{
356-
Bucket: "fakeBlobStorage",
356+
BucketUrl: "fakeBlobStorage",
357357
},
358358
cfg: &config.KubehoundConfig{
359359
Ingestor: config.IngestorConfig{

‎pkg/kubehound/storage/retrier.go

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"time"
66

77
"github.com/DataDog/KubeHound/pkg/config"
8+
"github.com/DataDog/KubeHound/pkg/telemetry/log"
89
)
910

1011
type Connector[T any] func(ctx context.Context, cfg *config.KubehoundConfig) (T, error)
@@ -17,6 +18,7 @@ func Retrier[T any](connector Connector[T], retries int, delay time.Duration) Co
1718
if err == nil || r >= retries {
1819
return provider, err
1920
}
21+
log.I.Warnf("Retrying to connect [%d/%d]", r+1, retries)
2022

2123
select {
2224
case <-time.After(delay):

‎test/system/kubehound_dump.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@ builder:
2929
# Ingestor configuration (for KHaaS)
3030
ingestor:
3131
blob:
32-
bucket: "" # (i.e.: s3://<your_bucket>)
32+
bucket_url: "" # (i.e.: s3://<your_bucket>)
3333
region: "" # (i.e.: us-west-2)
3434
temp_dir: "/tmp/kubehound"
3535
archive_name: "archive.tar.gz"
3636
max_archive_size: 2147483648 # 2GB
3737
api: # GRPC endpoint for the ingestor
3838
endpoint: "127.0.0.1:9000"
39-
insecure: true
39+
insecure: true

‎test/system/setup_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ func RunGRPC(ctx context.Context, runArgs *runArgs, p *providers.ProvidersFactor
193193
log.I.Fatal(err.Error())
194194
}
195195

196-
khCfg.Ingestor.Blob.Bucket = fmt.Sprintf("file://%s", fileFolder)
196+
khCfg.Ingestor.Blob.BucketUrl = fmt.Sprintf("file://%s", fileFolder)
197197
log.I.Info("Creating Blob Storage provider")
198198
puller, err := blob.NewBlobStorage(khCfg, khCfg.Ingestor.Blob)
199199
if err != nil {

0 commit comments

Comments
 (0)