From 59a804767d7217fceb178b91eedc4d5488aefcd3 Mon Sep 17 00:00:00 2001 From: Chief-Rishab Date: Sat, 13 May 2023 15:52:05 +0530 Subject: [PATCH] fix: use mutex locks while fetching resources in errgroup --- plugins/providers/bigquery/model_test.go | 71 ++++++++++++++++++++++++ plugins/providers/bigquery/provider.go | 3 + 2 files changed, 74 insertions(+) diff --git a/plugins/providers/bigquery/model_test.go b/plugins/providers/bigquery/model_test.go index eb8e447b0..3a707d62b 100644 --- a/plugins/providers/bigquery/model_test.go +++ b/plugins/providers/bigquery/model_test.go @@ -3,6 +3,7 @@ package bigquery_test import ( "testing" + "github.com/odpf/guardian/core/resource" "github.com/odpf/guardian/domain" "github.com/odpf/guardian/plugins/providers/bigquery" "github.com/stretchr/testify/assert" @@ -26,6 +27,39 @@ func TestDataSet(t *testing.T) { URN: "p_id:d_id", }, }, + { + ds: &bigquery.Dataset{ + ProjectID: "p_id", + DatasetID: "d_id", + Labels: nil, + }, + expectedResource: &domain.Resource{ + Type: bigquery.ResourceTypeDataset, + Name: "d_id", + URN: "p_id:d_id", + }, + }, + { + ds: &bigquery.Dataset{ + ProjectID: "p_id", + DatasetID: "d_id", + Labels: map[string]string{ + "key1": "value1", + }, + }, + expectedResource: &domain.Resource{ + Type: bigquery.ResourceTypeDataset, + Name: "d_id", + URN: "p_id:d_id", + Details: map[string]interface{}{ + resource.ReservedDetailsKeyMetadata: map[string]interface{}{ + "labels": map[string]string{ + "key1": "value1", + }, + }, + }, + }, + }, } for _, tc := range testCases { @@ -34,6 +68,7 @@ func TestDataSet(t *testing.T) { assert.Equal(t, tc.expectedResource.Type, actualResource.Type) assert.Equal(t, tc.expectedResource.Name, actualResource.Name) assert.Equal(t, tc.expectedResource.URN, actualResource.URN) + assert.Equal(t, tc.expectedResource.Details, actualResource.Details) } }) }) @@ -89,6 +124,41 @@ func TestTable(t *testing.T) { URN: "p_id:d_id.t_id", }, }, + { + tb: &bigquery.Table{ + TableID: "t_id", + ProjectID: "p_id", + DatasetID: "d_id", + Labels: nil, + }, + expectedResource: &domain.Resource{ + Type: bigquery.ResourceTypeTable, + Name: "t_id", + URN: "p_id:d_id.t_id", + }, + }, + { + tb: &bigquery.Table{ + TableID: "t_id", + ProjectID: "p_id", + DatasetID: "d_id", + Labels: map[string]string{ + "key1": "value1", + }, + }, + expectedResource: &domain.Resource{ + Type: bigquery.ResourceTypeTable, + Name: "t_id", + URN: "p_id:d_id.t_id", + Details: map[string]interface{}{ + resource.ReservedDetailsKeyMetadata: map[string]interface{}{ + "labels": map[string]string{ + "key1": "value1", + }, + }, + }, + }, + }, } for _, tc := range testCases { @@ -97,6 +167,7 @@ func TestTable(t *testing.T) { assert.Equal(t, tc.expectedResource.Type, actualResource.Type) assert.Equal(t, tc.expectedResource.Name, actualResource.Name) assert.Equal(t, tc.expectedResource.URN, actualResource.URN) + assert.Equal(t, tc.expectedResource.Details, actualResource.Details) } }) }) diff --git a/plugins/providers/bigquery/provider.go b/plugins/providers/bigquery/provider.go index c875fec6f..a7648f7bd 100644 --- a/plugins/providers/bigquery/provider.go +++ b/plugins/providers/bigquery/provider.go @@ -124,6 +124,7 @@ func (p *Provider) GetResources(pc *domain.ProviderConfig) ([]*domain.Resource, resources := []*domain.Resource{} eg, ctx := errgroup.WithContext(context.TODO()) eg.SetLimit(10) + var mu sync.Mutex datasets, err := client.GetDatasets(ctx) if err != nil { @@ -137,6 +138,8 @@ func (p *Provider) GetResources(pc *domain.ProviderConfig) ([]*domain.Resource, dataset.ProviderURN = pc.URN if containsString(resourceTypes, ResourceTypeDataset) { + mu.Lock() + defer mu.Unlock() resources = append(resources, dataset) }