Skip to content

Commit fe52f52

Browse files
irainiairainia
authored
fix: empty resource (#23)
* refactor: update naming within upstream for more readability * fix: remove upstream implementation to resource only * refactor: extract out group upstream extraction * test: add sorting actual results to ensure consistency * fix: ignore error caused by access denied or user permission issue * refactor: remove unnecessary implementation in test * refactor: move out ignorable error handling to extractor * fix: update to add logger to extractor
1 parent 76ba68d commit fe52f52

File tree

8 files changed

+270
-289
lines changed

8 files changed

+270
-289
lines changed

‎task/bq2bq/factory.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66

77
"cloud.google.com/go/bigquery"
88
"github.com/googleapis/google-cloud-go-testing/bigquery/bqiface"
9+
"github.com/hashicorp/go-hclog"
910
"golang.org/x/oauth2/google"
1011
"google.golang.org/api/drive/v2"
1112
"google.golang.org/api/option"
@@ -35,6 +36,6 @@ func (fac *DefaultBQClientFactory) New(ctx context.Context, svcAccount string) (
3536
type DefaultUpstreamExtractorFactory struct {
3637
}
3738

38-
func (d *DefaultUpstreamExtractorFactory) New(client bqiface.Client) (UpstreamExtractor, error) {
39-
return upstream.NewExtractor(client)
39+
func (d *DefaultUpstreamExtractorFactory) New(client bqiface.Client, logger hclog.Logger) (UpstreamExtractor, error) {
40+
return upstream.NewExtractor(client, logger)
4041
}

‎task/bq2bq/main.go

+6-8
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,11 @@ type ClientFactory interface {
4747
}
4848

4949
type UpstreamExtractor interface {
50-
ExtractUpstreams(ctx context.Context, query string, resourcesToIgnore []upstream.Resource) ([]*upstream.Upstream, error)
50+
ExtractUpstreams(ctx context.Context, query string, resourcesToIgnore []upstream.Resource) ([]upstream.Resource, error)
5151
}
5252

5353
type ExtractorFactory interface {
54-
New(client bqiface.Client) (UpstreamExtractor, error)
54+
New(client bqiface.Client, logger hclog.Logger) (UpstreamExtractor, error)
5555
}
5656

5757
type BQ2BQ struct {
@@ -224,10 +224,7 @@ func (b *BQ2BQ) GenerateDependencies(ctx context.Context, request plugin.Generat
224224
return response, fmt.Errorf("error extracting upstreams: %w", err)
225225
}
226226

227-
flattenedUpstreams := upstream.FlattenUpstreams(upstreams)
228-
uniqueUpstreams := upstream.UniqueFilterResources(flattenedUpstreams)
229-
230-
formattedUpstreams := b.formatUpstreams(uniqueUpstreams, func(r upstream.Resource) string {
227+
formattedUpstreams := b.formatUpstreams(upstreams, func(r upstream.Resource) string {
231228
name := fmt.Sprintf("%s:%s.%s", r.Project, r.Dataset, r.Name)
232229
return fmt.Sprintf(plugin.DestinationURNFormat, selfTable.Type, name)
233230
})
@@ -237,7 +234,7 @@ func (b *BQ2BQ) GenerateDependencies(ctx context.Context, request plugin.Generat
237234
return response, nil
238235
}
239236

240-
func (b *BQ2BQ) extractUpstreams(ctx context.Context, query, svcAccSecret string, resourcesToIgnore []upstream.Resource) ([]*upstream.Upstream, error) {
237+
func (b *BQ2BQ) extractUpstreams(ctx context.Context, query, svcAccSecret string, resourcesToIgnore []upstream.Resource) ([]upstream.Resource, error) {
241238
spanCtx, span := StartChildSpan(ctx, "extractUpstreams")
242239
defer span.End()
243240

@@ -247,7 +244,7 @@ func (b *BQ2BQ) extractUpstreams(ctx context.Context, query, svcAccSecret string
247244
return nil, fmt.Errorf("error creating bigquery client: %w", err)
248245
}
249246

250-
extractor, err := b.ExtractorFac.New(client)
247+
extractor, err := b.ExtractorFac.New(client, b.logger)
251248
if err != nil {
252249
return nil, fmt.Errorf("error initializing upstream extractor: %w", err)
253250
}
@@ -263,6 +260,7 @@ func (b *BQ2BQ) extractUpstreams(ctx context.Context, query, svcAccSecret string
263260
}
264261

265262
b.logger.Error("error extracting upstreams", err)
263+
return nil, err
266264
}
267265

268266
return upstreams, nil

‎task/bq2bq/main_test.go

+39-58
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99

1010
"github.com/googleapis/google-cloud-go-testing/bigquery/bqiface"
1111
"github.com/goto/optimus/sdk/plugin"
12+
"github.com/hashicorp/go-hclog"
1213
"github.com/stretchr/testify/assert"
1314
"github.com/stretchr/testify/mock"
1415

@@ -33,8 +34,8 @@ type extractorFactoryMock struct {
3334
mock.Mock
3435
}
3536

36-
func (e *extractorFactoryMock) New(client bqiface.Client) (UpstreamExtractor, error) {
37-
args := e.Called(client)
37+
func (e *extractorFactoryMock) New(client bqiface.Client, logger hclog.Logger) (UpstreamExtractor, error) {
38+
args := e.Called(client, logger)
3839

3940
r1, ok := args.Get(0).(UpstreamExtractor)
4041
if !ok {
@@ -48,10 +49,10 @@ type extractorMock struct {
4849
mock.Mock
4950
}
5051

51-
func (e *extractorMock) ExtractUpstreams(ctx context.Context, query string, resourcesToIgnore []upstream.Resource) ([]*upstream.Upstream, error) {
52+
func (e *extractorMock) ExtractUpstreams(ctx context.Context, query string, resourcesToIgnore []upstream.Resource) ([]upstream.Resource, error) {
5253
args := e.Called(ctx, query, resourcesToIgnore)
5354

54-
r1, ok := args.Get(0).([]*upstream.Upstream)
55+
r1, ok := args.Get(0).([]upstream.Resource)
5556
if !ok {
5657
return nil, args.Error(1)
5758
}
@@ -61,6 +62,7 @@ func (e *extractorMock) ExtractUpstreams(ctx context.Context, query string, reso
6162

6263
func TestBQ2BQ(t *testing.T) {
6364
ctx := context.Background()
65+
logger := hclog.NewNullLogger()
6466

6567
t.Run("GetName", func(t *testing.T) {
6668
t.Run("should return name bq2bq", func(t *testing.T) {
@@ -263,22 +265,21 @@ Select * from table where ts > "2021-01-16T00:00:00Z"`
263265

264266
extractor := new(extractorMock)
265267
extractor.On("ExtractUpstreams", mock.Anything, query, []upstream.Resource{destination}).
266-
Return([]*upstream.Upstream{
268+
Return([]upstream.Resource{
267269
{
268-
Resource: upstream.Resource{
269-
Project: "proj",
270-
Dataset: "dataset",
271-
Name: "table1",
272-
},
270+
Project: "proj",
271+
Dataset: "dataset",
272+
Name: "table1",
273273
},
274274
}, nil)
275275

276276
extractorFac := new(extractorFactoryMock)
277-
extractorFac.On("New", client).Return(extractor, nil)
277+
extractorFac.On("New", client, logger).Return(extractor, nil)
278278

279279
b := &BQ2BQ{
280280
ClientFac: bqClientFac,
281281
ExtractorFac: extractorFac,
282+
logger: logger,
282283
}
283284
got, err := b.GenerateDependencies(ctx, data)
284285
if err != nil {
@@ -335,45 +336,26 @@ Select * from table where ts > "2021-01-16T00:00:00Z"`
335336

336337
extractor := new(extractorMock)
337338
extractor.On("ExtractUpstreams", mock.Anything, query, []upstream.Resource{destination}).
338-
Return([]*upstream.Upstream{
339-
{
340-
Resource: upstream.Resource{
341-
Project: "proj",
342-
Dataset: "dataset",
343-
Name: "table1",
344-
},
345-
Upstreams: []*upstream.Upstream{
346-
{
347-
Resource: upstream.Resource{
348-
Project: "proj",
349-
Dataset: "dataset",
350-
Name: "table2",
351-
},
352-
},
353-
},
354-
},
355-
{
356-
Resource: upstream.Resource{
357-
Project: "proj",
358-
Dataset: "dataset",
359-
Name: "table2",
360-
},
361-
},
362-
{
363-
Resource: upstream.Resource{
364-
Project: "proj",
365-
Dataset: "dataset",
366-
Name: "table1",
367-
},
339+
Return([]upstream.Resource{
340+
{
341+
Project: "proj",
342+
Dataset: "dataset",
343+
Name: "table1",
344+
},
345+
{
346+
Project: "proj",
347+
Dataset: "dataset",
348+
Name: "table2",
368349
},
369350
}, nil)
370351

371352
extractorFac := new(extractorFactoryMock)
372-
extractorFac.On("New", client).Return(extractor, nil)
353+
extractorFac.On("New", client, logger).Return(extractor, nil)
373354

374355
b := &BQ2BQ{
375356
ClientFac: bqClientFac,
376357
ExtractorFac: extractorFac,
358+
logger: logger,
377359
}
378360
got, err := b.GenerateDependencies(ctx, data)
379361
if err != nil {
@@ -430,14 +412,15 @@ Select * from table where ts > "2021-01-16T00:00:00Z"`
430412

431413
extractor := new(extractorMock)
432414
extractor.On("ExtractUpstreams", mock.Anything, query, []upstream.Resource{destination}).
433-
Return([]*upstream.Upstream{}, nil)
415+
Return([]upstream.Resource{}, nil)
434416

435417
extractorFac := new(extractorFactoryMock)
436-
extractorFac.On("New", client).Return(extractor, nil)
418+
extractorFac.On("New", client, logger).Return(extractor, nil)
437419

438420
b := &BQ2BQ{
439421
ClientFac: bqClientFac,
440422
ExtractorFac: extractorFac,
423+
logger: logger,
441424
}
442425
got, err := b.GenerateDependencies(ctx, data)
443426
if err != nil {
@@ -494,22 +477,21 @@ Select * from table where ts > "2021-01-16T00:00:00Z"`
494477

495478
extractor := new(extractorMock)
496479
extractor.On("ExtractUpstreams", mock.Anything, query, []upstream.Resource{destination}).
497-
Return([]*upstream.Upstream{
480+
Return([]upstream.Resource{
498481
{
499-
Resource: upstream.Resource{
500-
Project: "proj",
501-
Dataset: "dataset",
502-
Name: "table1",
503-
},
482+
Project: "proj",
483+
Dataset: "dataset",
484+
Name: "table1",
504485
},
505486
}, nil)
506487

507488
extractorFac := new(extractorFactoryMock)
508-
extractorFac.On("New", client).Return(extractor, nil)
489+
extractorFac.On("New", client, logger).Return(extractor, nil)
509490

510491
b := &BQ2BQ{
511492
ClientFac: bqClientFac,
512493
ExtractorFac: extractorFac,
494+
logger: logger,
513495
}
514496
got, err := b.GenerateDependencies(ctx, data)
515497
if err != nil {
@@ -566,22 +548,21 @@ Select * from table where ts > "2021-01-16T00:00:00Z"`
566548

567549
extractor := new(extractorMock)
568550
extractor.On("ExtractUpstreams", mock.Anything, query, []upstream.Resource{destination}).
569-
Return([]*upstream.Upstream{
551+
Return([]upstream.Resource{
570552
{
571-
Resource: upstream.Resource{
572-
Project: "proj",
573-
Dataset: "dataset",
574-
Name: "table1",
575-
},
553+
Project: "proj",
554+
Dataset: "dataset",
555+
Name: "table1",
576556
},
577557
}, nil)
578558

579559
extractorFac := new(extractorFactoryMock)
580-
extractorFac.On("New", client).Return(extractor, nil)
560+
extractorFac.On("New", client, logger).Return(extractor, nil)
581561

582562
b := &BQ2BQ{
583563
ClientFac: bqClientFac,
584564
ExtractorFac: extractorFac,
565+
logger: logger,
585566
}
586567
got, err := b.GenerateDependencies(ctx, data)
587568
if err != nil {

0 commit comments

Comments
 (0)