refactor(gcp): provider v2 (#1334)

* refactor(gcp): refactor estimator to v2

* refactor(gcp): add Kind method to provider

* refactor(gcp): add discoverer

* refactor(gcp): move discoverer config one step higher

* refactor(gcp): refactor scanner logic

* refactor(gcp): add unit tests

* fix(gcp): run make fix

* fix(gcp): add headers to files

* fix(gcp): fix linter

* refactor(gcp): rename common to utils

* fix(gcp): import orders

provider/v2/gcp/config.go

@@ -0,0 +1,91 @@
+// Copyright © 2023 Cisco Systems, Inc. and its affiliates.
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gcp
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+)
+
+const (
+	DefaultEnvPrefix = "VMCLARITY_GCP"
+
+	projectID           = "project_id"
+	scannerZone         = "scanner_zone"
+	scannerSubnetwork   = "scanner_subnetwork"
+	scannerMachineType  = "scanner_machine_type"
+	scannerSourceImage  = "scanner_source_image"
+	scannerSSHPublicKey = "scanner_ssh_public_key"
+)
+
+type Config struct {
+	ProjectID           string `mapstructure:"project_id"`
+	ScannerZone         string `mapstructure:"scanner_zone"`
+	ScannerSubnetwork   string `mapstructure:"scanner_subnetwork"`
+	ScannerMachineType  string `mapstructure:"scanner_machine_type"`
+	ScannerSourceImage  string `mapstructure:"scanner_source_image"`
+	ScannerSSHPublicKey string `mapstructure:"scanner_ssh_public_key"`
+}
+
+func NewConfig() (*Config, error) {
+	// Avoid modifying the global instance
+	v := viper.New()
+
+	v.SetEnvPrefix(DefaultEnvPrefix)
+	v.AllowEmptyEnv(true)
+	v.AutomaticEnv()
+
+	_ = v.BindEnv(projectID)
+	_ = v.BindEnv(scannerZone)
+	_ = v.BindEnv(scannerSubnetwork)
+	_ = v.BindEnv(scannerMachineType)
+	_ = v.BindEnv(scannerSourceImage)
+	_ = v.BindEnv(scannerSSHPublicKey)
+
+	config := &Config{}
+	if err := v.Unmarshal(&config, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())); err != nil {
+		return nil, fmt.Errorf("failed to parse provider configuration. Provider=GCP: %w", err)
+	}
+	return config, nil
+}
+
+// nolint:cyclop
+func (c Config) Validate() error {
+	if c.ProjectID == "" {
+		return fmt.Errorf("parameter ProjectID must be provided by setting %v_%v environment variable", DefaultEnvPrefix, strings.ToUpper(projectID))
+	}
+
+	if c.ScannerZone == "" {
+		return fmt.Errorf("parameter ScannerZone must be provided by setting %v_%v environment variable", DefaultEnvPrefix, strings.ToUpper(scannerZone))
+	}
+
+	if c.ScannerSubnetwork == "" {
+		return fmt.Errorf("parameter ScannerSubnetwork must be provided by setting %v_%v environment variable", DefaultEnvPrefix, strings.ToUpper(scannerSubnetwork))
+	}
+
+	if c.ScannerMachineType == "" {
+		return fmt.Errorf("parameter ScannerMachineType must be provided by setting %v_%v environment variable", DefaultEnvPrefix, strings.ToUpper(scannerMachineType))
+	}
+
+	if c.ScannerSourceImage == "" {
+		return fmt.Errorf("parameter ScannerSourceImage must be provided by setting %v_%v environment variable", DefaultEnvPrefix, strings.ToUpper(scannerSourceImage))
+	}
+
+	return nil
+}

provider/v2/gcp/discoverer/discoverer.go

@@ -17,15 +17,200 @@ package discoverer
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"time"
 
+	compute "cloud.google.com/go/compute/apiv1"
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/api/iterator"
+
+	apitypes "github.com/openclarity/vmclarity/api/types"
+	"github.com/openclarity/vmclarity/core/to"
 	"github.com/openclarity/vmclarity/provider"
+	"github.com/openclarity/vmclarity/provider/v2/gcp/utils"
 )
 
-var _ provider.Discoverer = &Discoverer{}
+const (
+	maxResults = 500
+)
 
-type Discoverer struct{}
+type Discoverer struct {
+	DisksClient     *compute.DisksClient
+	InstancesClient *compute.InstancesClient
+	RegionsClient   *compute.RegionsClient
 
+	ProjectID string
+}
+
+// nolint: cyclop
 func (d *Discoverer) DiscoverAssets(ctx context.Context) provider.AssetDiscoverer {
-	// TODO implement me
-	panic("implement me")
+	assetDiscoverer := provider.NewSimpleAssetDiscoverer()
+
+	go func() {
+		defer close(assetDiscoverer.OutputChan)
+
+		regions, err := d.listAllRegions(ctx)
+		if err != nil {
+			assetDiscoverer.Error = fmt.Errorf("failed to list all regions: %w", err)
+			return
+		}
+
+		var zones []string
+		for _, region := range regions {
+			zones = append(zones, getZonesLastPart(region.Zones)...)
+		}
+
+		for _, zone := range zones {
+			assets, err := d.listInstances(ctx, nil, zone)
+			if err != nil {
+				assetDiscoverer.Error = fmt.Errorf("failed to list instances: %w", err)
+				return
+			}
+
+			for _, asset := range assets {
+				select {
+				case assetDiscoverer.OutputChan <- asset:
+				case <-ctx.Done():
+					assetDiscoverer.Error = ctx.Err()
+					return
+				}
+			}
+		}
+	}()
+
+	return assetDiscoverer
+}
+
+func (d *Discoverer) listInstances(ctx context.Context, filter *string, zone string) ([]apitypes.AssetType, error) {
+	var ret []apitypes.AssetType
+
+	it := d.InstancesClient.List(ctx, &computepb.ListInstancesRequest{
+		Filter:     filter,
+		MaxResults: to.Ptr[uint32](maxResults),
+		Project:    d.ProjectID,
+		Zone:       zone,
+	})
+	for {
+		resp, err := it.Next()
+		if errors.Is(err, iterator.Done) {
+			break
+		}
+		if err != nil {
+			_, err = utils.HandleGcpRequestError(err, "listing instances for project %s zone %s", d.ProjectID, zone)
+			return nil, err // nolint: wrapcheck
+		}
+
+		info, err := d.getVMInfoFromVirtualMachine(ctx, resp)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get vminfo from virtual machine: %w", err)
+		}
+		ret = append(ret, info)
+	}
+
+	return ret, nil
+}
+
+func (d *Discoverer) listAllRegions(ctx context.Context) ([]*computepb.Region, error) {
+	var ret []*computepb.Region
+
+	it := d.RegionsClient.List(ctx, &computepb.ListRegionsRequest{
+		MaxResults: to.Ptr[uint32](maxResults),
+		Project:    d.ProjectID,
+	})
+	for {
+		resp, err := it.Next()
+		if errors.Is(err, iterator.Done) {
+			break
+		}
+		if err != nil {
+			_, err := utils.HandleGcpRequestError(err, "list regions")
+			return nil, err // nolint: wrapcheck
+		}
+
+		ret = append(ret, resp)
+	}
+	return ret, nil
+}
+
+func (d *Discoverer) getVMInfoFromVirtualMachine(ctx context.Context, vm *computepb.Instance) (apitypes.AssetType, error) {
+	assetType := apitypes.AssetType{}
+	launchTime, err := time.Parse(time.RFC3339, *vm.CreationTimestamp)
+	if err != nil {
+		return apitypes.AssetType{}, fmt.Errorf("failed to parse time: %v", *vm.CreationTimestamp)
+	}
+	// get boot disk name
+	diskName := utils.GetLastURLPart(vm.Disks[0].Source)
+
+	var platform string
+	var image string
+
+	// get disk from gcp
+	disk, err := d.DisksClient.Get(ctx, &computepb.GetDiskRequest{
+		Disk:    diskName,
+		Project: d.ProjectID,
+		Zone:    utils.GetLastURLPart(vm.Zone),
+	})
+	if err != nil {
+		logrus.Warnf("failed to get disk %v: %v", diskName, err)
+	} else {
+		if disk.Architecture != nil {
+			platform = *disk.Architecture
+		}
+		image = utils.GetLastURLPart(disk.SourceImage)
+	}
+
+	err = assetType.FromVMInfo(apitypes.VMInfo{
+		InstanceProvider: to.Ptr(apitypes.GCP),
+		InstanceID:       *vm.Name,
+		Image:            image,
+		InstanceType:     utils.GetLastURLPart(vm.MachineType),
+		LaunchTime:       launchTime,
+		Location:         utils.GetLastURLPart(vm.Zone),
+		Platform:         platform,
+		SecurityGroups:   &[]apitypes.SecurityGroup{},
+		Tags:             to.Ptr(convertLabelsToTags(vm.Labels)),
+	})
+	if err != nil {
+		return apitypes.AssetType{}, provider.FatalErrorf("failed to create AssetType from VMInfo: %w", err)
+	}
+
+	return assetType, nil
+}
+
+// getZonesLastPart converts a list of zone URLs into a list of zone IDs.
+// For example input:
+//
+// [
+//
+//	https://www.googleapis.com/compute/v1/projects/gcp-etigcp-nprd-12855/zones/us-central1-c,
+//	https://www.googleapis.com/compute/v1/projects/gcp-etigcp-nprd-12855/zones/us-central1-a
+//
+// ]
+//
+// returns [us-central1-c, us-central1-a].
+func getZonesLastPart(zones []string) []string {
+	ret := make([]string, 0, len(zones))
+	for _, zone := range zones {
+		z := zone
+		ret = append(ret, utils.GetLastURLPart(&z))
+	}
+	return ret
+}
+
+func convertLabelsToTags(labels map[string]string) []apitypes.Tag {
+	tags := make([]apitypes.Tag, 0, len(labels))
+
+	for k, v := range labels {
+		tags = append(
+			tags,
+			apitypes.Tag{
+				Key:   k,
+				Value: v,
+			},
+		)
+	}
+
+	return tags
 }

provider/v2/gcp/discoverer/discoverer_test.go

@@ -0,0 +1,85 @@
+// Copyright © 2023 Cisco Systems, Inc. and its affiliates.
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discoverer
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	apitypes "github.com/openclarity/vmclarity/api/types"
+)
+
+func Test_getZonesLastPart(t *testing.T) {
+	type args struct {
+		zones []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "empty",
+			args: args{
+				zones: []string{},
+			},
+			want: []string{},
+		},
+		{
+			name: "get two zones",
+			args: args{
+				zones: []string{"https://www.googleapis.com/compute/v1/projects/gcp-etigcp-nprd-12855/zones/us-central1-c", "https://www.googleapis.com/compute/v1/projects/gcp-etigcp-nprd-12855/zones/us-central1-a"},
+			},
+			want: []string{"us-central1-c", "us-central1-a"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := getZonesLastPart(tt.args.zones)
+			if diff := cmp.Diff(tt.want, got); diff != "" {
+				t.Errorf("getZonesLastPart() mismatch (-want, +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func Test_convertLabelsToTags(t *testing.T) {
+	tests := []struct {
+		name string
+		args map[string]string
+		want []apitypes.Tag
+	}{
+		{
+			name: "sanity",
+			args: map[string]string{
+				"valid-tag": "valid-value",
+			},
+			want: []apitypes.Tag{{
+				Key: "valid-tag", Value: "valid-value",
+			}},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := convertLabelsToTags(tt.args); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("convertLabelsToTags() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

provider/v2/gcp/estimator/estimator.go

@@ -22,11 +22,8 @@ import (
 	"github.com/openclarity/vmclarity/provider"
 )
 
-var _ provider.Estimator = &Estimator{}
-
 type Estimator struct{}
 
 func (e *Estimator) Estimate(ctx context.Context, stats apitypes.AssetScanStats, asset *apitypes.Asset, template *apitypes.AssetScanTemplate) (*apitypes.Estimation, error) {
-	// TODO implement me
-	panic("implement me")
+	return &apitypes.Estimation{}, provider.FatalErrorf("Not Implemented")
 }