List instances to check for authorization issues

Author: AkarshES

pkg/cloudprovider/providers/oci/instances.go

@@ -243,7 +243,8 @@ func (cp *CloudProvider) InstanceExistsByProviderID(ctx context.Context, provide
 	}
 	instance, err := cp.client.Compute().GetInstance(ctx, instanceID)
 	if client.IsNotFound(err) {
-		return false, nil
+		return cp.checkForAuthorizationError(ctx, providerID)
+
 	}
 	if err != nil {
 		return false, err
@@ -252,6 +253,34 @@ func (cp *CloudProvider) InstanceExistsByProviderID(ctx context.Context, provide
 	return !client.IsInstanceInTerminalState(instance), nil
 }
 
+func (cp *CloudProvider) checkForAuthorizationError(ctx context.Context, instanceId string) (bool, error) {
+	cp.logger.With("instanceId", instanceId).Info("Received 404 for an instance, listing instances to check for authorization errors")
+	compartmentId, err := cp.getCompartmentIDByInstanceID(instanceId)
+	if err != nil {
+		return false, err
+	}
+	// to eliminate AD specific issues, list all ADs and make AD specific requests
+	availabilityDomains, err := cp.client.Identity().ListAvailabilityDomains(ctx, compartmentId)
+	for _, availabilityDomain := range availabilityDomains {
+		instances, err := cp.client.Compute().ListInstancesByCompartmentAndAD(ctx, compartmentId, *availabilityDomain.Name)
+		// if we are getting errors for ListInstances the issue can be authorization or other issues
+		// so to be safe we return the error back as we can't list instances in the compartment
+		if err != nil {
+			cp.logger.With("instanceId", instanceId).Info("Received error when listing instances to check for authorization errors")
+			return false, err
+		}
+
+		for _, instance := range instances {
+			if *instance.Id == instanceId {
+				// Can only happen if changes are done in policy in-between requests
+				return true, nil
+			}
+		}
+	}
+
+	return false, nil
+}
+
 // InstanceShutdownByProviderID returns true if the instance is shutdown in cloudprovider.
 func (cp *CloudProvider) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) {
 	//Please do not try to optimise it by using InstanceCache because we prefer correctness over efficiency here

pkg/cloudprovider/providers/oci/instances_test.go

@@ -310,6 +310,10 @@ func (MockOCIClient) Identity() client.IdentityInterface {
 // MockComputeClient mocks Compute client implementation
 type MockComputeClient struct{}
 
+func (c MockComputeClient) ListInstancesByCompartmentAndAD(ctx context.Context, compartmentId, availabilityDomain string) (response []core.Instance, err error) {
+	return nil, nil
+}
+
 func (MockComputeClient) GetInstance(ctx context.Context, id string) (*core.Instance, error) {
 	if instance, ok := instances[id]; ok {
 		return instance, nil

pkg/csi/driver/bv_controller_test.go

@@ -542,6 +542,10 @@ type MockComputeClient struct {
 	compute util.MockOCIComputeClient
 }
 
+func (c *MockComputeClient) ListInstancesByCompartmentAndAD(ctx context.Context, compartmentId, availabilityDomain string) (response []core.Instance, err error) {
+	return nil, nil
+}
+
 // GetInstance gets information about the specified instance.
 func (c *MockComputeClient) GetInstance(ctx context.Context, id string) (*core.Instance, error) {
 	if instance, ok := instances[id]; ok {

pkg/oci/client/compute.go

@@ -26,6 +26,7 @@ import (
 type ComputeInterface interface {
 	// GetInstance gets information about the specified instance.
 	GetInstance(ctx context.Context, id string) (*core.Instance, error)
+	ListInstancesByCompartmentAndAD(ctx context.Context, compartmentId, availabilityDomain string) (response []core.Instance, err error)
 
 	// GetInstanceByNodeName gets the OCI instance corresponding to the given
 	// Kubernetes node name.
@@ -53,6 +54,36 @@ func (c *client) GetInstance(ctx context.Context, id string) (*core.Instance, er
 	return &resp.Instance, nil
 }
 
+func (c *client) ListInstancesByCompartmentAndAD(ctx context.Context, compartmentID, availabilityDomain string) ([]core.Instance, error) {
+	var (
+		page      *string
+		instances []core.Instance
+	)
+	for {
+		if !c.rateLimiter.Reader.TryAccept() {
+			return nil, RateLimitError(false, "ListInstances")
+		}
+		resp, err := c.compute.ListInstances(ctx, core.ListInstancesRequest{
+			AvailabilityDomain: &availabilityDomain,
+			CompartmentId:      &compartmentID,
+			Page:               page,
+			RequestMetadata:    c.requestMetadata,
+		})
+		incRequestCounter(err, listVerb, instanceResource)
+
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		instances = append(instances, resp.Items...)
+		if page = resp.OpcNextPage; resp.OpcNextPage == nil {
+			break
+		}
+	}
+
+	return instances, nil
+}
+
 func (c *client) getInstanceByDisplayName(ctx context.Context, compartmentID, displayName string) (*core.Instance, error) {
 	var (
 		page      *string

pkg/volume/provisioner/block/block_test.go

@@ -227,6 +227,10 @@ func (c *MockFileStorageClient) AwaitMountTargetActive(ctx context.Context, logg
 
 type MockComputeClient struct{}
 
+func (c *MockComputeClient) ListInstancesByCompartmentAndAD(ctx context.Context, compartmentId, availabilityDomain string) (response []core.Instance, err error) {
+	return nil, nil
+}
+
 // GetInstance gets information about the specified instance.
 func (c *MockComputeClient) GetInstance(ctx context.Context, id string) (*core.Instance, error) {
 	return nil, nil

pkg/volume/provisioner/fss/fss_test.go

@@ -225,6 +225,10 @@ func (c *MockFileStorageClient) AwaitMountTargetActive(ctx context.Context, logg
 
 type MockComputeClient struct{}
 
+func (c *MockComputeClient) ListInstancesByCompartmentAndAD(ctx context.Context, compartmentId, availabilityDomain string) (response []core.Instance, err error) {
+	return nil, nil
+}
+
 // GetInstance gets information about the specified instance.
 func (c *MockComputeClient) GetInstance(ctx context.Context, id string) (*core.Instance, error) {
 	return nil, nil