diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 0dcb4ed26..0111303f9 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "net" + "net/url" "os/exec" "regexp" "strings" @@ -26,9 +27,8 @@ import ( ) const ( - defaultTimeout = 200 * time.Second - verifyNoneExistTimeout = 15 * time.Second - crdVersion = "v1alpha1" + defaultTimeout = 200 * time.Second + PolicyAPI = "policy/api/v1" ) type Status int @@ -340,22 +340,21 @@ func (data *TestData) waitForCRReadyOrDeleted(timeout time.Duration, cr string, return nil } -func (data *TestData) getCRProperties(timeout time.Duration, crType, crName, namespace, key string) (string, error) { +func (data *TestData) getCRPropertiesByJson(timeout time.Duration, crType, crName, namespace, key string) (string, error) { value := "" err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(ctx context.Context) (bool, error) { - cmd := fmt.Sprintf("kubectl get %s %s -n %s -o yaml | grep %s", crType, crName, namespace, key) + cmd := fmt.Sprintf("kubectl get %s %s -n %s -o json | jq '%s'", crType, crName, namespace, key) log.Printf("%s", cmd) rc, stdout, _, err := RunCommandOnNode(clusterInfo.masterNodeName, cmd) if err != nil || rc != 0 { return false, fmt.Errorf("error when running the following command `%s` on master Node: %v, %s", cmd, err, stdout) } else { - parts := strings.Split(stdout, ":") - if len(parts) != 2 { - return false, fmt.Errorf("failed to read attribute from output %s", stdout) - } else { - value = parts[1] - return true, nil + // check if 'null' in stdout + if strings.Contains(stdout, "null") { + return false, nil } + value = stdout + return true, nil } }) if err != nil { @@ -370,6 +369,10 @@ func (data *TestData) getCRResource(timeout time.Duration, cr string, namespace crs := map[string]string{} err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(ctx context.Context) (bool, error) { cmd := fmt.Sprintf("kubectl get %s -n %s", cr, namespace) + // check if name is nil + if cr == "namespaces" { + cmd = fmt.Sprintf("kubectl get %s %s", cr, namespace) + } log.Printf("%s", cmd) rc, stdout, _, err := RunCommandOnNode(clusterInfo.masterNodeName, cmd) if err != nil || rc != 0 { @@ -388,6 +391,9 @@ func (data *TestData) getCRResource(timeout time.Duration, cr string, namespace continue } uid_cmd := fmt.Sprintf("kubectl get %s %s -n %s -o yaml | grep uid", cr, parts[0], namespace) + if cr == "namespaces" { + uid_cmd = fmt.Sprintf("kubectl get %s %s -o yaml | grep uid", cr, parts[0]) + } log.Printf("trying to get uid for cr: %s", uid_cmd) rc, stdout, _, err := RunCommandOnNode(clusterInfo.masterNodeName, uid_cmd) if err != nil || rc != 0 { @@ -724,3 +730,36 @@ func (data *TestData) waitForResourceExistById(namespace string, resourceType st func (data *TestData) waitForResourceExistOrNot(namespace string, resourceType string, resourceName string, shouldExist bool) error { return data.waitForResourceExist(namespace, resourceType, "display_name", resourceName, shouldExist) } + +func (data *TestData) waitForResourceExistByPath(pathPolicy string, shouldExist bool) error { + err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, defaultTimeout, false, func(ctx context.Context) (bool, error) { + exist := true + + fullURL := PolicyAPI + pathPolicy + fullURL = strings.ReplaceAll(fullURL, "\"", "") + fullURL = strings.ReplaceAll(fullURL, "\n", "") + fullURL = strings.ReplaceAll(fullURL, "\r", "") + _, err := url.Parse(fullURL) + if err != nil { + fmt.Println("Invalid URL:", err) + return false, err + } + + resp, err := testData.nsxClient.Client.Cluster.HttpGet(fullURL) + if err != nil { + if !shouldExist { + return true, nil + } + return false, err + } + id, ok := resp["id"].(string) + if !ok || id == "" { + exist = false + } + if exist != shouldExist { + return false, nil + } + return true, nil + }) + return err +} diff --git a/test/e2e/manifest/testVPC/customize_networkconfig_updated.yaml b/test/e2e/manifest/testVPC/customize_networkconfig_updated.yaml new file mode 100644 index 000000000..a718cd518 --- /dev/null +++ b/test/e2e/manifest/testVPC/customize_networkconfig_updated.yaml @@ -0,0 +1,20 @@ +# This file is used in testing customized VPC case, +# it support customer to define its own VPC network config. +apiVersion: nsx.vmware.com/v1alpha1 +kind: VPCNetworkConfiguration +metadata: + name: selfdefinedconfig +spec: + defaultGatewayPath: /infra/tier-0s/PLR + # nsx-operator-ci would replace '{edge-cluster-id}' with real edge-cluster-id of testbed + edgeClusterPath: /infra/sites/default/enforcement-points/default/edge-clusters/{edge-cluster-id} + defaultIPv4SubnetSize: 26 + nsxtProject: /orgs/default/projects/nsx_operator_e2e_test + externalIPv4Blocks: + - /infra/ip-blocks/e2e_test_external_ip_blk + privateIPv4CIDRs: + - 172.29.0.0/16 + - 172.39.0.0/16 + - 172.49.0.0/16 + defaultSubnetAccessMode: Public + \ No newline at end of file diff --git a/test/e2e/manifest/testVPC/shared_ns.yaml b/test/e2e/manifest/testVPC/shared_ns.yaml new file mode 100644 index 000000000..c653e0acd --- /dev/null +++ b/test/e2e/manifest/testVPC/shared_ns.yaml @@ -0,0 +1,17 @@ +# This file is used in testing shared VPC case, +# it create a namespace with customized network config for VPC. +apiVersion: v1 +kind: Namespace +metadata: + annotations: + nsx.vmware.com/vpc_network_config: selfdefinedconfig + name: shared-vpc-ns-0 + +--- +apiVersion: v1 +kind: Namespace +metadata: + annotations: + nsx.vmware.com/vpc_network_config: selfdefinedconfig + nsx.vmware.com/shared_vpc_namespace: shared-vpc-ns-0 + name: shared-vpc-ns-1 diff --git a/test/e2e/manifest/testVPC/update_ns.yaml b/test/e2e/manifest/testVPC/update_ns.yaml new file mode 100644 index 000000000..fe2bccd36 --- /dev/null +++ b/test/e2e/manifest/testVPC/update_ns.yaml @@ -0,0 +1,8 @@ +# This file is used in testing shared VPC case, +# it create a namespace with customized network config for VPC. +apiVersion: v1 +kind: Namespace +metadata: + annotations: + nsx.vmware.com/vpc_network_config: selfdefinedconfig + name: update-ns \ No newline at end of file diff --git a/test/e2e/nsx_networkinfo_test.go b/test/e2e/nsx_networkinfo_test.go new file mode 100644 index 000000000..1d242b56d --- /dev/null +++ b/test/e2e/nsx_networkinfo_test.go @@ -0,0 +1,229 @@ +package e2e + +import ( + "log" + "path/filepath" + "strings" + "testing" +) + +const ( + NetworkInfoCRType = "networkinfoes" + NSCRType = "namespaces" + PrivateIPBlockNSXType = "IpAddressBlock" + + InfraVPCNamespace = "kube-system" + SharedInfraVPCNamespace = "kube-public" + + DefaultPrivateCIDR1 = "172.28.0.0" + DefaultPrivateCIDR2 = "172.38.0.0" + InfraPrivateCIDR1 = "172.27.0.0" + InfraPrivateCIDR2 = "172.37.0.0" + CustomizedPrivateCIDR1 = "172.29.0.0" + CustomizedPrivateCIDR2 = "172.39.0.0" + CustomizedPrivateCIDR3 = "172.39.0.0" +) + +func verifyCRCreated(t *testing.T, crtype string, ns string, expect int) (string, string) { + // there should be one networkinfo created + resources, err := testData.getCRResource(defaultTimeout, crtype, ns) + // only one networkinfo should be created under ns using default network config + if len(resources) != expect { + log.Printf("NetworkInfo list %s size not the same as expected %d", resources, expect) + panic("NetworkInfo CR creation verify failed") + } + assertNil(t, err) + + cr_name, cr_uid := "", "" + // waiting for CR to be ready + for k, v := range resources { + cr_name = k + cr_uid = strings.TrimSpace(v) + } + + return cr_name, cr_uid +} + +func verifyCRDeleted(t *testing.T, crtype string, ns string) { + res, _ := testData.getCRResource(defaultTimeout, crtype, ns) + assertTrue(t, len(res) == 0, "NetworkInfo CR %s should be deleted", crtype) +} + +func verifyPrivateIPBlockCreated(t *testing.T, ns, id string) { + err := testData.waitForResourceExistById(ns, PrivateIPBlockNSXType, id, true) + assertNil(t, err) +} + +func verifyPrivateIPBlockDeleted(t *testing.T, ns, id string) { + err := testData.waitForResourceExistById(ns, PrivateIPBlockNSXType, id, false) + assertNil(t, err) +} + +// Test Customized NetworkInfo +func TestCustomizedNetworkInfo(t *testing.T) { + // Create customized networkconfig + ncPath, _ := filepath.Abs("./manifest/testVPC/customize_networkconfig.yaml") + _ = applyYAML(ncPath, "") + nsPath, _ := filepath.Abs("./manifest/testVPC/customize_ns.yaml") + _ = applyYAML(nsPath, "") + + defer deleteYAML(nsPath, "") + defer deleteYAML(ncPath, "") + + ns := "customized-ns" + + networkinfo_name, _ := verifyCRCreated(t, NetworkInfoCRType, ns, 1) + _, ns_uid := verifyCRCreated(t, NSCRType, ns, 1) + + vpcPath, _ := testData.getCRPropertiesByJson(defaultTimeout, NetworkInfoCRType, networkinfo_name, ns, ".vpcs[0].vpcPath") + err := testData.waitForResourceExistByPath(vpcPath, true) + assertNil(t, err) + + //verify private ipblocks created for vpc + p_ipb_id1 := ns_uid + "_" + CustomizedPrivateCIDR1 + p_ipb_id2 := ns_uid + "_" + CustomizedPrivateCIDR2 + + verifyPrivateIPBlockCreated(t, ns, p_ipb_id1) + verifyPrivateIPBlockCreated(t, ns, p_ipb_id2) +} + +// Test Infra NetworkInfo +func TestInfraNetworkInfo(t *testing.T) { + // Check namespace cr existence + _, ns_uid := verifyCRCreated(t, NSCRType, InfraVPCNamespace, 1) + // Check networkinfo cr existence + networkinfo_name, _ := verifyCRCreated(t, NetworkInfoCRType, InfraVPCNamespace, 1) + + vpcPath, _ := testData.getCRPropertiesByJson(defaultTimeout, NetworkInfoCRType, networkinfo_name, InfraVPCNamespace, ".vpcs[0].vpcPath") + err := testData.waitForResourceExistByPath(vpcPath, true) + assertNil(t, err) + + //verify private ipblocks created for vpc + p_ipb_id1 := ns_uid + "_" + InfraPrivateCIDR1 + p_ipb_id2 := ns_uid + "_" + InfraPrivateCIDR2 + + verifyPrivateIPBlockCreated(t, InfraVPCNamespace, p_ipb_id1) + verifyPrivateIPBlockCreated(t, InfraVPCNamespace, p_ipb_id2) + + // kube-public vpcpath should be the same as kube-system vpcpath + networkinfo_name = SharedInfraVPCNamespace + vpcPath2, err := testData.getCRPropertiesByJson(defaultTimeout, NetworkInfoCRType, networkinfo_name, SharedInfraVPCNamespace, + ".vpcs[0].vpcPath") + assertNil(t, err) + assertTrue(t, vpcPath == vpcPath2, "vpcPath %s should be the same as vpcPath2 %s", vpcPath, vpcPath2) +} + +// Test Default NetworkInfo +func TestDefaultNetworkInfo(t *testing.T) { + // If no annotation on namespace, then NetworkInfo will use default network config to create vpc under each ns + ns := "networkinfo-default-1" + setupTest(t, ns) + defer teardownTest(t, ns, defaultTimeout) + + // Check namespace cr existence + _, ns_uid := verifyCRCreated(t, NSCRType, ns, 1) + // Check networkinfo cr existence + networkinfo_name, _ := verifyCRCreated(t, NetworkInfoCRType, ns, 1) + + vpcPath, _ := testData.getCRPropertiesByJson(defaultTimeout, NetworkInfoCRType, networkinfo_name, ns, ".vpcs[0].vpcPath") + err := testData.waitForResourceExistByPath(vpcPath, true) + assertNil(t, err) + + //verify private ipblocks created for vpc, id is nsuid + cidr + p_ipb_id1 := ns_uid + "_" + DefaultPrivateCIDR1 + p_ipb_id2 := ns_uid + "_" + DefaultPrivateCIDR2 + + verifyPrivateIPBlockCreated(t, ns, p_ipb_id1) + verifyPrivateIPBlockCreated(t, ns, p_ipb_id2) + + // delete namespace and check all resources are deleted + err = testData.deleteNamespace(ns, defaultTimeout) + assertNil(t, err) + verifyCRDeleted(t, NetworkInfoCRType, ns) + verifyCRDeleted(t, NSCRType, ns) + err = testData.waitForResourceExistByPath(vpcPath, false) + assertNil(t, err) + verifyPrivateIPBlockDeleted(t, ns, p_ipb_id1) + verifyPrivateIPBlockDeleted(t, ns, p_ipb_id2) +} + +// ns1 share vpc with ns, delete ns1, vpc should not be deleted +func TestSharedNetworkInfo(t *testing.T) { + ns := "shared-vpc-ns-0" + ns1 := "shared-vpc-ns-1" + + nsPath, _ := filepath.Abs("./manifest/testVPC/shared_ns.yaml") + _ = applyYAML(nsPath, "") + defer deleteYAML(nsPath, "") + + // Check namespace cr existence + _, ns_uid := verifyCRCreated(t, NSCRType, ns, 1) + _, _ = verifyCRCreated(t, NSCRType, ns1, 1) + // Check networkinfo cr existence + networkinfo_name, _ := verifyCRCreated(t, NetworkInfoCRType, ns, 1) + networkinfo_name_1, _ := verifyCRCreated(t, NetworkInfoCRType, ns1, 1) + + vpcPath, _ := testData.getCRPropertiesByJson(defaultTimeout, NetworkInfoCRType, networkinfo_name, ns, ".vpcs[0].vpcPath") + err := testData.waitForResourceExistByPath(vpcPath, true) + assertNil(t, err) + vpcPath1, _ := testData.getCRPropertiesByJson(defaultTimeout, NetworkInfoCRType, networkinfo_name_1, ns1, ".vpcs[0].vpcPath") + err = testData.waitForResourceExistByPath(vpcPath1, true) + assertNil(t, err) + + assertTrue(t, vpcPath == vpcPath1, "vpcPath %s should be the same as vpcPath2 %s", vpcPath, vpcPath1) + + //verify private ipblocks created for vpc, id is nsuid + cidr + p_ipb_id1 := ns_uid + "_" + CustomizedPrivateCIDR1 + p_ipb_id2 := ns_uid + "_" + CustomizedPrivateCIDR2 + + verifyPrivateIPBlockCreated(t, ns, p_ipb_id1) + verifyPrivateIPBlockCreated(t, ns, p_ipb_id2) + + // delete ns1 and check vpc not deleted + err = testData.deleteNamespace(ns1, defaultTimeout) + assertNil(t, err) + verifyCRDeleted(t, NetworkInfoCRType, ns1) + verifyCRDeleted(t, NSCRType, ns1) + err = testData.waitForResourceExistByPath(vpcPath, true) + assertNil(t, err) + verifyPrivateIPBlockCreated(t, ns, p_ipb_id1) + verifyPrivateIPBlockCreated(t, ns, p_ipb_id2) +} + +// update vpcnetworkconfig, and check vpc is updated +func TestUpdateVPCNetworkconfigNetworkInfo(t *testing.T) { + ns := "update-ns" + + nsPath, _ := filepath.Abs("./manifest/testVPC/update_ns.yaml") + _ = applyYAML(nsPath, "") + defer deleteYAML(nsPath, "") + + vncPathOriginal, _ := filepath.Abs("./manifest/testVPC/customize_networkconfig.yaml") + defer applyYAML(vncPathOriginal, "") + + // Check namespace cr existence + _, ns_uid := verifyCRCreated(t, NSCRType, ns, 1) + // Check networkinfo cr existence + networkinfo_name, _ := verifyCRCreated(t, NetworkInfoCRType, ns, 1) + + privateIPv4CIDRs, err := testData.getCRPropertiesByJson(defaultTimeout, NetworkInfoCRType, networkinfo_name, ns, ".vpcs[0].privateIPv4CIDRs") + assertTrue(t, strings.Contains(privateIPv4CIDRs, CustomizedPrivateCIDR1), "privateIPv4CIDRs %s should contain %s", privateIPv4CIDRs, CustomizedPrivateCIDR1) + assertTrue(t, strings.Contains(privateIPv4CIDRs, CustomizedPrivateCIDR2), "privateIPv4CIDRs %s should contain %s", privateIPv4CIDRs, CustomizedPrivateCIDR1) + assertNil(t, err) + + //verify private ipblocks created for vpc, id is nsuid + cidr + p_ipb_id1 := ns_uid + "_" + CustomizedPrivateCIDR1 + p_ipb_id2 := ns_uid + "_" + CustomizedPrivateCIDR2 + + verifyPrivateIPBlockCreated(t, ns, p_ipb_id1) + verifyPrivateIPBlockCreated(t, ns, p_ipb_id2) + + vncPath, _ := filepath.Abs("./manifest/testVPC/customize_networkconfig_updated.yaml") + _ = applyYAML(vncPath, "") + + privateIPv4CIDRs, err = testData.getCRPropertiesByJson(defaultTimeout, NetworkInfoCRType, networkinfo_name, ns, ".vpcs[0].privateIPv4CIDRs") + assertTrue(t, strings.Contains(privateIPv4CIDRs, CustomizedPrivateCIDR3), "privateIPv4CIDRs %s should contain %s", privateIPv4CIDRs, CustomizedPrivateCIDR3) + assertNil(t, err) + p_ipb_id3 := ns_uid + "_" + CustomizedPrivateCIDR3 + verifyPrivateIPBlockCreated(t, ns, p_ipb_id3) +} diff --git a/test/e2e/nsx_vpc_test.go b/test/e2e/nsx_vpc_test.go deleted file mode 100644 index 08c3669d8..000000000 --- a/test/e2e/nsx_vpc_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package e2e - -import ( - "log" - "path/filepath" - "strings" - "testing" -) - -const ( - VPCCRType = "vpcs" - VPCNSXType = "Vpc" - PrivateIPBlockNSXType = "IpAddressBlock" - - InfraVPCNamespace = "kube-system" - SharedInfraVPCNamespace = "kube-public" - - DefaultPrivateCIDR1 = "172.28.0.0" - DefaultPrivateCIDR2 = "172.38.0.0" - InfraPrivateCIDR1 = "172.27.0.0" - InfraPrivateCIDR2 = "172.37.0.0" - CustomizedPrivateCIDR1 = "172.29.0.0" - CustomizedPrivateCIDR2 = "172.39.0.0" -) - -var ( - verify_keys = []string{"defaultSNATIP", "lbSubnetCIDR", "lbSubnetPath", "nsxResourcePath"} -) - -func verifyVPCCRCreated(t *testing.T, ns string, expect int) (string, string) { - // there should be one vpc created - resources, err := testData.getCRResource(defaultTimeout, VPCCRType, ns) - // only one vpc should be created under ns using default network config - if len(resources) != expect { - log.Printf("VPC list %s size not the same as expected %d", resources, expect) - panic("VPC CR creation verify failed") - } - assertNil(t, err) - - var vpc_name, vpc_uid string = "", "" - // waiting for CR to be ready - for k, v := range resources { - vpc_name = k - vpc_uid = strings.TrimSpace(v) - } - - return vpc_name, vpc_uid -} - -func verifyPrivateIPBlockCreated(t *testing.T, ns, id string) { - err := testData.waitForResourceExistById(ns, PrivateIPBlockNSXType, id, true) - assertNil(t, err) -} - -func verifyVPCCRProperties(t *testing.T, ns, vpc_name string) { - for _, key := range verify_keys { - value, err := testData.getCRProperties(defaultTimeout, VPCCRType, vpc_name, ns, key) - assertNil(t, err) - if strings.TrimSpace(value) == "" { - log.Printf("failed to read key %s for VPC %s", key, vpc_name) - panic("failed to read attribute from VPC CR") - } - } -} - -// Test Customized VPC -func TestCustomizedVPC(t *testing.T) { - // Create customized networkconfig - ncPath, _ := filepath.Abs("./manifest/testVPC/customize_networkconfig.yaml") - _ = applyYAML(ncPath, "") - nsPath, _ := filepath.Abs("./manifest/testVPC/customize_ns.yaml") - _ = applyYAML(nsPath, "") - - defer deleteYAML(nsPath, "") - defer deleteYAML(ncPath, "") - - ns := "customized-ns" - - vpc_name, vpc_uid := verifyVPCCRCreated(t, ns, 1) - - err := testData.waitForCRReadyOrDeleted(defaultTimeout, VPCCRType, ns, vpc_name, Ready) - assertNil(t, err, "Error when waiting for VPC %s", vpc_name) - - verifyVPCCRProperties(t, ns, vpc_name) - - // Check nsx-t resource existing, nsx vpc is using vpc cr uid as id - err = testData.waitForResourceExistById(ns, VPCNSXType, vpc_uid, true) - assertNil(t, err) - - //verify private ipblocks created for vpc - p_ipb_id1 := vpc_uid + "_" + CustomizedPrivateCIDR1 - p_ipb_id2 := vpc_uid + "_" + CustomizedPrivateCIDR2 - - verifyPrivateIPBlockCreated(t, ns, p_ipb_id1) - verifyPrivateIPBlockCreated(t, ns, p_ipb_id2) -} - -// Test Infra VPC -func TestInfraVPC(t *testing.T) { - // there should be one shared vpc created under namespace kube-system - vpc_name, vpc_uid := verifyVPCCRCreated(t, InfraVPCNamespace, 1) - - err := testData.waitForCRReadyOrDeleted(defaultTimeout, VPCCRType, InfraVPCNamespace, vpc_name, Ready) - assertNil(t, err, "Error when waiting for VPC %s", vpc_name) - - verifyVPCCRProperties(t, InfraVPCNamespace, vpc_name) - - // Check nsx-t resource existing, nsx vpc is using vpc cr uid as id - err = testData.waitForResourceExistById(InfraVPCNamespace, VPCNSXType, vpc_uid, true) - assertNil(t, err) - - //verify private ipblocks created for vpc - p_ipb_id1 := vpc_uid + "_" + InfraPrivateCIDR1 - p_ipb_id2 := vpc_uid + "_" + InfraPrivateCIDR2 - - verifyPrivateIPBlockCreated(t, InfraVPCNamespace, p_ipb_id1) - verifyPrivateIPBlockCreated(t, InfraVPCNamespace, p_ipb_id2) - - // there should be no VPC exist under namespace kube-public - _, _ = verifyVPCCRCreated(t, SharedInfraVPCNamespace, 0) -} - -// Test Default VPC -func TestDefaultVPC(t *testing.T) { - // If no annotation on namespace, then VPC will use default network config to create - // VPC under each ns - ns := "vpc-default-1" - setupTest(t, ns) - defer teardownTest(t, ns, defaultTimeout) - - // Check vpc cr existence - vpc_name, vpc_uid := verifyVPCCRCreated(t, ns, 1) - - err := testData.waitForCRReadyOrDeleted(defaultTimeout, VPCCRType, ns, vpc_name, Ready) - assertNil(t, err, "Error when waiting for VPC %s", vpc_name) - - verifyVPCCRProperties(t, ns, vpc_name) - - // Check nsx-t resource existing, nsx vpc is using vpc cr uid as id - err = testData.waitForResourceExistById(ns, VPCNSXType, vpc_uid, true) - assertNil(t, err) - - //verify private ipblocks created for vpc - p_ipb_id1 := vpc_uid + "_" + DefaultPrivateCIDR1 - p_ipb_id2 := vpc_uid + "_" + DefaultPrivateCIDR2 - - verifyPrivateIPBlockCreated(t, ns, p_ipb_id1) - verifyPrivateIPBlockCreated(t, ns, p_ipb_id2) -}