Skip to content

Commit bdc9d36

Browse files
committed
WIP: e2e: add initial SGX EPC cgroups tests
Signed-off-by: Mikko Ylinen <[email protected]>
1 parent 0d1003c commit bdc9d36

File tree

3 files changed

+118
-3
lines changed

3 files changed

+118
-3
lines changed

Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ e2e-dlb:
172172
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:dlb.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
173173

174174
e2e-spr:
175-
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:(iaa|dsa)|Device:qat.*Mode:dpdk.*Resource:(cy|dc).*" -ginkgo.focus "Device:sgx.*|(SGX Admission)" -ginkgo.focus "Device:gpu.*Resource:i915" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
175+
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:(iaa|dsa)|Device:qat.*Mode:dpdk.*Resource:(cy|dc).*" -ginkgo.focus "Device:sgx.*|(SGX Admission)" -ginkgo.focus "Device:gpu.*Resource:i915" -ginkgo.skip "App:sgx-epc-cgroup" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
176176

177177
pre-pull:
178178
ifeq ($(TAG),devel)

scripts/set-version.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ if [ $# != 1 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then
1515
exit 1
1616
fi
1717

18-
files=$(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile test/e2e/*/*.go)
18+
files=$(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/stress-ng-gramine:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile test/e2e/*/*.go)
1919

2020
for file in $files; do
21-
sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
21+
sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/stress-ng-gramine:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
2222
done

test/e2e/sgx/sgx.go

+115
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"k8s.io/apimachinery/pkg/labels"
2929
"k8s.io/kubernetes/test/e2e/framework"
3030
e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
31+
e2ejob "k8s.io/kubernetes/test/e2e/framework/job"
3132
e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
3233
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
3334
admissionapi "k8s.io/pod-security-admission/api"
@@ -38,6 +39,10 @@ const (
3839
timeout = time.Second * 120
3940
kustomizationWebhook = "deployments/sgx_admissionwebhook/overlays/default-with-certmanager/kustomization.yaml"
4041
kustomizationPlugin = "deployments/sgx_plugin/base/kustomization.yaml"
42+
// TODO: move to epc-cgroups overlay once available.
43+
// kustomizationPlugin = "deployments/sgx_plugin/overlays/epc-cgroups/kustomization.yaml".
44+
stressNGImage = "intel/stress-ng-gramine:devel"
45+
stressNGEnclaveSize = 402653184
4146
)
4247

4348
func init() {
@@ -80,6 +85,9 @@ func describe() {
8085
})
8186

8287
ginkgo.Context("When SGX resources are available", func() {
88+
var nodeWithEPC string
89+
var epcCapacity int64
90+
8391
ginkgo.BeforeEach(func(ctx context.Context) {
8492
ginkgo.By("checking if the resource is allocatable")
8593
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "sgx.intel.com/epc", 150*time.Second); err != nil {
@@ -91,6 +99,20 @@ func describe() {
9199
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "sgx.intel.com/provision", 30*time.Second); err != nil {
92100
framework.Failf("unable to wait for nodes to have positive allocatable provision resource: %v", err)
93101
}
102+
103+
nodelist, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
104+
if err != nil {
105+
framework.Failf("failed to list Nodes: %v", err)
106+
}
107+
108+
// we have at least one node with sgx.intel.com/epc capacity
109+
for _, item := range nodelist.Items {
110+
if q, ok := item.Status.Allocatable["sgx.intel.com/epc"]; ok && q.Value() > 0 {
111+
epcCapacity = q.Value()
112+
nodeWithEPC = item.Name
113+
break
114+
}
115+
}
94116
})
95117

96118
ginkgo.It("deploys a sgx-sdk-demo pod requesting SGX enclave resources [App:sgx-sdk-demo]", func(ctx context.Context) {
@@ -120,6 +142,99 @@ func describe() {
120142
gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, pod.ObjectMeta.Name, "testcontainer"))
121143
})
122144

145+
ginkgo.It("deploys simultaneous SGX EPC stressor jobs with equal EPC limits but no memory limits [App:sgx-epc-cgroup]", func(ctx context.Context) {
146+
parallelism := int32(epcCapacity/stressNGEnclaveSize) + 1
147+
completions := int32(epcCapacity/stressNGEnclaveSize) + 1
148+
quantity := resource.NewQuantity(stressNGEnclaveSize/2, resource.BinarySI)
149+
150+
testArgs := []string{
151+
"stress-ng",
152+
"--verbose",
153+
"--vm",
154+
"1",
155+
"--vm-bytes",
156+
"20%",
157+
"--page-in",
158+
"-t",
159+
"30",
160+
}
161+
job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob", v1.RestartPolicyNever, parallelism, completions, nil, 0, nodeWithEPC)
162+
163+
job.Spec.Template.Spec.Containers[0].Image = stressNGImage
164+
job.Spec.Template.Spec.Containers[0].Args = testArgs
165+
job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
166+
Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
167+
Limits: v1.ResourceList{"sgx.intel.com/epc": *quantity},
168+
}
169+
170+
job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
171+
framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
172+
173+
err = e2ejob.WaitForJobComplete(ctx, f.ClientSet, f.Namespace.Name, job.Name, completions)
174+
framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
175+
})
176+
177+
ginkgo.It("deploys one SGX EPC stressor job with the EPC limit set to enclave size and the memory limit set very low [App:sgx-epc-cgroup]", func(ctx context.Context) {
178+
quantity := resource.NewQuantity(stressNGEnclaveSize, resource.BinarySI)
179+
180+
testArgs := []string{
181+
"stress-ng",
182+
"--verbose",
183+
"--vm",
184+
"1",
185+
"--vm-bytes",
186+
"20%",
187+
"--page-in",
188+
"-t",
189+
"30",
190+
}
191+
job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob", v1.RestartPolicyNever, 1, 1, nil, 0, nodeWithEPC)
192+
193+
job.Spec.Template.Spec.Containers[0].Image = stressNGImage
194+
job.Spec.Template.Spec.Containers[0].Args = testArgs
195+
job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
196+
Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
197+
Limits: v1.ResourceList{"sgx.intel.com/epc": *quantity,
198+
v1.ResourceMemory: resource.MustParse("42Mi")},
199+
}
200+
201+
job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
202+
framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
203+
204+
err = e2ejob.WaitForJobComplete(ctx, f.ClientSet, f.Namespace.Name, job.Name, 1)
205+
framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
206+
})
207+
208+
ginkgo.It("deploys one SGX EPC stressor job with EDMM that ramps EPC allocations and memory limit set to kill once enough EPC pages are reclaimed [App:sgx-epc-cgroup]", func(ctx context.Context) {
209+
quantity := resource.NewQuantity(epcCapacity/10, resource.BinarySI)
210+
211+
testArgs := []string{
212+
"stress-ng-edmm",
213+
"--verbose",
214+
"--bigheap",
215+
"1",
216+
"--bigheap-growth",
217+
"10m",
218+
"--page-in",
219+
"-t",
220+
"300",
221+
}
222+
job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob", v1.RestartPolicyNever, 1, 1, nil, 0, nodeWithEPC)
223+
224+
job.Spec.Template.Spec.Containers[0].Image = stressNGImage
225+
job.Spec.Template.Spec.Containers[0].Args = testArgs
226+
job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
227+
Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
228+
Limits: v1.ResourceList{"sgx.intel.com/epc": *quantity,
229+
v1.ResourceMemory: *quantity},
230+
}
231+
232+
job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
233+
framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
234+
err = e2ejob.WaitForJobFailed(f.ClientSet, f.Namespace.Name, job.Name)
235+
framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
236+
})
237+
123238
ginkgo.When("there is no app to run [App:noapp]", func() {
124239
ginkgo.It("does nothing", func() {})
125240
})

0 commit comments

Comments
 (0)