@@ -28,6 +28,7 @@ import (
28
28
"k8s.io/apimachinery/pkg/labels"
29
29
"k8s.io/kubernetes/test/e2e/framework"
30
30
e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
31
+ e2ejob "k8s.io/kubernetes/test/e2e/framework/job"
31
32
e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
32
33
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
33
34
admissionapi "k8s.io/pod-security-admission/api"
@@ -38,6 +39,10 @@ const (
38
39
timeout = time .Second * 120
39
40
kustomizationWebhook = "deployments/sgx_admissionwebhook/overlays/default-with-certmanager/kustomization.yaml"
40
41
kustomizationPlugin = "deployments/sgx_plugin/base/kustomization.yaml"
42
+ // TODO: move to epc-cgroups overlay once available.
43
+ // kustomizationPlugin = "deployments/sgx_plugin/overlays/epc-cgroups/kustomization.yaml".
44
+ stressNGImage = "intel/stress-ng-gramine:devel"
45
+ stressNGEnclaveSize = 402653184
41
46
)
42
47
43
48
func init () {
@@ -80,6 +85,9 @@ func describe() {
80
85
})
81
86
82
87
ginkgo .Context ("When SGX resources are available" , func () {
88
+ var nodeWithEPC string
89
+ var epcCapacity int64
90
+
83
91
ginkgo .BeforeEach (func (ctx context.Context ) {
84
92
ginkgo .By ("checking if the resource is allocatable" )
85
93
if err := utils .WaitForNodesWithResource (ctx , f .ClientSet , "sgx.intel.com/epc" , 150 * time .Second ); err != nil {
@@ -91,6 +99,20 @@ func describe() {
91
99
if err := utils .WaitForNodesWithResource (ctx , f .ClientSet , "sgx.intel.com/provision" , 30 * time .Second ); err != nil {
92
100
framework .Failf ("unable to wait for nodes to have positive allocatable provision resource: %v" , err )
93
101
}
102
+
103
+ nodelist , err := f .ClientSet .CoreV1 ().Nodes ().List (ctx , metav1.ListOptions {})
104
+ if err != nil {
105
+ framework .Failf ("failed to list Nodes: %v" , err )
106
+ }
107
+
108
+ // we have at least one node with sgx.intel.com/epc capacity
109
+ for _ , item := range nodelist .Items {
110
+ if q , ok := item .Status .Allocatable ["sgx.intel.com/epc" ]; ok && q .Value () > 0 {
111
+ epcCapacity = q .Value ()
112
+ nodeWithEPC = item .Name
113
+ break
114
+ }
115
+ }
94
116
})
95
117
96
118
ginkgo .It ("deploys a sgx-sdk-demo pod requesting SGX enclave resources [App:sgx-sdk-demo]" , func (ctx context.Context ) {
@@ -120,6 +142,99 @@ func describe() {
120
142
gomega .Expect (err ).To (gomega .BeNil (), utils .GetPodLogs (ctx , f , pod .ObjectMeta .Name , "testcontainer" ))
121
143
})
122
144
145
+ ginkgo .It ("deploys simultaneous SGX EPC stressor jobs with equal EPC limits but no memory limits [App:sgx-epc-cgroup]" , func (ctx context.Context ) {
146
+ parallelism := int32 (epcCapacity / stressNGEnclaveSize ) + 1
147
+ completions := int32 (epcCapacity / stressNGEnclaveSize ) + 1
148
+ quantity := resource .NewQuantity (stressNGEnclaveSize / 2 , resource .BinarySI )
149
+
150
+ testArgs := []string {
151
+ "stress-ng" ,
152
+ "--verbose" ,
153
+ "--vm" ,
154
+ "1" ,
155
+ "--vm-bytes" ,
156
+ "20%" ,
157
+ "--page-in" ,
158
+ "-t" ,
159
+ "30" ,
160
+ }
161
+ job := e2ejob .NewTestJobOnNode ("success" , "sgx-epc-stressjob" , v1 .RestartPolicyNever , parallelism , completions , nil , 0 , nodeWithEPC )
162
+
163
+ job .Spec .Template .Spec .Containers [0 ].Image = stressNGImage
164
+ job .Spec .Template .Spec .Containers [0 ].Args = testArgs
165
+ job .Spec .Template .Spec .Containers [0 ].Resources = v1.ResourceRequirements {
166
+ Requests : v1.ResourceList {"sgx.intel.com/epc" : * quantity },
167
+ Limits : v1.ResourceList {"sgx.intel.com/epc" : * quantity },
168
+ }
169
+
170
+ job , err := e2ejob .CreateJob (ctx , f .ClientSet , f .Namespace .Name , job )
171
+ framework .ExpectNoError (err , "failed to create job in namespace: %s" , f .Namespace .Name )
172
+
173
+ err = e2ejob .WaitForJobComplete (ctx , f .ClientSet , f .Namespace .Name , job .Name , completions )
174
+ framework .ExpectNoError (err , "failed to ensure job completion in namespace: %s" , f .Namespace .Name )
175
+ })
176
+
177
+ ginkgo .It ("deploys one SGX EPC stressor job with the EPC limit set to enclave size and the memory limit set very low [App:sgx-epc-cgroup]" , func (ctx context.Context ) {
178
+ quantity := resource .NewQuantity (stressNGEnclaveSize , resource .BinarySI )
179
+
180
+ testArgs := []string {
181
+ "stress-ng" ,
182
+ "--verbose" ,
183
+ "--vm" ,
184
+ "1" ,
185
+ "--vm-bytes" ,
186
+ "20%" ,
187
+ "--page-in" ,
188
+ "-t" ,
189
+ "30" ,
190
+ }
191
+ job := e2ejob .NewTestJobOnNode ("success" , "sgx-epc-stressjob" , v1 .RestartPolicyNever , 1 , 1 , nil , 0 , nodeWithEPC )
192
+
193
+ job .Spec .Template .Spec .Containers [0 ].Image = stressNGImage
194
+ job .Spec .Template .Spec .Containers [0 ].Args = testArgs
195
+ job .Spec .Template .Spec .Containers [0 ].Resources = v1.ResourceRequirements {
196
+ Requests : v1.ResourceList {"sgx.intel.com/epc" : * quantity },
197
+ Limits : v1.ResourceList {"sgx.intel.com/epc" : * quantity ,
198
+ v1 .ResourceMemory : resource .MustParse ("42Mi" )},
199
+ }
200
+
201
+ job , err := e2ejob .CreateJob (ctx , f .ClientSet , f .Namespace .Name , job )
202
+ framework .ExpectNoError (err , "failed to create job in namespace: %s" , f .Namespace .Name )
203
+
204
+ err = e2ejob .WaitForJobComplete (ctx , f .ClientSet , f .Namespace .Name , job .Name , 1 )
205
+ framework .ExpectNoError (err , "failed to ensure job completion in namespace: %s" , f .Namespace .Name )
206
+ })
207
+
208
+ ginkgo .It ("deploys one SGX EPC stressor job with EDMM that ramps EPC allocations and memory limit set to kill once enough EPC pages are reclaimed [App:sgx-epc-cgroup]" , func (ctx context.Context ) {
209
+ quantity := resource .NewQuantity (epcCapacity / 10 , resource .BinarySI )
210
+
211
+ testArgs := []string {
212
+ "stress-ng-edmm" ,
213
+ "--verbose" ,
214
+ "--bigheap" ,
215
+ "1" ,
216
+ "--bigheap-growth" ,
217
+ "10m" ,
218
+ "--page-in" ,
219
+ "-t" ,
220
+ "300" ,
221
+ }
222
+ job := e2ejob .NewTestJobOnNode ("success" , "sgx-epc-stressjob" , v1 .RestartPolicyNever , 1 , 1 , nil , 0 , nodeWithEPC )
223
+
224
+ job .Spec .Template .Spec .Containers [0 ].Image = stressNGImage
225
+ job .Spec .Template .Spec .Containers [0 ].Args = testArgs
226
+ job .Spec .Template .Spec .Containers [0 ].Resources = v1.ResourceRequirements {
227
+ Requests : v1.ResourceList {"sgx.intel.com/epc" : * quantity },
228
+ Limits : v1.ResourceList {"sgx.intel.com/epc" : * quantity ,
229
+ v1 .ResourceMemory : * quantity },
230
+ }
231
+
232
+ job , err := e2ejob .CreateJob (ctx , f .ClientSet , f .Namespace .Name , job )
233
+ framework .ExpectNoError (err , "failed to create job in namespace: %s" , f .Namespace .Name )
234
+ err = e2ejob .WaitForJobFailed (f .ClientSet , f .Namespace .Name , job .Name )
235
+ framework .ExpectNoError (err , "failed to ensure job completion in namespace: %s" , f .Namespace .Name )
236
+ })
237
+
123
238
ginkgo .When ("there is no app to run [App:noapp]" , func () {
124
239
ginkgo .It ("does nothing" , func () {})
125
240
})
0 commit comments