Skip to content

Commit 5bc4f5a

Browse files
justinmirjustinmir
committed
Support CPU options in machine pools
Expose CPU options on AWS launch templates so machine pools can configure nested virtualization and confidential compute. Include the options in launch template creation, observation, update detection, and v1beta1 round-trip conversion. Signed-off-by: Justin Miron <justin.miron@reddit.com>
1 parent e3ba355 commit 5bc4f5a

9 files changed

Lines changed: 239 additions & 1 deletion

‎config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -662,6 +662,40 @@ spec:
662662
"None": The instance may not make use of any Capacity Reservations. This is to conserve open reservations for desired workloads
663663
"CapacityReservationsOnly": The instance will only run if matched or targeted to a Capacity Reservation
664664
type: string
665+
cpuOptions:
666+
description: |-
667+
CPUOptions defines CPU-related settings for the instance, including the confidential computing policy.
668+
When omitted, this means no opinion and the AWS platform is left to choose a reasonable default.
669+
minProperties: 1
670+
properties:
671+
confidentialCompute:
672+
description: |-
673+
ConfidentialCompute specifies whether confidential computing should be enabled for the instance,
674+
and, if so, which confidential computing technology to use.
675+
Valid values are: Disabled, AMDEncryptedVirtualizationNestedPaging
676+
When set to Disabled, confidential computing will be disabled for the instance.
677+
When set to AMDEncryptedVirtualizationNestedPaging, AMD SEV-SNP will be used as the confidential computing technology for the instance.
678+
In this case, ensure the following conditions are met:
679+
1) The selected instance type supports AMD SEV-SNP.
680+
2) The selected AWS region supports AMD SEV-SNP.
681+
3) The selected AMI supports AMD SEV-SNP.
682+
More details can be checked at https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/sev-snp.html
683+
When omitted, this means no opinion and the AWS platform is left to choose a reasonable default,
684+
which is subject to change without notice. The current default is Disabled.
685+
enum:
686+
- Disabled
687+
- AMDEncryptedVirtualizationNestedPaging
688+
type: string
689+
nestedVirtualization:
690+
description: |-
691+
NestedVirtualization specifies whether to enable nested virtualization on the instance.
692+
Nested virtualization is supported on C8i, M8i, and R8i instance types.
693+
Valid values are: enabled, disabled
694+
enum:
695+
- enabled
696+
- disabled
697+
type: string
698+
type: object
665699
enclaveOptions:
666700
description: EnclaveOptions defines the options for Nitro Enclave
667701
support on the instance.

‎config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -671,6 +671,40 @@ spec:
671671
"None": The instance may not make use of any Capacity Reservations. This is to conserve open reservations for desired workloads
672672
"CapacityReservationsOnly": The instance will only run if matched or targeted to a Capacity Reservation
673673
type: string
674+
cpuOptions:
675+
description: |-
676+
CPUOptions defines CPU-related settings for the instance, including the confidential computing policy.
677+
When omitted, this means no opinion and the AWS platform is left to choose a reasonable default.
678+
minProperties: 1
679+
properties:
680+
confidentialCompute:
681+
description: |-
682+
ConfidentialCompute specifies whether confidential computing should be enabled for the instance,
683+
and, if so, which confidential computing technology to use.
684+
Valid values are: Disabled, AMDEncryptedVirtualizationNestedPaging
685+
When set to Disabled, confidential computing will be disabled for the instance.
686+
When set to AMDEncryptedVirtualizationNestedPaging, AMD SEV-SNP will be used as the confidential computing technology for the instance.
687+
In this case, ensure the following conditions are met:
688+
1) The selected instance type supports AMD SEV-SNP.
689+
2) The selected AWS region supports AMD SEV-SNP.
690+
3) The selected AMI supports AMD SEV-SNP.
691+
More details can be checked at https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/sev-snp.html
692+
When omitted, this means no opinion and the AWS platform is left to choose a reasonable default,
693+
which is subject to change without notice. The current default is Disabled.
694+
enum:
695+
- Disabled
696+
- AMDEncryptedVirtualizationNestedPaging
697+
type: string
698+
nestedVirtualization:
699+
description: |-
700+
NestedVirtualization specifies whether to enable nested virtualization on the instance.
701+
Nested virtualization is supported on C8i, M8i, and R8i instance types.
702+
Valid values are: enabled, disabled
703+
enum:
704+
- enabled
705+
- disabled
706+
type: string
707+
type: object
674708
enclaveOptions:
675709
description: EnclaveOptions defines the options for Nitro Enclave
676710
support on the instance.

‎exp/api/v1beta1/conversion.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ func (src *AWSMachinePool) ConvertTo(dstRaw conversion.Hub) error {
7676
if preference := restored.Spec.AWSLaunchTemplate.CapacityReservationPreference; preference != "" {
7777
dst.Spec.AWSLaunchTemplate.CapacityReservationPreference = preference
7878
}
79+
dst.Spec.AWSLaunchTemplate.CPUOptions = restored.Spec.AWSLaunchTemplate.CPUOptions
7980

8081
dst.Spec.DefaultInstanceWarmup = restored.Spec.DefaultInstanceWarmup
8182
dst.Spec.AWSLaunchTemplate.NonRootVolumes = restored.Spec.AWSLaunchTemplate.NonRootVolumes
@@ -143,6 +144,8 @@ func (src *AWSManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error {
143144
if preference := restored.Spec.AWSLaunchTemplate.CapacityReservationPreference; preference != "" {
144145
dst.Spec.AWSLaunchTemplate.CapacityReservationPreference = preference
145146
}
147+
148+
dst.Spec.AWSLaunchTemplate.CPUOptions = restored.Spec.AWSLaunchTemplate.CPUOptions
146149
}
147150
if restored.Spec.AvailabilityZoneSubnetType != nil {
148151
dst.Spec.AvailabilityZoneSubnetType = restored.Spec.AvailabilityZoneSubnetType

‎exp/api/v1beta1/zz_generated.conversion.go‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎exp/api/v1beta2/types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,11 @@ type AWSLaunchTemplate struct {
9797
// InstanceType is the type of instance to create. Example: m4.xlarge
9898
InstanceType string `json:"instanceType,omitempty"`
9999

100+
// CPUOptions defines CPU-related settings for the instance, including the confidential computing policy.
101+
// When omitted, this means no opinion and the AWS platform is left to choose a reasonable default.
102+
// +optional
103+
CPUOptions infrav1.CPUOptions `json:"cpuOptions,omitempty,omitzero"`
104+
100105
// RootVolume encapsulates the configuration options for the root volume
101106
// +optional
102107
RootVolume *infrav1.Volume `json:"rootVolume,omitempty"`

‎exp/api/v1beta2/zz_generated.deepcopy.go‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/cloud/services/ec2/launchtemplate.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,7 @@ func (s *Service) createLaunchTemplateData(scope scope.LaunchTemplateScope, imag
626626

627627
data := &types.RequestLaunchTemplateData{
628628
InstanceType: types.InstanceType(lt.InstanceType),
629+
CpuOptions: getLaunchTemplateCPUOptionsRequest(lt.CPUOptions),
629630
KeyName: sshKeyNamePtr,
630631
UserData: ptr.To[string](base64.StdEncoding.EncodeToString(userDataForLaunchTemplate)),
631632
}
@@ -712,6 +713,29 @@ func (s *Service) createLaunchTemplateData(scope scope.LaunchTemplateScope, imag
712713
return data, nil
713714
}
714715

716+
func getLaunchTemplateCPUOptionsRequest(cpuOptions infrav1.CPUOptions) *types.LaunchTemplateCpuOptionsRequest {
717+
request := &types.LaunchTemplateCpuOptionsRequest{}
718+
switch cpuOptions.ConfidentialCompute {
719+
case infrav1.AWSConfidentialComputePolicySEVSNP:
720+
request.AmdSevSnp = types.AmdSevSnpSpecificationEnabled
721+
case infrav1.AWSConfidentialComputePolicyDisabled:
722+
request.AmdSevSnp = types.AmdSevSnpSpecificationDisabled
723+
}
724+
725+
switch cpuOptions.NestedVirtualization {
726+
case infrav1.NestedVirtualizationPolicyEnabled:
727+
request.NestedVirtualization = types.NestedVirtualizationSpecificationEnabled
728+
case infrav1.NestedVirtualizationPolicyDisabled:
729+
request.NestedVirtualization = types.NestedVirtualizationSpecificationDisabled
730+
}
731+
732+
if *request == (types.LaunchTemplateCpuOptionsRequest{}) {
733+
return nil
734+
}
735+
736+
return request
737+
}
738+
715739
func getLaunchTemplateCapacityReservationSpecification(awsLaunchTemplate *expinfrav1.AWSLaunchTemplate) *types.LaunchTemplateCapacityReservationSpecificationRequest {
716740
if awsLaunchTemplate == nil {
717741
return nil
@@ -916,6 +940,7 @@ func (s *Service) SDKToLaunchTemplate(d types.LaunchTemplateVersion) (*expinfrav
916940
ID: v.ImageId,
917941
},
918942
InstanceType: string(v.InstanceType),
943+
CPUOptions: launchTemplateCPUOptionsFromSDK(v.CpuOptions),
919944
SSHKeyName: v.KeyName,
920945
SpotMarketOptions: SDKToSpotMarketOptions(v.InstanceMarketOptions),
921946
VersionNumber: d.VersionNumber,
@@ -1009,6 +1034,29 @@ func (s *Service) SDKToLaunchTemplate(d types.LaunchTemplateVersion) (*expinfrav
10091034
return i, decodedUserDataHash, launchTemplateUserDataSecretKey, bootstrapDataHash, nil
10101035
}
10111036

1037+
func launchTemplateCPUOptionsFromSDK(cpuOptions *types.LaunchTemplateCpuOptions) infrav1.CPUOptions {
1038+
if cpuOptions == nil {
1039+
return infrav1.CPUOptions{}
1040+
}
1041+
1042+
result := infrav1.CPUOptions{}
1043+
switch cpuOptions.AmdSevSnp {
1044+
case types.AmdSevSnpSpecificationEnabled:
1045+
result.ConfidentialCompute = infrav1.AWSConfidentialComputePolicySEVSNP
1046+
case types.AmdSevSnpSpecificationDisabled:
1047+
result.ConfidentialCompute = infrav1.AWSConfidentialComputePolicyDisabled
1048+
}
1049+
1050+
switch cpuOptions.NestedVirtualization {
1051+
case types.NestedVirtualizationSpecificationEnabled:
1052+
result.NestedVirtualization = infrav1.NestedVirtualizationPolicyEnabled
1053+
case types.NestedVirtualizationSpecificationDisabled:
1054+
result.NestedVirtualization = infrav1.NestedVirtualizationPolicyDisabled
1055+
}
1056+
1057+
return result
1058+
}
1059+
10121060
// LaunchTemplateNeedsUpdate checks if a new launch template version is needed.
10131061
//
10141062
// FIXME(dlipovetsky): This check should account for changed userdata, but does not yet do so.
@@ -1022,6 +1070,10 @@ func (s *Service) LaunchTemplateNeedsUpdate(scope scope.LaunchTemplateScope, inc
10221070
return true, services.LaunchTemplateNeedsUpdateReasonInstanceType, nil
10231071
}
10241072

1073+
if !cmp.Equal(incoming.CPUOptions, existing.CPUOptions) {
1074+
return true, services.LaunchTemplateNeedsUpdateReasonCPUOptions, nil
1075+
}
1076+
10251077
if !cmp.Equal(incoming.InstanceMetadataOptions, existing.InstanceMetadataOptions) {
10261078
return true, services.LaunchTemplateNeedsUpdateReasonInstanceMetadataOptions, nil
10271079
}

‎pkg/cloud/services/ec2/launchtemplate_test.go‎

Lines changed: 107 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,26 @@ func TestServiceSDKToLaunchTemplate(t *testing.T) {
369369
wantDataSecretKey: nil, // respective tag is not given
370370
wantBootstrapDataHash: nil, // respective tag is not given
371371
},
372+
{
373+
name: "CPU options",
374+
input: ec2types.LaunchTemplateVersion{
375+
LaunchTemplateName: aws.String("foo"),
376+
LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{
377+
CpuOptions: &ec2types.LaunchTemplateCpuOptions{
378+
AmdSevSnp: ec2types.AmdSevSnpSpecificationEnabled,
379+
NestedVirtualization: ec2types.NestedVirtualizationSpecificationEnabled,
380+
},
381+
},
382+
},
383+
wantLT: &expinfrav1.AWSLaunchTemplate{
384+
Name: "foo",
385+
CPUOptions: infrav1.CPUOptions{
386+
ConfidentialCompute: infrav1.AWSConfidentialComputePolicySEVSNP,
387+
NestedVirtualization: infrav1.NestedVirtualizationPolicyEnabled,
388+
},
389+
},
390+
wantUserDataHash: userdata.ComputeHash(nil),
391+
},
372392
{
373393
name: "spot market options",
374394
input: ec2types.LaunchTemplateVersion{
@@ -723,6 +743,32 @@ func TestServiceLaunchTemplateNeedsUpdate(t *testing.T) {
723743
want: true,
724744
wantNeedsUpdateReason: services.LaunchTemplateNeedsUpdateReasonInstanceType,
725745
},
746+
{
747+
name: "Should return true if incoming CPUOptions differ from existing CPUOptions",
748+
incoming: &expinfrav1.AWSLaunchTemplate{
749+
CPUOptions: infrav1.CPUOptions{
750+
NestedVirtualization: infrav1.NestedVirtualizationPolicyEnabled,
751+
},
752+
},
753+
existing: &expinfrav1.AWSLaunchTemplate{
754+
CPUOptions: infrav1.CPUOptions{
755+
NestedVirtualization: infrav1.NestedVirtualizationPolicyDisabled,
756+
},
757+
},
758+
want: true,
759+
wantNeedsUpdateReason: services.LaunchTemplateNeedsUpdateReasonCPUOptions,
760+
},
761+
{
762+
name: "Should return true if incoming CPUOptions remove existing CPUOptions",
763+
incoming: &expinfrav1.AWSLaunchTemplate{},
764+
existing: &expinfrav1.AWSLaunchTemplate{
765+
CPUOptions: infrav1.CPUOptions{
766+
NestedVirtualization: infrav1.NestedVirtualizationPolicyEnabled,
767+
},
768+
},
769+
want: true,
770+
wantNeedsUpdateReason: services.LaunchTemplateNeedsUpdateReasonCPUOptions,
771+
},
726772
{
727773
name: "new additional security group with filters",
728774
incoming: &expinfrav1.AWSLaunchTemplate{
@@ -1529,12 +1575,62 @@ var LaunchTemplateVersionIgnoreUnexported = cmpopts.IgnoreUnexported(
15291575
ec2types.LaunchTemplateIamInstanceProfileSpecificationRequest{},
15301576
ec2types.LaunchTemplateSpotMarketOptionsRequest{},
15311577
ec2types.LaunchTemplateInstanceMarketOptionsRequest{},
1578+
ec2types.LaunchTemplateCpuOptionsRequest{},
15321579
ec2types.Tag{},
15331580
ec2types.LaunchTemplateTagSpecificationRequest{},
15341581
ec2types.RequestLaunchTemplateData{},
15351582
ec2.CreateLaunchTemplateVersionInput{},
15361583
)
15371584

1585+
func TestGetLaunchTemplateCPUOptionsRequest(t *testing.T) {
1586+
testCases := []struct {
1587+
name string
1588+
cpuOptions infrav1.CPUOptions
1589+
expectedRequest *ec2types.LaunchTemplateCpuOptionsRequest
1590+
}{
1591+
{
1592+
name: "empty",
1593+
expectedRequest: nil,
1594+
},
1595+
{
1596+
name: "enabled nested virtualization",
1597+
cpuOptions: infrav1.CPUOptions{
1598+
NestedVirtualization: infrav1.NestedVirtualizationPolicyEnabled,
1599+
},
1600+
expectedRequest: &ec2types.LaunchTemplateCpuOptionsRequest{
1601+
NestedVirtualization: ec2types.NestedVirtualizationSpecificationEnabled,
1602+
},
1603+
},
1604+
{
1605+
name: "disabled nested virtualization",
1606+
cpuOptions: infrav1.CPUOptions{
1607+
NestedVirtualization: infrav1.NestedVirtualizationPolicyDisabled,
1608+
},
1609+
expectedRequest: &ec2types.LaunchTemplateCpuOptionsRequest{
1610+
NestedVirtualization: ec2types.NestedVirtualizationSpecificationDisabled,
1611+
},
1612+
},
1613+
{
1614+
name: "confidential compute and nested virtualization",
1615+
cpuOptions: infrav1.CPUOptions{
1616+
ConfidentialCompute: infrav1.AWSConfidentialComputePolicySEVSNP,
1617+
NestedVirtualization: infrav1.NestedVirtualizationPolicyEnabled,
1618+
},
1619+
expectedRequest: &ec2types.LaunchTemplateCpuOptionsRequest{
1620+
AmdSevSnp: ec2types.AmdSevSnpSpecificationEnabled,
1621+
NestedVirtualization: ec2types.NestedVirtualizationSpecificationEnabled,
1622+
},
1623+
},
1624+
}
1625+
1626+
for _, tc := range testCases {
1627+
t.Run(tc.name, func(t *testing.T) {
1628+
g := NewWithT(t)
1629+
g.Expect(getLaunchTemplateCPUOptionsRequest(tc.cpuOptions)).To(Equal(tc.expectedRequest))
1630+
})
1631+
}
1632+
}
1633+
15381634
func TestCreateLaunchTemplateVersion(t *testing.T) {
15391635
mockCtrl := gomock.NewController(t)
15401636
defer mockCtrl.Finish()
@@ -1559,8 +1655,15 @@ func TestCreateLaunchTemplateVersion(t *testing.T) {
15591655
marketType ec2types.MarketType
15601656
}{
15611657
{
1562-
name: "Should successfully creates launch template version",
1658+
name: "Should successfully create launch template version with CPU options",
15631659
awsResourceReference: []infrav1.AWSResourceReference{{ID: aws.String("1")}},
1660+
mpScopeUpdater: func(mps *scope.MachinePoolScope) {
1661+
spec := mps.AWSMachinePool.Spec
1662+
spec.AWSLaunchTemplate.CPUOptions = infrav1.CPUOptions{
1663+
NestedVirtualization: infrav1.NestedVirtualizationPolicyEnabled,
1664+
}
1665+
mps.AWSMachinePool.Spec = spec
1666+
},
15641667
expect: func(m *mocks.MockEC2APIMockRecorder) {
15651668
sgMap := make(map[infrav1.SecurityGroupRole]infrav1.SecurityGroup)
15661669
sgMap[infrav1.SecurityGroupNode] = infrav1.SecurityGroup{ID: "1"}
@@ -1569,6 +1672,9 @@ func TestCreateLaunchTemplateVersion(t *testing.T) {
15691672
expectedInput := &ec2.CreateLaunchTemplateVersionInput{
15701673
LaunchTemplateData: &ec2types.RequestLaunchTemplateData{
15711674
InstanceType: ec2types.InstanceTypeT3Large,
1675+
CpuOptions: &ec2types.LaunchTemplateCpuOptionsRequest{
1676+
NestedVirtualization: ec2types.NestedVirtualizationSpecificationEnabled,
1677+
},
15721678
IamInstanceProfile: &ec2types.LaunchTemplateIamInstanceProfileSpecificationRequest{
15731679
Name: aws.String("instance-profile"),
15741680
},

‎pkg/cloud/services/interfaces.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ const (
5252
LaunchTemplateNeedsUpdateReasonIamInstanceProfile LaunchTemplateNeedsUpdateReason = "IamInstanceProfile"
5353
// LaunchTemplateNeedsUpdateReasonInstanceType means a difference in the instance type was found.
5454
LaunchTemplateNeedsUpdateReasonInstanceType LaunchTemplateNeedsUpdateReason = "InstanceType"
55+
// LaunchTemplateNeedsUpdateReasonCPUOptions means a difference in the CPU options was found.
56+
LaunchTemplateNeedsUpdateReasonCPUOptions LaunchTemplateNeedsUpdateReason = "CPUOptions"
5557
// LaunchTemplateNeedsUpdateReasonInstanceMetadataOptions means a difference in the instance metadata options was found.
5658
LaunchTemplateNeedsUpdateReasonInstanceMetadataOptions LaunchTemplateNeedsUpdateReason = "InstanceMetadataOptions"
5759
// LaunchTemplateNeedsUpdateReasonSpotMarketOptions means a difference in the spot market options was found.

0 commit comments

Comments
 (0)