Skip to content

Commit 9d40730

Browse files
authored
Creates IAM roles for DP nodes (#65)
* Creates IAM roles for DP nodes, DNS serviceIP and Launch template discovery
1 parent 2fe10a9 commit 9d40730

File tree

20 files changed

+356
-81
lines changed

20 files changed

+356
-81
lines changed

operator/README.md

+3-2
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ KIT uses the [operator pattern](https://kubernetes.io/docs/concepts/extend-kuber
3030
aws cloudformation deploy \
3131
--template-file docs/kit.cloudformation.yaml \
3232
--capabilities CAPABILITY_NAMED_IAM \
33-
--stack-name kitControllerPolicy
33+
--stack-name kitControllerPolicy \
34+
--parameter-overrides ClusterName=${SUBSTRATE_CLUSTER_NAME}
3435
```
3536

3637
#### Associate the policy we just created to the kit-controller service account
@@ -40,7 +41,7 @@ KIT uses the [operator pattern](https://kubernetes.io/docs/concepts/extend-kuber
4041
--name kit-controller \
4142
--namespace kit \
4243
--cluster ${SUBSTRATE_CLUSTER_NAME} \
43-
--attach-policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/KitControllerPolicy \
44+
--attach-policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/KitControllerPolicy-${SUBSTRATE_CLUSTER_NAME} \
4445
--approve \
4546
--override-existing-serviceaccounts \
4647
--region=${AWS_REGION}

operator/charts/kit-operator/templates/data-plane-crd.yaml

+25
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,35 @@ spec:
3535
type: object
3636
spec:
3737
properties:
38+
allocationStrategy:
39+
description: AllocationStrategy helps user define the strategy to
40+
provision worker nodes in EC2, defaults to "lowest-price"
41+
type: string
3842
clusterName:
43+
description: ClusterName is used to connect the worker nodes to a
44+
control plane clusterName.
3945
type: string
46+
instanceTypes:
47+
description: InstanceTypes is an optional field thats lets user specify
48+
the instance types for worker nodes, defaults to instance types
49+
"t2.xlarge", "t3.xlarge" or "t3a.xlarge"
50+
items:
51+
type: string
52+
type: array
4053
nodeCount:
54+
description: NodeCount is the desired number of worker nodes for this
55+
dataplane.
4156
type: integer
57+
subnetSelector:
58+
additionalProperties:
59+
type: string
60+
description: SubnetSelector lets user define label key and values
61+
for kit to select the subnets for worker nodes. It can contain key:value
62+
to select subnets with particular label, or a specific key:"*" to
63+
select all subnets with a specific key. If no selector is provided,
64+
worker nodes are provisioned in the same subnet as control plane
65+
nodes.
66+
type: object
4267
type: object
4368
status:
4469
properties:

operator/cmd/controller/main.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@ import (
55
"fmt"
66

77
"github.com/awslabs/kit/operator/pkg/awsprovider"
8+
"github.com/awslabs/kit/operator/pkg/awsprovider/iam"
89
"github.com/awslabs/kit/operator/pkg/controllers"
910
"github.com/awslabs/kit/operator/pkg/controllers/controlplane"
1011
"github.com/awslabs/kit/operator/pkg/controllers/dataplane"
12+
"github.com/awslabs/kit/operator/pkg/kubeprovider"
1113
"github.com/awslabs/kit/operator/pkg/utils/scheme"
1214

1315
"github.com/go-logr/zapr"
@@ -51,7 +53,11 @@ func main() {
5153
})
5254
session := awsprovider.NewSession()
5355
err := manager.RegisterControllers(
54-
controlplane.NewController(manager.GetClient(), &awsprovider.AccountInfo{Session: session}),
56+
controlplane.NewController(manager.GetClient(),
57+
&awsprovider.AccountInfo{Session: session},
58+
iam.NewController(awsprovider.IAMClient(session),
59+
kubeprovider.New(manager.GetClient())),
60+
),
5561
dataplane.NewController(manager.GetClient(), session),
5662
).Start(controllerruntime.SetupSignalHandler())
5763
if err != nil {

operator/docs/kit.cloudformation.yaml

+16-27
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,14 @@
11
AWSTemplateFormatVersion: "2010-09-09"
22
Description: Resources used by https://github.com/awslabs/kit/operator
3+
Parameters:
4+
ClusterName:
5+
Type: String
6+
Description: "Substrate/host cluster name"
37
Resources:
4-
KitNodeInstanceProfile:
5-
Type: "AWS::IAM::InstanceProfile"
6-
Properties:
7-
InstanceProfileName: "KitNodeInstanceProfile"
8-
Path: "/"
9-
Roles:
10-
- Ref: "KitNodeRole"
11-
KitNodeRole:
12-
Type: "AWS::IAM::Role"
13-
Properties:
14-
RoleName: "KitNodeRole"
15-
Path: /
16-
AssumeRolePolicyDocument:
17-
Version: "2012-10-17"
18-
Statement:
19-
- Effect: Allow
20-
Principal:
21-
Service:
22-
!Sub "ec2.${AWS::URLSuffix}"
23-
Action:
24-
- "sts:AssumeRole"
25-
ManagedPolicyArns:
26-
- !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy"
27-
- !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKS_CNI_Policy"
28-
- !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
29-
- !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore"
308
KitControllerPolicy:
319
Type: "AWS::IAM::ManagedPolicy"
3210
Properties:
33-
ManagedPolicyName: "KitControllerPolicy"
11+
ManagedPolicyName: !Sub "KitControllerPolicy-${ClusterName}"
3412
PolicyDocument:
3513
Version: "2012-10-17"
3614
Statement:
@@ -49,10 +27,21 @@ Resources:
4927
- "autoscaling:DeleteAutoScalingGroup"
5028
- "autoscaling:UpdateAutoScalingGroup"
5129
- "autoscaling:SetDesiredCapacity"
30+
- "iam:CreateRole"
31+
- "iam:AddRoleToInstanceProfile"
32+
- "iam:CreateInstanceProfile"
33+
- "iam:AttachRolePolicy"
34+
- "iam:RemoveRoleFromInstanceProfile"
35+
- "iam:DeleteInstanceProfile"
36+
- "iam:DetachRolePolicy"
37+
- "iam:DeleteRole"
38+
- "iam:TagRole"
5239
# Read Operations
5340
- "ec2:DescribeInstances"
5441
- "ec2:DescribeLaunchTemplates"
5542
- "ec2:DescribeLaunchTemplateVersions"
5643
- "ec2:DescribeSubnets"
5744
- "ssm:GetParameter"
5845
- "autoscaling:DescribeAutoScalingGroups"
46+
- "iam:GetRole"
47+
- "iam:GetInstanceProfile"

operator/pkg/apis/controlplane/v1alpha1/controlplane.go

+7
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ limitations under the License.
1515
package v1alpha1
1616

1717
import (
18+
"context"
19+
1820
v1 "k8s.io/api/core/v1"
1921
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2022
)
@@ -82,3 +84,8 @@ type Instances struct {
8284
func (c *ControlPlane) ClusterName() string {
8385
return c.Name
8486
}
87+
88+
type ReconcileFinalize interface {
89+
Reconcile(context.Context, *ControlPlane) error
90+
Finalize(context.Context, *ControlPlane) error
91+
}

operator/pkg/awsprovider/awsprovider.go

+9
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/aws/aws-sdk-go/aws/session"
2828
"github.com/aws/aws-sdk-go/service/autoscaling"
2929
"github.com/aws/aws-sdk-go/service/ec2"
30+
"github.com/aws/aws-sdk-go/service/iam"
3031
"github.com/aws/aws-sdk-go/service/ssm"
3132
"github.com/aws/aws-sdk-go/service/ssm/ssmiface"
3233
)
@@ -83,6 +84,14 @@ func AutoScalingClient(sess *session.Session) *AutoScaling {
8384
return &AutoScaling{AutoScaling: autoscaling.New(sess)}
8485
}
8586

87+
type IAM struct {
88+
*iam.IAM
89+
}
90+
91+
func IAMClient(sess *session.Session) *IAM {
92+
return &IAM{IAM: iam.New(sess)}
93+
}
94+
8695
type AccountMetadata interface {
8796
ID() (string, error)
8897
}
+211
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
/*
2+
Licensed under the Apache License, Version 2.0 (the "License");
3+
you may not use this file except in compliance with the License.
4+
You may obtain a copy of the License at
5+
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
8+
Unless required by applicable law or agreed to in writing, software
9+
distributed under the License is distributed on an "AS IS" BASIS,
10+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
See the License for the specific language governing permissions and
12+
limitations under the License.
13+
*/
14+
15+
package iam
16+
17+
import (
18+
"context"
19+
"fmt"
20+
21+
"github.com/aws/aws-sdk-go/aws"
22+
"github.com/aws/aws-sdk-go/service/iam"
23+
apis "github.com/awslabs/kit/operator/pkg/apis/controlplane/v1alpha1"
24+
"github.com/awslabs/kit/operator/pkg/awsprovider"
25+
"github.com/awslabs/kit/operator/pkg/errors"
26+
"github.com/awslabs/kit/operator/pkg/kubeprovider"
27+
"go.uber.org/zap"
28+
29+
"knative.dev/pkg/ptr"
30+
)
31+
32+
type Controller struct {
33+
iam *awsprovider.IAM
34+
kubeClient *kubeprovider.Client
35+
}
36+
37+
// NewController returns a controller for managing IAM resources in AWS
38+
func NewController(iam *awsprovider.IAM, client *kubeprovider.Client) *Controller {
39+
return &Controller{iam: iam, kubeClient: client}
40+
}
41+
42+
var (
43+
kitNodeRolePolicies = []string{
44+
"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
45+
"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore",
46+
"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
47+
"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
48+
}
49+
)
50+
51+
func (c *Controller) Reconcile(ctx context.Context, controlPlane *apis.ControlPlane) error {
52+
role, err := c.getRole(ctx, KitNodeRoleNameFor(controlPlane.ClusterName()))
53+
if err != nil && !errors.IsIAMObjectDoNotExist(err) {
54+
return fmt.Errorf("getting IAM role for %v, %w", controlPlane.ClusterName(), err)
55+
}
56+
if role == nil {
57+
role, err = c.createRole(ctx, &iam.CreateRoleInput{
58+
AssumeRolePolicyDocument: aws.String(assumeRolePolicyDocument),
59+
Description: aws.String("Role assumed by dataplane nodes created by KIT operated"),
60+
RoleName: aws.String(KitNodeRoleNameFor(controlPlane.ClusterName())),
61+
Tags: generateRoleTags(controlPlane.ClusterName()),
62+
})
63+
if err != nil {
64+
return fmt.Errorf("creating IAM role for %v, %w", controlPlane.ClusterName(), err)
65+
}
66+
zap.S().Infof("[%s] Created IAM Role %v", controlPlane.ClusterName(), aws.StringValue(role.RoleName))
67+
}
68+
if err := role.addRoleToInstanceProfile(ctx, KitNodeRoleNameFor(controlPlane.ClusterName()),
69+
KitNodeInstanceProfileNameFor(controlPlane.ClusterName())); err != nil {
70+
return fmt.Errorf("adding instance profile to role, %w", err)
71+
}
72+
for _, policy := range kitNodeRolePolicies {
73+
if err := role.attachPolicy(ctx, policy, KitNodeRoleNameFor(controlPlane.ClusterName())); err != nil {
74+
return fmt.Errorf("attaching policies to role, %w", err)
75+
}
76+
}
77+
return nil
78+
}
79+
80+
func (c *Controller) Finalize(ctx context.Context, controlPlane *apis.ControlPlane) error {
81+
_, err := c.iam.RemoveRoleFromInstanceProfileWithContext(ctx, &iam.RemoveRoleFromInstanceProfileInput{
82+
InstanceProfileName: aws.String(KitNodeInstanceProfileNameFor(controlPlane.ClusterName())),
83+
RoleName: aws.String(KitNodeRoleNameFor(controlPlane.ClusterName())),
84+
})
85+
if err != nil && !errors.IsIAMObjectDoNotExist(err) {
86+
return fmt.Errorf("removing role from instance profile, %w", err)
87+
}
88+
_, err = c.iam.DeleteInstanceProfileWithContext(ctx, &iam.DeleteInstanceProfileInput{
89+
InstanceProfileName: aws.String(KitNodeInstanceProfileNameFor(controlPlane.ClusterName())),
90+
})
91+
if err != nil && !errors.IsIAMObjectDoNotExist(err) {
92+
return fmt.Errorf("deleting instance profile, %w", err)
93+
}
94+
95+
for _, policy := range kitNodeRolePolicies {
96+
if _, err = c.iam.DetachRolePolicyWithContext(ctx, &iam.DetachRolePolicyInput{
97+
PolicyArn: aws.String(policy),
98+
RoleName: aws.String(KitNodeRoleNameFor(controlPlane.ClusterName())),
99+
}); err != nil {
100+
return fmt.Errorf("detaching policy from role, %w", err)
101+
}
102+
}
103+
_, err = c.iam.DeleteRoleWithContext(ctx, &iam.DeleteRoleInput{
104+
RoleName: aws.String(KitNodeRoleNameFor(controlPlane.ClusterName())),
105+
})
106+
if err != nil && !errors.IsIAMObjectDoNotExist(err) {
107+
return fmt.Errorf("deleting role, %w", err)
108+
}
109+
zap.S().Infof("[%s] Deleted IAM Role %v and instance profile",
110+
controlPlane.ClusterName(), KitNodeRoleNameFor(controlPlane.ClusterName()))
111+
return nil
112+
}
113+
114+
func (c *Controller) getRole(ctx context.Context, roleName string) (*role, error) {
115+
roleOutput, err := c.iam.GetRoleWithContext(ctx, &iam.GetRoleInput{
116+
RoleName: aws.String(roleName),
117+
})
118+
if err != nil {
119+
return nil, fmt.Errorf("getting iam role %v, %w", roleName, err)
120+
}
121+
return &role{iam: c.iam, Role: roleOutput.Role}, nil
122+
}
123+
124+
func (c *Controller) createRole(ctx context.Context, roleInput *iam.CreateRoleInput) (*role, error) {
125+
roleOutput, err := c.iam.CreateRoleWithContext(ctx, roleInput)
126+
return &role{iam: c.iam, Role: roleOutput.Role}, err
127+
}
128+
129+
type role struct {
130+
iam *awsprovider.IAM
131+
*iam.Role
132+
}
133+
134+
func (r *role) addRoleToInstanceProfile(ctx context.Context, roleName, profileName string) error {
135+
profile, err := createInstanceProfile(ctx, r.iam, profileName)
136+
if err != nil {
137+
return fmt.Errorf("creating instance profile, %w", err)
138+
}
139+
if profile == nil {
140+
return fmt.Errorf("instance profile is NIL")
141+
}
142+
for _, role := range profile.Roles {
143+
if aws.StringValue(role.RoleName) == roleName {
144+
return nil
145+
}
146+
}
147+
_, err = r.iam.AddRoleToInstanceProfile(&iam.AddRoleToInstanceProfileInput{
148+
InstanceProfileName: aws.String(profileName),
149+
RoleName: aws.String(roleName),
150+
})
151+
return err
152+
}
153+
154+
func (r *role) attachPolicy(ctx context.Context, policyARN, roleName string) error {
155+
_, err := r.iam.AttachRolePolicy(&iam.AttachRolePolicyInput{
156+
PolicyArn: aws.String(policyARN),
157+
RoleName: aws.String(roleName),
158+
})
159+
if errors.IsIAMObjectAlreadyExist(err) {
160+
return nil
161+
}
162+
return err
163+
}
164+
165+
func createInstanceProfile(ctx context.Context, iamAPI *awsprovider.IAM, profileName string) (*iam.InstanceProfile, error) {
166+
profile, err := iamAPI.CreateInstanceProfileWithContext(ctx, &iam.CreateInstanceProfileInput{
167+
InstanceProfileName: aws.String(profileName),
168+
})
169+
if errors.IsIAMObjectAlreadyExist(err) {
170+
output, err := iamAPI.GetInstanceProfileWithContext(ctx, &iam.GetInstanceProfileInput{
171+
InstanceProfileName: aws.String(profileName),
172+
})
173+
if err != nil {
174+
return nil, err
175+
}
176+
return output.InstanceProfile, nil
177+
}
178+
if err != nil {
179+
return nil, fmt.Errorf("creating instance profile, %w", err)
180+
}
181+
return profile.InstanceProfile, nil
182+
}
183+
184+
func KitNodeRoleNameFor(clusterName string) string {
185+
return fmt.Sprintf("KitDataplaneNodes-%s-cluster", clusterName)
186+
}
187+
188+
func KitNodeInstanceProfileNameFor(clusterName string) string {
189+
return fmt.Sprintf("KitDataplaneNodesProfile-%s-cluster", clusterName)
190+
}
191+
192+
func generateRoleTags(clusterName string) []*iam.Tag {
193+
return []*iam.Tag{{
194+
Key: ptr.String(fmt.Sprintf("kubernetes.io/cluster/%s", clusterName)),
195+
Value: ptr.String("owned"),
196+
}}
197+
}
198+
199+
// KitNodeRole is assumed by the nodes provisioned by kit-operator for dataplane
200+
const assumeRolePolicyDocument = `{
201+
"Version": "2012-10-17",
202+
"Statement": [
203+
{
204+
"Action": "sts:AssumeRole",
205+
"Effect": "Allow",
206+
"Principal": {
207+
"Service": "ec2.amazonaws.com"
208+
}
209+
}
210+
]
211+
}`

0 commit comments

Comments
 (0)