kops-private-topology.yaml.gotmpl

apiVersion: kops/v1alpha2
kind: Cluster
metadata:
  name: {{ getenv "KOPS_CLUSTER_NAME" }}
spec:
  additionalPolicies:
    master: |
      [
        {
           "Effect": "Allow",
            "Action": [
               "ec2:DescribeTags"
             ],
              "Resource": "*"
        }
      ]
    bastion: |
      [
        {
           "Effect": "Allow",
            "Action": [
               "ec2:DescribeTags"
             ],
              "Resource": "*"
        }
      ]
    node: |
      [
        {
           "Effect": "Allow",
           "Action": [
               "ec2:DescribeTags"
           ],
           "Resource": "*"
        {{- if bool (getenv "KOPS_CLUSTER_AUTOSCALER_ENABLED" "false") }}
        },
        {
            "Effect": "Allow",
            "Action": [
                "autoscaling:DescribeAutoScalingGroups",
                "autoscaling:DescribeAutoScalingInstances",
                "autoscaling:DescribeTags",
                "autoscaling:SetDesiredCapacity",
                "autoscaling:TerminateInstanceInAutoScalingGroup"
            ],
            "Resource": "*"
        {{- end }}
        }
      ]
  api:
    loadBalancer:
      ## Public or Internal https://github.com/kubernetes/kops/blob/master/docs/topology.md#aws
      type: {{ getenv "KOPS_API_LOAD_BALANCER_TYPE" "Public" }}
      idleTimeoutSeconds: {{ getenv "KOPS_API_LOAD_BALANCER_IDLE_TIMEOUT_SECONDS" "600" }}
  kubeAPIServer:
  {{- if bool (getenv "KOPS_AUTHORIZATION_RBAC_ENABLED" "false") }}
    anonymousAuth: false
    authorizationMode: {{ getenv "KOPS_KUBE_API_SERVER_AUTHORIZATION_MODE" "RBAC" }}
  {{- end }}
  {{- if bool (getenv "KOPS_ADMISSION_CONTROL_ENABLED" "true") }}
    admissionControl:
    - NamespaceLifecycle
    - LimitRanger
    - ServiceAccount
    - DefaultStorageClass
    - DefaultTolerationSeconds
    - MutatingAdmissionWebhook
    - ValidatingAdmissionWebhook
    - ResourceQuota
    - NodeRestriction
    - Priority
    - Initializers
    - DenyEscalatingExec
  {{- end }}
  {{- if getenv "KOPS_OIDC_ISSUER_URL" }}
    oidcClientID: {{ getenv "KOPS_OIDC_CLIENT_ID" "kubernetes"}}
    oidcGroupsClaim: groups
    oidcGroupsPrefix: "oidc:"
    oidcIssuerURL: {{ getenv "KOPS_OIDC_ISSUER_URL" }}
    oidcUsernameClaim: email
  {{- end }}
  {{- if bool (getenv "KOPS_AWS_IAM_AUTHENTICATOR_ENABLED" "false") }}
  authentication:
    aws: {}
  {{- end }}
  authorization:
    {{- if bool (getenv "KOPS_AUTHORIZATION_RBAC_ENABLED" "false") }}
    rbac: {}
    {{- else }}
    alwaysAllow: {}
    {{- end }}
  channel: stable
  cloudLabels:
    Cluster: {{ getenv "KOPS_CLUSTER_NAME" }}
    {{- if bool (getenv "KOPS_CLUSTER_AUTOSCALER_ENABLED" "false") }}
    k8s.io/cluster-autoscaler/enabled: "true"
    kubernetes.io/cluster/{{ getenv "KOPS_CLUSTER_NAME" }}: "owned"
    {{- end }}
  cloudProvider: aws
  configBase: {{ getenv "KOPS_STATE_STORE" }}/{{ getenv "KOPS_CLUSTER_NAME" }}
  {{- if getenv "KOPS_DNS_ZONE" }}
  dnsZone: {{ getenv "KOPS_DNS_ZONE" }}
  {{- else }}
  dnsZone: {{ getenv "KOPS_CLUSTER_NAME" | regexp.Replace "^[^.]+\\." "" }}
  {{- end }}
  etcdClusters:
  - etcdMembers:
    {{- range (getenv "KOPS_AVAILABILITY_ZONES" | strings.Split ",") }}
    - encryptedVolume: true
      instanceGroup: master-{{.}}
      name: {{ . | regexp.Replace "^.*[0-9]" "" }}
    {{- end }}
    name: main
  - etcdMembers:
    {{- range (getenv "KOPS_AVAILABILITY_ZONES" | strings.Split ",") }}
    - encryptedVolume: true
      instanceGroup: master-{{.}}
      name: {{ . | regexp.Replace "^.*[0-9]" "" }}
    {{- end }}
    name: events
  hooks:
  # Mitigate CVE-2019-5736
  - before:
    - docker.service
    manifest: |
      Type=oneshot
      ExecStart=/usr/bin/chattr +i /usr/bin/docker-runc
    name: cve-2019-5736.service
{{- if bool (getenv "KIAM_INSTALLED" "true") }}
  # Install iptable entry for kiam which, when kiam is not present, will prevent any access to EC2 metadata
  # NOTE: the interface name (after -i) must be the same as the interface name passed to kiam
  #       and if kiam is not listening on the default 8181 port, you must replace 8181 with the correct port number
  - after:
    - network.target
    manifest: |
      Type=oneshot
      ExecStart=/bin/sh -c '/sbin/iptables -t nat -A PREROUTING -d 169.254.169.254/32 \
          -i cali+ -p tcp -m tcp --dport 80 -j DNAT \
          --to-destination $(curl -s http://169.254.169.254/latest/meta-data/local-ipv4):8181'
    name: kiam-iptables.service
    roles: [Node]
{{- end }}
{{- if getenv "TELEPORT_PROXY_DOMAIN_NAME" }}
  - name: teleport-node.service
    roles: [Node, Master, Bastion]
    requires: [network.target, teleport-node-install.service]
    after: [network.target]
    manifest: |
      Type=simple
      Restart=on-failure
      ExecStart=/bin/bash -c '/usr/local/bin/teleport start --config=/etc/teleport.yaml \
      --pid-file=/var/run/teleport.pid \
      --auth-server="{{- print (getenv "TELEPORT_AUTH_DOMAIN_NAME" | default (print "auth." (getenv "TELEPORT_PROXY_DOMAIN_NAME"))) ":3025" -}}" \
      --advertise-ip="$(curl -sS http://169.254.169.254/latest/meta-data/local-ipv4)" \
      --labels "id=$(curl -sS http://169.254.169.254/latest/meta-data/instance-id),role=$(/bin/sh /usr/local/bin/k8s-role-label.sh)"'
      ExecReload=/bin/kill -HUP $MAINPID
      PIDFile=/var/run/teleport.pid
  - name: teleport-node-install.service
    roles: [Node, Master, Bastion]
    before: [teleport-node.service]
    requires: [network.target]
    after: [network.target]
    manifest: |
      Type=oneshot
      ExecStart=/bin/bash -c 'set -e -o pipefail && \
      wget -q -t 3 -O /tmp/teleport.tar.gz \
      https://get.gravitational.com/teleport-ent-v{{- getenv "TELEPORT_VERSION" -}}-linux-amd64-bin.tar.gz && \
      tar  -xzf /tmp/teleport.tar.gz -C /tmp && \
      cp -f /tmp/teleport-ent/teleport /usr/local/bin/teleport && \
      chmod 551 /usr/local/bin/teleport'
  fileAssets:
    - name: teleport.yaml
      # path is actually the path plus the filename
      path: /etc/teleport.yaml
      roles: [Node, Master, Bastion]
      content: |
        teleport:
          auth_token: {{ getenv "TELEPORT_NODE_TOKEN" | default "node-not-completely-secure-but-not-a-huge-risk" }}
          log:
            output: stdout
            severity: INFO
        ssh_service:
          enabled: true
        auth_service:
          enabled: false
        proxy_service:
          enabled: false
    - name: k8s-role-label.sh
      # path is actually the path plus the filename
      path: /usr/local/bin/k8s-role-label.sh
      roles: [Node, Master, Bastion]
      content: |
        #!/bin/sh
        aws ec2 describe-tags \
          --filters "Name=resource-id,Values=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" \
          --region $(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document \
                     | grep region | cut -d\" -f4) \
          --query 'Tags[*].{tag:Key,value:Value}' \
          --output text | sed 's/\t/=/' | grep k8s.io/role | sed 's%k8s.io/role/%%' | cut -f1 -d=
{{- end }}
{{- if bool (getenv "KOPS_AUTHORIZATION_RBAC_ENABLED" "false") }}
  kubelet:
    anonymousAuth: false
{{- end }}
  kubernetesApiAccess:
  - 0.0.0.0/0
  kubernetesVersion: {{ getenv "KUBERNETES_VERSION" }}
  masterPublicName: api.{{ getenv "KOPS_CLUSTER_NAME" }}
  networkCIDR: {{ getenv "KOPS_NETWORK_CIDR" "172.20.0.0/16" }}
  {{- if getenv "KOPS_SHARED_VPC_ID" }}
  networkID: {{ getenv "KOPS_SHARED_VPC_ID" }}
  {{- end }}
  networking:
    calico: {}
  nonMasqueradeCIDR: {{ getenv "KOPS_NON_MASQUERADE_CIDR" "100.64.0.0/10" }}
  sshAccess:
  - 0.0.0.0/0
  subnets:
  {{- range $index, $value := (getenv "KOPS_AVAILABILITY_ZONES" | strings.Split ",") }}
  {{- $privateSubnets := getenv "KOPS_PRIVATE_SUBNETS" | required "KOPS_PRIVATE_SUBNETS must be defined" }}
  {{- $privateSubnetIDs := getenv "KOPS_SHARED_PRIVATE_SUBNET_IDS" }}
  {{- $natGateways := getenv "KOPS_SHARED_NAT_GATEWAYS" }}
  {{- $subnet := strings.Split "," $privateSubnets }}
  {{- $subnetID := strings.Split "," $privateSubnetIDs }}
  {{- $nat := strings.Split "," $natGateways }}
  - cidr: {{ index $subnet $index }}
    {{- if $privateSubnetIDs }}
    id: {{ index $subnetID $index }}
    {{- end }}
    name: {{ $value }}
    {{- if $natGateways }}
    egress: {{ index $nat $index }}
    {{- end }}
    type: Private
    zone: {{ $value }}
  {{- end }}
  {{- range $index, $value := (getenv "KOPS_AVAILABILITY_ZONES" | strings.Split ",") }}
  {{- $utilitySubnets := getenv "KOPS_UTILITY_SUBNETS" }}
  {{- $utilitySubnetIDs := getenv "KOPS_SHARED_UTILITY_SUBNET_IDS" }}
  {{- $subnet := strings.Split "," $utilitySubnets }}
  {{- $subnetID := strings.Split "," $utilitySubnetIDs }}
  - cidr: {{ index $subnet $index }}
    {{- if $utilitySubnetIDs }}
    id: {{ index $subnetID $index }}
    {{- end }}
    name: utility-{{ $value }}
    type: Utility
    zone: {{ $value }}
  {{- end }}
  topology:
    bastion:
      bastionPublicName: {{ getenv "KOPS_BASTION_PUBLIC_NAME" }}.{{ getenv "KOPS_CLUSTER_NAME" }}
    dns:
      type: Public
    masters: private
    nodes: private

---
apiVersion: kops/v1alpha2
kind: InstanceGroup
metadata:
  labels:
    kops.k8s.io/cluster: {{ getenv "KOPS_CLUSTER_NAME" }}
  name: bastions
spec:
  image: {{ getenv "KOPS_BASE_IMAGE" }}
  machineType: {{ getenv "BASTION_MACHINE_TYPE" }}
  maxSize: 1
  minSize: 1
  role: Bastion
  subnets:
  {{- range (getenv "KOPS_AVAILABILITY_ZONES" | strings.Split ",") }}
  - utility-{{ . }}
  {{- end }}

{{ range (getenv "KOPS_AVAILABILITY_ZONES" | strings.Split ",") }}

---
apiVersion: kops/v1alpha2
kind: InstanceGroup
metadata:
  labels:
    kops.k8s.io/cluster: {{ getenv "KOPS_CLUSTER_NAME" }}
  name: master-{{ . }}
spec:
  detailedInstanceMonitoring: {{ getenv "KOPS_CLOUDWATCH_DETAILED_MONITORING" "false" }}
  associatePublicIp: false
  image: {{ getenv "KOPS_BASE_IMAGE" }}
  machineType: {{ getenv "MASTER_MACHINE_TYPE" }}
  maxSize: 1
  minSize: 1
  role: Master
  subnets:
  - {{ . }}
{{- end }}

---
apiVersion: kops/v1alpha2
kind: InstanceGroup
metadata:
  labels:
    kops.k8s.io/cluster: {{ getenv "KOPS_CLUSTER_NAME" }}
  name: nodes
spec:
  detailedInstanceMonitoring: {{ getenv "KOPS_CLOUDWATCH_DETAILED_MONITORING" "false" }}
  associatePublicIp: false
  image: {{ getenv "KOPS_BASE_IMAGE" }}
  machineType: {{ getenv "NODE_MACHINE_TYPE" }}
  maxSize: {{ getenv "NODE_MAX_SIZE" }}
  minSize: {{ getenv "NODE_MIN_SIZE" }}
  role: Node
  subnets:
  {{- range (getenv "KOPS_AVAILABILITY_ZONES" | strings.Split ",") }}
  - {{ . }}
  {{- end }}

{{/* Allow the manifest to be extended via a datasource */}}
{{- if (datasourceExists "extensions") -}}
---
{{ include "extensions" }}
{{- end -}}