From 2ed2045600bc12ffd543f19cbe1c12fb25236d0b Mon Sep 17 00:00:00 2001 From: Luis Arcega Date: Tue, 11 Jun 2024 18:26:51 -0500 Subject: [PATCH] Adding ALB troubleshooting scenario and Troubleshooting Methodologies --- .../alb/.workshop/terraform/main.tf | 176 +++++++++++++ .../alb/.workshop/terraform/outputs.tf | 13 + .../alb/.workshop/terraform/template/break.sh | 155 ++++++++++++ .../terraform/template/other_issue.json | 207 +++++++++++++++ .../alb/.workshop/terraform/vars.tf | 35 +++ .../alb/creating-alb/fix_ingress/ingress.yaml | 21 ++ .../fix_ingress/kustomization.yaml | 4 + .../creating-alb/fix_ui/kustomization.yaml | 6 + .../alb/creating-alb/fix_ui/service.yaml | 16 ++ .../alb/creating-alb/ingress.yaml | 21 ++ .../alb/creating-alb/kustomization.yaml | 7 + .../alb/creating-alb/service.yaml | 16 ++ website/docs/troubleshooting/_category_.json | 3 + website/docs/troubleshooting/alb/alb_fix_1.md | 165 ++++++++++++ website/docs/troubleshooting/alb/alb_fix_5.md | 91 +++++++ website/docs/troubleshooting/alb/alb_fix_7.md | 236 ++++++++++++++++++ .../alb/alb_fix_wrapping_up.md | 20 ++ .../alb/assets/IntScen-ALBC-1-2.webp | Bin 0 -> 1144 bytes .../troubleshooting/alb/assets/alb-503.webp | Bin 0 -> 4288 bytes .../alb/assets/alb-does-not-exist.webp | Bin 0 -> 4468 bytes .../alb/assets/alb-working.webp | Bin 0 -> 26192 bytes .../troubleshooting/alb/assets/ingress.webp | Bin 0 -> 15150 bytes website/docs/troubleshooting/alb/index.md | 73 ++++++ .../alb/tests/hook-add-ingress.sh | 32 +++ .../troubleshooting/alb/tests/hook-suite.sh | 11 + website/docs/troubleshooting/index.md | 30 +++ website/docusaurus.config.js | 6 + website/sidebars.js | 1 + website/test-durations.json | 3 +- 29 files changed, 1347 insertions(+), 1 deletion(-) create mode 100644 manifests/modules/troubleshooting/alb/.workshop/terraform/main.tf create mode 100644 manifests/modules/troubleshooting/alb/.workshop/terraform/outputs.tf create mode 100755 manifests/modules/troubleshooting/alb/.workshop/terraform/template/break.sh create mode 100644 manifests/modules/troubleshooting/alb/.workshop/terraform/template/other_issue.json create mode 100644 manifests/modules/troubleshooting/alb/.workshop/terraform/vars.tf create mode 100644 manifests/modules/troubleshooting/alb/creating-alb/fix_ingress/ingress.yaml create mode 100644 manifests/modules/troubleshooting/alb/creating-alb/fix_ingress/kustomization.yaml create mode 100644 manifests/modules/troubleshooting/alb/creating-alb/fix_ui/kustomization.yaml create mode 100644 manifests/modules/troubleshooting/alb/creating-alb/fix_ui/service.yaml create mode 100644 manifests/modules/troubleshooting/alb/creating-alb/ingress.yaml create mode 100644 manifests/modules/troubleshooting/alb/creating-alb/kustomization.yaml create mode 100644 manifests/modules/troubleshooting/alb/creating-alb/service.yaml create mode 100644 website/docs/troubleshooting/_category_.json create mode 100644 website/docs/troubleshooting/alb/alb_fix_1.md create mode 100644 website/docs/troubleshooting/alb/alb_fix_5.md create mode 100644 website/docs/troubleshooting/alb/alb_fix_7.md create mode 100644 website/docs/troubleshooting/alb/alb_fix_wrapping_up.md create mode 100644 website/docs/troubleshooting/alb/assets/IntScen-ALBC-1-2.webp create mode 100644 website/docs/troubleshooting/alb/assets/alb-503.webp create mode 100644 website/docs/troubleshooting/alb/assets/alb-does-not-exist.webp create mode 100644 website/docs/troubleshooting/alb/assets/alb-working.webp create mode 100644 website/docs/troubleshooting/alb/assets/ingress.webp create mode 100644 website/docs/troubleshooting/alb/index.md create mode 100644 website/docs/troubleshooting/alb/tests/hook-add-ingress.sh create mode 100644 website/docs/troubleshooting/alb/tests/hook-suite.sh create mode 100644 website/docs/troubleshooting/index.md diff --git a/manifests/modules/troubleshooting/alb/.workshop/terraform/main.tf b/manifests/modules/troubleshooting/alb/.workshop/terraform/main.tf new file mode 100644 index 000000000..7e913aa8b --- /dev/null +++ b/manifests/modules/troubleshooting/alb/.workshop/terraform/main.tf @@ -0,0 +1,176 @@ +terraform { + required_providers { + # kubectl = { + # source = "gavinbunney/kubectl" + # version = ">= 1.14" + # } + } +} + + + +provider "aws" { + region = "us-east-1" + alias = "virginia" +} + +locals { + tags = { + module = "troubleshooting" + } +} + +data "aws_vpc" "selected" { + tags = { + created-by = "eks-workshop-v2" + env = var.addon_context.eks_cluster_id + } +} + +data "aws_subnets" "public" { + tags = { + created-by = "eks-workshop-v2" + env = var.addon_context.eks_cluster_id + } + + filter { + name = "tag:Name" + values = ["*Public*"] + } +} + + +resource "time_sleep" "blueprints_addons_sleep" { + depends_on = [ + module.eks_blueprints_addons + ] + + create_duration = "15s" + destroy_duration = "15s" +} + + +resource "null_resource" "break_public_subnet" { + triggers = { + #cluster_id = var.addon_context.eks_cluster_id + public_subnets = join(" ", data.aws_subnets.public.ids) + timestamp = timestamp() + } + count = length(data.aws_subnets.public) + + provisioner "local-exec" { + when = destroy + command = "aws ec2 create-tags --resources ${self.triggers.public_subnets} --tags Key=kubernetes.io/role/elb,Value='1'" + } + + provisioner "local-exec" { + command = "aws ec2 delete-tags --resources ${self.triggers.public_subnets} --tags Key=kubernetes.io/role/elb,Value='1'" + } +} + + +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "1.16.2" + + enable_aws_load_balancer_controller = true + aws_load_balancer_controller = { + wait = true + } + + cluster_name = var.addon_context.eks_cluster_id + cluster_endpoint = var.addon_context.aws_eks_cluster_endpoint + cluster_version = var.eks_cluster_version + oidc_provider_arn = var.addon_context.eks_oidc_provider_arn + + tags = merge( + var.tags, + local.tags + ) + + depends_on = [null_resource.break_public_subnet] + +} + + +# create a new policy from json file +resource "aws_iam_policy" "issue" { + name = "issue" + path = "/" + policy = file("${path.module}/template/other_issue.json") +} + +# attach issue policy to role +resource "aws_iam_role_policy_attachment" "issue_policy_attachment" { + role = module.eks_blueprints_addons.aws_load_balancer_controller.iam_role_name + policy_arn = aws_iam_policy.issue.arn + depends_on = [module.eks_blueprints_addons, time_sleep.blueprints_addons_sleep] +} + +resource "null_resource" "detach_existing_policy" { + triggers = { + role_name = module.eks_blueprints_addons.aws_load_balancer_controller.iam_role_name, + timestamp = timestamp() + } + + provisioner "local-exec" { + command = "aws iam detach-role-policy --role-name ${self.triggers.role_name} --policy-arn ${module.eks_blueprints_addons.aws_load_balancer_controller.iam_policy_arn}" + when = create + } + + depends_on = [aws_iam_role_policy_attachment.issue_policy_attachment] +} + +resource "null_resource" "kustomize_app" { + triggers = { + always_run = timestamp() + } + + provisioner "local-exec" { + command = "kubectl apply -k ~/environment/eks-workshop/modules/troubleshooting/alb/creating-alb" + when = create + } + + depends_on = [aws_iam_role_policy_attachment.issue_policy_attachment] +} + + + +# Example to now how to get variables from add ons outputs DO-NOT-DELETE; AddOns and helms documentaitons does not show exactly the output variables returned +#resource "null_resource" "blue_print_output" { +# for_each = module.eks_blueprints_addons.aws_load_balancer_controller +# triggers = { +# +# timestamp = timestamp() +# } +# +# #count = length(module.eks_blueprints_addons.aws_load_balancer_controller) +# provisioner "local-exec" { +# command = "mkdir -p /eks-workshop/logs; echo \" key: ${each.key} Value:${each.value}\" >> /eks-workshop/logs/action-load-balancer-output.log" +# } +# +# depends_on = [module.eks_blueprints_addons,time_sleep.blueprints_addons_sleep] +#} + +#option to run a bash script file +#resource "null_resource" "break2" { +# provisioner "local-exec" { +# command = "${path.module}/template/break.sh ${path.module} mod2" +# } +# +# triggers = { +# always_run = timestamp() +# } +# depends_on = [module.eks_blueprints_addons,time_sleep.blueprints_addons_sleep] +#} + +#option to run a kubectl manifest +#resource "kubectl_manifest" "alb" { +# yaml_body = templatefile("${path.module}/template/ingress.yaml", { +# +# }) +# +# depends_on = [null_resource.break_policy] +#} + + diff --git a/manifests/modules/troubleshooting/alb/.workshop/terraform/outputs.tf b/manifests/modules/troubleshooting/alb/.workshop/terraform/outputs.tf new file mode 100644 index 000000000..8669df9bd --- /dev/null +++ b/manifests/modules/troubleshooting/alb/.workshop/terraform/outputs.tf @@ -0,0 +1,13 @@ +output "environment_variables" { + description = "Environment variables to be added to the IDE shell" + value = merge({ + VPC_ID = data.aws_vpc.selected.id, + LOAD_BALANCER_CONTROLLER_ROLE_NAME = module.eks_blueprints_addons.aws_load_balancer_controller.iam_role_name, + LOAD_BALANCER_CONTROLLER_POLICY_ARN_FIX = module.eks_blueprints_addons.aws_load_balancer_controller.iam_policy_arn, + LOAD_BALANCER_CONTROLLER_POLICY_ARN_ISSUE = aws_iam_policy.issue.arn, + LOAD_BALANCER_CONTROLLER_ROLE_ARN = module.eks_blueprints_addons.aws_load_balancer_controller.iam_role_arn + }, { + for index, id in data.aws_subnets.public.ids : "PUBLIC_SUBNET_${index + 1}" => id + } + ) +} \ No newline at end of file diff --git a/manifests/modules/troubleshooting/alb/.workshop/terraform/template/break.sh b/manifests/modules/troubleshooting/alb/.workshop/terraform/template/break.sh new file mode 100755 index 000000000..02011b4e9 --- /dev/null +++ b/manifests/modules/troubleshooting/alb/.workshop/terraform/template/break.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +#. .env + +set -e + +mkdir -p /eks-workshop/logs +log_file=/eks-workshop/logs/action-$(date +%s).log + +exec 2>&1 + +logmessage() { + echo "$@" >&7 + echo "$@" >&1 +} +export -f logmessage + +# Function to get the role name from a role ARN +get_role_name_from_arn() { + local role_arn=$1 + + # Extract the role name from the ARN + role_name=$(logmessage "$role_arn" | awk -F'/' '{print $NF}') + + if [ -n "$role_name" ]; then + logmessage "$role_name" + else + logmessage "Failed to retrieve role name from ARN: $role_arn" + return 1 + fi +} + +# Function to get the Kubernetes role attached to a service account +get_service_account_role() { + local namespace=$1 + local service_account=$2 + + # Get the role ARN associated with the service account + role_arn=$(kubectl get serviceaccount "$service_account" -n "$namespace" -o jsonpath="{.metadata.annotations['eks\.amazonaws\.com\/role-arn']}") + + if [ -n "$role_arn" ]; then + logmessage "Service Account: $service_account" + logmessage "Namespace: $namespace" + logmessage "Role ARN: $role_arn" + get_role_name_from_arn "$role_arn" + return 0 + else + logmessage "Failed to retrieve role for service account '$service_account' in namespace '$namespace'" + return 1 + fi + +} + +# Function to get the first policy ARN attached to a role ARN +get_first_policy_arn_from_role_arn() { + local role_arn=$1 + + # Get the list of policies attached to the role + policy_arn=$(aws iam list-attached-role-policies --role-name "$role_arn" --query 'AttachedPolicies[0].PolicyArn' --output text) + + if [ -n "$policy_arn" ]; then + logmessage "First Policy ARN attached to role '$role_arn':" + logmessage "Policy: $policy_arn" + return 0 + else + logmessage "Failed to retrieve policy ARN for role '$role_arn'" + return 1 + fi +} + +# Function to update the policy with new statement +update_policy_with_new_statement() { + local policy_arn=$1 + local new_statement=$2 + + logmessage "PolicyARN: $policy_arn" + logmessage "Statement: $new_statement" + aws iam create-policy-version --policy-arn $policy_arn --policy-document $new_statement --set-as-default + +} + +# Function to remove an action from a policy statement +remove_action_from_policy_statement() { + local policy_name=$1 + local action_to_remove=$2 + + # Get the current policy document + policy_document=$(aws iam get-policy-version --policy-arn "$policy_arn" --query 'PolicyVersion.Document' --version-id v1 --output json) + + # Remove the specified action from the statements + new_statements=$(logmessage "$policy_document" | jq ".Statement[] | select(.Action[] | contains('$action_to_remove')) | .Action = [.Action[] | select(. != '$action_to_remove')]") + new_policy_document=$(logmessage '{"Version": "2012-10-17", "Statement": '"$new_statements"'}') ++ + # Update the policy with the modified document + logmessage "Policy Document" + logmessage $new_policy_document + #aws iam create-policy-version --policy-arn "$policy_arn" --policy-document "$new_policy_document" --set-as-default + + if [ $? -eq 0 ]; then + logmessage "Action removed from policy statement successfully." + return 0 + else + logmessage "Failed to remove action from policy statement." + return 1 + fi +} + +# Function to remove tags from subnets ids +remove_tags_from_subnets() { + local tag_key="Key=kubernetes.io/role/elb,Value=1" + + logmessage "retrive subnets ids with tag key assigned to specific vpc_id via aws cli" + logmessage "getting public subnets from VPC: $vpc_id " + + + subnets_vpc=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$vpc_id" --query 'Subnets[*].SubnetId' --output text) + logmessage "subnets_vpc: $subnets_vpc" + + +#remove tag from subnets with AWS cli + for subnet_id in $subnets_vpc; do + logmessage "public subnets: $subnet_id" + aws ec2 delete-tags --resources "$subnet_id" --tags "Key=$tag_key" || logmessage "Failed to remove tag from subnet $subnet_id" + done + return 0 +} + +# Getting the service role +path_tofile=$1 +mode=$2 +vpc_id=$3 +public_subnets=$4 +namespace="kube-system" +service_account="aws-load-balancer-controller-sa" +#new_statement="file://$path_tofile/template/iam_policy_incorrect.json" +new_statement="file://$path_tofile/template/other_issue.json" + +logmessage "path_sent: $path_tofile" + + +# validate if mode is equal to mod1 +logmessage "mode: $mode" +if [ "$mode" == "mod1" ]; then + logmessage "Removing subnet tags" + remove_tags_from_subnets +else + logmessage "Removing permissions" + get_service_account_role "$namespace" "$service_account" + get_first_policy_arn_from_role_arn "$role_name" + update_policy_with_new_statement "$policy_arn" "$new_statement" + +fi + + + + diff --git a/manifests/modules/troubleshooting/alb/.workshop/terraform/template/other_issue.json b/manifests/modules/troubleshooting/alb/.workshop/terraform/template/other_issue.json new file mode 100644 index 000000000..4503e4e9b --- /dev/null +++ b/manifests/modules/troubleshooting/alb/.workshop/terraform/template/other_issue.json @@ -0,0 +1,207 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["iam:CreateServiceLinkedRole"], + "Resource": "*", + "Condition": { + "StringEquals": { + "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeVpcs", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:GetCoipPoolUsage", + "ec2:DescribeCoipPools", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "cognito-idp:DescribeUserPoolClient", + "acm:ListCertificates", + "acm:DescribeCertificate", + "iam:ListServerCertificates", + "iam:GetServerCertificate", + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + "shield:GetSubscriptionState", + "shield:DescribeProtection", + "shield:CreateProtection", + "shield:DeleteProtection" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": ["ec2:CreateSecurityGroup"], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": ["ec2:CreateTags"], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "StringEquals": { + "ec2:CreateAction": "CreateSecurityGroup" + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": ["ec2:CreateTags", "ec2:DeleteTags"], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + "ec2:DeleteSecurityGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": ["elasticloadbalancing:CreateTargetGroup"], + "Resource": "*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteRule" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:DeleteTargetGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets" + ], + "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:SetWebAcl", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:ModifyRule" + ], + "Resource": "*" + } + ] +} diff --git a/manifests/modules/troubleshooting/alb/.workshop/terraform/vars.tf b/manifests/modules/troubleshooting/alb/.workshop/terraform/vars.tf new file mode 100644 index 000000000..812087dc5 --- /dev/null +++ b/manifests/modules/troubleshooting/alb/.workshop/terraform/vars.tf @@ -0,0 +1,35 @@ +# tflint-ignore: terraform_unused_declarations +variable "eks_cluster_id" { + description = "EKS cluster name" + type = string +} + +# tflint-ignore: terraform_unused_declarations +variable "eks_cluster_version" { + description = "EKS cluster version" + type = string +} + +# tflint-ignore: terraform_unused_declarations +variable "cluster_security_group_id" { + description = "EKS cluster security group ID" + type = any +} + +# tflint-ignore: terraform_unused_declarations +variable "addon_context" { + description = "Addon context that can be passed directly to blueprints addon modules" + type = any +} + +# tflint-ignore: terraform_unused_declarations +variable "tags" { + description = "Tags to apply to AWS resources" + type = any +} + +# tflint-ignore: terraform_unused_declarations +variable "resources_precreated" { + description = "Have expensive resources been created already" + type = bool +} diff --git a/manifests/modules/troubleshooting/alb/creating-alb/fix_ingress/ingress.yaml b/manifests/modules/troubleshooting/alb/creating-alb/fix_ingress/ingress.yaml new file mode 100644 index 000000000..521dbf7c1 --- /dev/null +++ b/manifests/modules/troubleshooting/alb/creating-alb/fix_ingress/ingress.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ui + namespace: ui + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/healthcheck-path: /actuator/health/liveness +spec: + ingressClassName: alb + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ui + port: + number: 80 diff --git a/manifests/modules/troubleshooting/alb/creating-alb/fix_ingress/kustomization.yaml b/manifests/modules/troubleshooting/alb/creating-alb/fix_ingress/kustomization.yaml new file mode 100644 index 000000000..972f3ed06 --- /dev/null +++ b/manifests/modules/troubleshooting/alb/creating-alb/fix_ingress/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ingress.yaml diff --git a/manifests/modules/troubleshooting/alb/creating-alb/fix_ui/kustomization.yaml b/manifests/modules/troubleshooting/alb/creating-alb/fix_ui/kustomization.yaml new file mode 100644 index 000000000..5400397eb --- /dev/null +++ b/manifests/modules/troubleshooting/alb/creating-alb/fix_ui/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../../../../base-application/ui +patches: + - path: service.yaml diff --git a/manifests/modules/troubleshooting/alb/creating-alb/fix_ui/service.yaml b/manifests/modules/troubleshooting/alb/creating-alb/fix_ui/service.yaml new file mode 100644 index 000000000..8a78041ce --- /dev/null +++ b/manifests/modules/troubleshooting/alb/creating-alb/fix_ui/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: ui + labels: + helm.sh/chart: ui-0.0.1 + app.kubernetes.io/name: ui + app.kubernetes.io/instance: ui + app.kubernetes.io/component: service + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/created-by: eks-workshop +spec: + selector: + app.kubernetes.io/name: ui + app.kubernetes.io/instance: ui + app.kubernetes.io/component: service diff --git a/manifests/modules/troubleshooting/alb/creating-alb/ingress.yaml b/manifests/modules/troubleshooting/alb/creating-alb/ingress.yaml new file mode 100644 index 000000000..7f4d06f31 --- /dev/null +++ b/manifests/modules/troubleshooting/alb/creating-alb/ingress.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ui + namespace: ui + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/healthcheck-path: /actuator/health/liveness +spec: + ingressClassName: alb + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: service-ui + port: + number: 80 diff --git a/manifests/modules/troubleshooting/alb/creating-alb/kustomization.yaml b/manifests/modules/troubleshooting/alb/creating-alb/kustomization.yaml new file mode 100644 index 000000000..439dd9a06 --- /dev/null +++ b/manifests/modules/troubleshooting/alb/creating-alb/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../../../base-application/ui + - ingress.yaml +patches: + - path: service.yaml diff --git a/manifests/modules/troubleshooting/alb/creating-alb/service.yaml b/manifests/modules/troubleshooting/alb/creating-alb/service.yaml new file mode 100644 index 000000000..5d02e9441 --- /dev/null +++ b/manifests/modules/troubleshooting/alb/creating-alb/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: ui + labels: + helm.sh/chart: ui-0.0.1 + app.kubernetes.io/name: ui + app.kubernetes.io/instance: ui + app.kubernetes.io/component: service + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/created-by: eks-workshop +spec: + selector: + app.kubernetes.io/name: ui-app + app.kubernetes.io/instance: ui + app.kubernetes.io/component: service diff --git a/website/docs/troubleshooting/_category_.json b/website/docs/troubleshooting/_category_.json new file mode 100644 index 000000000..0d4475828 --- /dev/null +++ b/website/docs/troubleshooting/_category_.json @@ -0,0 +1,3 @@ +{ + "collapsed": false +} diff --git a/website/docs/troubleshooting/alb/alb_fix_1.md b/website/docs/troubleshooting/alb/alb_fix_1.md new file mode 100644 index 000000000..fab629245 --- /dev/null +++ b/website/docs/troubleshooting/alb/alb_fix_1.md @@ -0,0 +1,165 @@ +--- +title: "Section 1 - Fixing Tag Issue" +sidebar_position: 30 +--- + +The task for you in this troubleshooting scenario is to investigate the deployment for AWS Load Balancer Controller as well as the ingress object created by following the prompts with the script. At the end of this session, you should be able to see the ui app on your EKS cluster using ALB ingress through the browsers as depicted in the image. + +![ingress](./assets/ingress.webp) + +## Let's start the troubleshooting + +### Step 1: + +First, we need to verify the status of our pods and get ingress for ingress object creation. To do so, we will use `kubectl` tool. + +```bash +$ kubectl get pod -n ui +NAME READY STATUS RESTARTS AGE +ui-68495c748c-jkh2z 1/1 Running 0 85s +``` + +### Step 2: + +In _Step 1_, we checked the pods status for our application and aws-load-balancer-controller. The _aws-load-balancer-controller_ deployment is responsible for ALB creation for any ingress objects applied to the cluster. + +Upon looking for ingress object, did you observe any ALB DNS name to access your application with the ingress object? You can also verify ALB creation in the AWS Management Console. In a successful installation scenario, the ingress object should have an ALB DNS name shown like the example below. However in this case, the ADDRESS section where the ALB DNS should have populated is empty. + +```bash +$ kubectl get ingress/ui -n ui +NAME CLASS HOSTS ADDRESS PORTS AGE +ui alb * 80 105s + +#---This is the expected output when the ingress was deployed correctly-- +NAME CLASS HOSTS ADDRESS PORTS AGE +ingress-2048 * k8s-ui-ingress2-xxxxxxxxxx-yyyyyyyyyy.region-code.elb.amazonaws.com 80 2m32s +``` + +### Step 3: + +Check further into the ingress for any events indicating why we do not see the ALB DNS. You can retrieve those logs by running the following command. The event logs should point you towards what the issue might be with ingress creation. + +```bash +$ kubectl describe ingress/ui -n ui +Name: ui +Labels: +Namespace: ui +Address: +Ingress Class: alb +Default backend: +Rules: + Host Path Backends + ---- ---- -------- + * + / service-ui:80 () +Annotations: alb.ingress.kubernetes.io/healthcheck-path: /actuator/health/liveness + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning FailedBuildModel 2m23s (x16 over 5m9s) ingress Failed build model due to couldn't auto-discover subnets: unable to resolve at least one subnet (0 match VPC and tags: [kubernetes.io/role/elb]) + +``` + +Refer the documentation on prerequisites for setting up ALB with EKS: https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/deploy/subnet_discovery/ + +### Step 4: + +_Step 3_ points to issues with the subnet auto-discovery for load balancer controller deployment. Ensure that all the public subnets have correct tags `tag:kubernetes.io/role/elb,Values=1'` + +:::info +Keep in mind that public subnet means the route table for the subnet has an Internet Gateway allowing traffic to and from the internet. +::: + +1. To find the all subnets through the command line, filter through existing ones with the following tag "Key: `alpha.eksctl.io/cluster-name` Value: `${EKS_CLUSTER_NAME}`". There should be four subnets. **Note:** _For your convenience we have added the cluster name as env variable with the variable `$EKS_CLUSTER_NAME`._ + +```bash +$ aws ec2 describe-subnets --filters "Name=tag:alpha.eksctl.io/cluster-name,Values=${EKS_CLUSTER_NAME}" --query 'Subnets[].SubnetId[]' +[ + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx" +] +``` + +2. Then by adding in the subnet ID into the route tables CLI filter one at a time, `--filters 'Name=association.subnet-id,Values=subnet-xxxxxxxxxxxxxxxxx'`, identify which subnets are public. + +``` +aws ec2 describe-route-tables --filters 'Name=association.subnet-id,Values=' --query 'RouteTables[].Routes[].[DestinationCidrBlock,GatewayId]' +``` + +Here a script that will help to iterate over the list of subnets + +```bash + +$ for subnet_id in $(aws ec2 describe-subnets --filters "Name=tag:alpha.eksctl.io/cluster-name,Values=${EKS_CLUSTER_NAME}" --query 'Subnets[].SubnetId[]' --output text); do echo "Subnect: ${subnet_id}"; aws ec2 describe-route-tables --filters "Name=association.subnet-id,Values=${subnet_id}" --query 'RouteTables[].Routes[].[DestinationCidrBlock,GatewayId]'; done +``` + +If the output shows `0.0.0.0/0` route to an Internet gateway ID, this is a public subnet. See below example. + +``` +WSParticipantRole:~/environment $ aws ec2 describe-route-tables --filters "Name=association.subnet-id,Values=subnet-xxxxxxxxxxxxx0470" --query 'RouteTables[].Routes[].[DestinationCidrBlock,GatewayId]' +[ + [ + "10.42.0.0/16", + "local" + ], + [ + "0.0.0.0/0", + "igw-xxxxxxxxxxxxxxxxx" + ] +] +``` + +3. Once you have all the public subnet ID's, describe subnets with the appropriate tag and confirm that the public subnet ID's that you identified are missing. In our case, none of our subnets have the correct tags. + +```bash +$ aws ec2 describe-subnets --filters 'Name=tag:kubernetes.io/role/elb,Values=1' --query 'Subnets[].SubnetId' +[] +``` + +4. Then add the correct tags. To help you a little bit, we have added the 3 public subnets to the `env` variables with the names `PUBLIC_SUBNET_1, PUBLIC_SUBNET_2 and PUBLIC_SUBNET_3` + +``` +aws ec2 create-tags --resources subnet-xxxxxxxxxxxxxxxxx subnet-xxxxxxxxxxxxxxxxx subnet-xxxxxxxxxxxxxxxxx --tags 'Key="kubernetes.io/role/elb",Value=1' +``` + +```bash +$ aws ec2 create-tags --resources $PUBLIC_SUBNET_1 $PUBLIC_SUBNET_2 $PUBLIC_SUBNET_3 --tags 'Key="kubernetes.io/role/elb",Value=1' +``` + +5. Confirm the tags are created. You should see the public subnet ID's populated following the command below. + +```bash +$ aws ec2 describe-subnets --filters 'Name=tag:kubernetes.io/role/elb,Values=1' --query 'Subnets[].SubnetId' +[ + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx" +] +``` + +6. Now restart the controller deployment using the kubectl rollout restart command: + +```bash timeout=180 +$ kubectl -n kube-system rollout restart deploy aws-load-balancer-controller +deployment.apps/aws-load-balancer-controller restarted +``` + +7. Now, check again the ingress deployment: + +```bash expectError=true +$ kubectl describe ingress/ui -n ui + Warning FailedDeployModel 68s ingress Failed deploy model due to AccessDenied: User: arn:aws:sts::xxxxxxxxxxxx:assumed-role/alb-controller-20240611131524228000000002/1718115201989397805 is not authorized to perform: elasticloadbalancing:CreateLoadBalancer on resource: arn:aws:elasticloadbalancing:us-west-2:xxxxxxxxxxxx:loadbalancer/app/k8s-ui-ui-5ddc3ba496/* because no identity-based policy allows the elasticloadbalancing:CreateLoadBalancer action + status code: 403, request id: b862fb9c-480b-44b5-ba6f-426a3884b6b6 + Warning FailedDeployModel 26s (x5 over 66s) ingress (combined from similar events): Failed deploy model due to AccessDenied: User: arn:aws:sts::xxxxxxxxxxxx:assumed-role/alb-controller-20240611131524228000000002/1718115201989397805 is not authorized to perform: elasticloadbalancing:CreateLoadBalancer on resource: arn:aws:elasticloadbalancing:us-west-2:xxxxxxxxxxxx:loadbalancer/app/k8s-ui-ui-5ddc3ba496/* because no identity-based policy allows the elasticloadbalancing:CreateLoadBalancer action + status code: 403, request id: 197cf2f7-2f68-44f2-92ae-ff5b36cb150f +``` + +:::tip +In AWS generally for creation/deletion/update of any resource, you will observe a corresponding API call which are recorded in CloudTrail. Look for any CloudTrail events for CreateLoadBalancer API calls. Do you observe any such calls in the last 1 hour of this lab setup? +::: diff --git a/website/docs/troubleshooting/alb/alb_fix_5.md b/website/docs/troubleshooting/alb/alb_fix_5.md new file mode 100644 index 000000000..3cf1d840d --- /dev/null +++ b/website/docs/troubleshooting/alb/alb_fix_5.md @@ -0,0 +1,91 @@ +--- +title: "Section 2 - Fixing Policy Issue" +sidebar_position: 31 +--- + +### Step 5 + +With this setup, we’re leveraging IAM Roles for Service Accounts, which essentially allows pods to assume IAM roles using service accounts in Kubernetes and OIDC provider associated with your EKS cluster. Locate the service account that load balancer controller is using and find out the IAM role associated with it, to identify the IAM entity that would make API calls to provision your load balancer. +Try running: + +```bash +$ kubectl get serviceaccounts -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller -o yaml +``` + +```yaml {8} +apiVersion: v1 +items: + - apiVersion: v1 + automountServiceAccountToken: true + kind: ServiceAccount + metadata: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::xxxxxxxxxxxx:role/alb-controller-20240611131524228000000002 + meta.helm.sh/release-name: aws-load-balancer-controller + meta.helm.sh/release-namespace: kube-system + creationTimestamp: "2024-06-11T13:15:32Z" + labels: + app.kubernetes.io/instance: aws-load-balancer-controller + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: aws-load-balancer-controller + app.kubernetes.io/version: v2.7.1 + helm.sh/chart: aws-load-balancer-controller-1.7.1 + name: aws-load-balancer-controller-sa + namespace: kube-system + resourceVersion: "4950707" + uid: 6d842045-f2b4-4406-869b-f2addc67ff4d +kind: List +metadata: + resourceVersion: "" +``` + +:::tip +Can you verify if there’s a call in your CloudTrail events with the IAM role listed in the output for above command? If not, take a look at the logs from your controller. +::: + +### Step 6 + +You can check the logs from controller pods to find additional details which could be preventing the load balancer to create. Let's check the logs using the command below. + +```bash +$ kubectl logs -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller +``` + +For example the output may show something similar to the below output. + +``` +{"level":"error","ts":"2024-06-11T14:24:24Z","msg":"Reconciler error","controller":"ingress","object":{"name":"ui","namespace":"ui"},"namespace":"ui","name":"ui","reconcileID":"49d27bbb-96e5-43b4-b115-b7a07e757148","error":"AccessDenied: User: arn:aws:sts::xxxxxxxxxxxx:assumed-role/alb-controller-20240611131524228000000002/1718115201989397805 is not authorized to perform: elasticloadbalancing:CreateLoadBalancer on resource: arn:aws:elasticloadbalancing:us-west-2:xxxxxxxxxxxx:loadbalancer/app/k8s-ui-ui-5ddc3ba496/* because no identity-based policy allows the elasticloadbalancing:CreateLoadBalancer action\n\tstatus code: 403, request id: a24a1620-3a75-46b7-b3c3-9c80fada159e"} +``` + +As you can see the error indicates the IAM role does not have the correct permissions, in this case the permissions to create the load balancer `elasticloadbalancing:CreateLoadBalancer`. + +:::tip +Verify the correct permissions required by the IAM role in the documentations here [[1]](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/deploy/installation/#setup-iam-manually) where you can find the latest IAM permissions json file required for the LB Controller. After the changes, you have to wait a few minutes for the changes to reflect, since IAM uses an eventual consistency model. To make the changes, locate the IAM role through the AWS console and add the missing permissions that are shown in the log. In this case CreateLoadBalancer is missing. +::: + +Now let's fix it. To avoid conflicts with the automation of the workshop, we have already provisioned the correct permissions into the account and added the environment variable `LOAD_BALANCER_CONTROLLER_ROLE_NAME` that contains the role name and `LOAD_BALANCER_CONTROLLER_POLICY_ARN_FIX` which contains the correct IAM policy arn, and `LOAD_BALANCER_CONTROLLER_POLICY_ARN_ISSUE` that contains the incorrect IAM policy arn. + +So, to fix it we will just need to attach the correct IAM policy, as follows: + +```bash +$ aws iam attach-role-policy --role-name ${LOAD_BALANCER_CONTROLLER_ROLE_NAME} --policy-arn ${LOAD_BALANCER_CONTROLLER_POLICY_ARN_FIX} +``` + +and detach the incorrect IAM policy from the role: + +```bash +$ aws iam detach-role-policy --role-name ${LOAD_BALANCER_CONTROLLER_ROLE_NAME} --policy-arn ${LOAD_BALANCER_CONTROLLER_POLICY_ARN_ISSUE} +``` + +Try accessing the new Ingress URL in the browser as before to check if you can access the UI app: + +```bash +$ kubectl get ingress -n ui ui -o jsonpath="{.status.loadBalancer.ingress[*].hostname}{'\n'}" +k8s-ui-ui-5ddc3ba496-1208241872.us-west-2.elb.amazonaws.com +``` + +:::tip +It can take a couple of minutes for the Load Balancer to be available once created. +::: + +Also, feel free to go to CloudTrail again and verify the API call for CreateLoadBalancer is there. diff --git a/website/docs/troubleshooting/alb/alb_fix_7.md b/website/docs/troubleshooting/alb/alb_fix_7.md new file mode 100644 index 000000000..ded94a1af --- /dev/null +++ b/website/docs/troubleshooting/alb/alb_fix_7.md @@ -0,0 +1,236 @@ +--- +title: "Section 3 - Fixing Manifest Configs" +sidebar_position: 32 +--- + +### Step 7 + +Even though the ingress creation succeeded, when you try accessing the app in browser there is an error stating, "Backend service does not exist". + +![ALb-Backend-DoesNotExist](./assets/alb-does-not-exist.webp) + +Since ingress is created, that would mean that there is an issue with communication from the Kubernetes ingress to the service. Check the deployment and service using: + +```bash +$ kubectl -n ui get service/ui -o yaml +``` + +```yaml {27} +apiVersion: v1 +kind: Service +metadata: + annotations: + ... + labels: + app.kubernetes.io/component: service + app.kubernetes.io/created-by: eks-workshop + app.kubernetes.io/instance: ui + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: ui + helm.sh/chart: ui-0.0.1 + name: ui + namespace: ui + resourceVersion: "4950875" + uid: dc832144-b2a1-41cd-b7a1-8979111da677 +spec: + ... + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + selector: + app.kubernetes.io/component: service + app.kubernetes.io/instance: ui + app.kubernetes.io/name: ui-app + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +``` + +And now check the ingress configuration: + +```bash +$ kubectl get ingress/ui -n ui -o yaml +``` + +```yaml {23} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + alb.ingress.kubernetes.io/healthcheck-path: /actuator/health/liveness + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip + ... + finalizers: + - ingress.k8s.aws/resources + generation: 1 + name: ui + namespace: ui + resourceVersion: "4950883" + uid: 327b899c-405e-431b-8d67-32578435f0b9 +spec: + ingressClassName: alb + rules: + - http: + paths: + - backend: + service: + name: service-ui + port: + number: 80 + path: / + pathType: Prefix +... +``` + +From the outputs, observe the ingress spec and the service name `name: service-ui` that it is pointing to versus what the service name should be. + +We will need to edit the ingress spec to point to correct service name using the command below, which contains the fix: + +```bash +$ kubectl apply -k ~/environment/eks-workshop/modules/troubleshooting/alb/creating-alb/fix_ingress +``` + +To look like: + +```yaml {10} +spec: + ingressClassName: alb + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ui + port: + number: 80 +``` + +Try accessing the ALB again using the domain name shared in the get ingress output and check if you can access the app now? + +### Step 8 + +Now we observe a 503 error when accessing the ALB: + +![ALb-503-ERROR](./assets/alb-503.webp) + +503 would suggest a server-side issue, specifically with the service being unavailable. But we ensured that the service was running on the cluster when we ran get service command in _Step 7_. + +In Kubernetes, a service is just a construct to expose deployments either externally or within the cluster. Services rely on selectors to be able to send traffic to the correct backend deployment. To verify that we have our service pointing to the correct deployment, check the endpoints that are dynamically configured by kube-proxy on service creation. Run the following command: + +```bash +$ kubectl -n ui get endpoints ui +NAME ENDPOINTS AGE +ui 13d +``` + +The endpoints in command above should be pointing to IPs of the app pods running in _ui_ namespace. Can you identify if the selectors are setup correctly in service? + +### Step 9: + +Taking a look at the deployment spec using command below, verify the selector value being used versus the one used in your service. + +```bash +$ kubectl -n ui get deploy/ui -o yaml +``` + +```yaml {34} +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + ... + name: ui + namespace: ui + .. +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: service + app.kubernetes.io/instance: ui + app.kubernetes.io/name: ui + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/path: /actuator/prometheus + prometheus.io/port: "8080" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: service + app.kubernetes.io/created-by: eks-workshop + app.kubernetes.io/instance: ui + app.kubernetes.io/name: ui + spec: + containers: +... + +``` + +And + +```bash +$ kubectl -n ui get svc ui -o yaml +``` + +```yaml {22} +apiVersion: v1 +kind: Service +metadata: + annotations: + ... + labels: + app.kubernetes.io/component: service + app.kubernetes.io/created-by: eks-workshop + app.kubernetes.io/instance: ui + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: ui + helm.sh/chart: ui-0.0.1 + name: ui + namespace: ui + resourceVersion: "5000404" + uid: dc832144-b2a1-41cd-b7a1-8979111da677 +spec: + ... + selector: + app.kubernetes.io/component: service + app.kubernetes.io/instance: ui + app.kubernetes.io/name: ui-app + sessionAffinity: None + type: ClusterIP +... +``` + +Notice what the `service/ui` selector is using and what the actual `deployment/ui` labels are. To fix the issue, we need to update the `service/ui` selector `app.kubernetes.io/name: ui-app` to `app.kubernetes.io/name: ui`. + +:::tip +You can either update the service selector with: + +- `kubectl edit service -n ` or +- `kubectl patch service -n --type='json' -p='[{"op": "replace", "path": "/spec/selector", "value": {"key1": "value1", "key2": "value2"}}]'` + ::: + +for your convenience, we have added a kustomize script that update the selector, just execute the following command: + +```bash +$ kubectl apply -k ~/environment/eks-workshop/modules/troubleshooting/alb/creating-alb/fix_ui +``` + +Now refresh the browsers and you should see the ui application: + +![ALB-UI-APP](./assets/alb-working.webp) + +### Go ahead and enjoy a break, you’ve earned it!! diff --git a/website/docs/troubleshooting/alb/alb_fix_wrapping_up.md b/website/docs/troubleshooting/alb/alb_fix_wrapping_up.md new file mode 100644 index 000000000..9dcee5a2a --- /dev/null +++ b/website/docs/troubleshooting/alb/alb_fix_wrapping_up.md @@ -0,0 +1,20 @@ +--- +title: "Wrapping it up" +sidebar_position: 33 +--- + +## Wrapping it up + +Here’s the general flow of how Load Balancer Controller works: + +1. The controller watches for [ingress events](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-controllers) from the API server. When it finds ingress resources that satisfy its requirements, it begins the creation of AWS resources. + +2. An [ALB](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html) (ELBv2) is created in AWS for the new ingress resource. This ALB can be internet-facing or internal. You can also specify the subnets it's created in using annotations. + +3. [Target Groups](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-target-groups.html) are created in AWS for each unique Kubernetes service described in the ingress resource. + +4. [Listeners](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-listeners.html) are created for every port detailed in your ingress resource annotations. When no port is specified, sensible defaults (80 or 443) are used. Certificates may also be attached via annotations. + +5. [Rules](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/listener-update-rules.html) are created for each path specified in your ingress resource. This ensures traffic to a specific path is routed to the correct Kubernetes Service. + +--- diff --git a/website/docs/troubleshooting/alb/assets/IntScen-ALBC-1-2.webp b/website/docs/troubleshooting/alb/assets/IntScen-ALBC-1-2.webp new file mode 100644 index 0000000000000000000000000000000000000000..c7cf4200fe115335de92ff18cc0db4975816ed0f GIT binary patch literal 1144 zcmV-;1c&=lNk&F+1ONb6MM6+kP&goD1ONa~8~~jGDmDQ_06x)Xs!*sUqoSlSO4$G! z32AQpJ^^Ag`TquM$Z0;>IH%;i=WO`zwcVGfq=}!*_yc%nsYm!R+02PqjiTSj=2_F^D5fW8 z@ELvIRlEhL?B{6Wg#ax_gxo+hDD-gDDDoV+V73lYKQgWv|8Wm|3DdXWStkZ!IrMJ$ z8r^gj{}kTMG+E5w8Y7~7sFYAv|F%|eM~Hd#zzbrLr9NZ`1X?$KqNhkDs#r!ZG;oILxm?Tp*@?ua|q?DQ0C!PUbo*LA*Xs?DxQUj&sJ4*Q4%i; zzu7rE=b-D+;$yR!skXyzl`-&Db2ZJ^LE9*h1GixZ0*vBjtK*Is?Ck#p0P(xL3R9YZ ze1R+hM%c*E{U^VV-tQJmm!Lu-?xL*B^pltW12H=RBRnlL2 z)SNv9Z>ON94QalJ5ugQ~wFm$JWc)+9RqnE@B6We074zUo7sL>tKH6ud*2fbYO?WhN zCLRq`yz4TRfMgG3D9uz)uAxG4V7bwmSx|+oIe!LAmQAHONAvT)F_BQA_GezkQEyk9 z@yyG{Anu!kwLHF~aAyL7wlNUq#x%!y2PQ;$6|YT|iUrHG4udK8)GKfmtb#;;_PXH= z-YcID67uj0B|&KO+K@}1j4` z^G-adz%e#89L?xNoDfpTjdL={y`oW~^j@LCJiy=0v$3kxrc(e)U#V*PIW)19*_V_a z%Ug@EXw-`>)Up_JL~Srr3zA)|U}-4Q>NrVzETo4<1g|Ic@(sgq`L*s{-+TY+V;Qhj zYlKn^Oyfm>z)>Gu;*a5%rX$hINSyqtfY2bB`)z9~%TXUyn3q)^_Pa#RY&Du)w8kPP ztChiTZ0C*(Fg$aP33YINI(sX*1b{;pbQrPYg1v!9#%4``zUXP-89S}N`-Pq2OvBNm z|LCkQc%nUZwN6^8#ZI61JeIH6??%4DYDecs`P}(%lg69CAm>hV2tbeZXe+S>U-l=f zD)cap^DW#vvw(40tBE!Jh-I;q65TA&h|`}jV(puc$GknCGRINg%JIQ9Q^gOhws2D7 zS$S(OfR((>yZZdz*y;ht?&FE<5G|}XD>@MulzNl9(wO)vR~+2?WOi^5ub*_X8{Tf$ zTLn7P73AbnZzQRV(a9{Q^+oPq$IWtDZ&V;JMTP0ntQqn%_p~({`R1WF6^u0T%;rLAX_pmSXPw?NbzU@ESc<%id z{THxr>!IiMQEL3@GQsakH+t7!%1ht6z~&Jr!ooF$wz}bpx4Clwzd*S-aKc14 zS0fZCrlc@Dumw!-Jq3JrTY#j0P|63^&w9s76vI z?4zEXJrB;D*xBt4yZ)KI&-%v8hZEj6prZCH-N1tN>_9&?R9#zNn9Bue1mCawMHZfu zU6cbg%|_nlP6UN~P?HE;36-d=!dS<+V(QPJA%DbPH@THqD)%z01zzS=V5{89tPB7E z{{MJEE&Nok*VkSlZqzl{6@ctLk~c@(xPRzncNtM6{>^*Rs?tYxTlRvxfeM=m55V=RsJ6e4kT7~v{J!;382t*9@3rhpP|D|noa(fR{j zbyoN2bB!hpMU&UgiKZrb* z3hD9ID7tM|x|dNQQ~C$0N>P3mI{(Zi{V!4Uryv-s9HLRnGXVYMP(E)O6}2Ma9mC7z zJ~}^_r)!Y|a-UP77uJoL#4E2`=6=vb{pM3^c69^6&fG66hSyvp6~1AtT0wAEZVklp z=QBs|U3fz(x!;@T5AdWuoNm@ds|`oY)Ixd_i>@TFD^aiPzg!5xx_F)?&TXsMW>1 z+EY?Pn90eXrK`zwyr|FbVC=a13A|*Es7E+!JX@4|iB!tL50fEfayM=0Fj~(@5fs@7 zeP0t96W($32>v^`CdBbOKap}GZ%zO|6|5Eynne#_0n{$HW@2O#2z-$+_t3OPYBaiP1{#0}%x2gsPFz@chtQ_rpMDmt8C;pLOl6}KZ*^%rc>*wY? z@olitkn9L-TA)WQ0fydz#%gjH8%ELq-PCV36BaZO%T1Kae0^#;wgj{GaEU{CyeRLa z{89g66~5@nz&IB8vGP3x4dLV3rs0F6xk;Nlv;^ZujtS(M+9F28)CplRgIKbTg9L*rb)BK+keTIamQeee0_BSUOvN{`!4(^^JSDfp*&9TiI+huE=-k zvJ4<^Z8lU5{Rw^X`Z-M2E`MB4uL=$|um==v(Z3QgWeP{s266B-w@Lj$+O8)E_kd zv2><9u-_!&>ek+Tk?$R8Rd<`l&OrXWVMu zXm$?U+8QZT%SwZckKd{EO>C#s2|L)PJfJ3Wk{?JJ)Kp=a;f; zId{N(dn|xLSg^yCsD_tVBZu0r(UrKz7X%7gaMSj0mfR_bwSuA6N(zrE@6cpnfVek7;s|*c~@Pt=RPx%?HEuRS3VW>h|LJ3TmLXio8qeT}~@u zJJgosJ>JN=laU`-2*1IGsg56Zil6Z+`+$4ry(|2e-_tGW`(`hQD^^Bz7{47YN552IrKCAEM&Ug-ll4>C1xXJNeg|__Zo|3pp>W=8Q zY}NdF8F8TyF8PMt*~r*XzhrK2RIt%HH8|jNbhsCpWl=j~4)sjU!_P(5=wLN5apY5! z1>pF;5T~)UjWj##;rJ3a8yaJiJ$JQsWzuZU$G>x?L8sS#0yj4ox^>IYF{76sKkqp> zXDt9@rg&y`6wjiFdJl`XBQ$c-+t>cvi{r#fkLYRfa>8D9%D0R)hj+iUVG7&lqxC;b z_dTMFqmo+dhre<9w6iWYiLQ2QX4!n7F$THnJ`bR`ixoJP=WO3uDLesOsSbFG9fdqE zi9Oe%q|Bp8%HV)@;Lfb7GdxhpV0pq^_4p1TCrOIWs%b+NXLe&pC8w+~5EwqT9{VyY zqS23xvjPrN#VG;^;4)Lz>0weF;tN7CV732&_I?rfpNbtNN;`3}Q>@DHd?rG|>q`o% z1KD!gj(s7Gx|q|+xnv--=j`G@`SoC}vW^(6S}bvp3%58ErV^MSzaRd$AcQ**t8zpS{#pF~)Du_zN@c*7wJSAOWE%mGbJeWVgIO z{j_Mvfj%-Gmq@2t#h?Uge?B%0vTC?~O^}PnBK$6Zn)i@E_!C-y5cObv9mz(s)Sl2= ztX=p=gW)n37g|_VP#)XM8scf?imH=)05OU4uBq0w=+}cil-s4FVDhpJ;)p|H#NwW& ztz}ADIPR69+S&kQ6tCFx#{ZlHW&ymZ;<#MvW$o#tlsh@gWtZtW#0N&S_$hS&D*4{Cda$#iDtCpfXuHa5tA~8|wl4EG!j*4-V z`k!Z+$ugtkuhjcWiz<^HSkZT1|Cm+S&OLlF;2z{Z<^csm7EOc0t`=tt{=Wwv5r&_I zYdClGO^zqfJ5jpJD&gi*$DxQ>MewnDTk*H@qc}&>sII<1fTV9AP!6ysZ13|f$bZgKj*+p!7I0JmVu!8OuJ!oW$ZD(-q8VO;oz2Lb^ zmjD2oU(YYaN?k%P15Ko8Q}o2(oFLGy_NR|cSCJUezlSWn56CG6*>q*-dZ2oW~` zg|e5U&Ad)eFvtf-!tHtm(%0n&+k-eyb#yJQ1)TG4<(_oaMg70sVEhd&oF}* za;tmSUNgZvt4(k)IA7QBUBCe1Hdrsi99N5N4WBg#O$&Mk>;tOY`_Vnng|kpOk=Us0 zf`GWEM3X#{row0)YT)I+!>~0b8;Xc-+RZYP|0dG7<0&B3~|r(^CAM3jkQ;x8c>2 zla=R8$q9tQekmyjaOFEELf;SVh5-&Cd?mSiQq%Or}ncjFAeC{y2O; zt~+e04NB+vAn}&NgFEdPA-+jbAm4o^>qP9Q^l8o-ouf)1JDk8PL)A)FzzW8)mzVZI z# zq4JlX!T+*qb%6AaxbOq(kZ^5zZX>X1*55m*G*#$Y<~4I##TGvjI=~js2Y2NcLRX3j zy5=hT+jB}f@BL`+c133*4M9r_t!nx}!P2OK_Qc8r_e9>GrZ;;>6}}$GaGj#Nsx%?M zF}qS`ub5`^nw(rV_2YQHmsboef#C1y1jqGHJ`a@rr*P#`WC%?gXMf8A?LZ#6%;s_v zILniQ<+pm*crN-3e*=WG(4r|V3fEHncSb3;6u6>M)VY)R;oL15A8XL)$i(U0F>2U? zXgG%Xym7nz1b~rC$X9NR= zg{79bu~Ooe3p*rv&RloXZt1UPpB{@4fkSukM}SRbEhoecx(! z>y%)W+pFs&tvgZ}UO<+ckAkR=ILV<=++_keyg(z^?K9)ow^UZ3NM*bH0P3Am-= iuX2yvahm1NL2ZDJX|10001qxqEp4 literal 0 HcmV?d00001 diff --git a/website/docs/troubleshooting/alb/assets/alb-does-not-exist.webp b/website/docs/troubleshooting/alb/assets/alb-does-not-exist.webp new file mode 100644 index 0000000000000000000000000000000000000000..3bc725a2e4335fe056aa74de8e34b23552d2ceb5 GIT binary patch literal 4468 zcmcJR)j|`10zgNNE(s|?x=XsGCfy;UK}uRex;q9VrMp8BMt4d{BhpAoIbsMnV!QY4 ze!)FY=lsDjP*+iLBL@IX-YDt4)suW_^IyNT1r%ZMcw&A9yk2J+peF4RYN1)txDS;O3a*l<~3vLI1NXt8OBvZx{3ST>cmVY#c6ft~AU)h++qD@k{b7Kb*By#O;enhSdQ1 zoB%!1EbocRcn!ew<30Fie&zN~?-^6^Puq=e$(EwaaAdcvcGS(GuU(W17j>+hgK6Ze zL9;R*Jy<7aPDKQ@ZI|i88G-MB?@|&6AJMcK;8sswNY4k97 zU{AnXPK3;(9aj2C1}`!w;*2~^LZW2;RCWDB-BEaqf7J;WxFVX?K|xE@SUikEjpp>kf6K)W}b~sxcNA9Xnev)PF5{ zVjjgf$UiK1$W83-(Qa)t&1zH!(bT%lgNv*?6S;k>KLtNm2dRlu;>Z+4#RMdfImUX= z_j`|h`MGEpT-Kv}IW8d0hIb5){g#=*Psb#CS%>pszSQC+lDEF+ECHLk_5yFPckBC#e%$VXq2Q=HoO8m>=i@!n?X!1R>cMhAccOLLR+0CP^D|`y z%HnbVkDOKjpwJjb@;DC8QwZv37dC>=iK7WUx6&bCY<>FT&=B!87D5Ru>P6kEYz&(Q zX(|;OjOR0&%}%pHSkAM~ak#~3tLox-S&TWu{Y+CLv$nGOGalS;C9c1?-@^;-<{2deR z)Fo;qjb#A36xBceoC!1j&(HZ_?;xT>1ahv7{SMX;O=Y(Lov6&`H#N*j>#EL;<}@mR zcGbk~P=Y=R$xdN=EG=TvazKE^;ZM)Z)n`ni{-7_V zWs$-|GfWnLC34xNGth-|Y({buc5=>oNJ)Im)kbgvo=DWs8kC@93KKrkyWK3!c)%Ni z<;<{w>2>#)uUObkkq?YkrG-T1e6Q}fY)#K%M|b!9TaZH>XTChI7|M-2U!G5<1;xR< zVbmmT02KzO8=JW@-mqWCbzjNf+It5qOZ6x^$4!fGuLr6Cy<|$4b5PO#iEPc77?T#7 zZY;afzMJ|s>kO3n?VQqLYaC@unuV-QYQRRnD)oZI8(63$-2`QZx)L(#cx(9zWQ7e< z@)MeHvvg>OgIw^!5AX|VA1iRZ5dKnvwzP-G5^3ci4v}7ATG}a^&<#4m*PnskMOeqq zy*X#!R;)8e+(D9DlX!pM^cWKVL^%2kUq4nj`s23#zE5bX#~n@b!As0y-fEXpRVg-n zYdnfW0UfaLBxpWc6U_uPa4sHx63mTal&RMJd$S%Gpqg%uNO(xLdX0ii}mQce*f;)Hy7k(8Ebqc!o&C z$W)Wgt<06d3E`=BM|FYgZwI-PFX*t{t2T})jo8LA3K)W$e-}kjjo01S=6L;~htAj=FwOAy6(~$4?G5vM7iR^scp&qf*2w2wI3w}ALtA-L z1{|Wj=W~9dwyk$;mxyam@hsTP>M?C4JV*)H7>ODSF!WU6kD(Q1P`@H;8`nG!a7yg)N@5%67^HHQc~@9o9RR4X>+0Op=$rAYWHO#i5_GqU>6W=2 z*ERvt7!B_E{>TFpU*rnAgkKL%JXJ&t?~BJbBa!5s_Yc(Gs_?q+@zS-S^tO;#=*HN> zU__5@BI)U`Pa#-I`_D|3b6F%Eq+M_7gX5|?7^q~v$JK_y6g2QuQ%$hvzLHLygZ?=Ua7PQ(5=r|dyeG9cS(Jby5An_m zOlNN%%-OoD+-S+G%EAcY0iF6hEa!n7&60K^Ye6SPB5W-q-xUo}dKBjr{aefO!bvXC z%>a+fqgFNB^U3MYo32vf`EgsV?Bp`G3bMa`83)dUSWx)#w1*6HqOarR@AsrdOv6oM zleu;0d{pSPTGC=&)9Wh$F7*}CjNF)%kOiy# zji^H(s9K18yuNOKGK6B;CfSrMsUWdt`nQi#hVInf%^lPbb&U**=O#F=2EDAR>B+D{ zGAGnHv2_C2+!iO}V*{SAD=wRPl!6qptKjr(?4#2Rec1X=wwb~D3A|5h9$}?KhH;C4 zHQM3Nx^%q=sejWlySOqWos#iei}A(wlv@92@X#=wb*T=*LaTJg{K@Yny+#?;pSfWD z<5^(=J|CcoQk5Lj(<_fKFkilZ@F$7jej*t{Dj$oEd{WMhLoywkaEg5mGu1)?b9#2@ z<*GFjXU{1HQ!t}Cb9lvR7~WDp7DV^yd|bhje5Zm|dPU>3UB!`&xr+8WCSP5lJ*Ek1 zo!$Ny|y0RX+assH9M!E|JQI;CW6!Grl2W z)i8z@5}pN+V8{yH2Qfsij^+3-;%9CXupZOPgiDef%&#T;KKkkUl^G+^fZAHxAVR2DxjG&5{g!0V zxGw5-+;y;rfcYo~2vO93(jb4uR+r$-Ki5aP3BwqPw`8A<$}w7c9*`Qs^Y5}X`FYp} zg#)LmSJO!-Gf*|MePS2U9VaUd&=Mp6>n~n{7Z$jMLg##XWVVhH#v%ISv^Xr-P^Z+^7js??k@vx^_}1R@snS- zwsy}OH>w*@YpOdj;b259y5od_1v~tLmJ`%je4=GBP3{_juWs@ga?Kgfm}n79ky!%0 zpRTGtc@ZMyx-z=ht+5OT{rZo+o%7lapqb@cGrdbdmkGEQn1{mhlB>Lc8->Lw z@XjGNkZDWdSyinIp?L=H%7>l)7gMGJx%7Wp3ucfYhjh_gz+_Da(6cNMk5tyW?r)=y zgd~SoBKdwQQEQ%#diKzd6)pWru_8K!cIo_Qf`~^J)6h{dm)?QHFx+x;YVDogLEKV9 zvB^PizoY7d4p$ssu1zU}FH1Ipuh#~z)(LBQ#xSmx@-W4EZqVfK(GpPv38CAL&P+Ss z^)x~`ejrREh>+hk^?FGK%m?FN&3bK1sl{r1ttPIQWMn~9F>0u z8X(Nzt+B>&`P<9aOpvN|wgb4~e5QQR!WIKEi^ieS*n^WvYyYr$ZF`oYK0Wc_}qB2A80d5Rds?2GBGi=YAmi-kQhT>;?8ejk2zmo5nAw+q2=N zO?gs7zRUBG2t{YGUebSkl>_6RwPKueQDlkJ4QXm+Lwya<0+d=dB^|NxxLPblP7%v8 zls7nXtbnv4V_l2n!^)iciJaDcjI394lc;MeRgQGsrmtKjOMbI+`HExch_9`;UzAW_ z95Lnp&mu!$Fs+I`ah2GZ#mRCcugUE@)&1>smL#ly)~~C{%FWg@XCX#iD)f%Kf5XvE z5(Wx4oYG3cBm5184SQo5|Bfx1O|TF>c{!y&%!D_xxs}|lx?SerQ^*hO2H1;WxJI1&Ug%Dp=3-$HKB3DzT&^n? z0t|$@$54fWbY&SV#~t8Do(ivoyJuf<1c+cP$>a?wu1DH12f`!ZU(zaOk)ZPk_L6!? z%5bf4-9V(ClWTEvjswHEkC<9q?w-o4nGJ_jd@d)MuUjVE$j^3>t$&AC)3k#7Y78t` z1X0O>WqNIImnR`q6*vhOd)~r|;ZA~CPi|Bn3V%YdrtG#sGb1UY{sbM8bq+0vR>Qqs zPuie{7;c8v4ctX01hiv2Xx!}>NdH^+TQ4T#^{&F_|E?mm*HN2Dnsme?LeuZdFl2RA z?z&%1UPDkT`q=)|9uz_RvZT2Q1{s)g2iQ`p6scup0lls+Qu3qgbDn#{jZYyYq4NZ~gr1es|IbcW%ZznYh`UA_2slkdB1W z`}58%cbcw<7Y*;s&*|qSkL;r4)cMcoXk)Y8=Ne7;slQc#$4J|iFvMWnsfIAoPYPhF z?T?*2S4I?EHIaKYqgAEb0G?#=8~t@Dr}8J&w=DYRCB6E-@Yt5cy5wos;6<$?Ek{s6 zU65H!p%q;DlbJ$^E6ERG#_WwTK6%-a5(}$9aBmyG1;v?qM8q8h9(n5KxhS!)pm?XL zL^a03j$Srq%cCzR5GlrNz~L-$jnebtwEVtA)nx%fLO$N$&;6-(Po3%JEgwF&S2@Gd zw&3r?%v}(fUTwE-iUVH7h#~zG}^&3&Fnib?fJ;EaC#-MLK!q M_97wf3V{FdA5YYx8~^|S literal 0 HcmV?d00001 diff --git a/website/docs/troubleshooting/alb/assets/alb-working.webp b/website/docs/troubleshooting/alb/assets/alb-working.webp new file mode 100644 index 0000000000000000000000000000000000000000..1d3bfb1b485586e2d1f678f545086c90ea4cabe6 GIT binary patch literal 26192 zcmbrkW0Y;dvL#%$Z5yX-+qPYG%C>FWwsFd~ZQHi1&wc&w`yOuh7~kmpG4~p|GBYA0 zBVy*>u@xmmMa4}40MtYT%v8Sn`LwDpLd1Irk44hyOs_8kblR%$;su2-Ep0#^WT#O{ad+t@?WO ztn{q-o_@dG5`Wo#vgN>E<9FWN?!0sfe~*7(c~<_6eO0|!e>E;5f1rQz{d_O)ynH43 z&ibzW{9NmNnE(8&G``gCMSjeD!*9)4^8Nh0boKK6e0V&DJ{x`6mfBMD1%C65<#P9M ze51bztu?axw0yHg){goP{UBZ$eZ+p|uzx--Dy{L|d>XrE0oX45^#ANUihtB{c5i(E z`AR*qeV0b~R(yc@%KwO+K^DlZ^c;D@eh+)Neh_|LJX>Mfe{W694s5h}V0gOH3BlVO zof=$icERwJqIcfd;KSboD=W6d{TW4rv>-q}=g-g}p3_aOW*rpgS-v)@R=^5nSWL7< zV4av43P3tXP=`o;ZOMZ%>UzPqC4|DX3 zyT}&A33Wj!GnY_*Z;k~qeaY167V0X5J7{2+ z8ICCoU8OfsaX#7`i#=4+D6~ospr{0q^uTL_!2+SCKODKIu9|C>?n6-tAmxGA4uJ(i zO>;1MLtXy8LChaP)rXW5S|bGJ4>dKcf%!=PSGf?p|9PfYZ+>o&CI8~ZKMuR2)5d22 zP2tt-UwJ}D>i^#L3(v=Y@sNM^{~rYY+q(Zg^&dmXHC!f`{@uC$#Z=E>@4tHdr}6(M zRsZ!62{|*dy((Bn!dW=XQBeFyct08X=ac=>dYy3Y|F)~F{S$YAPPfUbk1(^yXuWc^ zk!mD}MBfLMmI-xd<6iI@otI@Y^ygx`8-Wrvt#(2x$;v6g$H;pWI_zH5qcEID#dMdV zrHbC1c5Qkv^K0J($PlAVeRuvmgUQ;`JMh>31|Uy~Wc&OhX6wzMpa$Q2=A>uS#>OKh zS8a3#lHg*#izDRO1`(-UbX(oz)?$MWb4L4DzhPpJe2Ul^PlNQYN#NeU%K!5i2aF2& z;qxyU*2~wwNhXW}=yqRq)VXWT&ffD=#2ZRKEkr&?+cA8q;VxDo)M7B;$Wzhk!9mpb zE9KN&yG~$v(^b5HU~J8@^e)_&>yVvn!RD_n%IB3rDd&oI%{720hU8--U|Fh*;0F=h z5S?d!14OGW?X9)CO^t(;AkM z&Dt-n4m>V~xbY;2?9kI@=%nePqxqTXSb!6ey$q&Hq3fk zA2QPmQ!@ZSbk>*QAW+6N*vw?zyOZ7Q1cxlqF^1L^UuY-z2?^hF>eU|pSo?rng}_ShWLHW-b#^9&b*QDtzOD07^&k@4d)b-74yj zkS0d@gJZ5!I~XK`4`KRG`IKh|C=GIMYF|}Tces5{$M82B{%35&tTQE@SL#Y^5`?c| z0@15n%UwsEPdo?hIu;7tpSd4vDDPCV7(N%q4<48wVc=gWWrWZ@l1@L6G>5_6(P;W0 zEqf*r033HZbO4ilEBD@NUH;s?#pnK~B>zv_y&wdyq@xx4k6F3I{R=MWNG*u?>RM7X z=^p^A(`S-&h}t&GO3`VUeE)Bx@prwn5|A%wUEbf?mNg>w(m?{GGsFVNxWB4Xb^n+qx6oUM@qBug--ot6`dW7=eQ8VK&c0708mEU@h1Z|*wZ7}M#FTPGog z5q|8*)=R8%4n1Lbkpbux`C~msILO@Vo)`?;kpa768@<{RtS(xzB$MxQC)!%2RR2yo zqUk?wI{(dqcx7M#9z4G{{^p?nY=D0TaP6K$WbCk@QbR?t6k{t=c#+q3e+kuZ3+UXD z_i!?r&QxTK9LugqoTk#bCZ1SLe(dpQn1QayIA)#8EW4z&61YSi**+>(w0k|QczaSG z#79P}TzMXists|IbTl*;#$iU@8kyc!I9lbeOz^{mV?x0ghsSbR_)Yr8PVIU$9x5J7 z^v|>^EYDUr)sPA#VhI_H4F|$Iscdv8)re*OS_F*d+nSN{s)0oO(V9ll)r^ElBa;t7 zRCw*(wE|7!v#b!53jm}G>6Uu*Tv=#R${ucZ)k<2SUWW(+c92GX974unSaqcvDqFMP zro0!XOpqohR11cp;MQYZB1x5n^WA!IAwvi7xa9kfmZ%bcCK6ktCgrWyL9Ee#&fq_R zSjL}yY9Bi;!ppJ8ZBm#7gc6R`b`SJuaFT4+C$)@iZDKl;^ngsvjSHy^7Y!;T~4)dI$aGygzY#-*ab~^ z7$=2BQ&zgO-p`WzV1nike#K7FlB*5>$^hE&69eiRSf z^h|4DKO8khODTuc!=F5!S)1yE9w-O#M3*L~c-WY`QKc^pbcx@y5pY~U9KUsCbV}+# z?rUV|`l73|HvUYPyFCTdUS5|9$W(g|@&J@rFeU0>Xh~LDDUuG_YU<;NDdwhq0J9rM z|2XjqlS1ss;>5;noFb6fhI<$AZ!pm;-D+GJ6-J$#;<$FTeiyA>1ZQ8 zJNP?U%nH4kXjR1-1Zj!FP534|8-@KEkFf;o=-28*Ze_@@z@>Rel`Y}jD;iQL461S! zK<4u|xpNc)Rpc7OJ&*1AB;F@}c+KIOA|qf?*V?>IYNmCJHPQ!(G`U2;MMgnBMw7iY zW^9lm1o?3`Sb6qvAY~s5AXv;{VXFrYz5kMX$EVb{Zk<9SN|un8k-D5E1XY$X^591s z3QhM{JPMGzje8Kq9-^p^BP-~Q(rZwrBD~|U`Jf$DT@+5%O-R=;v97tO%{VcKe+XiL#gXdPlg25uC@nks$N9(xj z88+NZC+{1b-X+S^dwIbunGQ*5Y$KxHYLfTbzfxQzL%d@*dcHbA@0dW{2HER+F05M+ z8!Y;ro0LT`yeLk8_I$7|?MCo4l*pPIcsn~)O2>olet@AP^*7m8$@ zShy&Xzh;}ZsFu3aEGAx^jPCZJBzs9pJJrx|ndV#cUUT65Pwch1o#XFw+@e^2g*MSo zScEl~M3NnqVO=HdAH|XfsZNt_Wz8{K6IgpGc(A`6Qmk5!{I*sFERKr`WEsgSG_PI#}OiCI`BhZ99oAs%U&>&xuW^gS1# zq1)oY`d1H~u4+NlVo_6 z?{HsbMJ81TC+3ttzmcJv_>gH0tu{(_^D4%!`@rdEmfjg=oTVR*J6PqD^Y+EiOoxkW zSx#u_OPG!|YfXEl8Vj|>Ub%#h^enb|eqY1U@u-^A?1$4yF9vPPN*S@>kDabFO2_4| zLm!Qnwv6xgei8wNOVM9$01hd8@i>R1Zaesp zwup>&#yxm5yuQ&0L`_vZluaTi!P%ArN+&vHMU5B_j)1mAmCkjK#BshUPnM5xgqitM zF;w>+0~E-q@rP3XXP#5#2DnZZ8)v?56zULkBGfoCuJm(ruZj^CWH7RvF;<#sgm0eD zKJD1yLx!a@G1=Q-sMwru0zP~gZ0V!~t#A*gHAN-jiMhM~>NhM863w~{$Njv6t&&1GjE+!M!J0L2yX9bh*9_Hny#_r?r+%ym?#xm3|LzF`R~ zuB@k!W55=tO)yPasPXhAePuw|0tcvYKkNGowr-qu zeXtmaIbk6%di?1}ie>yFdyl$kpru>Gb_no+dqoJyGn^hD6CfplIV5|>jKz(mENt`ae~Bx5POqT<7$PwrlM6LGK0(KkMYj0 zH_oxH?C`NYG_FR_9&R-)2l_5ow&-v&z*0A|+pmAR1^gzY{pz0xXg(g|IjPO{9|OSG`y7sxQI*a8sMAb5 zeUMY-j3&}dTQYEpSpW0|)7I7e_=vtC97b@zFX9I{%amg{(rj6YewH+7GIayd?gi|3 zx+7`u;J`7I6D6pR>E+jpyJv$7P2c6cZ7)(|4Jo7zh{nj7GD{b-%<>;((-o8KCWSsUBV>wGLtKgwwvAl zXP<0Fu^*`BN_eb<7LZ@_2VmC+vX1(xgAbwura`w!S>W}xrN&aPgn#|hbp6h~e(#>B zy!}8TpeOv~Aekgioi_!fPPfyL*`-Huth(B;2>{^x9MF#dz*rvu03uwBxf1E7M2FfM ztWPAl?L)XIr+{pjrJfHvun%Zp&)A1#H^z zu)uv=rHkrKOeukw7PwPp`a_~GZG0X(t^G-;B2bbI7a-Rq!@JmiTnn1;fiqNwHg>0L zGw&gH0P{JSK_o_`Cs>h$_CzK$AfdihCrNJbwaNP9B6?9r&!-GpbnuO0mjz=>6EZGY zjY!)xifQ1^F7Uxx7%P}e3LbXRoy421^<+ZBbSj8F96i%7v*xmpQ{@U3nENK7Z?x&8 z0nW8k0d#U=V}tmQ_ZYj^#rc*K8Fs6jj4We~y$w3f27b<&i=&Kf--jClj0}!d|7FfY z0yq^CT(yCSU{ox-6`ue-#37ABtzaInSwY?LnAev}`!~qbJmO_=T%Z`pSl&JPCN6}U zR>*r+eQJ6)RkgBmRM#JWiFw9OAoIV(&NVWn7EaEmCo?J;cz#hlBHl5I87q z)qMvYL}yHXk~HU(@9fUhx{^MeOn(wR^wqLlDKX_-guVHofud(KzE0sPeMW`?LinG+ z2`PqAlwdC!+$QpO$(A4TYwdRYLoHenrwWn0!s}AV1*5dT{0CW!-6I;v3HsAKnuC6|-GlmX+UK^!b zZL#9HWr$@biS&wz%9D+)2MQ|};BX<)VXUHf z4HY$2danx+mf@u|UV}7yv)ki8VGn5BRp2R)tgL3{Icv+yHFDlS21~t?euw*{{V2i3 z%zm&imxJ{QZwh!S9o%{1vei_fEfg3^DV=f25d{`fp2eTTZXOxy#0?I04479ZS*E}0 z9~{cEz{yx9QA*`RXizqnJRcjER>Pmg99itl-&Kt-K{MY@Q*^r`hI)MuW5sB7r|^D= z4v^2Msb*>@iU50S!?CJNMo5|(y1fS{Phx-HcXHo%ll7rvC4cZR4U@9f_M;V7yelS% znEeFANzmL^Am-4m?FXo6*UO7j3wes;)cSeq;a}J+RBZ4NmyMbL#~oDizVpgYgqh=i>xwqhqpopAG4vpd zD&uxWpguug?zZ}{l8>`MQdiCc04-}oaq8}eOVv6QeV$?+B{J&;ctaNat_pSzsgv3` zqeTZKnw1(ioF>udzy!as-d#nAsTikT$ZCs=k5t{Mo~P5N)Fcl+r-+WVEm5`JI#SQo zGs@$_*Qx@UB(p%at{IrDA&B9PTg=BAP}uUx6n*9^ntdSHKrUB0$f*WtX60z4)@}gv z!S*)Jlz1D*ZN=v9@odyyL9{mXihJjL`+Xjrm-}$B2LWupls!56v5>H<*30$y+?Rae`TVJG zB-Hg_MPnT$7TI>ZwbkrS(QXZ-z)-B_GtGKI=*B_K(cR>Uf9-uML_=G6@M^C$^>&Mn zv^PtcQ;wyTL|RNeuuqAuFCqfNBLU;D!)A89H0SYHJqbl}Iu*}O6GDBrB@w85mi;h) z>qAEbuq{Jq-{CQrj||e}7|Tj%Z@uVUYO-98S4VicX=Bp36#ke#jpP8QWVke_9DsZ* zn4PwaG&Rp%`orr@tDsn;gxk60Ch=?zgl8HuqfmCN&G))C>nc!gFPrRMC`C>?I&FcE zG;Www65f99_dDCh)U*l_IDPC8@jYy$%%b!2Jx$dOU$lr@%x?9bV#j%=K=9WnIu+`~ zS3)Po(EXUst_zWI+b4ZJ#X&pEJllT z(YtX+d={+z^w7nT?p`QXEE)uK2V|xIyNE4sH@-G2*UL12{BB37bk+>yQ3SQ8=R(|C zxOP;o?%TkX$}hE_>^&_EdFFmBS0a0b44KQJxX{D9(URCgW^}+Qo{zc*Mrx~R|%#;N#X{Qu# z=wc(y!7cFZL`B8;K$Bp(yPJ62a5OU(8OMoS&|Sx=w|aHq?GuiZaxY+`;ojq9yz3)P zRK$W2aIo;@Xp?QuH*-n$lSD+87gLMi$+>q%SrbW@p!DlIHk6|(?=fV%{_a*U`!no2Fgi=Ezk^2-|%IEGn-e8f?CpaXo%-zY|?R?q&3s; zeBNOzs^QJ_;4pF?My$0^mTT8+E*>c)PmC0337F=j9za6%U&xkcB}6Q|9^<8`d;7_Y z_>z$|P`H&K9=Y0SFpT^ypJmbrf z?U7A6=8hiu{;HIS`vZ96wt{LdULy!yrLleacU%9dL5Q?@V!*+i|Dy}?!0#_w=*j;@ zpe6pl2;uk_091Qd{~}PT{$B)VL4QRp@iqd^ni$e$oWk}P4F9Bo7oha-%rX{V<-+Y7 zl59gZK3?z)Uu=4&=I;=c>T+ek4@sTuW&E{pq}RILfWWqi@1^6 zcOZKEk_#jKC2J^xzwb*Ok%T6}okTs;5k_|y$0^-z;8xBL=P44-6^2@u$6hNGe4eR`ZsVe205jF z2JJAxq_O;PO6`dApE^ODQIy?8FyGuzpG;WZCI-=M{scndNO1D)5hZ_K( zhVs-%>9m4ZWMyVSd0f^Y_ZLjw>a*wo4e)P^1|AQ5wBKd4u%1cxFk{dZw;a6KL z*igB3XOCf{mtSY_cNTaYwqMl1zzn4bmld)gzKV1Vcn3(c{#&(^2&zA+QX@#y&0``2 z=}WiESPE|%7=El!5U9t42EUz%(G2O9AwAo!h5z8XWWZ>Y}xe78W_0%TUVYW!42+h-(NoWWYu zPr#c6Bjss}I#`hFnD3c2fJAYrnq2-#?Js%VFu~oDyJ+DuK@8fz40zLb$PCk<8j&)S zEyxC&?ZFIshy%(ddPUA@!=ZpJ#pE>t<+2HDr&Tjx`Xc)ySwRYVCpk0V$@?~jV!*MS zH4c;MYPZ5H z(E(TZg6Je%d}mD-#C5&%m-(c3gn1H%BS-GPXuz6_J*zP4gmLWq?0&lym0i8M9hh!-SA zPelBMd=&!+)xfN!fEs@bexf?$xUqiAFS_9Y9EtwEKcF%;9uP06p{C~`O*bTcnTW+I z-_B={y(IC4txqQPwoAs+))X=-1JvFh(^m)try?IE zlpQjK2#1H1%_VZQ50}vv+Yb3c+Zcp*KU4{3FyVCZgi0|Wg@c}{8)+C-O>J`2AYZ8% z-i{CC(Mm|N8M;#$bEnHrB8Wtw6Hp$`SRS;O1;d9*Jk)a+`~VzO0K|sVHH)3L8|E%_ zjpPA4n(f**e}?=|&x1RYV$_~8T6tJiCb?yvK_AFNmPxmCj|z@m1~GMb_3fK{Ci1W0 zlTBDA;!B1vy@kP3X-vTdtH+1$5eH3Jb!ylv^y?9pjm_8S(u{r!cud+romfiK3&rG` zq~tfDYZF9pttWFvsnC;@VYCI4ENc2 z_Pdm$JB^3FF!<35ov>v;F9~#)kBE~D_|hrt+r#CWz^}^DkzFjEA{fRR1S>I7JNaFR zaA+#KOFsR*fGrZa4g3cAlT41ZlV<-yoQZd?iK6)MzFFjqoD4YK#hLMwB_O$AEUiHK zMn6Q>P7QXUXNS3tmj3d=_P#`)GD#3xEY? zy~JZXI8eRgpM~vb+aQCc-A;4TY7r^r=Y!RDe_TF&iCl93@(oYQ#9@v9JGzfq#BGve3Y3CF#_FFlX% z=){AbDV5FK_wRWLgssFb>0dG6lBYiL6i5Z5duzMN+Fe)fg+F}%h%{nrUD5(OM819F+FN?#$o;orgX=%Qk_H_hD`a zJ7yEPA$FA^j+4Ky(->j=7HRUKHRnU$ec!JMhjWS(rI7_C@9HQZ>4$y#3<@ zbHY6*k%^mHHAMjcH@~ktBgZIlWzEjFtSR{B_>C{2N8%?qF6~e05%I~oL`4+gcM#M~ z;H@msJ8+GHlUZ&u@e*6K$hC}rn8h4ZR&tbY?k1z_A+iZuQu$xeInXOlyPayGHc?qY zb%s(6eL$QI)93WEaI0bRIpUn&2eQ2KWhJ(sjkFF`hv9n}6RkXZm&3mtuzE5o2;Kqf z0Tt`-csj(qUyO~8RNc9W`6$?6wY&Ssez6*iCS-B(KCcC0l-osv?M!l%fiRfEAaC|f z#6=sQf9btIM^S+>b2SgZ@gaWu9|{gJZxXWWE^3dUb*r+&eh-!B(y))><6jN9N5 z9};khi7<86TBx;1M2pm0>?HFqzIe{>6`vGpot&27#2S0a%A=&K51tSvk40iCvE7CS z7{`oWF_09}RrE(DffnsWh&B0zy;Ar7k)T|C_frexfr8twwGQ)VSF0cd8dFQD_sm<} zt1HY1w|%md=ldtDv}~G3edXZ(&(a2X9KPXpS{`~~#zOdc4-cDz>tm!$-?&Fyiy}{} zPvOGPN2^xd<4O9S29q|f9CaZyS1KE9=`h)W(8PM3bWKaA)QLFDy{b+jwUPB1>ISwQ z*CrA4w=`hCMw;fK$C}~D9;{lGy}ySkIhE$~i&p!?Y8%)U{_8+yOwMl-6Jy4%smcE6sAfoLYS7xl&=dh}~VdU4uo zT^LBsa2f`|;usE(Eq5vNE~C%JF;A4r>5+}fKjVpApZYYAt}^-4&@yZWq7s8U7tMNg z-K5u5)aUPe>6J{2Q_VOUPZ(SQUum9f4%aU3`my z&6(7PwN9rm%}T*cMlP^&zEOw$4(1^cA)n1wG4Bi?y^&>dtw_Q)tCw4Wu=R6*eF|rr zW^=hbg#PEH;wpX$>{0!=D^=KP0Amw+IQ7>tB-bF+*R6x2hkTuFm#Xf5#!ZzVqc0T6z0@=0@a z)1ig+;#nn1CUQN$p_*JGF+PqLKX0%U&0eAWo3l^g0z(1wY}JVL75qBoyRx*LfVI&w z0l=Ni4G>%7lKVSMi5~I_O9uVGglTReJt)RvWYr{2D$NRl=?X+}WJ$#>PMTzUPJkF3 z!A>0+!FPixMZsH9*6p43#hRnS&3_A?KsN&I#e0eBoN}A|!Q~Xj-;|^B*I|B)G(zhI zIKH~R%IlhOST5@X$hi|I#RqXj6KmkTr7eYnBVfSCMBpegz!5eLtzw5#n|qg+zSzdX zerhpg|B}V3_y*7c4(oVc+hxLRxC+bMzJlFQj>18@%^?Jq)<20`DL?gPZw*~rBs^Br z&ObbIk>5+Bt6Au^rY?9~-QzB|&w%!BYJ;1UlLfB0^Dg&A6^uNwv%jrm0b zL9Hoy?o^vjo_793+DV~DTF`_q_G`YzwN2GqCu3hxeU)I`QPU;A5#B9A!F<1>%$Gwo zenKmC(Xs8>7{gmB(c;~7@p>w7xD~QbO>AXz9#lzh!cIW2|5`@NNmHx9LcnY+fUiGm z{E^&4sM*Bkt##a22CTtEF-8!x$_xURxgO)<{>k$9sz0lRSGbDwteqybs)F?dp&#}r zl)Bi9@agA#Bx5FUwuT>vyGaCsa5eTT75z*xH0Ci_H3Oyq)4Z@}))#r>x1DFKcK&g$ z`EMAe3cMRjE+gjMPve`0aZpc^jE}YM6ZGsBO)AX$^5qHJ-YtQsv3e%O=#k>Ik}{mi z^o?1shocVfT@L$N=GE%H=Xr;RX$!W@Cl&_5 z2eNt2{UOm^fcch|M8&6Y){G=k6pKJ}fb7qbItQOh+Lg~0LJ9QmP6&Ils6)aaUbOhG zu$KfuN=$nFW(lA>}^g} zk&zuHwqmn`jj%_a@NC7tNpKKhMFj|NU+ccE=;12yHQE?&c0ed6VR^6d>M^JG-vSSPL=bvA`0ML<2!jI^#1)OaXfKVV3ty?V_;GUGH>! z=w;u^$|LEMnCsx9?JX#=Bb9i1g9LNngphrj*H+{dTX;y>!%j7hLcPx;M|lo%gihWbUpb_^|8p ztxIk8QxSf26FYVg?yK}JQyn1e2TG=2Dqd0WC81z&ZqHI*CTS!;ujt{_)1%cB^A!=w z@1ObWn;nY1ru0Jrag~36he9mEM#UH%(ZAJuERAFJAD)L>@yVm;923Pry3I^&Mvt~z zNLl2tyHcc$o%FvtwUoj#xSu*cRu{bYhc7#;Sys1Oz{9?Hr`W7<8PW9NX>; zC?RP8cMG(?GuW$SE}HqVEo#eJ%_##7W{UUaBGSU==Z6AQ7hM7@|I}bL7Jl2eGFeEW6F^n2PfVv zHZ@rd;zXE6`AKAD{YnUf!+JstO#4TTqgehOa19jw(-LMB2&p3eFhtFu-!Tk?JVN+O zzJ1xyuWZz)WvuO!%!ND6B9~LOMdSSQ2tmmc5eLRXPThthC{R{W2R0jWorU1*{lpLL z8dy{V)2Wn+WRvSb>rOdVShuqgDjTKg{L7{Mxw9qoM0gJznK!n%mSvG&SMjEOof2+G ztdGrst^H~6#jt^>qJT4#ii9_c+7>@wYO?9=S{i_RbJ}yKUi`OvDl7Px(7K+Z*V7ft znGrSlJfq$#^^0&u+7A7;1vMajPy}#Vd{0;fLBFMY#`6{|B<_*0_vO~G8JW?Fb=%Ax z?dt-D++2*c)%taz`4R@v_jQ)m$K(?iiBBV3gC@uz_XpMh)KQ-&VZ7I5m6(RX^6lVh zl#r_PFG6)^*2408PM4ys*O%_j!-}Sf`(Nz!JCS7GE%FFdrsk-)Z|VOO{HA5H?&(h; zJ@RJ)L*KRyl`F!{)dP~9!t9&A$H>bycy0?V8^)J%zC7Z-+pEtYIGf%cWe_9xbUR`J zJ>m^^4GmVsxxV9YBmW0m0I+e(D&6|qiRJgafl(kfk2FwOt;kBP_+H4_D4*X9fdn^#lP8jGQ@S)lb z2szFy>#>B_utP>QK=L9#0Tbl9?ge9+^`Adgmu9jfQZfX?s^x^gw5Hk-Z$3@*XN`af zj>=;sqE)TGjdX!v88a4tYxIO(`0q&Tf&&1rLqC!86Z{}hX|-U-ilXYUcy@}$DL)(3GXL%^8BrN`TVLj2f>S`lrQ~K(@h;-;cODmgB#uZ) zH6DsNte|}rO(!j5I7w>}$KYdyJX&3P+K=L3RR5F62-}fBZt(;Q!NcBko0-QWw|1un zrt$oEBQiW}Tg99jK%}QoP#oTyUF(vq6Fnql`Rz#K05Ar#g+bo(osX=?znPvJk9V}N z_p5L>o#4!D?er2ene)vPoMfeG`|=eg5|#_Nc;X0MOXG4_ESP-5;bt+0B1V4-0MT`1 zpQR0oQ&z-T7yrIJ-V@U6v*kkng1P_vQRyDWb&6x&@;GKtU6g;uW_37Aw;G4jQaL4V?a9pHNEfO8VlendN3(j!(76hZj0#^+;MQkeiD*hvqatG{E;%a z)FcE&xrC3DK-t9N&RE>FMS-|TiRv?fYpK})x2llzYs}Xfk3w5F2>m<+UK-h7ulMv^ zwvjeA!yfLqRbqs6XkAXXWIYm=lQIKqRweW$ar3@itK%k!>n=pt9bXzCfg zLx!*=itpLhZ#RSzl#FS)_-VtcgGiVO5*u^9Dx=i>f47PJ>DKqe~*blE$fAcO$3Gq!* z*AU1|om1!bYoXG|((fJHmqUFEKzMFb9uTP%SPLsZ$UPU-soROc1V3devgeu&?3u5ulsge`}H~~fT0+{nU(7peZocE^olK<=pP(2f%)Xj(66@NK*UcW zWQAy(NR!+^Jj#x`9+3+y`WuF@8Di@+(%4zD8IDwPJ!33R>if@NuX~= zG8};}4dZ^&@&B|+Z+K%WRrWx;;%hg7n`Y>vHNWPdN{lwt%ZRi9it+hVw|k(?4(ROj z>(>1(5TMpcTH&sdH*8TTC<;uIzX@$|bGz2weE4D)Cj`dzDnhfn-xM5S96a*!NO03) zBrXA;9Pgsmd>GvMMYQ@Lcop^%JZKrM2Ok@Eu_E9k+Hjkl9^f0yn{!&uq@S4xW{*|5 zc1b-mb(gy>P(UpR`q|0Vno25dzKW|f&+yJHAF=75SuqM)2h#lgGnIk$7SRuBZpim_%uZXcM^!6ff-qdiL zACBu3mkGWBgHVDx94Pc`Uz0&ag?52?lfs<}aWtXp>@L=om`CN%>_$cUfhF*_K9mRh zPSD zwaW`6EXw48Et>UVidileF0#Y=KS(_sv`I^H=9u3nEt)2&AE?7p8Mc@O*Vj#y4DbRD zOC>N;ns{gY&GNn6BY?Ds(3ABP0{+FQfw3+sxTQ)FtAm=Ho4p4uQkPZ5H@mZZyvR;M z1?eL3ADU+xMZf9CM(rzVuaFG9L2tE&wA1%_^ONl!llhnPn(i zUXU(cC;T|d&25Q&At%KPVhk(W>fI?gw6c`+D1psN%cu}vMmzKY`Mc^-o#}EjU)2m|P1_%}j^v0V*u-LwAv*ul0iNkikiZ z(5EmNXDfCGTCH<+PFob}%9~3#NFv z_uGOI4t1Z|Q_d1+Sz^6@%}NZUNbcLVNXCLhCwi9`k}K!SGI6tZ2_ORw!}i9zzDf}~ zfy7`^IuJNpK9CLtJ%g^TiAs%dD-yK$`L6G)VamHH->!*>P|5q}Hx4U2PO{=d?{3`|1UKJm0ltJ9=j-zziWqU) zF*XwTJYMGm-ZK99m4#7jp*p#M2`g!-L88s0f?2zu7KB(iTOP*ykb>rhUa#%6VYge$ zlDLJ^=nw%CJnD_!!W%MTSa3gaQg}A=PJ`k+PZMO%Gf=>DRFE2F#TLEWCnB#pz9l^d+gzr&#Ov%x`=ig&##Pw>b(a;Gbp(+78%#5}%aNWNS|?`;6je4A zhw(POwiXNo-#+(ONU``kGQCR0hpx1tNoa!Xw+kSOfwvNRNOeoHH2=qV+r^Q$W;S0` z5;v0P%VIKkRr1GZvOCe^)~S{|f*(-}nFc zSRefhK*qa&5i51^F9Q7ex5_t8@83t%I@W*mOpF&6oPl)wyn_eEklq^Ru1(yu8Tp1g zFo)I)W&>tL6JE-f@|zKhd!kUzVvf$^4M919{Xe{9-2I2p|Js0jz0HmTd_XEkfriI4 z0E3@!+Jg#DboAImLEW#fA9$N_2df=_>6Usqwp!nPD;+AacqMt-wbu@+TFF-o?07kr@)E@IeOQxdWH)=eUz~TM{0( zhgW9b{DBV}Sn-=xw0jZx;W4TFlqZB$GdUJJt~aXnHSo9ca#>}tk%Pv~^97U51s`f4 zpo}4&*;kan9uWmn$_;hqn{$q#7^PVLH@r!6g)~&>E|*a?OhklEo$tJt8lDp_D~iRf6J=u19BvDGNtfz=1!snD9iahQJQWc35eVEW2fK zNV?f}Kr`KCXG|jO7S*T6pd8q*rYDgPKXYa(D$lY(L%Jk@Vs@np;VQup zn`vA-Nb1ca?ZYmOcI7t!q}o_MVl5mNk@*c1aBUz15X#|fw$lxPRIk>kZGIdk3S?f!n5c(3x)=7)yIRxz$7v9jKa@e$ks3_ipcUbSvnPMYa^$GRJMrDUZA? zAg7)k{WOl$)Y8}`pW)!4P5!B46V!>%N`D!fu`s~`6vQ)MVrl+tY%_KkFvks;Eq4GnX}c9@q3-zeDF=RaQd?|W~|$Satp zL`*UQ)=M`9o>iT3b>L?qtLP0kTC*jhK0b=71my+MmgK(tF&yiM1?rbzMTPHrfAxU) zD2Um)6+JeB??(u0P%2d7IF77Z%6md7^GRxtj+jNow$1@*c6o_0DIKu!hXa971oea4 z)i#hpSlHzZ$ou*bHwAOU4n)9WqARdpwMrE8{RCIt;|t?x6?TNJVOf?>2bC%tm4}ak zRz9pg-Ebva-L4$@yy{7uUoXEomt`U#ZVg&)NpKikH@I-S(M8MiX<-Yr#1jmWbb7^Z zH7HPob;?j+d}s)W)# zyM&WR`WhP6#fXX&8RUb@X{jD@A2bNr*p*Llwg?r7un3pRu8 z*tI9HmqNRIel*e0GJHor^Jnnw5nN$P^Vg{r7z@4dD*c`LJ^Wlje@Ud!Pqij9g`Dis zbTDX5#`6n{QtL(cxp=OPOZ8TE$zjP1D_1I^5o!)8G7_VypQn7joTV1tk@TFml-kq4 z!$hh5^yhk^Bu}@mY-;ou&CTx}<*8 zoE6c^4{rTnK8@r%`)?qV%t+cuu;5VVWv7GUjDFqQcjv$bt^lkEAV`H#hfo17kjMd0 zi@5(4g(7<0Rtd2>_o94}B2Qa3VMK}_Tl}7Oa zg$nW{!dA7UC1k|1nY|fJdZk@UA;$cU*0c(8B{*V4W(I8Xojk?%L`)AgC`i)Vf$CF} zl87Uico2OCgM?vCNgn?!Qr)SJVUAB<7IcM}2&n{w;H+H3w1JFRzQ*7$l%JmV>>=Zi zKe9;>T{Q%&apI1`C_t)=@3WjpXW!?uc$=S z<@5>P!%2w6P=OUsvLkQdGEO9I!PT>Km^RmmMn_$2FTqh*S^x=6FMoRM+1(A zsGu+R0GwQrOynBz8J0F*uy5uXW!V@6+$mxFp8~12oJp9{AX5EUOAt4;4G8HcCqkIB z&gWZ8?}y6Hf^s=462}S2beyqRaT@CXC8%rV3T<+021?sUtmb=(bO-si-tt*?7UxFJ$&Bz5tu)}_C#Gbpdz9pmhE2{fYg2Q)KpAx6@YAee*{{X^mFPS z9(KqLZzZU+t=Q~~<9Q6;dbj`aUyR*DJ*o$n^^jJ12m)@Rc;C)Gy6YHP*L)$`77kXI zA3sR88BhI1XHE8XQLU{XV;VC{>g1G%a}lx{m9_$hb2vK_QB-A1pI%{2)ENM3 zny&2boEdD^^o!#?SM!nsQ4PY}J`S^{eL>X~dbUK)c^^@PezK4%35X|6Z?}i)(i|Za zwkgbfY=thYNJk_{LA-;(k>Cq4MQ}zTj)RJQLhv(T70Y#Ed zwMv7;OuS5ocPQK_7`CCYsfpeJH(Pjw2%&J=W_uaSZ;l*f6_oeH+RzZtn&RIiu+a0e ziEf+-77KA?9=Ob;_Ed>5ncojBT{Gb*4_oZgaXL&Yj_SBZ?qV>g6uik>;|=kM59(8Y zXfle5Z~99=(E7(Kx?0`z=eAT$nUvkJhtA(1)onf*M7Rsn zx{fN6428 z^h@wA?Gu-?49lUam?!waV9M{aqpjr#s8HSXgp|n-(rRbzY)KD>#6|(!45J_HYvh zfCnmY9lz)QPk*(V0=qB(j>nvkAL4bm2aDg8LEM9LD6>(VdDIchFG@Bt8Os8qKfhP9 zmkeXMI#VKV!0mdFT;^PhE_eK!!MG=3V_gRS0-O;OSSAwrcG2okfi@1{+>{Eyj8G=VpQ zKM&w*o;rWAE(ZdnU^Hsv>aoAlbJH~9;pTQ-bYBu=Fun92e*GmWY*lMLRMz3*FKI0y z;!u~$r_MH=!V-~4(^=iLG#6ertzsHE$Q;6Ks3j{eGYwcUl<1cw8{Dr^yl^U{gL~wnR3;8X0 ze!75Q0=6vGy+{!ypSw}P{`w~IoYVG4Fr{ELg{<;cJ3GEBXC^{e<^U(e`i6RZW;s|a z0Yo)LPOchNH|w}nq-mcAj>t06^^G6Tt3tVOR3iB=pu=bG-rT~#Ir|4hoTi%@QM?kN z$XlQUv2a-*%3Qkk+GXnMK}-9z>W6gx7TbInMCB>xT>pGiY|k(rouHSQ=NZg&Iu@P^Cx*pwAkf8i+K_8c z%+0IP2RwB=p>#bbWUB|^5IkZv1`&~D=rZ?JA5p>41-qS z14H1xX?vT264y?1ggHVL^(F}xR@Y!{(4QZnfTP-=;;z9@ z%9{-l9NkSF!`4&K0wpr&Qw7)YrHD@2$-X57es3-CG!jh0-#3(bDIOWunO#Mo7yCiB z5ZL%7^)2%qcDnK!JmOL+%@efY#}AD4YiWNG86H*((r1WbLoGZ`9u*afXu7 zLxbUXIZ12PuVaWEqmxR3`YESqM$dhZIAWG3kZPfRzvg_S0l_; zMx=QoEZwF~ta@G8G^Yj9@VKKGw?>@Jc{O$>^SSkcwP#*=eMgQSIhCh2F)>xY#Pbn^ z>0jd|^Gc6kI=glQRBWW5;M-il6rxARiKV8@?E+ezTh9%EG_>0C_|p~NY^XPS+x?$9 z1oby95;+YOa9Uu-TcqXcVSW+0Y?3RbM%KJ61I*RG%R8b0(mOu9$P&bo97fF+;p!Jg z>^;tifoEdy`qsl*`7HbdXl>u;j5@~*uLyBaBAP(JQn0EBc*2ZIrxd%Bw(mj5C3v1{*c?8V%Z=@p=TPnYMj6mYp6wJ=^E!~sn+qU=n?(YI;ZNVYBeEw#rF*Zj_K8#k%7pCx1 z9BOMJ!RrF1@R?RRort{i>fv!iU$=jO4G4*Hz%z8 zt+QC=W0qMQY%qpQKt?4;%p1gZPxZ?(uZluQq zBVSY~S@OjQj^3Mf8}@INE6Tv_e}6YUfKT0-;Fbz^Ay?rjnM^MkApY@MsaYi-Sn8gh zwrNZ)N~HMiS%8b1*bIt9s3NeI013;{Db#gs;ks~~P#tu5W!F129@#rZA77Mmbz0{w zRFNyWC>xWdUHNTB7#B;3ItE{&*SLI2Amh#$4v`Wl&p(kC(P|)Ax=4ZUy;~dkdNa~4 zYC;q?A|w%aP8e!wkgU;-69=aakYm#;Al}# z9g}{#CE%iGT;Xt5n~MIY2J?VuyO$3_dtbiloM0y{*019Ou($4;U$KBU1e)C%m#B7u zdQlL%FDiA5z`2L4A@n?&UijEA$3ci7vclf>QOJq0VKWzu0&5#z2TkdRZ``EBehs#t zP{^o#LQ=VD`VjZgtpL^eRbl-72B>t_%UF;++CA-AmMa*pfuGQ{nd?Mt3jFRv4O1z# z;AI#5iODx&2Otv-oOh5{q1 z3Amp=_s+&>f}*~?1?UwJ(Xc996ESPOQ(kE1##O=8wUJ2`&;6V6pIu2+3PJjLeEL-@ zE~0Wub?_%vwhE6W(YMeS0cI&_nQ-hFcKF39qa$C5Tw@2bYkI~o=q{v|Z|wJCCi{IO zJ&J)Snr@9gK+%mVh{fu3*qD@zTiSI+Z#QB+qGDp`k+f>rJ@lr%mfhx!VZe{082AsN zi00vurKbRF$tYL}CpXzH7J)LJU-vMRwjp#-%znK?&$7NWxlo37#X85U5{ym4BwIm3 z+0BfWMMhsvIGf%dH#{805BfjetF$i0BJ;ycZm4DszGZf7p3i&)psb~9{j?q1;YW4K zLxek!?#qbe?H%|F#I*neJX=PA7e5_dTqR!y6RS&Ub%&qJrwJR-6~ zoj}tE3({~TY1J?wx!AcGZV*TqCKr*Ajl^cd`83`tX3irOq_;`i+vf@sD{MCvLRmV2 zb~#8+N$Y4;HjI_kitxl<{0Qcv_tH@Crhs%~^AXC^51P@s1BJC=#U6XwfRixa#t+_e zcfTSB*+D7JO}{<4wVDoUYT1w$mGE2tI>!?EI!g`1AL?`w&ahE4w4A(Q6rY{OEU;wi z6SZ3sckejLuZwiu%G==Ai8&)kf`OZWZZ7*{xvOxuIAW}Q8i89d}4kx)@ zJ1lWmO-kqW#zjc{WptsJo)`OpJdyFN%#p2FFa-qN^CphL^|u+{Tk zSEUuOeqQW}y*cTr%U-Cijh_8j%2?fx#OQx%HodyW>4Mz?)d>A*I*}wx--5thn#E zIdZ~^jreARaUB;Rm=y+&_9J|`Cu+d?*?+QlG^lfD z>CfOaYwZ}u&IK^e2?tY zg{&0qb4#$QQKdmY+tegVow;=_*m%C zX8EMSn8kV9ufk}G$V$Cya4&w9PXE`(lj=CoE2aao%gvN48@1A{IT;7&rX>X8>_L!6 zyWESwzRt{&E^w@mBqr%EtXQWQcsf-sLpgBx5?y9V_L!XqNy>t5cOr!W4W3B=`I~fm^Y+GMvZ6+av1D z&1RqFt@v-3%^;eVcxKp`)_ef?Ch~~cL{(z@o;k2U)-#q@f(1BC3Z;Jio;O18E2M98 z)nLuH4$@6Lp=1Nd5048;;f{=jMOw)O{NR9bkF2zHk0P@l0$y%mhV(OZ6Pxt_#6BTP zH_=+ZYM%(=my;18^v1?A@VsFpPZLd2hCj6!MQf8x>3o5Su3bsbyLFxQ&&RLNy z&7qnQ3PH?CMr68$l=`9s)j%Ayk*+}bg6~^Rg+O^xuO!g%uh%0YZTqIQlnE-{9P41A-jVZ{wkKJ@VH`Bu~-YLrHb%hH!6^Dszz?Dbt-8tP1o6*?$B7L;9-N z1~gq{-hqO(T7~pB@)!ZM=+G_|4e+rG+M9Coq?)ACzL&!m@G4B+zW#|#D-87rk(6S| z8rs9e5_YOLhx{fA@9xe2e8+d-6`1_R%za#(IA0)1Xm^jM?id#S+onyXJiT8}?Az%v zP>?>`1AcBviQ@|8K?&;@{cs9`!k)zdQhey+BY-+}?vLDeZkkqf;U$~9*LYT(EHg@`7PN$*k0OGLDn5TBsbum0_r zRIk?9K3^X9xAEkl^P|je&T_evPKs+;StZC-^Wg{>if;Rd%RNL@zLKOp%vtD zx)S1O@g&dDY(euYUUJ`YWolVSF9AF;L5ZcjHpPt^!gDFfP%WqFU%AX~ILD;i3^k+t zIDwSU^+Z|nUVhS;&4ocVW5W8l!&=6ftwUMHvGb9PG+ zIxe2(i_0^mIx>hLPWfh4F5%0x*PUlSuf z)naw*;^C*v_<^rUgaxB9<@aJ=6D$2Vh-Oeq1w&d6fqcV2J0`PfOnH*A=Dn zHRug?n#CSH;JqNf<9u;|Ka>t;@L&;S6H5S0C5K$h4KUpZs)*X#5)855v~z@^f1zwi z_^%T0RTkKS7)6vU{>@I~QT%IrgdsdPOx4!m?TN#c>-iJCogzNJ7E?GZdnP@!HC0~T zc_c1ws;K|R6YJD9vq@GeFaD-y2Sy2pi9co!0nvGuC+nQ($Wuo-#{~v8>dWMEpcu#t zT|JT~)JvBFSh39En6Z|Q#a-qcUN&dq=@g@-1DFve@=$Gk$ECjO-tswmsx zuMI?u|6Tb!y!Dk#^}G5)Q>NG`2jV5|EPR|rnwmBxt@7IbS=&9!)>Bv(ziO5JJuU;X zNOypqY)1Cn4x^{zcMwNjyzpMVTw?Yqwc|luXLD)FFNBKH|op`X_+QWUt4VzvQA~c(wr!}7dPDO zi{+WK9@NuE)>2zV&~XhI^H7W%jeM?&$fiS?@ZH3Iih!uLS58flRQ_gk#PQzBlATSq z-IZOSo;?!Mb)=OT00Cl(yQ?xYC+>&I32uqAhG&L`$`BAu(E&SDch&J-Y8tHT$n5!!_YX;quS0w!Mcs_<%v1Pv~+8Z|3WXC#o>9YMDejr5Yr>k zhk#1TFv-^J>31%@?v4CylT7nNXli&Rq`!G9`J6cWtu$hYwcS|XJl(6{kufRWoUrZC z%N@}J&VugNBZG!^-jg56wr6k#{(4ZFo)pp^x`HCW;}?UOUrXrS>XHc*t1p!=JLdm< z(&aIYBtSOBB0Hwcn0)itLzhsO+hj=@RKE0_xEee&1nNrcI2w~PO?oewXVv9rgy7+; z42nwkK--)W&oEyEbu-mP#=aoLIN6u4OAM7#A6~HhN1cyPFjVdZ(2jV00dc`Qq$or6kfnoRO`}&;)bQ+}3cB z=gk}1?_lr*6PaB|g}1`kRF=W^jJJk1I$n~_%r$^xqzYBXu?CM|==@a{lN8F>cK@h< z9PnBhhzDA%bFOIN#-um0l$$Nl!vir|m9e0bHNpHpnUTMe?xHBJs|7;Iek=o^+8me6 zro`tRW|G6HemLiD^UJr#4L-N>{n(o3LjI{*wuu<5X$y4Wii*ujj_S_q4Mg6FIl1>Y zPm|C@B5uls1jL6tun@hz9$^q4adj{gUM2WV_qO`ZtyR||I$bUX-nLtBUixATl30F~<;4Xz*0okIcrp||bY~ne){@dcdNK5a*&0r1e`- zqDUcsf+r>n)26Q|`RpAbj{W4of0g0`2t;b=l_E`*pYc8<7qcB(57ay+4S*E|cMY%= zuAG(vucnUR61%tW4oX@Jv##UC*mr-{YAROUICQ?zjwl*OX#h&OTO{8*U4U zBU=PJTz+C+W&$*M?q2RI$fehB<0fhK#1S$aja_a%%RsO+R}!e%LUYYTQjZR)3}Wf9 zMjv}r-C!MOL=2B+)!jFO;{FGxodH(P2vc)kzO7P{24HV2G?4IF?i*1Yj>I+K0VDrL z3uxRNA;eRJE_7C%<7M7FJJ1c3&H1h~;Z^DMJ}=zcXK#}6ueb_Y!^CT?=c`+79LUJl z*6!*{*QxtE5BYUY);-Q6WPgt znrw#KGE1H2ft$WjUz+Xymfcr*Z)cxjNU_aC35(=Ro6xzV)wU@m?~F(fl2+NEzTz(W zIaScqXaN<#U2KF(@RIYuF5c`K(*H3CR8MQkbWffPo5klg;f$)`FWx5lV1V;QqE~p1 zmlw$vVMDpWOsJ1L^odsmp#I5i8YZjRzzmZS1eFho<)&L`jr+|g4w(zB;SiJcjOndz zX3cL(9mpr2jc8LOu7uQPfG29+Slw`HXQCaB)Q~`UoZ^i}U2=#L+8x~In=iL{(16Qy z-$n@UwR1T)H3sXr{Vmjld)G8R)^A)Pv`mIFt~B5g9%bCDOQ@mgOf0P?m}%{Z8oKz0 za`_ypeN1nUPoC$E0dM?9UBe9UWPt1sb18Re9Nn*cMADZm*E1zr5&z_NTAtdZ4<7Al z&s62qwTBv_-5C!te1md;%&jv|R*r>)p7r`~&)^~bUo27tn}wEQIVJunGfk~O85h+T z^pybRvHWaW!YfpARTrVvHRyMtVEtY5B{`c^<=FT|Nh|kbJr{RIZL&p6RxoF(KjTFDQX0^?OC0$ zk^qn+<}VJQD-0}4!(@csnvdMknPH8`7rwRMMkyy=nqraD(1sP zDwB??b3%c7s!pbb1c#jBGN&7Y!A<_h3YA{i|T!l*Yvh$77YarfaYVir`b+J*GUq!2d?`XP-IvaHeS(C*LejU0c=GwQ{KiW zI4BUgplaaSj)pN&GWAHKziNsML0{BZ zt2aAhn-?O&(Z@&VSSwknBW-eWo$O-ac2TVace;A#+P}=i$}F{v$D%9>ZIqUA~Q16lzi0WmW;)r z-Qlf9_MVLawgO5y9n7|O(1TE}`Qlt*1R6F%D(6qGWYHqm97E% zHkPM3K_7DZU5ybx$y#-ULdubh2*zb4gMVt*yrJ=`(>*wcnDBTNA!7Xd&6D}KLdjGq z3aetb4LLJg**Daa!Fj73hL>kLe9DMpsUxz(p=!nep{azat?@n+?QZl{Op(WOETt0F zl{0FyWF8<6E_VS1K+SFku+%B3hFUTmNPHlPL07UZqq4S4%Nd;_S?yvg?LA2Q1VOW{ zb#TU7jZo!G7Ddy%+Qg?gB}<)a-Qg!RHJa8YBWG_;rx)lFkpha{enU5_+?k!(oRV`` zYBX@V-%iHe*oRE6(+5s*a_`+r z2I9_|QkYG?15@#fKKAXry?LtCD`Zcu{jt*3T>7)EOWPW1H0=Cl&0TxpD(ulh#WN%# z%OSL-k+RnQ!kloI%w-Fi@@))ErIWWvu84KRsuWXr;yh|m(X5R1h$ESm5Z_@?qk zb;TJ8^i(;5zUx~1K3gonR;I}kT9=;S&OCk2z{l{A-Ca-v6P7>jMYs43FCgo|-=Ycz zU1qplH(Z-=^tE-G=$?cW*TQ@08b%1?4(d)S1M#;Xtm;XI&>zJlS>;_wCd*K`X4x9pFDWbX#T7yPuU_`<&~0QnV+O6wOOKE#YO4HzsgW< z3Z=ug-R`29mmE8U!EG|dQr%Q%^0wDu_*po}yp1O%gO@2iw)7o}@?A=WGE2r>h5c3@hwX|+MI&3L53XVhby8lNM(!mr z`L<_V?Et1zNr(BKmWJB)$N5atWBtf9u|*F%V`c`}a$A1X%PQ(hPKCX7cxPnJTa^s~ z+ygE5d9{tVGy&`l4Y#4zGC7qyP+fdMcerCDXOUzZIV3`BH6xh^a05Q9UB0K%{GQHq zR2&z*EU&bkcV{MgcEKK01Q{>m$RB#S{jqGf{)z zdaC70y6pZeTX)_4MbF3RwK3z)`!*Ky)~VS@tXv*nS1vv}bLidGaA2>v#cipqma$aZ zlKh;{$0^Hj9DFHO)?!|}@_XChM#|qO$I)I37||85<*I+SdXqDXEUQ0MdAun{lA}pR z?~+>PpG|k#)GKa&uA7bEZ6IBl3pJYYovPhmoiQb3OOol6;&gbps0AgHn8tx&I^jj% z))te2PCL+ldpcEZ{hB8&FVp3%Q7fAwe(v{awVrhQr5h{ngkZ~dZZ}ztbf2l(zug&L z%@$yy#M#uuhQh$)>{2WPhyB=0IzO;$j-vqOuI0YlJ^@4&^a<4*%VK;=8nnEdkP7{UXl|LkxkOY=|ViEDY-Q8z-8An}buXijwKo3m2yj_eO>rRjkLf`&g| z{*J`EN0tF;!ygYDIwMw26r&(P;M$4+&&gd&Y(T?3>nG})E}=d02!0Yp7E7~AP{H7t zo^}$qA_u2P_wJkY6>*T)Rn~Z+kooes3)7pSgsPJvZTd6koedPgr^iD-t%jNu|=s%cwIYrh4YbBCRKkc@iJ`!d|un)G5TE( zt689(`s42F&zZmbQ6L@rx%Q^;t^{Gi+9r;cKTY&!Ta}RWoA%lBCe&iZ z*1&I--c`6WTDc>`0Hy;cI_+b}W%c6SGVN{45t_*>xQj2*tCnQ`$V{W--_r4~>F~qyZXQl~a5?@rk7wl|Du4IR(^$ zO0S}Z!x-Z^Ud3~I>5ZR4WztUQ*1-;@{6I@`F% zLgBQbi-`SE+C?86L$`!M6;pv5llL3Utftp-#Y))pIaY33RqNfywE9oPHC^)|-! z6aAqUH{s8riRXf|lCz%K`!96WF32#KdM=&-jqHE;oyyfO@lB6D9Pz6B8xk6;C`Ao6 zs<&1?qM? zNjc&iQXGm{3LZ6dh?wO+$ws{gmItz#F-$`pxVBaZJ-sky8Zn$~LO8|P0IHdBeDG#C z0r&=HACy3br`vREW?HOJHlc^&=qZGli{tv$K9@g18PQz+9kI6*556Wp~KEZl+kZBi3JYpiRm1&deq@yOI;G#q12 ze_;(PqhgZ0ebowkeSJqm2c#6k=>Lt}t7feugO4tl=W+-jvl<75Ts^R@W)Q)qSB(p} zdEviV{*%f7&n>?85P^R1?6xhrsEE2=;JmdxN{^adpgJXcDGaUlTwZ)CjqbC^Pi^;&pQ%zfgZj4Yz^@S2(;I%g7Fa$YC^}0+ zK|Mm7wDl`yZZCu++JDPEiIB0DTogJ+1^nGc%A~I|W9L%_RZ%I2){dQg9#w{%+5khM z13_x)n^Cv3%nm3P!q}HnZTG!iMFBY)Oou2A2M+Sf3b{`nhE0}&0&EJ&fGrpHXL!*f zer-8*a>Yw^HqVaeV4+i8+V;FlDZ83GHw%jhNkk`t#Mq$ID1yor`dLS4%CHiQ8SeM| zU6Tw6=_;i^b@4FX`sKp~dZPVKtmJbyNQF&_?PguN>MV7$N?k}ygTl0r(8K;gF|q{V zHs3mIh>H~K?ooEvw$1`vI$K2pMXFl#rxMy98n(?B>b*uZweAy^bgG&6H zG-|K11ooIRp$B@sN752Th;a94bSWK=7@K}f=uo_y0r0%y1&F-&4G1v=)!7ykp$Nwx zX<#UAgOab$@k&<|JUHZwP_nKcDNv+;-GaNaVYibImTQ5C7ck4Pfk@2g+JgT+EGDoh zG;{#4iON=K$k1K*)P5c?TJKBN7wNp_D^mXAKAjtRgX5Z!`BFQS$0Mp0vp(C0hz26a zSKS;4D}G(v{u*(7)oR|q4s`~Xk$J3nYcW9mivZE~yjNtEXkM#{nd>S4VBi|Bl*#RN zai4P|-vjjs@TsVZv4nv4?V*bKn>o>Bg18`$fo5M0UI3_TSg;zuRRxsVrzt`=6q)K_ z^sMPHY z#@3(rObrK$OyxynJG=Ot%5?4^>l;NVzPqW5F&mvuYfQH&`pN$9>+RD zG~ZKbq5L0&oFG6uH!(n0D)T|UO6@w-DNNrELe0h`kBc1^WNe=oobhwO@|}RBCcq*8 zZP(p15Ak-#&uW_y^yo5Fyy~gU9kRggYuFmAs~xwYTgvq{9V7UDxnHN^tGVND^{giK z`om*-(3>LZ*!Ydpl0qjs=q=)e%o0LEB>TZid&a5xIT-*&{#@nq*2W!OvHjtEc?J=W zH`U%U4G#>jT={%vWOtrvTd#y3^yxgui|=vq5Hiz3Wa}(V74pdG9kcG_0w&|PU4iVgJInhKyj|pHJLeXgqSl{ ze6;OueMCuCS-W1N5y7%NAa916bIj#D&%6cwSoUtj*Qn>hK`029qY-G>f55nfgE^i^ z(ZHU_`B=c=R%uG!%4Ixy8$$KCf#V_<3>+Rt`3Ysom&urq#m}`r?)K6ZL*YyX&}xEPInyVhpqxE3iut7c0fAM> z)Rs4{MZmm^InKP=l?c8kC`WND*DB1Pgc_4Z^CRFHkULht*6h|4+MMZl>gZI+`&!zp zxM%^Vom;!(*KWf5I|h~{c2I?w91{43OH3!k&L+(P2>?al6{d^4D2rF2151_LRKy-c zE)_??zVIM5UiRqwu>(pu)o;3}7@RF7$=k66+FNquVWfj>Mj{Bpl4$~sgX-rR>?Hg&4rr0Tov6<1}O zf4Lk+pM|y}xyui~H#7Hw+qzPw@$#d%yUK(?V6d2b2s`t3bo1zJAge<7G)v10kIT1Wdw$Qr&lqEx}o8lrA z*Gdoc6P`S3m>q1Bla&IeT3+bhT=XO(TvCv9BR~&% zm*vtD?LnGQ0vz#iSV{;8(@$Au$3t+XG%xA~U9#F@(K^H|%p-+(^hp~VL>J*!8!URN zH{62ckoy`gm<{V*g6I}UZ5!Z#$Vj@Bl3*l~h|j&pV^Yw74Ub3xJ~-u#S-M0v}Ji?1vXaN-)klp#D%o$4x~ zo#V^Rj9)}AeAg2&8%OM6R$wJMLxgveK2j$Ms9m67Wz?iSbdqY3TvEf)Q6jE)c{_2~ zB@L9C+?rOJQdq|&4SYf5w+WHbe=z2ON_9CIzA%YHJu0lRKxt@80i#9A;jF1=j`Tov zDQF5kEw3bkd6E8bXgzc=mszJKT6cL$xUEyaGFRcyK4g`C5qWWFJs8N;SjXke$S_`R zD;eT5{UJ+2xrBR6aTG#P1E#t@49gjwbE?toT^PP9GlN*U_QUmV662o8WkzQ?LmzJ9 zt(@u9-blG~X6pTX02UNmF~Q#4^wVBhoKI3Ir$i>bmuU^dkHlTTRt%hr}x+Vc^YA0nf@!t#V5IOrP+a0wda)N1BS-C}BltTYM zYvT~SgD1*xQ=WL_KGG|fFg(~WuTWh0ngcFC0`WP_jsy2QJeZJ%Qs@@&gWMBYh|hES zK&gNgN<8ELu2`A5M9kAY%{oh1h27FJ{YeFpESN$#?F*t-rp7>N z<(#-$hAFT<@%A7u=$1GlyAM$W$SO-2&xOl0cW(zHLeT%_wlX7uU>@57JQMrCH!PYwmzR*y97 zxGJdkzt#1O_2Mn)ivxF(>-1gcw=%xm_?I^Ss@QAz?0GGo>I3Y#n627O)1&V!qeoi?ha;R7)W`nSFR z$26XaW5D;ZBd^LOERE;^mM%*>V2QKK?_}EdBoXo_Qc! zKp#lwUfxY(U@;LTkwpJ!x?N-Cr1jFDj!e&p2eeSb2{XK8eVyIE_Jj;rJ2Dt3CEV=O zBIW}ze+ddcbaN9jK6RN|(88}9-9G5;)Zl+9Y0bfLo%ZLOU|N9d(xW;pnJltlhyERt0*MfEJ+JuwXY26QMSQ=hJi=t%9!T<$ z>yc<&#^ywcSBr*dTou&Nbub_5JU4CTJQ4~j$)!88|L7*;M4u}BMu{1ZE%zDv#e}Sz z8HMF?o)!9}j!7u!q7}MWs`3HI!jY_&`IS>)?Xe1*DJx!`E#Q;vU?;z%xzVinP$`}2 zUzJ%V13a-y^CBx2DqC2}eW@SR6b6p*VEcN4Q!~shaXypKlX3DbKxYH=OSq0h3WGeW zsov?OO}&m=!A0rb7KnuTzGE#B)prs%r`uUP-mWlBH5rmy(LnIo05iLiK#KY6LPRWL z4gMb56%9z!3NWV?8jd7swe?lf;4l@-tGq(?hBQ8zVH5PV(sK5na$;$_i0t)-;x1FU zPGR$>S?9!#!tNgU;UvRCWv}0>LUe{OOo3?BHDYayE9**28G>%~AR!qZ%|ZNTd$s|oNrO)C387fW~G6iLBE{y_IO!{Mc%OIVm$QkG*X@VXXNMWQa( z%+^Y{ty(g1G#wi+-4ieKK}QWJLmjpz$=kZp6AL zJBNjH?(KG_Xi)Y=bVLUxlmJ%Zz9B_Q28C*y$$As33~o)i18{iWJ2M)(3RA^8V%6#l zK#mfB`+z6kw>o)}VcOxC=2^Rx$a_O3ax`j)R#ke_;k z6d2#VzZCXZ0aMYV8@`J!SXIz0`YnymP4ieY?Jwj=NN3JJpd6iw^m4Bi7WIYlHB*sG zpPg&cYh6gVDt`0yWYrGeEeiZ>2()0qEFMTRMWo|e2-NvEn0D3R@K(;7r68(tAiA5Ok6sd>&@yx8Vm z6Z7L0^ncI+2mT+f{F~3|Z-mg|_sBBoyM#pM6gim74gVY7A|adq1;fAS!sx^7ILPxK z(EI}mf(x3z`2Uaie`DzU^dFb--*^25hIW|k=aP0|k+`=1zt|7}_^d|7l(WwN+Aez7 zYrDvW5i2s!v55(adn(#{!n(v^Z$*!Ep|~J+yKH#*(KxovgbC$rw?L`k&-f?t%^YN0 zF_UenD>mI7Cg>+EQq$UNyNXBa&M^x!>&fi)`}M4p{&Ie~g5Vb})7Pnm9-W<3jr}s) z8mAn_(x{$KK7sR@pZjRV%?e@J|MNJa^ew|Qk75MRWF+wqW_cNL(X5}2$966Z{L$%a z{qYB;%Zph~%;<(d*=wEFIc}_K5pc$1Iw6w8QlSmk6e1gGs#oFZdf}*O!3dKEi4THL z&R4bkw*l-1QID%8w(6EPfDz_TU<1E#*an%CQEA8cpFp?rOTg?shjao(RuQY_Gik8< zVJgUOJ0_v=LnG>}pgxp_3|HN~RgU!i_V4(XcTl+4KH_Os4S6i1LURjBw2?C-@t=pS znr}jvv{nwbS4*x;20U}Ys{5Q;0?$Xp5LgL+KRW>Y3ilipMQVRt?j2T1aVi$CQl51i zOeLkVx|82lbR71*;;D;e)=LYn!n9QJ94IRD)?yy(Us*esF8cOF_RRl=B-f(4pMbIk zXI(91u7(mcP*O;{dbWL~N1L^)Ci6}A5|4nFu9Xj{gAm>gjqymmyZ;l6o#nBel;e^z zWT%~_5`b)onN+uZdI*p{30YI8^8PU3$*Zu$*AxNi!?ZfAyD^B9)J&Aqd-oh9T8RtbIZgf=Q{N{u#F92`X_o;TS(dN7|xg zgi5Sq272=_4bheCG*cIwA`hzS&+A%5Q#-NAuets*Xq;udPEDuy`*xNT@gcvfCcWIO z6=pAPv47?3g9c*kZXv1pt^ZzHhY9w{`<|@*f^x~Z9$_WsZDO0LA~Ypg&HIg7(jhJt zRs-HdR{$Dw(BnRblJZv-7RK@i!od+Kcml52WYpM(_lx~L%jl9u$cCg7Nrm-rT1S96 zx)3st%)1m4( zI$$^bV*I0$v_hZ9&(+wBhYNAg`Uz6q$Yn4Ks#}y3v7=59efgH?J#}gV3UOcKdeYcn zug4|m3RM>`&YZ9|!J%(3;jGljSd!tV(3K=8o(x4?4~~|}*^Xc0 zL=2uwBj&Vy6yk~_!sxRsw?tUuxg`m}AzV#jlv9qw{b=enhYwyZm9Ke;#1N8nw?HF+ zqrj7zBQZM`C?MymaYp#Nr%E_O?ridg7x4hwu@&odl!%k(;>`*!!WO#68%%XJYmC$q zc+_}@?X*~GmC4((JpWsQ*2C3YBIg3H7a3JZ@~w|!t}X0)(wfaGvhDqy`%iKjh+Evi zqt>0!PMAs*dJu-C50sN4+8=P1cA8|_gM&;#T_#rN@5UR{rdl777nS4eywTlAniQp$ z%du>U9kjnqL_A%--=2@C9*vXdkOqxeziC)tvNd#I!6kbQ_Fl0|{sJp?OGaM21t{h5 zvE`$-kAiL;=AV6M?`{es%)(zbfd+AqOzrUI`rR|E2kc=WsOfOEM=g!P$fq&$pO_F$qB6lv;E;-*IL$ZLsiVzWwy6z|16F5D- z#b@rH$-HF=4Uu#y^q}PU2=K+E>5l{En~ryX%}$@0b#eD`&v58ID`QVMT~h%UwaZ%f zoHfvaOD;JET?Ae3<286+M=M3`B^AjsilRCewHPRsVX?*_OjXbKf*K9<8_JU)8k{aq z!>))h#}F@amP_o$&D+~iMM%6UTyKdN0B|QR0061}kK#BlY~SY;O7w0oQC>#JLyA*!djWK3ASw_RYV|663KURbo0GP+CDo9RDOJT%B za$P9g0_&O_C7+IhMvbAU`C!K_vn3VTu8@riph~c~z9NBizL-5PK&+>Odp2#B@eN&0 z?ZE2y$wh9s_^BCzS;F2^`P$TIQI>oJ<3}A_MWa2E1W_Sy6ZDG4&ER@5xH<^0L}CAia1#D!Yzf0apNC$J!kE` z9zxH);8WpP#g7zy*3_+d%61rO=K_9e95i*78ZFIuO+6IY-+ZVBz>8(;|?jkUCAeZ7%-<1UY&~L$5YC*>Zo-hQA zyvqX7MX&^Z#tRxs#@p%t8b+@uJwA6yo zQm2#UdFSMgkQ5BL-1^6$j>-crWiNikk%i+tb^ zi`+_s7Td%{^xC{++8el16-Ksg)&XnLl6WqE$GwHz&Tiz`d1^vQuS>*jN=p^g>a63p2R zd%-s$@{{%CPtsH{08yhK0$Fvm`y4R0%ZbpFZoT~3mo2%(=pf^f+~rPhtRv%-N$_%5 zhpr}0Se5<}v45c3FH)aMVz0lSq-kc*1lyBG(opIQ2R2|>_p{hajdO! z!uEMKVo241UwDp$yyKunag|hiBH21l4tw45^=9nfg}N^8RKKHRc~Y0oCxhZcr(`?w z^>HC7bcQ&7gUiKanStYRcrIN(Q3wJRn2xQOH!VYVIWWj7Xsat03LJZoX{cL$Yt2q^ zlq(~go1rEN^Z!xm*!(QmJO}RI9em6f;VajR7|7LUdaJ1-zIXj=r}OyE;)~qgn-i?! zt-Ap|Q<04Si-;TFL6N-aT)Xt?<}jn&-;nHUVkSIM`jlWED^9Yt%xv!~D@)A21F=dM z=7zLAvi(!OQkQs3X(zsZLrv%H_sv%+Rl5PF%pa%HS}uJ2nKsFFQ5p+)X+zm6mLs8@ZGQQQlyd zWs3ooHVxML0FI4x>PBpS8rLgUAvIOBvCH%w{sD8d)#0)=8k0pDU6S(+&8nMF)c_Lw z=TU>TES|*Y4ZCiuTt~}ze#j#0b`v1sT=thWOFy5fDFI1&$ayx|U_@3-Yf>L3j+4Wv zEovj?(|5TPaAR&*E_>%?U^}#wRSpVTw9k%e%c)ZUJgk-N#-EcO=<-_i6#*@&GbGek ztI4MLVgZ-h?|X1JSkYD!zs=s{=pLQ314?Ncl68aQ0q$%mwkbE37M0UWPAAZ(B9dkxfO-hV!~ zFU53-h0CK$ojd~p(JXjvQ8$C+&6$WIz1rfVDcR96TRmYQ@5pfA3S?glkQqLdK_h(q$qpl3o&l)!~iC!=

r zp)R6nRH&Q2cU!p98H25fE3hE}Qm}i=1xwTWTtO!&FN!@P-}GN<>0SU|Qdw+Y1{K*a z(ccUGLhvHHh=sNo4p06(>WUu$_FU+!sg&8&Myd^|$PL<7ev)>NQhM%Op0Y_cnQ_o% z#e!@^f|YrH^e*znyG|{?Vo^A_*#a}$WW6uU4{OhR9(@8mPfacfYyXd4@4I{07qRR>a|cLw9CUdp~sW$M6S-y0FDweb;Eb)i2n*^!S)_cd;6 zeH}UuwG^9w;D*(P@d>fFCh~q<@We}xCAOmSW10GI`#=LwFv1^h9O_|%liCED7{VQxDwHVFK;Be z*!dj-fRE7xK!|k2k+=elzf9yig$1`1SmRP02riOiu1+l`_T>Jka(6|PiC#pjPqYhF zDCw>c#7)yqG91eQfAk-WzdC>BJbJ-ze4m&bKc>WBXY16C}NU1f$&vqZAu-J(cbX zE9Qm6hgnw{)C(~?(HIc)avHBzb|YjKhO&*Y>F&&xf34B zy57TO^k5SwuDi0#ki+UG0Bc*!Y>XuZQZ*<1D7ck|>#a!s;NEWD$m3_skemxrs)ID~ zpybTOpXoNA9fsViW8jT@h*CR26zvQ3va)YoSqm8l-y4J(g)7v24x~+61GojHh%TL?u0)$49l!<~-aPVzsH}^CiUue8iZ9QG$m&;Dld4ycyO9 zR+r_oIYEOuS zlJ+e^Zt)y9UgGr}%!(qkz`01-Nzbwru3YNfZwY(X(_rj2f=yCE^w{sNBnbmF{qH=D zK|03o>|xo^YG@+@m<1q!8RTSzw|famxry4`W<&&4;aMi791HR_z>z<&u(!{>4g?EG zQ65hodrOG31tE6EWh4h+rkNFC^{C3J3F;sfK*%bPuKwFBSeKIuPK)YMQYPn*KaX9? z{SZS=PMggZehCNG{;yo%4$r6kC~&3X+7kpV##vZXB#o0)4yJ}+6#)-0AyOW?xre(M zNT`ogK_>tlO_(@Cnxccdz%z>Ta4}XLI7d+zd=(9J1F&oN^%)S8JV~^2DBp~uT=PXO zQo1^Fl)Fe|^Qck~sW`Z1zSQM&?a`J!&48`M3Hz&R4vZf*mY)2I$6$XKOdR1Xwa{;~ zOCsVEi-b4b)8-q$uhV+4)xK1oMHfcHZf3nMS_?o)NAiC+HsbCx51UMONzydFk))Gj zDvPqFE|WyU@-iXIGv>HrvPcFW+{|xAe|Ps#9`wN>+$SrI6zMQy9X`E8dV5%Wj4q*- zSnVkG2AVHNXjwB84L?2d_qrLOMrg7VJ=!fBKbiDX;19kAf#1$TtY*LCM7Xloh(w!F zgoPrR1-v5P2w_jDIOtfHoSa8}amxJs42F=6EQk0)i+Yb=RBjIyc_Hg#l`hE&Z6t<= zC)RitEl(7Zu&9MBhZrA-j8+mwMFoi`hp~K^NFBiJXe03B6-x5x?8ue&hML;|Q1EQB zednfAA=0;e6+7;^jtv&%jCkcj|J|Um@9u5+LmeF!*)3s=o?;}N*bhe;^ou@EZ=%61 zk?qNzX+DG0{i5qTK`xcOF*{xH;)Q;{N`q_gDCNIs}R72%#&1Rfshi0*GQwgT6b)YvXlf zJwH|&w3$t<1DD9qq`ZcLIUKL#9r6I%J*1S^)cIKrB3%~xJ#5$$DF415VA6ocyHlHy zg%$Z19C&6`dh;1=Vt^P>jRco2Y#|I-Cf;gtYB@e9r(JUEliQK!A@XGmi~>g$YG}c1sT>R zI64_G$Ton((vTkYiufTh=S>1t(V=->1!}X*5n7ay(q$hjT3=?lVE>u78EHwgHI8f( zsm(WyuIVuvBV^i&mC*Xa_Y;BSMq`Zo^VIYHBFu$ zazYOwp@+K;cU7)%@p;6`vHy!uJ?ilI;W%8^=9E&TBY1n4aDuI-4$u-PIM5g70ANR$E+7s8Rp3BO} zckkdJ3kW13KKeUFD7~&?%x}qs2SF(%?|IZ+B;fIJm3Tb(Gy%raUy2A+ONy0@y^8D0 z5di^b4dCv$NeQU=4JHahzCMZ;_6z;*Ztka5T(x<*wfJ|+IP3QUP3?CL*C zRpI2URm*(Vrduc23W9M%GkdYlObRqm^w3&8+^LgKvWK+7$pUt~r%-EBMl<5=J6AA9WPbt$j2nFhIX=%J__PTdwEg5jw#i`~1j0&rw;+ywdZ%;v zc2JXklced#+*E1oNiH!=Up98m@A*ZC>oUK=ULGm?{kiW}?`s6ITCpa=rOoL}`87 zqm~FB`62t1aYZg06uvAg+_TAQ`DWwUW3((pN!yK7;d{?i`@9Nmy%@C=Su10<)VLVl zD!klS`8yta=Ldw~TYY=InYvUBA7+42kHe6Cg}e4zEQiMnx>-WW?d>7PouZlAXO1yQ zoBIziY(k*L+KN|oVd?9`y>I4KVU znwKk^QpR!eWwp~}Fw4ZsBofh&65DHdEnk>oeXrt+6KiEl!QvEMw?9O!GBQOrc3PVK zwiaYCOc9nn@5i%2iRS+ZOq~Aii%YYy_%6xRKI_@I==G+$MG-@ZUe^r{HJ%Zv-UZ);Or;*NFS`11BpdHU2WaBL%G za&W#_ys;bPS{U`#ua+4MFpClbn&sH}2qHSY13v1%XgPUWm;F^A)thKs1uWrRwBC50 z!PmGLM4G`bl(%2M9N%^?)C_s>?h}&F1!_u&zqUe9U8nw==xKY`{Y?P>2Q% 80/TCP 12d +``` + +and + +```bash +$ kubectl get ingress -n ui +NAME CLASS HOSTS ADDRESS PORTS AGE +ui alb * 80 11m + +``` + +Now, do not panic!! the output is expected since it is supposed the ingress/alb shouldn't be created. Let's verify the load balancer was indeed not created: + +```bash +$ aws elbv2 describe-load-balancers --query 'LoadBalancers[?contains(LoadBalancerName, `k8s-ui-ui`) == `true`]' +[] +``` + +If you get the same outputs, it means you are ready to start the troubleshooting. So please, continue with the next page. diff --git a/website/docs/troubleshooting/alb/tests/hook-add-ingress.sh b/website/docs/troubleshooting/alb/tests/hook-add-ingress.sh new file mode 100644 index 000000000..e573766a3 --- /dev/null +++ b/website/docs/troubleshooting/alb/tests/hook-add-ingress.sh @@ -0,0 +1,32 @@ +set -Eeuo pipefail + +before() { + echo "noop" +} + +after() { + sleep 20 + + export ui_endpoint=$(kubectl -n kube-system get ingress -n ui ui -o json | jq -r '.status.loadBalancer.ingress[0].hostname') + + if [ -z "$ui_endpoint" ]; then + >&2 echo "Failed to retrieve hostname from Ingress" + exit 1 + fi + + EXIT_CODE=0 + + timeout -s TERM 400 bash -c \ + 'while [[ "$(curl -s -o /dev/null -L -w ''%{http_code}'' ${ui_endpoint}/home)" != "200" ]];\ + do sleep 20;\ + done' || EXIT_CODE=$? + + echo "Timeout completed" + + if [ $EXIT_CODE -ne 0 ]; then + >&2 echo "Ingress did not become available after 400 seconds" + exit 1 + fi +} + +"$@" diff --git a/website/docs/troubleshooting/alb/tests/hook-suite.sh b/website/docs/troubleshooting/alb/tests/hook-suite.sh new file mode 100644 index 000000000..8b5a4baea --- /dev/null +++ b/website/docs/troubleshooting/alb/tests/hook-suite.sh @@ -0,0 +1,11 @@ +set -e + +before() { + echo "noop" +} + +after() { + prepare-environment +} + +"$@" diff --git a/website/docs/troubleshooting/index.md b/website/docs/troubleshooting/index.md new file mode 100644 index 000000000..847e6758a --- /dev/null +++ b/website/docs/troubleshooting/index.md @@ -0,0 +1,30 @@ +--- +title: "Troubleshooting Scenarios" +sidebar_position: 1 +weight: 40 +--- + +Even with careful planning and preparation, unexpected issues can sometimes arise when working with technology or completing complex tasks. This module provides examples of common troubleshooting scenarios to issues reported to AWS support, along with step-by-step guidance on how to diagnose and resolve the problems. + +Keep in mind that we will use previous concepts from the other chapters while going through each scenario. + +### These are the scenarios covered in this module: + +- **AWS Load Balancer Controller** +- **Node not ready (Coming soon)** +- others.. + +:::info Troubleshooting Methodologies +As you progress through the scenarios, we will be introducing an overview of different troubleshooting methodologies. For example, all our scenarios are based in the **Reproductions method**. + +#### Reproductions Method + +Systems and applications come in varying sizes and complexities, which means that you cannot always rely on a full-scale reproduction. We recommend starting with a cut-down reproduction, focusing solely on the components involved. There are times where the issue is specific to the environment and there is a combination of factors necessary for it to occur - in this case, you may need a more complex or even full-scale reproduction, but in our experience that is far less common. + +Being able to reproduce an issue allows you: + +- Observe and experiment in a controlled environment, without affecting users of the system. +- Allows your team to hand over the problem to the team responsible for the failing components, so your team can continue to focus on more pressing matters and mitigation. +- Provide reproduction instructions to the designers or builders of that component, they can perform a deep, targeted investigation. + +::: diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 9b2339322..897e63b82 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -157,6 +157,12 @@ const config = { position: "left", label: "AIML", }, + { + type: "doc", + docId: "troubleshooting/index", + position: "left", + label: "Troubleshooting", + }, { href: "https://github.com/aws-samples/eks-workshop-v2", position: "right", diff --git a/website/sidebars.js b/website/sidebars.js index 7da64994c..4375d0340 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -22,6 +22,7 @@ const sidebars = { observability: [{ type: "autogenerated", dirName: "observability" }], automation: [{ type: "autogenerated", dirName: "automation" }], aiml: [{ type: "autogenerated", dirName: "aiml" }], + troubleshooting: [{ type: "autogenerated", dirName: "troubleshooting" }], }; module.exports = sidebars; diff --git a/website/test-durations.json b/website/test-durations.json index 41279d196..2b4c0de6a 100644 --- a/website/test-durations.json +++ b/website/test-durations.json @@ -185,5 +185,6 @@ "/security/secrets-management/secrets-manager/create-secret.md": 1273, "/security/secrets-management/secrets-manager/external-secrets.md": 14963, "/security/secrets-management/secrets-manager/index.md": 281009, - "/security/secrets-management/secrets-manager/mounting-secrets.md": 16049 + "/security/secrets-management/secrets-manager/mounting-secrets.md": 16049, + "troubleshooting/alb/index.md": 16049 }