diff --git a/.github/workflows/deploy-do-web.yml b/.github/workflows/deploy-do-web.yml deleted file mode 100644 index 8735046a..00000000 --- a/.github/workflows/deploy-do-web.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Deploy DO Web - -on: workflow_dispatch - -env: - FILENAME: Dockerfile.web - IMAGE_NAME: gitcoinco/indexer-web - IMAGE_TAG: ${{ github.sha }} - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Build the Docker image - run: docker build -f "$FILENAME" -t "$IMAGE_NAME:$IMAGE_TAG" . # build the Docker image using envs defined above - - # login to dockerhub then push the image to the dockerhub repo - - name: Push Docker image - run: |- - echo ${{secrets.DOCKERHUB_PASS}} | docker login -u ${{secrets.DOCKERHUB_USER}} --password-stdin - docker push "$IMAGE_NAME:$IMAGE_TAG" diff --git a/README_KUBERNETES.md b/README_KUBERNETES.md new file mode 100644 index 00000000..32c8c708 --- /dev/null +++ b/README_KUBERNETES.md @@ -0,0 +1,233 @@ +## TLDR + +### Manual WebServer Deployment to DigitalOcean: + +``` +# build docker image +docker build -f Dockerfile.web -t gitcoinco/indexer-index:latest . + +# login into docker (engineering account - 1Password) +docker login + +# push the image to DockerHub +docker push gitcoinco/indexer-web:latest + +# if you didn't do it before, generate a token https://cloud.digitalocean.com/account/api/tokens +# and authenticate with it in doctl +doctl auth init -t {token} + +# save locally kubernetes cluster config to use kubectl +doctl kubernetes cluster kubeconfig save k8s-indexer-web + +# create config map with the env file (production envs) +# to update it, delete it and create a new one with that name +kubectl create configmap indexer-web-envs --from-env-file=.env + +# deploy to kubernetes cluster +kubectl apply -f ./k8s/web-deployment.yaml +``` + +### Force restart and pull of latest image version + +``` +kubectl rollout restart deployment indexer-web -n backend-web +``` + +### Getting data and logs from the cluster examples + +- `kubectl get nodes` + +- `kubectl get deployment indexer-web -n backend-web` + +- `kubectl get hpa -n backend-web` + +- `kubectl get pods -A` +- `kubectl get pods -n backend-web` + + - `kubectl logs {podname} -n backend-web` + +- `kubectl get svc` + +- `kubectl get all -n ingress-nginx` + +- `kubectl get all -n backend-web` + +- `kubectl get secrets -n backend-web` + +- `kubectl get crds` + + - `kubectl get crds | grep cert-manager` + + - `kubectl get order -n backend-web` + + - `kubectl get certificaterequest -n backend-web` + + - `kubectl get certificate -n backend-web` + +- `helm repo list` + +# How did we do it? + +We followed this tutorials as reference: + +- https://www.digitalocean.com/community/tutorials/deploying-an-express-application-on-a-kubernetes-cluster +- https://www.digitalocean.com/community/tech-talks/securing-your-kubernetes-ingress-with-lets-encrypt + +## Requirements + +- docker +- kubectl (https://kubernetes.io/docs/tasks/tools) +- doctl (https://docs.digitalocean.com/reference/doctl/how-to/install) +- helm (https://helm.sh/docs/intro/install) + +## Building Docker image and pushing to DockerHub + +This step is needed everytime we make changes in the repo + +### Build docker image of Dockerfile.web + +``` +docker build -f Dockerfile.web -t gitcoinco/indexer-index:latest . +``` + +### Login into Dockerhub with Engineering account + +``` +docker login +``` + +### Push the image to DockerHub + +``` +docker push gitcoinco/indexer-web:latest +``` + +## Kubernetes Cluster Setup + +### Create the cluster in DigitalOcean with `doctl` + +Run: + +``` +doctl kubernetes cluster create k8s-indexer-web \ + --region nyc1 \ + --tag indexer-do,indexer-do-web \ + --node-pool "name=pool-indexer-web;size=s-2vcpu-4gb;count=2;auto-scale=true;min-nodes=2;max-nodes=5;tag=indexer-do;tag=indexer-do-web" +``` + +This commands creates a kubernetes cluster in DigitalOcean with this specs: + +- name: `k8s-indexer-web` +- region: New York - 1 +- tags: `indexer-do` & `indexer-do-web` +- a pool of 2 nodes initially with auto-scaling set to minimum 2 nodes and maximum 5 +- each node runs in a vm machine type `s-2vcpu-4gb` +- each node with tag values `indexer-do` & `indexer-do-web` + +### Save the created cluster config in your local kubectl + +After the creation of the cluster has finished, run: + +``` +doctl kubernetes cluster kubeconfig save k8s-indexer-web +``` + +### Create a config map with env values (kubernetes doesn't support .env file): + +- Create a config map (with .env having production values): + + ``` + # Check nodes + kubectl get nodes + + # Create namespace backend-web + kubectl create ns backend-web + + kubectl create configmap indexer-web-envs --from-env-file=.env -n backend-web + ``` + +- `indexer-web-envs` is referenced in `k8s/web-deployment.yaml` and **needed for deployment** + +### Deploy web server + +``` +# Deploy web server +kubectl apply -f ./k8s/web-deployment.yaml + +# Deploy horizontal pod auto-scaling +kubectl apply -f ./k8s/web-hpa-autoscaling.yaml + +# Check pods +kubectl get pods -n backend-web +``` + +### SSL, Ingress-nginx, LetsEncrypt + +#### Install ingress-nginx + +``` +# Create namespace ingress-nginx +kubectl create ns ingress-nginx + +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx + +helm repo update + +helm install ingress-nginx ingress-nginx/ingress-nginx --namespace ingress-nginx + +# upgrade with nginx-values with commented service and config +helm upgrade ingress-nginx ingress-nginx/ingress-nginx -f ./k8s/nginx-values.yaml -n ingress-nginx + +# Check resources in namespace ingress-nginx +# Check until the load balancer has external ip +kubectl get all -n ingress-nginx +``` + +Point the domain to the load balancer from DO UI -> networking -> domains -> create A record pointing to the load balancer + +#### Apply web-ingress config + +web-ingress.yml should have `annotations` and `tls` sections commented at this point + +``` +kubectl apply -f k8s/web-ingress.yaml +``` + +#### Install cert-manager + +``` +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.2/cert-manager.crds.yaml + +helm repo add jetstack https://charts.jetstack.io --force-update + +helm repo update jetstack + +helm install cert-manager jetstack/cert-manager --version v1.15.2 -f ./k8s/cert-manager-values.yaml --namespace cert-manager --create-namespace +``` + +#### Apply cert-issuer config + +``` +kubectl apply -f ./k8s/cert-issuer.yaml +``` + +#### Uncomment annotations and tls sections in web-ingress config and apply the changes + +``` +kubectl apply -f ./k8s/web-ingress.yaml +``` + +#### Wait until the certificate has been emited (READY = True) + +``` +✗ kubectl get certificate -n backend-web + +NAME READY SECRET AGE +letsencrypt-nginx True letsencrypt-nginx 80m +``` + +#### Upgrade ingress-nginx with nginx-values config file + +``` +helm upgrade ingress-nginx ingress-nginx/ingress-nginx -f ./k8s/nginx-values.yaml -n ingress-nginx +``` diff --git a/READ_DOCKER_KB.md b/READ_DOCKER_KB.md deleted file mode 100644 index 2d0e1484..00000000 --- a/READ_DOCKER_KB.md +++ /dev/null @@ -1,70 +0,0 @@ -We followed this tutorial: https://www.digitalocean.com/community/tutorials/deploying-an-express-application-on-a-kubernetes-cluster - -- To build the docker images: - - ``` - docker build -f Dockerfile.index -t indexer/index . - docker build -f Dockerfile.web -t indexer/web . - ``` - -- To push the images to docker hub: - - ``` - docker tag indexer/index:latest gitcoinco/indexer-index:latest - docker push gitcoinco/indexer-index:latest - - docker tag indexer/web:latest gitcoinco/indexer-web:latest - docker push gitcoinco/indexer-web:latest - ``` - -### SOLUTION FOR ENVIRONMENT VARIABLES (kubernetes doesn't support .env file): - -- Created a config map: - - ``` - kubectl create configmap indexer-web-config --from-env-file=.env - ``` - -- Reference the config map in kb-deployment.yml with: - - ``` - envFrom: - - configMapRef: - name: indexer-web-config - ``` - -- NOT DONE - For sensitive data we could have creaded a secret with: - - ``` - kubectl create secret generic indexer-web-secrets --from-env-file=.env - ``` - -- NOT DONE - And referenced the secret in kb-deployment.yml with: - - ``` - envFrom: - - secretRef: - name: indexer-web-secrets - ``` - -### Logs - -Get the current instances of server: - -``` -kubectl get pods -``` - -Possible response: - -``` -NAME READY STATUS RESTARTS AGE -indexer-web-554574455d-lg98l 1/1 Running 0 41m -indexer-web-554574455d-t4tgj 1/1 Running 0 41m -``` - -Then to see the logs of the first instance: - -``` -kubectl logs indexer-web-554574455d-lg98l -``` diff --git a/k8s/cert-issuer.yaml b/k8s/cert-issuer.yaml new file mode 100644 index 00000000..01df8e9b --- /dev/null +++ b/k8s/cert-issuer.yaml @@ -0,0 +1,15 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: letsencrypt-nginx + namespace: backend-web +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: engineering@gitcoin.co + privateKeySecretRef: + name: letsencrypt-nginx-private-key + solvers: + - http01: + ingress: + class: nginx diff --git a/k8s/cert-manager-values.yaml b/k8s/cert-manager-values.yaml new file mode 100644 index 00000000..b316a527 --- /dev/null +++ b/k8s/cert-manager-values.yaml @@ -0,0 +1,3 @@ +installCRDs: false +prometheus: + enabled: false diff --git a/k8s/nginx-values.yaml b/k8s/nginx-values.yaml new file mode 100644 index 00000000..f60231b3 --- /dev/null +++ b/k8s/nginx-values.yaml @@ -0,0 +1,19 @@ +## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/index.md + +controller: + replicaCount: 2 + resources: + requests: + cpu: 100m + memory: 90Mi + service: # commented before certificate + annotations: + service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: "true" + service.beta.kubernetes.io/do-loadbalancer-tls-passthrough: "true" + config: # commented before certificate + use-proxy-protocol: "true" + keep-alive-requests: "10000" + upstream-keep-alive: "1000" + worker-processes: "auto" + max-worker-connections: "65535" + use-gzip: "true" diff --git a/k8s/web-deployment.yaml b/k8s/web-deployment.yaml new file mode 100644 index 00000000..57414077 --- /dev/null +++ b/k8s/web-deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-web + namespace: backend-web +spec: + replicas: 1 + selector: + matchLabels: + app: indexer-web + template: + metadata: + labels: + app: indexer-web + spec: + containers: + - name: backend-web + image: "gitcoinco/indexer-web:latest" + ports: + - name: http + containerPort: 8080 + envFrom: + - configMapRef: + name: indexer-web-envs + resources: + requests: + memory: "1Gi" # Request 1GB of memory + cpu: "500m" # Request 0.5 vCPU + limits: + memory: "2Gi" # Limit to 2GB of memory + cpu: "1000m" # Limit to 1 vCPU +--- +apiVersion: v1 +kind: Service +metadata: + name: indexer-web + namespace: backend-web +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app: indexer-web diff --git a/k8s/web-hpa-autoscaling.yaml b/k8s/web-hpa-autoscaling.yaml new file mode 100644 index 00000000..baf29066 --- /dev/null +++ b/k8s/web-hpa-autoscaling.yaml @@ -0,0 +1,20 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: indexer-web-hpa + namespace: backend-web + +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: indexer-web + minReplicas: 1 + maxReplicas: 3 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 # percentage diff --git a/k8s/web-ingress.yaml b/k8s/web-ingress.yaml new file mode 100644 index 00000000..25ca356f --- /dev/null +++ b/k8s/web-ingress.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-web + namespace: backend-web + annotations: #### commented before cert-manager was installed + cert-manager.io/issuer: letsencrypt-nginx +spec: + tls: #### commented before cert-manager was installed + - hosts: + - do.gitcoin-indexer.xyz + secretName: letsencrypt-nginx + rules: + - host: do.gitcoin-indexer.xyz + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: indexer-web + port: + number: 8080 + ingressClassName: nginx diff --git a/kb-deployment.yaml b/kb-deployment.yaml deleted file mode 100644 index 997b4423..00000000 --- a/kb-deployment.yaml +++ /dev/null @@ -1,22 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: indexer-web -spec: - replicas: 2 - selector: - matchLabels: - app: indexer-web - template: - metadata: - labels: - app: indexer-web - spec: - containers: - - name: indexer-web - image: "gitcoinco/indexer-web:latest" - ports: - - containerPort: 80 - envFrom: - - configMapRef: - name: indexer-web-config diff --git a/kb-service.yaml b/kb-service.yaml deleted file mode 100644 index 6cde5a7c..00000000 --- a/kb-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: load-balancer - labels: - app: indexer-web -spec: - type: LoadBalancer - ports: - - name: http - port: 80 - targetPort: 8080 - - name: https - port: 443 - targetPort: 8080 - selector: - app: indexer-web