Merge pull request #1627 from Mohanraj209/MOSIP-38032

[MOSIP-38032] Deployment of External services with helmsman using git…

.github/workflows/helmsman_external.yml

@@ -0,0 +1,147 @@
+name: Deploy External services of mosip using Helmsman
+
+on:
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "Choose Helmsman mode: dry-run or apply"
+        required: true
+        default: "dry-run"
+        type: choice
+        options:
+          - dry-run
+          - apply       
+  push:
+    paths:
+      - deployment/v3/helmsman/dsf/*
+
+jobs:
+  set-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout repository with full history
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Get full commit history
+
+      - name: Generate workflow matrix
+        id: set-matrix
+        run: |
+          matrix_json='{"include":[]}'
+          
+          if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then
+            matrix_json='{"include":[
+              {"dsf_files":"prereq-dsf.yaml","wg_conf":"wg0"},
+              {"dsf_files":"external-dsf.yaml","wg_conf":"wg1"}
+            ]}'
+          else
+            # Handle different event types properly
+            if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then
+              # For push events, use GitHub's provided SHAs
+              base_sha="${{ github.event.before}}"
+              head_sha="${{ github.sha }}"
+            elif [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+              # For PRs, compare against target branch
+              base_sha="${{ github.event.pull_request.base.sha }}"
+              head_sha="${{ github.event.pull_request.head.sha }}"
+            fi
+
+            # Get changed files safely
+            changed_files=$(git diff --name-only "$base_sha" "$head_sha" -- 'deployment/v3/helmsman/dsf/' || echo "")
+            
+            entries=()
+            
+            # Check for exact file paths
+            if echo "$changed_files" | grep -qx 'deployment/v3/helmsman/dsf/prereq-dsf.yaml'; then
+              entries+=('{"dsf_files":"prereq-dsf.yaml","wg_conf":"wg0"}')
+            fi
+      
+            if echo "$changed_files" | grep -qx 'deployment/v3/helmsman/dsf/external-dsf.yaml'; then
+              entries+=('{"dsf_files":"external-dsf.yaml","wg_conf":"wg1"}')
+            fi
+
+            if [ ${#entries[@]} -gt 0 ]; then
+              matrix_json="{\"include\":[$(IFS=,; echo "${entries[*]}")]}"
+            fi
+          fi
+
+          echo "matrix=$matrix_json" >> $GITHUB_OUTPUT
+  deploy:
+    runs-on: ubuntu-latest
+    needs: set-matrix
+    strategy:
+      matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
+    steps:
+      - name: Show matrix values
+        run: |
+          echo "Processing ${{ matrix.dsf_files }}"
+          echo "Using WireGuard config: ${{ matrix.wg_conf }}"
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set Default Mode
+        run: |
+          if [ -z "${{ github.event.inputs.mode }}" ]; then
+            echo "mode=apply" >> $GITHUB_ENV
+          else
+            echo "mode=${{ github.event.inputs.mode }}" >> $GITHUB_ENV
+          fi        
+
+      - name: Setup ufw firewall
+        run: |
+          sudo ufw enable
+          sudo ufw allow ssh
+          sudo ufw allow 51820/udp
+          sudo ufw status  
+
+      - name: Install WireGuard
+        run: sudo apt-get install -y wireguard
+
+      - name: Configure WireGuard
+        run: |
+          echo "${{ secrets.CLUSTER_WIREGUARD_WG0 }}" | sudo tee /etc/wireguard/wg0.conf
+          echo "${{ secrets.CLUSTER_WIREGUARD_WG1 }}" | sudo tee /etc/wireguard/wg1.conf
+
+      - name: Start WireGuard
+        run: |
+          sudo chmod 600 /etc/wireguard/${{ matrix.wg_conf }}.conf
+          sudo chmod 700 /etc/wireguard/
+          sudo chmod 644 /lib/systemd/system/[email protected]
+          sudo systemctl daemon-reload
+          sudo wg-quick up ${{ matrix.wg_conf }}
+          sudo wg show ${{ matrix.wg_conf }}
+
+      - name: Setup Helm
+        run: |
+          curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+          sudo chmod 700 get_helm.sh
+          sudo ./get_helm.sh
+          helm version --client
+
+      - name: Install Helmsman
+        run: |
+          curl -L https://github.com/Praqma/helmsman/releases/download/v3.17.1/helmsman_3.17.1_linux_amd64.tar.gz -o helmsman.tar.gz
+          tar xzf helmsman.tar.gz
+          sudo mv helmsman /usr/local/bin          
+
+      - name: Initiate helmsman to apply the DSF configurations.
+        env:
+          KUBECONFIG: ${{ secrets.KUBECONFIG }}
+        run: |
+          curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.22.0 TARGET_ARCH=x86_64 sh -
+          export PATH=$PWD/istio-1.22.0/bin:$PATH
+          curl -LO https://dl.k8s.io/release/v1.31.3/bin/linux/amd64/kubectl
+          chmod +x kubectl
+          mkdir -p ~/.local/bin
+          mv ./kubectl ~/.local/bin/kubectl
+          kubectl version --client        
+          mkdir -p $HOME/.kube
+          echo "${{ secrets.KUBECONFIG }}" > $HOME/.kube/config
+          chmod 400 $HOME/.kube/config
+          export KUBECONFIG=$HOME/.kube/config
+          kubectl config view
+          kubectl get nodes
+          helmsman --debug --${{ env.mode }} -f deployment/v3/helmsman/dsf/${{ matrix.dsf_files }} 

deployment/README.md

@@ -2,3 +2,4 @@
 
 * [Single click installer - V2](sandbox-v2)
 * [Production grade deployment - V3](v3)
+* [Helmsman-deployment - v3](v3/helmsman) : To-Do - Experimental

deployment/v3/helmsman/README.md

@@ -0,0 +1,67 @@
+# Helmsman
+
+Helmsman is a tool which is used to manage kubernetes deployment with helm chart, so basically it provides us a way to manage helm releases including installing, deleting and upgrading helm releases based on our requirement using helmsman configuration file/desired State File (DSF).
+
+### Explanation:
+Imagine that we have different applications that we want to run on kubernetes which has it’s own process on how to set it up and run it, so helmsman helps us to keep the instructions within the Desired State File (DSF) and follows the instructions present in configuration file to make sure that the applications/services are setup correctly.
+
+Like explaned in the above point we tell the helmsman what applications we want to run and how we want them to set up using simple Desired State File (DSF). helmsman uses a simple declarative TOML file to allow you to describe a desired state for your k8s applications. Alternatively YAML declaration is also acceptable.
+
+The Desired State File (DSF) tells the helmsman on how the deployments should be handled like “install this app”, “use this version of app” or “make sure this app is running on these many number of instances”.
+
+Helmsman sees what you desire, validates that your desire makes sense, compares it with the current state of Helm and figures out what to do to make your desire come true.
+
+
+### Working:
+Helmsman gets its directions to navigate from a declarative file called Desired State File (DSF) maintained by the user (Kubernetes admin) and is usually version controlled. DSFs follow a specification which allows user to define how to connect to a Kubernetes cluster, what namespaces to use/create, what Helm repos to use for finding charts, and what instances (aka releases) of the chart to be installed/deleted/rolled back/upgraded and with what input parameters.
+
+Helmsman interprets your wishes from the DSF and compares it to what’s running in the designated cluster. It is smart enough to figure out what changes need to be applied to make your wishes come true without maintaining/storing any additional information anywhere.
+
+Note: To get more information about desired state file(DSF) please check the README.md file located in dsf directory.
+
+### Installation:
+
+Please make sure the following are installed prior to using helmsman as a binary:
+
+* [kubectl](https://github.com/kubernetes/kubectl)
+* [helm](https://github.com/helm/helm) (helm >=v2.10.0 for helmsman >= 1.6.0, helm >=v3.0.0 for helmsman >=v3.0.0)
+* [helm-diff](https://github.com/databus23/helm-diff) (helmsman >= 1.6.0)
+
+Check the [releases](https://github.com/Praqma/Helmsman/releases) page for the different versions.
+```
+# on Linux
+curl -L https://github.com/Praqma/helmsman/releases/download/v3.17.0/helmsman_3.17.0_linux_amd64.tar.gz | tar zx
+# on MacOS
+curl -L https://github.com/Praqma/helmsman/releases/download/v3.17.0/helmsman_3.17.0_darwin_amd64.tar.gz | tar zx
+
+mv helmsman /usr/local/bin/helmsman
+```
+
+### Helmsman commands:
+
+The below commands can be exicuted manually via cmd terminal
+
+To plan without executing:
+
+```helmsman -f example.yaml```
+
+To plan and execute the plan:
+
+```helmsman --apply -f example.yaml```
+
+To show debugging details:
+
+```helmsman --debug --apply -f example.yaml```
+
+To run a dry-run:
+
+```helmsman --debug --dry-run -f example.yaml```
+
+To limit execution to specific application:
+
+```helmsman --debug --dry-run --target artifactory -f example.yaml```
+
+> **Note:**
+> - This directory is a **work-in-progress** and currently **experimental**.
+> - It is subject to changes as we continue to refine the deployment process.
+> - Contributions and feedback are welcome as part of ongoing development!

deployment/v3/helmsman/dsf/README.md

@@ -0,0 +1,34 @@
+# Desired state file (DSF)
+
+The helmsman configuration file is also called as “Desired State File (DSF)” which tells the helmsman what applications we want to run and how we want them to set up within kubernetes cluster. when we create a DSF for helmsman we’re essentially providing a set of rules that helmsman should follow to manage our applications on kubernetes cluster.
+
+The Instructions that include within DSF:
+
+* which helm chart to use.
+* configuration values
+* dependencies between the helm charts.
+* desired state.
+* environment specific configuration.
+
+### The desired state file consists of:
+
+* Metadata [Optional] -- metadata for any human reader of the desired state file.
+* Certificates [Optional] -- only needed when you want Helmsman to connect kubectl to your cluster for you.
+* Context [optional] -- define the context in which a DSF is used.
+* Settings [Optional] -- data about your k8s cluster and how to deploy Helm on it if needed.
+* Namespaces -- defines the namespaces where you want your Helm charts to be deployed.
+* Helm repos [Optional] -- defines the repos where you want to get Helm charts from.
+* Apps -- defines the applications/charts you want to manage in your cluster.
+
+### Deploy Pre-requisites and External services of mosip
+
+To deploy Pre-requisites and External services of mosip we have two dsf files i,e 
+
+* `prereq-dsf.yaml`: Installs Pre-requisites such as monitoring, logging, alerting, istio, httpbin and global_configmap.  
+* `external-dsf.yaml`: Intsalls all the External services of mosip.
+
+Make sure to update the above two dsf files with the required configuration changes as per the environement and also update the `global_configmap.yaml` file with the required domain's then initiate the `helmsman_external.yml` workflow file.
+
+### Note: 
+* Commit and push changes to this `deployment/v3/helmsman/dsf/` directory will automatically trigger the workflow to fetch the latest changes and apply to the cluster. 
+* Make sure to maintain seperate dsf files per each environment for maintainence and reproducability.