feat: private submodule repository (#348)

docs/profiles.md

@@ -148,3 +148,23 @@ that are potentially useful are:
    ```bash
     git config -f .gitmodules --unset submodule.xxx.branch
     ```
+
+## Private repository profiles
+
+To use private repositories as submodules, the pipeline must have access to
+the repository. This can be achieved by configuring the git credentials in the
+vault path `/secret/git` under a key named `credentials`. These credentials are
+usually in the form of a personal access token (PAT), which is put in the URL,
+which points to the base host of the repository, as in this example:
+`https://user:[email protected]`. The URL is set as the value for the
+`credentials` key:
+
+![vault git credentials](vault-git-credentials.png)
+
+If private repositories across multiple GIT servers are used, the credentials
+URL for each one must be listed in the value, separated by space.
+
+The recommended settings for the PAT are:
+
+- For `GitHub`: use fine-grained token, providing access only to the profile
+repositories with read permissions for: `Contents`, `Commit statuses`, and `Metadata`.

terraform/gitlab/ci-templates/k8s-cluster/.gitlab-ci.yml

@@ -38,6 +38,7 @@ variables:
   K8S_TEMPLATE_PATH: terraform/k8s
   TMP_TEMPLATES_DIR: /tmp/iac-templates
   TMP_GIT_REPO: /tmp/git-iac-templates
+  GIT_SUBMODULE_STRATEGY: none
   IGNORE_TF_DEPRECATION_WARNING: true
   KUBE_IN_CLUSTER_CONFIG: true
   VAULT_ADDR: http://vault-active.vault.svc.cluster.local:8200
@@ -57,22 +58,26 @@ cache:
 
 .source:
   script:
+    - export VAULT_TOKEN="$(vault write -field=token auth/${VAULT_AUTH_PATH}/login role=$VAULT_AUTH_ROLE jwt=$VAULT_ID_TOKEN)"
+    - export GIT_CREDENTIALS="$(vault kv get -field=credentials ${KV_SECRET_PATH}/git)"
+    - if [ "$GIT_CREDENTIALS" != "" ]; then echo "$GIT_CREDENTIALS" | tr ' ' '\n' > ~/.gitcredentials.store; fi
+    - git config --global credential.helper 'store --file ~/.gitcredentials.store'
+    - git submodule sync --recursive
+    - git submodule update --init --recursive
     - source $(which gitlab-terraform)
     - .gitlab/scripts/config-merge.sh
     - yq eval '.' $CONFIG_PATH/cluster-config.yaml -o=json > cluster-config.json
     - yq eval '.' $CONFIG_PATH/addons-vars.yaml -o=json > addons-vars.yaml
     - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./cluster-config.json); do export $var; done
     - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./addons-vars.yaml); do export $var; done
-    - export VAULT_TOKEN="$(vault write -field=token auth/${VAULT_AUTH_PATH}/login role=$VAULT_AUTH_ROLE jwt=$VAULT_ID_TOKEN)"
     - export ENV_VAULT_TOKEN="$(vault kv get -field=value ${KV_SECRET_PATH}/${CI_PROJECT_NAME}/env_token)"
     - export $cloud_platform_client_secret_name="$(vault kv get -field=value ${KV_SECRET_PATH}/${CI_PROJECT_NAME}/cloud_platform_client_secret)"
     - export GITLAB_CI_PAT="$(vault kv get -field=value ${KV_SECRET_PATH}/gitlab/gitlab_ci_pat)"
     - export NETBIRD_GW_SETUP_KEY="$(vault kv get -field=value ${KV_SECRET_PATH}/${CI_PROJECT_NAME}/netbird_gw_setup_key)"
     - export NETBIRD_K8S_SETUP_KEY="$(vault kv get -field=value ${KV_SECRET_PATH}/${CI_PROJECT_NAME}/netbird_k8s_setup_key)"
     - export KUBERNETES_OIDC_CLIENT_ID="$(vault kv get -field=value ${KV_SECRET_PATH}/${CI_PROJECT_NAME}/kubernetes_oidc_client_id)"
     - source setcivars.sh
-    - echo "https://${PRIVATE_REPO_USER}:${PRIVATE_REPO_TOKEN}@${PRIVATE_REPO}" > ~/.gitcredentials.store
-    - git config --global credential.helper 'store --file ~/.gitcredentials.store'
+    - echo "https://${PRIVATE_REPO_USER}:${PRIVATE_REPO_TOKEN}@${PRIVATE_REPO}" >> ~/.gitcredentials.store
     - git config --global advice.detachedHead false
     - source set-ansible-destroy-preq-vars.sh
     - export TERRAGRUNT_EXCLUDE_DIR="/dev/null"
@@ -234,7 +239,7 @@ refresh-deploy-infra:
   script:
     - !reference [.source, script]
     - .gitlab/scripts/refresh-repo.sh $cluster_name $TMP_GIT_REPO $TMP_TEMPLATES_DIR $GITLAB_CI_PAT $iac_terraform_modules_tag
-    - git checkout "$CI_COMMIT_REF_NAME" && git pull --recurse-submodules
+    - git checkout "$CI_COMMIT_REF_NAME" && git pull
     - !reference [.source, script]
     - terragrunt run-all apply --terragrunt-non-interactive -input=false
     - .gitlab/scripts/pushtorepo.sh $CI_PROJECT_PATH $CI_SERVER_HOST $CI_COMMIT_REF_NAME $GITOPS_BUILD_OUTPUT_DIR $GITLAB_CI_PAT $ARGO_CD_ROOT_APP_PATH

terraform/gitops/generate-files/templates/mcm/values-mcm.yaml.tpl

@@ -10,7 +10,7 @@ db:
 api:
   image:
     name: ghcr.io/pm4ml/connection-manager-api
-    version: v2.1.0
+    version: v2.4.0
   url: https://${mcm_fqdn}
   extraTLS:
     rootCert:
@@ -76,6 +76,8 @@ ui:
     clientId: ${oauth_key}
     clientSecretName: ${oauth_secret_secret}
     clientSecretKey: ${oauth_secret_secret_key}
+  image:
+    version: 1.8.4
 
 ingress:
 %{ if istio_create_ingress_gateways ~}

terraform/gitops/generate-files/templates/pm4ml/values-pm4ml.yaml.tpl

@@ -129,7 +129,7 @@ scheme-adapter:
       DFSP_ID: *dfspId
       CACHE_URL: redis://${redis_host}:${redis_port}
       AUTO_ACCEPT_QUOTES: false
-      AUTO_ACCEPT_PARTY: false
+      AUTO_ACCEPT_PARTY: ${auto_accept_party}
       AUTO_ACCEPT_R2P_PARTY: false
       AUTO_ACCEPT_R2P_BUSINESS_QUOTES: false
       AUTO_ACCEPT_R2P_DEVICE_OTP: false

terraform/gitops/pm4ml/pm4ml.tf

@@ -63,6 +63,7 @@ module "generate_pm4ml_files" {
     pm4ml_external_switch_client_secret_vault_value = "value"
     istio_external_gateway_name                     = var.istio_external_gateway_name
     cert_man_vault_cluster_issuer_name              = var.cert_man_vault_cluster_issuer_name
+    auto_accept_party                               = each.value.auto_accept_party
     enable_sdk_bulk_transaction_support             = each.value.enable_sdk_bulk_transaction_support
     kafka_host                                      = "kafka"
     kafka_port                                      = "9092"
@@ -233,6 +234,11 @@ variable "pm4ml_external_switch_client_secret" {
   default     = "pm4ml-external-switch-client-secret"
 }
 
+variable "auto_accept_party" {
+  type        = bool
+  description = "auto_accept_party"
+  default     = false
+}
 variable "enable_sdk_bulk_transaction_support" {
   type        = bool
   description = "enable_sdk_bulk_transaction_support"

terraform/k8s/default-config/cluster-config.yaml

@@ -8,8 +8,8 @@ cloud_platform_client_secret_name: AWS_SECRET_ACCESS_KEY
 k8s_cluster_module: base-k8s
 cloud_region: eu-west-1
 k8s_cluster_type: microk8s
-ansible_collection_tag: v5.3.9-rc01
-iac_terraform_modules_tag: v5.3.9-rc02
+ansible_collection_tag: v5.4.0-rc2
+iac_terraform_modules_tag: v5.4.0-rc2
 letsencrypt_email: [email protected]
 dns_zone_force_destroy: true
 longhorn_backup_object_store_destroy: true

terraform/k8s/default-config/mojaloop-vars.yaml

@@ -1,6 +1,6 @@
 bulk_enabled: false
 third_party_enabled: false
-mcm_chart_version: 1.1.0
+mcm_chart_version: 1.2.4
 mojaloop_chart_version: 16.0.0
 ttk_testcases_tag: 16.1.0
 currency: ${currency}

terraform/k8s/default-config/pm4ml-vars.yaml

@@ -6,6 +6,7 @@ pm4ml_external_switch_oidc_token_route: realms/dfsps/protocol/openid-connect/tok
 pm4ml_external_switch_client_secret_vault_path: "mcmdev_client_secret"
 pm4ml_ttk_enabled: true
 ttk_testcases_tag: v16.1.0
+auto_accept_party: false
 enable_sdk_bulk_transaction_support: false
 opentelemetry_enabled: false
 opentelemetry_namespace_filtering_enable: false