diff --git a/codefresh/.ci/values/defaults-hpa.yaml b/codefresh/.ci/values/defaults-hpa.yaml index 2b5a4bc0f..0c97b12b7 100644 --- a/codefresh/.ci/values/defaults-hpa.yaml +++ b/codefresh/.ci/values/defaults-hpa.yaml @@ -75,3 +75,7 @@ argo-platform: promotion-orchestrator: hpa: enabled: true + +mongodb: + migration: + enabled: true diff --git a/codefresh/Chart.lock b/codefresh/Chart.lock index 98cc11773..32cdef874 100644 --- a/codefresh/Chart.lock +++ b/codefresh/Chart.lock @@ -49,7 +49,7 @@ dependencies: version: 1.23.1 - name: cfsign repository: oci://quay.io/codefresh/charts - version: 1.8.8 + version: 1.8.9 - name: tasker-kubernetes repository: oci://quay.io/codefresh/charts version: 1.26.18 @@ -64,58 +64,58 @@ dependencies: version: 1.14.22 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfapi repository: oci://quay.io/codefresh/charts - version: 21.279.2 + version: 21.279.3 - name: cfui repository: oci://quay.io/codefresh/charts version: 14.98.27 @@ -142,16 +142,16 @@ dependencies: version: 0.8.10 - name: cf-platform-analytics repository: oci://quay.io/codefresh/charts - version: 0.49.87 + version: 0.50.2 - name: cf-platform-analytics repository: oci://quay.io/codefresh/charts - version: 0.49.87 + version: 0.50.2 - name: argo-platform repository: oci://quay.io/codefresh/charts - version: 1.3506.0 + version: 1.3524.0-onprem-ae70d8a - name: argo-hub-platform repository: oci://quay.io/codefresh/charts - version: 0.1.23 + version: 0.1.24 - name: cf-oidc-provider repository: oci://quay.io/codefresh/charts version: 0.0.16 @@ -167,5 +167,5 @@ dependencies: - name: salesforce-reporter repository: oci://quay.io/codefresh/charts version: 1.30.11 -digest: sha256:814b879b8e7b0b276c66b821c69c2c22febbbec4a30fed89117d50530ae0ea5e -generated: "2025-05-19T23:11:03.858637+03:00" +digest: sha256:657e1a67d919daf178a83ddd81736519ea6bdc9a1c0bfb2364dad583586c1c1b +generated: "2025-06-02T16:12:53.480633+03:00" diff --git a/codefresh/Chart.yaml b/codefresh/Chart.yaml index 78ea431b0..3de8d030d 100644 --- a/codefresh/Chart.yaml +++ b/codefresh/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: Helm Chart for Codefresh On-Prem name: codefresh -version: 2.8.0-rc.1 +version: 2.8.0-rc.2 keywords: - codefresh home: https://codefresh.io/ @@ -15,11 +15,15 @@ appVersion: 2.7.0 annotations: artifacthub.io/prerelease: "true" artifacthub.io/alternativeName: "codefresh-onprem" - artifacthub.io/containsSecurityUpdates: "true" + # artifacthub.io/containsSecurityUpdates: "true" # supported kinds are added, changed, deprecated, removed, fixed and security. artifacthub.io/changes: | - - kind: changed - description: "Initial 2.8.0 release" + - kind: fixed + description: "Fix dependencies version" + - kind: added + description: "(cf-api): Add ability to assign admin/platform permissions for user for specified group during sync" + - kind: fixed + description: "(platform-analytics/postgresql): Fix pg_partman configuration" dependencies: - name: cf-common repository: oci://quay.io/codefresh/charts @@ -97,7 +101,7 @@ dependencies: repository: oci://quay.io/codefresh/charts condition: context-manager.enabled - name: pipeline-manager - version: "~1.139.0" + version: "~3.139.0" repository: oci://quay.io/codefresh/charts condition: pipeline-manager.enabled - name: gitops-dashboard-manager @@ -211,7 +215,7 @@ dependencies: repository: oci://quay.io/codefresh/charts condition: cf-broadcaster.enabled - name: helm-repo-manager - version: "~0.21.0" + version: "*" repository: oci://quay.io/codefresh/charts condition: helm-repo-manager.enabled - name: hermes @@ -237,7 +241,7 @@ dependencies: repository: oci://quay.io/codefresh/charts condition: argo-platform.enabled - name: argo-platform - version: "~1.3507.0" + version: "1.3524.0-onprem-ae70d8a" repository: oci://quay.io/codefresh/charts condition: argo-platform.enabled - name: argo-hub-platform diff --git a/codefresh/README.md b/codefresh/README.md index bae555802..742e464ca 100644 --- a/codefresh/README.md +++ b/codefresh/README.md @@ -1,6 +1,6 @@ ## Codefresh On-Premises -![Version: 2.8.0](https://img.shields.io/badge/Version-2.8.0-informational?style=flat-square) ![AppVersion: 2.7.0](https://img.shields.io/badge/AppVersion-2.7.0-informational?style=flat-square) +![Version: 2.8.0-rc.2](https://img.shields.io/badge/Version-2.8.0--rc.2-informational?style=flat-square) ![AppVersion: 2.7.0](https://img.shields.io/badge/AppVersion-2.7.0-informational?style=flat-square) Helm chart for deploying [Codefresh On-Premises](https://codefresh.io/docs/docs/getting-started/intro-to-codefresh/) to Kubernetes. @@ -67,7 +67,7 @@ Helm chart for deploying [Codefresh On-Premises](https://codefresh.io/docs/docs/ - GCR Service Account JSON `sa.json` (provided by Codefresh, contact support@codefresh.io) - Firebase [Realtime Database URL](https://firebase.google.com/docs/database/web/start#create_a_database) with [legacy token](https://firebase.google.com/docs/database/rest/auth#legacy_tokens). See [Firebase Configuration](#firebase-configuration) - Valid TLS certificates for Ingress -- When [external](#external-postgressql) PostgreSQL is used, `pg_cron` and `pg_partman` extensions **must be enabled** for [analytics](https://codefresh.io/docs/docs/dashboards/home-dashboard/#pipelines-dashboard) to work (see [AWS RDS example](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/PostgreSQL_pg_cron.html#PostgreSQL_pg_cron.enable)) +- When [external](#external-postgressql) PostgreSQL is used, `pg_cron` and `pg_partman` extensions **must be enabled** for [analytics](https://codefresh.io/docs/docs/dashboards/home-dashboard/#pipelines-dashboard) to work (see [AWS RDS example](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/PostgreSQL_pg_cron.html#PostgreSQL_pg_cron.enable)). The `pg_cron` extension should be the 1.4 version or higher for Azure Postgres DB. ## Get Repo Info @@ -119,6 +119,9 @@ global: # name: my-secret # key: firebase-secret + # -- Enable auto-index creation in MongoDB + # This is required for first-time installations! + # For upgrades, you should set it to `false`! env: MONGOOSE_AUTO_INDEX: "true" MONGO_AUTOMATIC_INDEX_CREATION: "true" @@ -1202,7 +1205,7 @@ cfapi: ### Projects pipelines limit ```yaml -cfapi: +pipeline-manager: env: # Determines project's pipelines limit (default: 500) PROJECT_PIPELINES_LIMIT: 500 @@ -2032,7 +2035,7 @@ cfapi: #### Auto-index creation in MongoDB -[Auto-index creation in MongoDB](#enabling-auto-index-creation) +[Auto-index creation in MongoDB](#auto-index-creation-in-mongodb) #### ⚠️ New indexes in MongoDB @@ -2092,7 +2095,7 @@ Default MongoDB image is changed from 6.x to 7.x. If you run external MongoDB (i.e. [Atlas](https://cloud.mongodb.com)), it is **required** to upgrade it to 7.x after upgrading Codefresh On-Prem to 2.8.x. -For backward compatibility (in case you need to rollback to 6.x), you can set [`featureCompatibilityVersion`](https://www.mongodb.com/docs/v6.0/reference/command/setFeatureCompatibilityVersion/) to `6.0` in your values file. +- **Before the upgrade**, for backward compatibility (in case you need to rollback to 6.x), you should set [`featureCompatibilityVersion`](https://www.mongodb.com/docs/v6.0/reference/command/setFeatureCompatibilityVersion/) to `6.0` in your values file. ```yaml mongodb: @@ -2101,38 +2104,177 @@ mongodb: featureCompatibilityVersion: "6.0" ``` +- Perform Codefresh On-Prem upgrade to 2.8.x. Make sure all systems are up and running. + +- **After the upgrade**, if all system are stable, you need to set `featureCompatibilityVersion` to `7.0` in your values file and re-deploy the chart. + +```yaml +mongodb: + migration: + enabled: true + featureCompatibilityVersion: "7.0" +``` + +⚠️ ⚠️ ⚠️ If FCV (FeatureCompatibilityVersion) is managed by MongoDB itself (i.e. Atlas), you can disable it completely (that is default value in Helm chart) + +```yaml +mongodb: + migration: + enabled: false +``` + +#### ⚠️ New indexes in MongoDB + +If you maintain indexes manually (i.e. [Auto-index creation](#enabling-auto-index-creation) is off) you must create the following indexes **before** the upgrade: + +- [Database: `codefresh`, collection: `users`, index: `account_1__id_1`](https://github.com/codefresh-io/codefresh-onprem-helm/tree/release-2.8/indexes/codefresh/users.json#L2-L9) +- [Database: `codefresh`, collection: `users`, index: `role_1_account_1__id_1`](https://github.com/codefresh-io/codefresh-onprem-helm/tree/release-2.8/indexes/codefresh/users.json#L10-L17) + ### PostgreSQL update Default PostgreSQL image is changed from 13.x to 17.x If you run external PostgreSQL, follow the [official instructions](https://www.postgresql.org/docs/17/upgrading.html) to upgrade to 17.x. -⚠️ ⚠️ ⚠️ If you run built-in PostgreSQL `bitnami/postgresql` subchart, direct upgrade is not supported. You need to backup your data, delete the old PostgreSQL StatefulSet with PVCs and restore the data into a new PostgreSQL StatefulSet. +⚠️ ⚠️ ⚠️ If you run built-in PostgreSQL `bitnami/postgresql` subchart, direct upgrade is not supported due to **incompatible breaking changes** in the database files. You will see the following error in the logs: +``` +postgresql 17:36:28.41 INFO ==> ** Starting PostgreSQL ** +2025-05-21 17:36:28.432 GMT [1] FATAL: database files are incompatible with server +2025-05-21 17:36:28.432 GMT [1] DETAIL: The data directory was initialized by PostgreSQL version 13, which is not compatible with this version 17.2. +``` +You need to backup your data, delete the old PostgreSQL StatefulSet with PVCs and restore the data into a new PostgreSQL StatefulSet. + +- **Before the upgrade**, backup your data on a separate PVC + +- Create PVC with the same or bigger size as your current PostgreSQL PVC: + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgresql-dump +spec: + storageClassName: + resources: + requests: + storage: + volumeMode: Filesystem + accessModes: + - ReadWriteOnce +``` + +- Create a job to dump the data from the old PostgreSQL StatefulSet into the new PVC: + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: postgresql-dump +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + containers: + - name: postgresql-dump + image: quay.io/codefresh/postgresql:17 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "1" + env: + - name: PGUSER + value: "" + - name: PGPASSWORD + value: "" + - name: PGHOST + value: "" + - name: PGPORT + value: "" + command: + - "/bin/bash" + - "-c" + - | + pg_dumpall --verbose > /opt/postgresql-dump/dump.sql + volumeMounts: + - name: postgresql-dump + mountPath: /opt/postgresql-dump + securityContext: + runAsUser: 0 + fsGroup: 0 + volumes: + - name: postgresql-dump + persistentVolumeClaim: + claimName: postgresql-dump + restartPolicy: Never +``` + +- Delete old PostgreSQL StatefulSet and PVC ```console -PGUSER=postgres -PGHOST=cf-postgresql -PGPORT=5432 -PGPASSWORD=postgres -BACKUP_DIR=/tmp/pg_backup -BACKUP_SQL=backup.sql -TIMESTAMP=$(date +%Y%m%d%H%M%S) -NAMESPACE=codefresh - -# Backup PostgreSQL data -pg_dumpall --verbose > "$BACKUP_DIR/$BACKUP_SQL.$TIMESTAMP" 2>> "$LOG_FILE" - -# Delete old PostgreSQL StatefulSet STS_NAME=$(kubectl get sts -n $NAMESPACE -l app.kubernetes.io/instance=$RELEASE_NAME -l app.kubernetes.io/name=postgresql -o jsonpath='{.items[0].metadata.name}') PVC_NAME=$(kubectl get pvc -n $NAMESPACE -l app.kubernetes.io/instance=$RELEASE_NAME -l app.kubernetes.io/name=postgresql -o jsonpath='{.items[0].metadata.name}') kubectl delete sts $STS_NAME -n $NAMESPACE kubectl delete pvc $PVC_NAME -n $NAMESPACE +``` -# Perform Codefresh On-Prem upgrade to 2.8.x +- Peform the upgrade to 2.8.x with PostgreSQL seed job enabled to re-create users and databases -# Restore PostgreSQL data -psql -U -f "$BACKUP_DIR/$BACKUP_SQL.$TIMESTAMP" >> "$LOG_FILE" 2>&1 +```yaml +seed: + postgresSeedJob: + enabled: true +``` + +- Create a job to restore the data from the new PVC into the new PostgreSQL StatefulSet: + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: postgresql-restore +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + containers: + - name: postgresql-restore + image: quay.io/codefresh/postgresql:17 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "1" + env: + - name: PGUSER + value: "" + - name: PGPASSWORD + value: "" + - name: PGHOST + value: "" + - name: PGPORT + value: "" + command: + - "/bin/bash" + - "-c" + - | + psql -f /opt/postgresql-dump/dump.sql + volumeMounts: + - name: postgresql-dump + mountPath: /opt/postgresql-dump + securityContext: + runAsUser: 0 + fsGroup: 0 + volumes: + - name: postgresql-dump + persistentVolumeClaim: + claimName: postgresql-dump + restartPolicy: Never ``` ### RabbitMQ update @@ -2247,7 +2389,7 @@ After platform upgrade, Consul fails with the error `refusing to rejoin cluster | argo-platform.runtime-monitor | object | See below | runtime-monitor Don't enable! Not used in onprem! | | argo-platform.ui | object | See below | ui | | argo-platform.useExternalSecret | bool | `false` | Use regular k8s secret object. Keep `false`! | -| builder | object | `{"affinity":{},"container":{"image":{"registry":"docker.io","repository":"library/docker","tag":"28.0-dind"}},"enabled":true,"imagePullSecrets":[],"initContainers":{"register":{"image":{"registry":"us-docker.pkg.dev/codefresh-inc/public-gcr-io","repository":"codefresh/curl","tag":"8.11.1"}}},"nodeSelector":{},"podSecurityContext":{},"resources":{},"tolerations":[]}` | builder | +| builder | object | `{"affinity":{},"container":{"image":{"registry":"docker.io","repository":"library/docker","tag":"28.2-dind"}},"enabled":true,"imagePullSecrets":[],"initContainers":{"register":{"image":{"registry":"us-docker.pkg.dev/codefresh-inc/public-gcr-io","repository":"codefresh/curl","tag":"8.11.1"}}},"nodeSelector":{},"podSecurityContext":{},"resources":{},"tolerations":[]}` | builder | | cf-broadcaster | object | See below | broadcaster | | cf-oidc-provider | object | See below | cf-oidc-provider | | cf-platform-analytics-etlstarter | object | See below | etl-starter | diff --git a/codefresh/README.md.gotmpl b/codefresh/README.md.gotmpl index 2d3038ebe..2fc1306ac 100644 --- a/codefresh/README.md.gotmpl +++ b/codefresh/README.md.gotmpl @@ -68,7 +68,7 @@ Helm chart for deploying [Codefresh On-Premises](https://codefresh.io/docs/docs/ - GCR Service Account JSON `sa.json` (provided by Codefresh, contact support@codefresh.io) - Firebase [Realtime Database URL](https://firebase.google.com/docs/database/web/start#create_a_database) with [legacy token](https://firebase.google.com/docs/database/rest/auth#legacy_tokens). See [Firebase Configuration](#firebase-configuration) - Valid TLS certificates for Ingress -- When [external](#external-postgressql) PostgreSQL is used, `pg_cron` and `pg_partman` extensions **must be enabled** for [analytics](https://codefresh.io/docs/docs/dashboards/home-dashboard/#pipelines-dashboard) to work (see [AWS RDS example](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/PostgreSQL_pg_cron.html#PostgreSQL_pg_cron.enable)) +- When [external](#external-postgressql) PostgreSQL is used, `pg_cron` and `pg_partman` extensions **must be enabled** for [analytics](https://codefresh.io/docs/docs/dashboards/home-dashboard/#pipelines-dashboard) to work (see [AWS RDS example](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/PostgreSQL_pg_cron.html#PostgreSQL_pg_cron.enable)). The `pg_cron` extension should be the 1.4 version or higher for Azure Postgres DB. ## Get Repo Info @@ -120,6 +120,9 @@ global: # name: my-secret # key: firebase-secret + # -- Enable auto-index creation in MongoDB + # This is required for first-time installations! + # For upgrades, you should set it to `false`! env: MONGOOSE_AUTO_INDEX: "true" MONGO_AUTOMATIC_INDEX_CREATION: "true" @@ -795,7 +798,7 @@ Auto-index creation in MongoDB is disabled by default in Codefresh On-Prem to pr It is critical to ensure that your MongoDB indexes are always aligned with the latest recommended state for your Codefresh On-Prem version. Outdated or missing indexes can lead to degraded performance, slow queries, and increased resource consumption. Always review release notes and update or create indexes as specified during upgrades or when new collections/fields are introduced. Regularly auditing and maintaining your indexes helps ensure optimal system reliability and scalability. -The indexes list is located at the [codefresh-io/codefresh-onprem-helm](https://github.com/codefresh-io/codefresh-onprem-helm/tree/onprem-2.8.0/indexes) repository. +The indexes list is located at the [codefresh-io/codefresh-onprem-helm](https://github.com/codefresh-io/codefresh-onprem-helm/tree/release-2.8/indexes) repository. The indexes are stored in JSON files with keys and options specified. The directory structure is: @@ -837,6 +840,11 @@ Ref: - [Create an Index in Atlas DB](https://www.mongodb.com/docs/atlas/atlas-ui/indexes/#create-an-index) - [Create an Index with mongosh](https://www.mongodb.com/docs/manual/reference/method/db.collection.createIndex/) +##### ⚠️ Upgrading when the indexes are maintained manually + +If you maintain indexes manually and you upgrade your On-Prem installation you must create all indexes which were introduced from your current version up to the version you upgrade to manually. + +You can follow [Upgrading section](#upgrading) to see what changes were made for indexes in every specific release. ### High Availability @@ -1202,7 +1210,7 @@ cfapi: ### Projects pipelines limit ```yaml -cfapi: +pipeline-manager: env: # Determines project's pipelines limit (default: 500) PROJECT_PIPELINES_LIMIT: 500 @@ -1751,7 +1759,7 @@ nomios: ... ``` -### To 2.0.12 +### To 2-0-12 #### ⚠️ Legacy ChartMuseum subchart deprecation @@ -2038,6 +2046,12 @@ cfapi: [Auto-index creation in MongoDB](#auto-index-creation-in-mongodb) +#### ⚠️ New indexes in MongoDB + +If you maintain indexes manually (i.e. [Auto-index creation](#enabling-auto-index-creation) is off) you must create the following index **before** the upgrade: + +- [Database: `read-models`, collection: `images-binaries`, index: `accountId_1_imageName_1`](https://github.com/codefresh-io/codefresh-onprem-helm/tree/release-2.8/indexes/read-models/images-binaries.json#L75-L94) + ### To 2-7-0 ### [What's new in 2.7.x](https://codefresh.io/docs/docs/whats-new/on-prem-release-notes/#on-premises-version-27) @@ -2069,6 +2083,15 @@ global: - "value" ``` +#### ⚠️ New indexes in MongoDB + +If you maintain indexes manually (i.e. [Auto-index creation](#enabling-auto-index-creation) is off) you must create the following indexes **before or right after** the upgrade: + +> ⚠ **Note!** In case if you create indexes **before** the upgrade, please, create `codefresh.feature-store-versioned` collection manually in advance. + +- [Database: `codefresh`, collection: `feature-store-versioned`, index: `createdAt_1`](https://github.com/codefresh-io/codefresh-onprem-helm/tree/release-2.8/indexes/codefresh/feature-store-versioned.json#L2-L9) +- [Database: `codefresh`, collection: `feature-store-versioned`, index: `LDRedisStoreVersion_1__id_-1`](https://github.com/codefresh-io/codefresh-onprem-helm/tree/release-2.8/indexes/codefresh/feature-store-versioned.json#L10-L17) + ### To 2-8-0 ### [What's new in 2.8.x](https://codefresh.io/docs/docs/whats-new/on-prem-release-notes/#on-premises-version-28) @@ -2081,7 +2104,7 @@ Default MongoDB image is changed from 6.x to 7.x. If you run external MongoDB (i.e. [Atlas](https://cloud.mongodb.com)), it is **required** to upgrade it to 7.x after upgrading Codefresh On-Prem to 2.8.x. -For backward compatibility (in case you need to rollback to 6.x), you can set [`featureCompatibilityVersion`](https://www.mongodb.com/docs/v6.0/reference/command/setFeatureCompatibilityVersion/) to `6.0` in your values file. +- **Before the upgrade**, for backward compatibility (in case you need to rollback to 6.x), you should set [`featureCompatibilityVersion`](https://www.mongodb.com/docs/v6.0/reference/command/setFeatureCompatibilityVersion/) to `6.0` in your values file. ```yaml mongodb: @@ -2090,38 +2113,179 @@ mongodb: featureCompatibilityVersion: "6.0" ``` +- Perform Codefresh On-Prem upgrade to 2.8.x. Make sure all systems are up and running. + +- **After the upgrade**, if all system are stable, you need to set `featureCompatibilityVersion` to `7.0` in your values file and re-deploy the chart. + +```yaml +mongodb: + migration: + enabled: true + featureCompatibilityVersion: "7.0" +``` + +⚠️ ⚠️ ⚠️ If FCV (FeatureCompatibilityVersion) is managed by MongoDB itself (i.e. Atlas), you can disable it completely (that is default value in Helm chart) + +```yaml +mongodb: + migration: + enabled: false +``` + +#### ⚠️ New indexes in MongoDB + + +If you maintain indexes manually (i.e. [Auto-index creation](#enabling-auto-index-creation) is off) you must create the following indexes **before** the upgrade: + + +- [Database: `codefresh`, collection: `users`, index: `account_1__id_1`](https://github.com/codefresh-io/codefresh-onprem-helm/tree/release-2.8/indexes/codefresh/users.json#L2-L9) +- [Database: `codefresh`, collection: `users`, index: `role_1_account_1__id_1`](https://github.com/codefresh-io/codefresh-onprem-helm/tree/release-2.8/indexes/codefresh/users.json#L10-L17) + ### PostgreSQL update Default PostgreSQL image is changed from 13.x to 17.x If you run external PostgreSQL, follow the [official instructions](https://www.postgresql.org/docs/17/upgrading.html) to upgrade to 17.x. -⚠️ ⚠️ ⚠️ If you run built-in PostgreSQL `bitnami/postgresql` subchart, direct upgrade is not supported. You need to backup your data, delete the old PostgreSQL StatefulSet with PVCs and restore the data into a new PostgreSQL StatefulSet. +⚠️ ⚠️ ⚠️ If you run built-in PostgreSQL `bitnami/postgresql` subchart, direct upgrade is not supported due to **incompatible breaking changes** in the database files. You will see the following error in the logs: +``` +postgresql 17:36:28.41 INFO ==> ** Starting PostgreSQL ** +2025-05-21 17:36:28.432 GMT [1] FATAL: database files are incompatible with server +2025-05-21 17:36:28.432 GMT [1] DETAIL: The data directory was initialized by PostgreSQL version 13, which is not compatible with this version 17.2. +``` +You need to backup your data, delete the old PostgreSQL StatefulSet with PVCs and restore the data into a new PostgreSQL StatefulSet. + +- **Before the upgrade**, backup your data on a separate PVC + +- Create PVC with the same or bigger size as your current PostgreSQL PVC: + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgresql-dump +spec: + storageClassName: + resources: + requests: + storage: + volumeMode: Filesystem + accessModes: + - ReadWriteOnce +``` + +- Create a job to dump the data from the old PostgreSQL StatefulSet into the new PVC: + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: postgresql-dump +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + containers: + - name: postgresql-dump + image: quay.io/codefresh/postgresql:17 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "1" + env: + - name: PGUSER + value: "" + - name: PGPASSWORD + value: "" + - name: PGHOST + value: "" + - name: PGPORT + value: "" + command: + - "/bin/bash" + - "-c" + - | + pg_dumpall --verbose > /opt/postgresql-dump/dump.sql + volumeMounts: + - name: postgresql-dump + mountPath: /opt/postgresql-dump + securityContext: + runAsUser: 0 + fsGroup: 0 + volumes: + - name: postgresql-dump + persistentVolumeClaim: + claimName: postgresql-dump + restartPolicy: Never +``` + +- Delete old PostgreSQL StatefulSet and PVC ```console -PGUSER=postgres -PGHOST=cf-postgresql -PGPORT=5432 -PGPASSWORD=postgres -BACKUP_DIR=/tmp/pg_backup -BACKUP_SQL=backup.sql -TIMESTAMP=$(date +%Y%m%d%H%M%S) -NAMESPACE=codefresh - -# Backup PostgreSQL data -pg_dumpall --verbose > "$BACKUP_DIR/$BACKUP_SQL.$TIMESTAMP" 2>> "$LOG_FILE" - -# Delete old PostgreSQL StatefulSet STS_NAME=$(kubectl get sts -n $NAMESPACE -l app.kubernetes.io/instance=$RELEASE_NAME -l app.kubernetes.io/name=postgresql -o jsonpath='{.items[0].metadata.name}') PVC_NAME=$(kubectl get pvc -n $NAMESPACE -l app.kubernetes.io/instance=$RELEASE_NAME -l app.kubernetes.io/name=postgresql -o jsonpath='{.items[0].metadata.name}') kubectl delete sts $STS_NAME -n $NAMESPACE kubectl delete pvc $PVC_NAME -n $NAMESPACE +``` + +- Peform the upgrade to 2.8.x with PostgreSQL seed job enabled to re-create users and databases + +```yaml +seed: + postgresSeedJob: + enabled: true +``` -# Perform Codefresh On-Prem upgrade to 2.8.x +- Create a job to restore the data from the new PVC into the new PostgreSQL StatefulSet: -# Restore PostgreSQL data -psql -U -f "$BACKUP_DIR/$BACKUP_SQL.$TIMESTAMP" >> "$LOG_FILE" 2>&1 +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: postgresql-restore +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + containers: + - name: postgresql-restore + image: quay.io/codefresh/postgresql:17 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "1" + env: + - name: PGUSER + value: "" + - name: PGPASSWORD + value: "" + - name: PGHOST + value: "" + - name: PGPORT + value: "" + command: + - "/bin/bash" + - "-c" + - | + psql -f /opt/postgresql-dump/dump.sql + volumeMounts: + - name: postgresql-dump + mountPath: /opt/postgresql-dump + securityContext: + runAsUser: 0 + fsGroup: 0 + volumes: + - name: postgresql-dump + persistentVolumeClaim: + claimName: postgresql-dump + restartPolicy: Never ``` ### RabbitMQ update diff --git a/codefresh/files/mongoSetCompatibilityVersion.sh b/codefresh/files/mongoSetCompatibilityVersion.sh index b539b9e6f..b7a7ab3ba 100644 --- a/codefresh/files/mongoSetCompatibilityVersion.sh +++ b/codefresh/files/mongoSetCompatibilityVersion.sh @@ -16,6 +16,11 @@ waitForMongoDB() { done } +getMongoDBVersion() { + local version=$(mongosh ${MONGODB_ROOT_URI} --eval "db.version()" 2>&1) + echo $version +} + parseMongoURI() { local proto="$(echo $1 | grep :// | sed -e's,^\(.*://\).*,\1,g')" local parameters="$(echo $1 | grep '?' | cut -d '?' -f2)"; if [[ -n $parameters ]]; then parameters="?${parameters}"; fi @@ -33,4 +38,10 @@ parseMongoURI $MONGO_URI waitForMongoDB -mongosh ${MONGODB_ROOT_URI} --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"$MONGODB_COMPAT_VERSION\" } )" +MONGO_VERSION=$(getMongoDBVersion) + +if [[ $MONGO_VERSION == 7* ]]; then + mongosh ${MONGODB_ROOT_URI} --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"$MONGODB_COMPAT_VERSION\", confirm: true } )" +else + mongosh ${MONGODB_ROOT_URI} --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"$MONGODB_COMPAT_VERSION\" } )" +fi diff --git a/codefresh/templates/hooks/pre-upgrade/delete-consul-svc-job.yaml b/codefresh/templates/hooks/pre-upgrade/delete-consul-svc-job.yaml index 1e01d429a..4bc915e8f 100644 --- a/codefresh/templates/hooks/pre-upgrade/delete-consul-svc-job.yaml +++ b/codefresh/templates/hooks/pre-upgrade/delete-consul-svc-job.yaml @@ -38,7 +38,7 @@ spec: - /bin/sh - -c - | - set -e + set -x CONSUL_SVC_HEADLESS=$(kubectl get svc -n {{ .Release.Namespace }} \ -l app.kubernetes.io/instance={{ include "codefresh.fullname" . }}\ -l app.kubernetes.io/name=consul \ diff --git a/codefresh/values.yaml b/codefresh/values.yaml index 9a4b9df36..0d46ca0c9 100644 --- a/codefresh/values.yaml +++ b/codefresh/values.yaml @@ -487,11 +487,11 @@ runtimeImages: COMPOSE_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/compose:v2.32.2-1.5.2 CONTAINER_LOGGER_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/cf-container-logger:1.12.5 DIND_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/dind:26.1.4-1.28.8 - DOCKER_BUILDER_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/cf-docker-builder:1.4.4 + DOCKER_BUILDER_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/cf-docker-builder:1.4.6 DOCKER_PULLER_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/cf-docker-puller:8.0.20 DOCKER_PUSHER_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/cf-docker-pusher:6.0.17 - DOCKER_TAG_PUSHER_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/cf-docker-tag-pusher:1.3.15 - ENGINE_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/engine:1.177.8 + DOCKER_TAG_PUSHER_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/cf-docker-tag-pusher:1.3.17 + ENGINE_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/engine:1.178.0 FS_OPS_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/fs-ops:1.2.10 GIT_CLONE_IMAGE: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/cf-git-cloner:10.2.0 KUBE_DEPLOY: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/cf-deploy-kubernetes:16.2.6 @@ -733,7 +733,8 @@ cf-platform-analytics-etlstarter: container: image: - registry: us-docker.pkg.dev/codefresh-enterprise/gcr.io + registry: us-docker.pkg.dev/codefresh-inc/gcr.io + repository: codefresh/cf-platform-analytics env: ENV: production DATASET_ID: etl @@ -974,8 +975,8 @@ helm-repo-manager: ingress: enabled: false image: - repository: us-docker.pkg.dev/codefresh-inc/public-gcr-io/codefresh/chartmuseum - tag: 8795e993 + repository: ghcr.io/helm/chartmuseum + tag: v0.16.3 resources: requests: cpu: 100m @@ -1102,7 +1103,7 @@ mongodb: cpu: 200m memory: 256Mi migration: - enabled: true + enabled: false featureCompatibilityVersion: "6.0" # -- nats @@ -1251,7 +1252,7 @@ builder: image: registry: docker.io repository: library/docker - tag: 28.0-dind + tag: 28.2-dind affinity: {} nodeSelector: {} podSecurityContext: {} @@ -1273,7 +1274,7 @@ runner: image: registry: docker.io repository: library/docker - tag: 28.0-dind + tag: 28.2-dind affinity: {} nodeSelector: {} podSecurityContext: {} @@ -1776,6 +1777,8 @@ argo-platform: MONGODB_READMODELS_URI: '{{ include "argo-patform-libs.env-vars.mongodb-readmodels-uri-env-var-value" . }}' RABBITMQ_URLS: $(RABBITMQ_PROTOCOL)://$(RABBITMQ_USER):$(RABBITMQ_PASSWORD)@$(RABBITMQ_HOST) secrets: + v1-host: http://{{ .Release.Name }}-{{ .Values.global.cfapiEndpointsService }} + v1-internal-host: http://{{ .Release.Name }}-{{ .Values.global.cfapiService }}:{{ .Values.global.cfapiInternalPort }} mongodb-protocol: *mongodb-protocol mongodb-options: *mongodb-options mongodb-host: *mongodb-host diff --git a/indexes/codefresh/users.json b/indexes/codefresh/users.json new file mode 100644 index 000000000..64ea6d343 --- /dev/null +++ b/indexes/codefresh/users.json @@ -0,0 +1,19 @@ +[ + { + "v": 2.0, + "key": { + "account": 1.0, + "_id": 1.0 + }, + "name": "account_1__id_1" + }, + { + "v": 2.0, + "key": { + "role": 1.0, + "account": 1.0, + "_id": 1.0 + }, + "name": "role_1_account_1__id_1" + } +]