From 425518297fcf941d74a34dd6c0e07747f255f6b6 Mon Sep 17 00:00:00 2001 From: "alexey.lazarenko" Date: Mon, 29 Aug 2022 17:26:36 +0300 Subject: [PATCH 1/3] JE-62800 [DBaaS recovery MS2] MySQL&MariaDB cluster health check and recovery: subscribe cluster status validation on CS events --- addons/recovery/manifest.yml | 91 ++++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 20 deletions(-) diff --git a/addons/recovery/manifest.yml b/addons/recovery/manifest.yml index 8f19c8f4..4da589ed 100644 --- a/addons/recovery/manifest.yml +++ b/addons/recovery/manifest.yml @@ -7,11 +7,11 @@ description: short: The add-on performs a comprehensive diagnostic of your database cluster and can perform automatic recovery. logo: /images/database-recovery.png -baseUrl: https://raw.githubusercontent.com/jelastic-jps/mysql-cluster/JE-63333/addons/recovery +baseUrl: https://raw.githubusercontent.com/jelastic-jps/mysql-cluster/JE-62800-test/addons/recovery targetNodes: nodeGroup: sqldb - + mixins: - https://raw.githubusercontent.com/jelastic-jps/mysql-cluster/v2.5.0/scripts/common.yml @@ -19,6 +19,14 @@ globals: ssh_root: /root/.ssh/ ssh-key: id_rsa_db_monitoring tmp: /tmp/ + DIAGNOSTIC_FAILED: |- + Errors were discovered during the Database Cluster diagnostic. + Please check the **/var/log/db_recovery.log** log file for details. Click the "Cluster Recovery" button in the add-on's tab for automatic recovery. + Follow the [Manual Recovery](https://github.com/jelastic-jps/mysql-cluster/blob/master/addons/recovery/docs/ManualRecoveryGuide.md) guide to recover the cluster manually. + RECOVERY_FAILED: |- + An error occurs while recovering the Database Cluster. + Please check the **/var/log/db_recovery.log** log file for details. + Follow the [Manual Recovery](https://github.com/jelastic-jps/mysql-cluster/blob/master/addons/recovery/docs/ManualRecoveryGuide.md) guide to recover the cluster manually. buttons: - name: diagnostic @@ -50,16 +58,64 @@ settings: inputType: password required: true -onAfterScaleOut[sqldb]: generate-share-keys +onAfterScaleOut[sqldb]: generate-share-keys -onAfterRedeployContainer[sqldb]: generate-share-keys +onBeforeStart: + if (!${event.params.internal:false}): + - api: env.control.StartEnv + internal: true + - diagnostic: + event: true + - if (response.isDiagnosticsResultOk == false): + stopEvent: + type: warning + message: ${globals.DIAGNOSTIC_FAILED} + +onBeforeRestartNode[sqldb]: + if (!${event.params.internal:false}): + - api: env.control.RestartNodes + nodeGroup: sqldb + internal: true + - diagnostic: + event: true + - if (response.isDiagnosticsResultOk == false): + stopEvent: + type: warning + message: ${globals.DIAGNOSTIC_FAILED} + +onBeforeChangeTopology: + if (!${event.params.internal:false}): + - api: env.control.ChangeTopology + envName: ${env.name} + env: ${event.params.env} + nodes: ${event.params.nodes} + internal: true + - diagnostic: + event: true + - if (response.isDiagnosticsResultOk == false): + stopEvent: + type: warning + message: ${globals.DIAGNOSTIC_FAILED} -onAfterClone: - install: ${baseUrl}/manifest.yml?_r=${fn.random} - envName: ${event.response.env.envName} - nodeGroup: ${targetNodes.nodeGroup} - settings: - user: ${settings.user} +onBeforeRedeployContainer[sqldb]: + - generate-share-keys + - if (!${event.params.internal:false}): + - api: env.control.RedeployContainers + nodeGroup: sqldb + internal: true + - diagnostic: + event: true + - if (response.isDiagnosticsResultOk == false): + stopEvent: + type: warning + message: ${globals.DIAGNOSTIC_FAILED} + +onAfterClone: + install: ${baseUrl}/manifest.yml?_r=${fn.random} + envName: ${event.response.env.envName} + nodeGroup: ${targetNodes.nodeGroup} + settings: + user: ${settings.user} password: ${settings.password} onInstall: @@ -79,18 +135,12 @@ responses: message: Database Cluster recovery has been finished successfully! 98: - type: warning - message: | - An error occurs while recovering the Database Cluster. - Please check the **/var/log/db_recovery.log** log file for details. - Follow the [Manual Recovery](https://github.com/jelastic-jps/mysql-cluster/blob/master/addons/recovery/docs/ManualRecoveryGuide.md) guide to recover the cluster manually. + type: success + message: ${globals.RECOVERY_FAILED} 99: - type: warning - message: | - Errors were discovered during the Database Cluster diagnostic. - Please check the **/var/log/db_recovery.log** log file for details. Click the "Cluster Recovery" button in the add-on's tab for automatic recovery. - Follow the [Manual Recovery](https://github.com/jelastic-jps/mysql-cluster/blob/master/addons/recovery/docs/ManualRecoveryGuide.md) guide to recover the cluster manually. + type: success + message: ${globals.DIAGNOSTIC_FAILED} actions: clusterValidate: @@ -124,6 +174,7 @@ actions: diagnostic: script: /scripts/db-recovery.js exec: ' --diagnostic' + event: ${this.event:} recovery: script: /scripts/db-recovery.js From b214df8101f1b34d154a0e1b60f95b73ee0bcce2 Mon Sep 17 00:00:00 2001 From: "alexey.lazarenko" Date: Mon, 29 Aug 2022 17:27:02 +0300 Subject: [PATCH 2/3] JE-62800 [DBaaS recovery MS2] MySQL&MariaDB cluster health check and recovery: subscribe cluster status validation on CS events --- addons/recovery/scripts/db-recovery.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/addons/recovery/scripts/db-recovery.js b/addons/recovery/scripts/db-recovery.js index e5294621..fdb2b27c 100644 --- a/addons/recovery/scripts/db-recovery.js +++ b/addons/recovery/scripts/db-recovery.js @@ -1,10 +1,11 @@ var SQLDB = "sqldb", AUTH_ERROR_CODE = 701, UNABLE_RESTORE_CODE = 98, - FAILED_CLUSTER_CODE = 99, + FAILED_CLUSTER_CODE = 97, RESTORE_SUCCESS = 201, envName = "${env.name}", exec = getParam('exec', ''), + isEvent = getParam('event', false), failedNodes = [], isMasterFailed = false, GALERA = "galera", @@ -95,6 +96,10 @@ if (isRestore) { } } else { + if (isEvent) { + resp.result = 0; + resp.isDiagnosticsResultOk = false; + } return resp; } From 00595ec392f03eb3b810dce33ab05c97607f3aaf Mon Sep 17 00:00:00 2001 From: "alexey.lazarenko" Date: Mon, 29 Aug 2022 17:27:39 +0300 Subject: [PATCH 3/3] Update manifest.yml --- addons/recovery/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/recovery/manifest.yml b/addons/recovery/manifest.yml index 4da589ed..23b99343 100644 --- a/addons/recovery/manifest.yml +++ b/addons/recovery/manifest.yml @@ -7,7 +7,7 @@ description: short: The add-on performs a comprehensive diagnostic of your database cluster and can perform automatic recovery. logo: /images/database-recovery.png -baseUrl: https://raw.githubusercontent.com/jelastic-jps/mysql-cluster/JE-62800-test/addons/recovery +baseUrl: https://raw.githubusercontent.com/jelastic-jps/mysql-cluster/JE-62800/addons/recovery targetNodes: nodeGroup: sqldb