DPE-3389 support rollback incompat data dir (#385)

* rollback from incompatible data_dir

* port test

* remove vm fixture

* missing test parameter

* no need to patch for oci

* fixes

* remove dup error and properly translating exception

* always replace layer and fixed call ordering

* complete test and complete upgrade process

* fix/add test

* big timeout for recovery

* continuous writes are testes on other tests

* mitigate false-positive failure

* more robust test

* missing retry action

* fix list to exclude unit by name

Author: paulomach

src/charm.py

@@ -18,8 +18,6 @@
     BYTES_1MB,
     MySQLAddInstanceToClusterError,
     MySQLCharmBase,
-    MySQLConfigureInstanceError,
-    MySQLConfigureMySQLUsersError,
     MySQLCreateClusterError,
     MySQLGetClusterPrimaryAddressError,
     MySQLGetMemberStateError,
@@ -72,11 +70,7 @@
 )
 from k8s_helpers import KubernetesHelpers
 from log_rotate_manager import LogRotateManager
-from mysql_k8s_helpers import (
-    MySQL,
-    MySQLCreateCustomConfigFileError,
-    MySQLInitialiseMySQLDError,
-)
+from mysql_k8s_helpers import MySQL
 from relations.mysql import MySQLRelation
 from relations.mysql_provider import MySQLProvider
 from relations.mysql_root import MySQLRootRelation
@@ -276,7 +270,7 @@ def _is_unit_waiting_to_join_cluster(self) -> bool:
             and not self.unit_peer_data.get("unit-initialized")
         )
 
-    def _join_unit_to_cluster(self) -> None:
+    def join_unit_to_cluster(self) -> None:
         """Join the unit to the cluster.
 
         Try to join the unit from the primary unit.
@@ -485,45 +479,45 @@ def _open_ports(self) -> None:
             except ops.ModelError:
                 logger.exception("failed to open port")
 
-    def _configure_instance(self, container) -> bool:
-        """Configure the instance for use in Group Replication."""
-        try:
-            # Run mysqld for the first time to
-            # bootstrap the data directory and users
-            logger.debug("Initializing instance")
-            self._mysql.fix_data_dir(container)
-            self._mysql.initialise_mysqld()
-
-            # Add the pebble layer
-            logger.debug("Adding pebble layer")
-            container.add_layer(MYSQLD_SAFE_SERVICE, self._pebble_layer, combine=False)
-            container.restart(MYSQLD_SAFE_SERVICE)
-
-            logger.debug("Waiting for instance to be ready")
-            self._mysql.wait_until_mysql_connection(check_port=False)
-
-            logger.info("Configuring instance")
-            # Configure all base users and revoke privileges from the root users
-            self._mysql.configure_mysql_users()
-            # Configure instance as a cluster node
-            self._mysql.configure_instance()
+    def _write_mysqld_configuration(self):
+        """Write the mysqld configuration to the file."""
+        memory_limit_bytes = (self.config.profile_limit_memory or 0) * BYTES_1MB
+        new_config_content, _ = self._mysql.render_mysqld_configuration(
+            profile=self.config.profile,
+            memory_limit=memory_limit_bytes,
+        )
+        self._mysql.write_content_to_file(path=MYSQLD_CONFIG_FILE, content=new_config_content)
 
-            if self.has_cos_relation:
-                if container.get_services(MYSQLD_EXPORTER_SERVICE)[
-                    MYSQLD_EXPORTER_SERVICE
-                ].is_running():
-                    # Restart exporter service after configuration
-                    container.restart(MYSQLD_EXPORTER_SERVICE)
-                else:
-                    container.start(MYSQLD_EXPORTER_SERVICE)
-        except (
-            MySQLConfigureInstanceError,
-            MySQLConfigureMySQLUsersError,
-            MySQLInitialiseMySQLDError,
-            MySQLCreateCustomConfigFileError,
-        ) as e:
-            logger.debug("Unable to configure instance: {}".format(e))
-            return False
+    def _configure_instance(self, container) -> None:
+        """Configure the instance for use in Group Replication."""
+        # Run mysqld for the first time to
+        # bootstrap the data directory and users
+        logger.debug("Initializing instance")
+        self._mysql.fix_data_dir(container)
+        self._mysql.initialise_mysqld()
+
+        # Add the pebble layer
+        logger.debug("Adding pebble layer")
+        container.add_layer(MYSQLD_SAFE_SERVICE, self._pebble_layer, combine=True)
+        container.restart(MYSQLD_SAFE_SERVICE)
+
+        logger.debug("Waiting for instance to be ready")
+        self._mysql.wait_until_mysql_connection(check_port=False)
+
+        logger.info("Configuring instance")
+        # Configure all base users and revoke privileges from the root users
+        self._mysql.configure_mysql_users()
+        # Configure instance as a cluster node
+        self._mysql.configure_instance()
+
+        if self.has_cos_relation:
+            if container.get_services(MYSQLD_EXPORTER_SERVICE)[
+                MYSQLD_EXPORTER_SERVICE
+            ].is_running():
+                # Restart exporter service after configuration
+                container.restart(MYSQLD_EXPORTER_SERVICE)
+            else:
+                container.start(MYSQLD_EXPORTER_SERVICE)
 
         self._open_ports()
 
@@ -535,8 +529,6 @@ def _configure_instance(self, container) -> bool:
             # Do not block the charm if the version cannot be retrieved
             pass
 
-        return True
-
     def _mysql_pebble_ready_checks(self, event) -> bool:
         """Executes some checks to see if it is safe to execute the pebble ready handler."""
         if not self._is_peer_data_set:
@@ -560,17 +552,13 @@ def _on_mysql_pebble_ready(self, event) -> None:
             event.defer()
             return
 
+        if not self.upgrade.idle:
+            # when upgrading pebble ready is
+            # task delegated to upgrade code
+            return
+
         container = event.workload
-        try:
-            memory_limit_bytes = (self.config.profile_limit_memory or 0) * BYTES_1MB
-            new_config_content, _ = self._mysql.render_mysqld_configuration(
-                profile=self.config.profile,
-                memory_limit=memory_limit_bytes,
-            )
-            self._mysql.write_content_to_file(path=MYSQLD_CONFIG_FILE, content=new_config_content)
-        except MySQLCreateCustomConfigFileError:
-            logger.exception("Unable to write custom config file")
-            raise
+        self._write_mysqld_configuration()
 
         logger.info("Setting up the logrotate configurations")
         self._mysql.setup_logrotate_config()
@@ -585,15 +573,13 @@ def _on_mysql_pebble_ready(self, event) -> None:
         self.unit.status = MaintenanceStatus("Initialising mysqld")
 
         # First run setup
-        if not self._configure_instance(container):
-            raise
+        self._configure_instance(container)
 
         if not self.unit.is_leader():
             # Non-leader units should wait for leader to add them to the cluster
             self.unit.status = WaitingStatus("Waiting for instance to join the cluster")
             self.unit_peer_data.update({"member-role": "secondary", "member-state": "waiting"})
-
-            self._join_unit_to_cluster()
+            self.join_unit_to_cluster()
             return
 
         try:
@@ -607,7 +593,6 @@ def _on_mysql_pebble_ready(self, event) -> None:
             self.app_peer_data["units-added-to-cluster"] = "1"
 
             state, role = self._mysql.get_member_state()
-
             self.unit_peer_data.update(
                 {"member-state": state, "member-role": role, "unit-initialized": "True"}
             )
@@ -702,7 +687,7 @@ def _on_update_status(self, _: Optional[UpdateStatusEvent]) -> None:
         if not self.unit.is_leader() and self._is_unit_waiting_to_join_cluster():
             # join cluster test takes precedence over blocked test
             # due to matching criteria
-            self._join_unit_to_cluster()
+            self.join_unit_to_cluster()
             return
 
         if self._is_cluster_blocked():
@@ -748,7 +733,7 @@ def _on_peer_relation_changed(self, event: RelationChangedEvent) -> None:
             return
 
         if self._is_unit_waiting_to_join_cluster():
-            self._join_unit_to_cluster()
+            self.join_unit_to_cluster()
 
     def _on_database_storage_detaching(self, _) -> None:
         """Handle the database storage detaching event."""

src/mysql_k8s_helpers.py

@@ -16,6 +16,7 @@
     MySQLConfigureMySQLUsersError,
     MySQLExecError,
     MySQLGetClusterEndpointsError,
+    MySQLServiceNotRunningError,
     MySQLStartMySQLDError,
     MySQLStopMySQLDError,
 )
@@ -61,10 +62,6 @@ class MySQLInitialiseMySQLDError(Error):
     """Exception raised when there is an issue initialising an instance."""
 
 
-class MySQLServiceNotRunningError(Error):
-    """Exception raised when the MySQL service is not running."""
-
-
 class MySQLCreateCustomConfigFileError(Error):
     """Exception raised when there is an issue creating custom config file."""
 
@@ -229,10 +226,13 @@ def wait_until_mysql_connection(self, check_port: bool = True) -> None:
         Retry every 5 seconds for 30 seconds if there is an issue obtaining a connection.
         """
         if not self.container.exists(MYSQLD_SOCK_FILE):
-            raise MySQLServiceNotRunningError()
+            raise MySQLServiceNotRunningError
 
-        if check_port and not self.check_mysqlsh_connection():
-            raise MySQLServiceNotRunningError("Connection with mysqlsh not possible")
+        try:
+            if check_port and not self.check_mysqlsh_connection():
+                raise MySQLServiceNotRunningError("Connection with mysqlsh not possible")
+        except MySQLClientError:
+            raise MySQLServiceNotRunningError
 
         logger.debug("MySQL connection possible")
 
@@ -852,3 +852,7 @@ def set_cluster_primary(self, new_primary_address: str) -> None:
         """Set the cluster primary and update pod labels."""
         super().set_cluster_primary(new_primary_address)
         self.update_endpoints()
+
+    def fetch_error_log(self) -> Optional[str]:
+        """Fetch the MySQL error log."""
+        return self.read_file_content("/var/log/mysql/error.log")

src/relations/mysql_provider.py

@@ -216,6 +216,12 @@ def _on_update_status(self, _) -> None:
         if self.charm._is_cluster_blocked():
             return
 
+        if self.charm.upgrade.state == "failed":
+            # skip updating endpoints if upgrade failed
+            # unit pod still will be labeled from another unit
+            logger.debug("Skip labelling pods on failed upgrade")
+            return
+
         container = self.charm.unit.get_container(CONTAINER_NAME)
         if (
             not container.can_connect()

src/rotate_mysql_logs.py

@@ -43,7 +43,12 @@ def __init__(self, charm: "MySQLOperatorCharm"):
 
     def _rotate_mysql_logs(self, _) -> None:
         """Rotate the mysql logs."""
-        if self.charm.peers is None or self.charm.unit_peer_data.get("unit-initialized") != "True":
+        if (
+            self.charm.peers is None
+            or self.charm.unit_peer_data.get("unit-initialized") != "True"
+            or not self.charm.upgrade.idle
+        ):
+            # skip when not initialized or during an upgrade
             return
 
         try:

src/upgrade.py

@@ -15,19 +15,24 @@
 )
 from charms.mysql.v0.mysql import (
     MySQLGetMySQLVersionError,
+    MySQLRebootFromCompleteOutageError,
     MySQLRescanClusterError,
+    MySQLServerNotUpgradableError,
+    MySQLServiceNotRunningError,
     MySQLSetClusterPrimaryError,
     MySQLSetVariableError,
 )
-from ops import JujuVersion
+from ops import Container, JujuVersion
 from ops.model import BlockedStatus, MaintenanceStatus, RelationDataContent
+from ops.pebble import ChangeError
 from pydantic import BaseModel
 from tenacity import RetryError, Retrying
 from tenacity.stop import stop_after_attempt
 from tenacity.wait import wait_fixed
 from typing_extensions import override
 
 import k8s_helpers
+from constants import MYSQLD_SAFE_SERVICE
 
 if TYPE_CHECKING:
     from charm import MySQLOperatorCharm
@@ -218,27 +223,45 @@ def _on_pebble_ready(self, event) -> None:
         if self.state not in ["upgrading", "recovery"]:
             return
 
+        container = event.workload
+        self.charm._write_mysqld_configuration()
+
+        logger.info("Setting up the logrotate configurations")
+        self.charm._mysql.setup_logrotate_config()
+
         try:
-            self.charm.unit.set_workload_version(self.charm._mysql.get_mysql_version() or "unset")
-        except MySQLGetMySQLVersionError:
-            # don't fail on this, just log it
-            logger.warning("Failed to get MySQL version")
-        try:
-            failure_message = "unknown error"
+            self.charm._reconcile_pebble_layer(container)
             self._check_server_upgradeability()
             self.charm.unit.status = MaintenanceStatus("recovering unit after upgrade")
             if self.charm.app.planned_units() > 1:
                 self._recover_multi_unit_cluster()
             else:
                 self._recover_single_unit_cluster()
             self._complete_upgrade()
-        except Exception:
-            failure_message = "Unit failed to rejoin the cluster after upgrade"
-            logger.error(failure_message)
+        except MySQLRebootFromCompleteOutageError:
+            logger.error("Failed to reboot single unit from outage after upgrade")
             self.set_unit_failed()
             self.charm.unit.status = BlockedStatus(
                 "upgrade failed. Check logs for rollback instruction"
             )
+        except (
+            RetryError,
+            MySQLServerNotUpgradableError,
+            MySQLServiceNotRunningError,
+            ChangeError,
+        ):
+            # Failed to recover unit
+            if (
+                not self._check_server_unsupported_downgrade()
+                or self.charm.app.planned_units() == 1
+            ):
+                # don't try to recover single unit cluster or errors other then downgrade
+                logger.error("Unit failed to rejoin the cluster after upgrade")
+                self.set_unit_failed()
+                return
+            logger.warning("Downgrade is incompatible. Resetting workload")
+            self._reset_on_unsupported_downgrade(container)
+            self._complete_upgrade()
 
     def _recover_multi_unit_cluster(self) -> None:
         logger.debug("Recovering unit")
@@ -265,6 +288,11 @@ def _recover_single_unit_cluster(self) -> None:
     def _complete_upgrade(self):
         # complete upgrade for the unit
         logger.debug("Upgraded unit is healthy. Set upgrade state to `completed`")
+        try:
+            self.charm.unit.set_workload_version(self.charm._mysql.get_mysql_version() or "unset")
+        except MySQLGetMySQLVersionError:
+            # don't fail on this, just log it
+            logger.warning("Failed to get MySQL version")
         self.set_unit_completed()
         if self.charm.unit_label == f"{self.charm.app.name}/1":
             # penultimate unit, reset the primary for faster switchover
@@ -301,3 +329,27 @@ def _check_server_upgradeability(self) -> None:
         instance = self.charm._get_unit_fqdn(f"{self.charm.app.name}/0")
         self.charm._mysql.verify_server_upgradable(instance=instance)
         logger.debug("MySQL server is upgradeable")
+
+    def _check_server_unsupported_downgrade(self) -> bool:
+        """Check error log for unsupported downgrade.
+
+        https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html
+        """
+        if log_content := self.charm._mysql.fetch_error_log():
+            return "MY-013171" in log_content
+
+        return False
+
+    def _reset_on_unsupported_downgrade(self, container: Container) -> None:
+        """Reset the cluster on unsupported downgrade."""
+        container.stop(MYSQLD_SAFE_SERVICE)
+        self.charm._mysql.reset_data_dir()
+        self.charm._write_mysqld_configuration()
+        self.charm._configure_instance(container)
+        # reset flags
+        self.charm.unit_peer_data.update({"member-role": "secondary", "member-state": "waiting"})
+        # rescan is needed to remove the instance old incarnation from the cluster
+        leader = self.charm._get_primary_from_online_peer()
+        self.charm._mysql.rescan_cluster(from_instance=leader, remove_instances=True)
+        # rejoin after
+        self.charm.join_unit_to_cluster()