feat(mysql): tendbcluster元数据检查 #9214

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/mysql_cluster_topo/tendbcluster/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from .check import health_check

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/mysql_cluster_topo/tendbcluster/access_relate.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from typing import List
+
+from django.utils.translation import ugettext_lazy as _
+
+from backend.db_meta.enums import InstanceInnerRole, TenDBClusterSpiderRole
+from backend.db_meta.models import Cluster
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.check_response import CheckResponse
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.decorator import checker_wrapper
+from backend.db_report.enums import MetaCheckSubType
+
+
+@checker_wrapper
+def _cluster_spider_access_remote(c: Cluster) -> List[CheckResponse]:
+    """
+    master spider 只能访问 remote master
+    slave spider 只能访问 remote slave
+    mnt master spider 只能访问 remote master
+    mnt slave spider 只能访问 remote slave
+    """
+    bad = []
+    for pi in c.proxyinstance_set.all():
+        if pi.tendbclusterspiderext.spider_role in [
+            TenDBClusterSpiderRole.SPIDER_MASTER,
+            TenDBClusterSpiderRole.SPIDER_MNT,
+        ]:
+            can_access_remote_role = InstanceInnerRole.MASTER
+        elif pi.tendbclusterspiderext.spider_role in [
+            TenDBClusterSpiderRole.SPIDER_SLAVE,
+            TenDBClusterSpiderRole.SPIDER_SLAVE_MNT,
+        ]:
+            can_access_remote_role = InstanceInnerRole.SLAVE
+        else:
+            continue
+
+        for si in pi.storageinstance.all():
+            if si.instance_inner_role != can_access_remote_role:
+                bad.append(
+                    CheckResponse(
+                        msg=_(
+                            "{} 关联到 {}: {}".format(
+                                pi.tendbclusterspiderext.spider_role, si.instance_inner_role, si.ip_port
+                            )
+                        ),
+                        check_subtype=MetaCheckSubType.ClusterTopo,
+                        instance=pi,
+                    )
+                )
+
+    return bad

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/mysql_cluster_topo/tendbcluster/check.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from typing import List
+
+from backend.db_meta.enums import ClusterType
+from backend.db_meta.models import Cluster
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.check_response import CheckResponse
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.tendbcluster.access_relate import (
+    _cluster_spider_access_remote,
+)
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.tendbcluster.entry_bind import (
+    _cluster_entry_on_spider,
+    _cluster_entry_on_storage,
+)
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.tendbcluster.status import (
+    _cluster_master_remote_count,
+    _cluster_master_spider_count,
+)
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.tendbha.replicate import (
+    cluster_master_as_ejector,
+    cluster_replicate_out,
+    cluster_slave_as_receiver,
+)
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.tendbha.status import (
+    cluster_instance_status,
+    cluster_master_entry_count,
+    cluster_master_status,
+    cluster_standby_slave_status,
+    cluster_status,
+)
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.tendbha.unique_cluster import (
+    cluster_instance_unique_cluster,
+)
+
+
+def health_check(cluster_id: int) -> List[CheckResponse]:
+    """
+    集群状态正常
+    主入口数 >= 1
+    主 spider >= 2
+    master 状态正常
+    master 实例数和分片数一致
+    每个 master 实例唯一 standby slave
+    standby slave 状态正常
+    主/从入口 bind 的 spider 必须和正常 spider 数量一致
+    master spider 只能访问 remote master
+    slave spider 只能访问 remote slave
+    mnt master spider 只能访问 remote master
+    mnt slave spider 只能访问 remote slave
+    master 只能作为 ejector
+    slave 只能作为 receiver
+    不允许有到集群外部的同步关系
+    """
+    qs = Cluster.objects.filter(cluster_type=ClusterType.TenDBCluster).prefetch_related(
+        "clusterentry_set__proxyinstance_set",
+        "clusterentry_set__storageinstance_set",
+        "proxyinstance_set__storageinstance",
+        "storageinstance_set__as_receiver__ejector__cluster",
+        "storageinstance_set__as_ejector__receiver__cluster",
+        "storageinstance_set__cluster",
+        "proxyinstance_set__cluster",
+        "tendbclusterstorageset_set",
+    )
+    cluster_obj = qs.get(id=cluster_id)
+
+    res = []
+    # unique
+    res.extend(cluster_instance_unique_cluster(cluster_obj))
+    # status
+    res.extend(cluster_status(cluster_obj))
+    res.extend(cluster_instance_status(cluster_obj))
+    res.extend(cluster_master_entry_count(cluster_obj))
+    res.extend(_cluster_master_spider_count(cluster_obj))
+    res.extend(cluster_master_status(cluster_obj))
+    res.extend(_cluster_master_remote_count(cluster_obj))
+    res.extend(cluster_standby_slave_status(cluster_obj))
+    # bind
+    res.extend(_cluster_entry_on_spider(cluster_obj))
+    res.extend(_cluster_entry_on_storage(cluster_obj))
+    # access relate
+    res.extend(_cluster_spider_access_remote(cluster_obj))
+    # replicate
+    res.extend(cluster_master_as_ejector(cluster_obj))
+    res.extend(cluster_slave_as_receiver(cluster_obj))
+    res.extend(cluster_replicate_out(cluster_obj))
+    return res

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/mysql_cluster_topo/tendbcluster/entry_bind.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from typing import List
+
+from django.utils.translation import ugettext_lazy as _
+
+from backend.db_meta.enums import ClusterEntryRole, InstancePhase, InstanceStatus, TenDBClusterSpiderRole
+from backend.db_meta.models import Cluster
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.check_response import CheckResponse
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.decorator import checker_wrapper
+from backend.db_report.enums import MetaCheckSubType
+
+
+@checker_wrapper
+def _cluster_entry_on_spider(c: Cluster) -> List[CheckResponse]:
+    """
+    访问入口 bind 到 spider 的数量必须和集群正常 spider 相等
+    """
+    bad = []
+    for ce in c.clusterentry_set.all():
+
+        if ce.role == ClusterEntryRole.MASTER_ENTRY:
+            spider_role = TenDBClusterSpiderRole.SPIDER_MASTER
+        else:
+            spider_role = TenDBClusterSpiderRole.SPIDER_SLAVE
+
+        cnt = 0
+        for pi in c.proxyinstance_set.all():
+            if (
+                pi.status == InstanceStatus.RUNNING
+                and pi.phase == InstancePhase.ONLINE
+                and pi.tendbclusterspiderext.spider_role == spider_role
+            ):
+                cnt += 1
+
+        if cnt != ce.proxyinstance_set.count():
+            bad.append(
+                CheckResponse(
+                    msg=_("访问入口 {} 关联 {} 和集群 {} 数量不相等".format(ce.entry, spider_role, spider_role)),
+                    check_subtype=MetaCheckSubType.ClusterTopo,
+                )
+            )
+
+    return bad
+
+
+@checker_wrapper
+def _cluster_entry_on_storage(c: Cluster) -> List[CheckResponse]:
+    """
+    访问入口不能 bind 到存储
+    """
+    bad = []
+    for ce in c.clusterentry_set.all():
+        for si in ce.storageinstance_set.all():
+            bad.append(
+                CheckResponse(
+                    msg=_("访问入口 {} 关联到存储实例".format(ce.entry)), check_subtype=MetaCheckSubType.ClusterTopo, instance=si
+                )
+            )
+
+    return bad

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/mysql_cluster_topo/tendbcluster/status.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from typing import List
+
+from django.utils.translation import gettext_lazy as _
+
+from backend.db_meta.enums import InstanceInnerRole, InstancePhase, InstanceStatus, TenDBClusterSpiderRole
+from backend.db_meta.models import Cluster
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.check_response import CheckResponse
+from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.decorator import checker_wrapper
+from backend.db_report.enums import MetaCheckSubType
+
+
+@checker_wrapper
+def _cluster_master_spider_count(c: Cluster) -> List[CheckResponse]:
+    """
+    至少 2 个存活的 master spider
+    """
+    cnt = 0
+    for pi in c.proxyinstance_set.all():
+        if (
+            pi.status == InstanceStatus.RUNNING
+            and pi.phase == InstancePhase.ONLINE
+            and pi.tendbclusterspiderext.spider_role == TenDBClusterSpiderRole.SPIDER_MASTER
+        ):
+            cnt += 1
+
+    bad = []
+    if cnt < 2:
+        bad.append(CheckResponse(msg=_("正常 spider master 不足 2 个"), check_subtype=MetaCheckSubType.ClusterTopo))
+
+    return bad
+
+
+@checker_wrapper
+def _cluster_master_remote_count(c: Cluster) -> List[CheckResponse]:
+    """
+    master remote 数量等于分片数
+    """
+    bad = []
+
+    remote_master_count = 0
+    for si in c.storageinstance_set.all():
+        if si.instance_inner_role == InstanceInnerRole.MASTER:
+            remote_master_count += 1
+
+    shard_count = c.tendbclusterstorageset_set.count()
+    if shard_count != remote_master_count:
+        bad.append(
+            CheckResponse(
+                msg=_("分片数 {} != remote master 数 {}".format(shard_count, remote_master_count)),
+                check_subtype=MetaCheckSubType.ClusterTopo,
+            )
+        )
+
+    return bad
+
+
+@checker_wrapper
+def _cluster_one_standby_slave_each_shard(c: Cluster) -> List[CheckResponse]:
+    """
+    每个 shard 的 standby slave 是唯一的
+    """
+    bad = []
+
+    for si in c.storageinstance_set.all():
+        if si.instance_inner_role == InstanceInnerRole.MASTER:
+            m = []
+            for tp in si.as_ejector.all():
+                if tp.receiver.is_stand_by:
+                    m.append(tp.receiver)
+
+            if len(m) <= 0:
+                bad.append(
+                    CheckResponse(
+                        msg=_("无 standby slave"),
+                        check_subtype=MetaCheckSubType.ClusterTopo,
+                        instance=si,
+                    )
+                )
+
+            if len(m) > 1:
+                bad.append(
+                    CheckResponse(
+                        msg=_("standby slave 多余 1 个: {}".format(",".join([ele.ip_port for ele in m]))),
+                        check_subtype=MetaCheckSubType.ClusterTopo,
+                        instance=si,
+                    )
+                )
+
+    return bad