Skip to content

Commit

Permalink
Enhance the default configurations for TiFlash (#4358)
Browse files Browse the repository at this point in the history
  • Loading branch information
@KanShiori
KanShiori authored Jan 11, 2022
1 parent 8e12a86 commit 9eb0658
Show file tree
Hide file tree
Showing 7 changed files with 664 additions and 12 deletions.
2 changes: 1 addition & 1 deletion pkg/manager/member/tiflash_member_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -611,7 +611,7 @@ func flashVolumeClaimTemplate(storageClaims []v1alpha1.StorageClaim) ([]corev1.P
}

func getTiFlashConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) {
config := getTiFlashConfig(tc)
config := GetTiFlashConfig(tc)

configText, err := config.Common.MarshalTOML()
if err != nil {
Expand Down
14 changes: 3 additions & 11 deletions pkg/manager/member/tiflash_upgrader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ import (
"github.com/pingcap/tidb-operator/pkg/controller"
mngerutils "github.com/pingcap/tidb-operator/pkg/manager/utils"
"github.com/pingcap/tidb-operator/pkg/tiflashapi"
"github.com/pingcap/tidb-operator/pkg/util/cmpver"

"github.com/Masterminds/semver"
"github.com/pingcap/advanced-statefulset/client/apis/apps/v1/helper"
apps "k8s.io/api/apps/v1"
"k8s.io/klog/v2"
Expand All @@ -31,8 +31,7 @@ import (
var (
// the first version that tiflash support `tiflash/store-status` api.
// https://github.com/pingcap/tidb-operator/issues/4159
tiflashEqualOrGreaterThanV512, _ = semver.NewConstraint(">=v5.1.2-0")
tiflashVersionsNeedCheckStatus = map[string]struct{}{"lastest": {}, "nightly": {}}
tiflashEqualOrGreaterThanV512, _ = cmpver.NewConstraint(cmpver.GreaterOrEqual, "v5.1.2")
)

type tiflashUpgrader struct {
Expand Down Expand Up @@ -113,14 +112,7 @@ func (u *tiflashUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Statefu
return controller.RequeueErrorf("tidbcluster: [%s/%s]'s upgraded TiFlash pod: [%s], store state is not UP", ns, tcName, podName)
}

needCheckStatus := false
tiflashVersion := tc.TiFlashVersion()
if _, ok := tiflashVersionsNeedCheckStatus[tiflashVersion]; ok {
needCheckStatus = true
} else if ver, err := semver.NewVersion(tiflashVersion); err == nil && tiflashEqualOrGreaterThanV512.Check(ver) { // NOTE: if parse image version failed, will skip this check
needCheckStatus = true
}
if needCheckStatus {
if larger, err := tiflashEqualOrGreaterThanV512.Check(tc.TiFlashVersion()); err == nil && larger {
status, err := u.deps.TiFlashControl.GetTiFlashPodClient(tc.Namespace, tc.Name, podName, tc.IsTLSClusterEnabled()).GetStoreStatus()
if err != nil {
return controller.RequeueErrorf("tidbcluster: [%s/%s]'s upgraded TiFlash pod: [%s], get store status failed: %s", ns, tcName, podName, err)
Expand Down
12 changes: 12 additions & 0 deletions pkg/manager/member/tiflash_upgrader_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@ func TestTiFlashUpgraderUpgrade(t *testing.T) {
store := tc.Status.TiFlash.Stores["3"]
store.LeaderCount = 0
tc.Status.TiFlash.Stores["3"] = store
fakeClient := NewFakeTiKVClient(tiflashControl, tc, "upgrader-tiflash-2")
fakeClient.AddReaction(tiflashapi.GetStoreStatusActionType, func(action *tiflashapi.Action) (interface{}, error) {
return tiflashapi.Running, nil
})
},
changeOldSet: func(oldSet *apps.StatefulSet) {
mngerutils.SetStatefulSetLastAppliedConfigAnnotation(oldSet)
Expand All @@ -162,6 +166,14 @@ func TestTiFlashUpgraderUpgrade(t *testing.T) {
store := tc.Status.TiFlash.Stores["2"]
store.LeaderCount = 0
tc.Status.TiFlash.Stores["2"] = store
fakeClient := NewFakeTiKVClient(tiflashControl, tc, "upgrader-tiflash-2")
fakeClient.AddReaction(tiflashapi.GetStoreStatusActionType, func(action *tiflashapi.Action) (interface{}, error) {
return tiflashapi.Running, nil
})
fakeClient = NewFakeTiKVClient(tiflashControl, tc, "upgrader-tiflash-1")
fakeClient.AddReaction(tiflashapi.GetStoreStatusActionType, func(action *tiflashapi.Action) (interface{}, error) {
return tiflashapi.Running, nil
})
},
changeOldSet: func(oldSet *apps.StatefulSet) {
mngerutils.SetStatefulSetLastAppliedConfigAnnotation(oldSet)
Expand Down
117 changes: 117 additions & 0 deletions pkg/manager/member/tiflash_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (

"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/util/cmpver"

corev1 "k8s.io/api/core/v1"
)

Expand All @@ -30,6 +32,11 @@ const (
defaultServerLog = "/data0/logs/server.log"
)

var (
// the first version that tiflash change default config
tiflashEqualOrGreaterThanV540, _ = cmpver.NewConstraint(cmpver.GreaterOrEqual, "v5.4.0")
)

func buildTiFlashSidecarContainers(tc *v1alpha1.TidbCluster) ([]corev1.Container, error) {
spec := tc.Spec.TiFlash
config := spec.Config.DeepCopy()
Expand Down Expand Up @@ -99,6 +106,116 @@ func buildSidecarContainer(name, path, image string,
}
}

func GetTiFlashConfig(tc *v1alpha1.TidbCluster) *v1alpha1.TiFlashConfigWraper {
version := tc.TiFlashVersion()
if ok, err := tiflashEqualOrGreaterThanV540.Check(version); err == nil && ok {
return getTiFlashConfigV2(tc)
}
return getTiFlashConfig(tc)
}

func getTiFlashConfigV2(tc *v1alpha1.TidbCluster) *v1alpha1.TiFlashConfigWraper {
config := tc.Spec.TiFlash.Config.DeepCopy()
if config == nil {
config = v1alpha1.NewTiFlashConfig()
}
if config.Common == nil {
config.Common = v1alpha1.NewTiFlashCommonConfig()
}
if config.Proxy == nil {
config.Proxy = v1alpha1.NewTiFlashProxyConfig()
}
common := config.Common
proxy := config.Proxy

name := tc.Name
ns := tc.Namespace
clusterDomain := tc.Spec.ClusterDomain
ref := tc.Spec.Cluster.DeepCopy()
noLocalPD := tc.HeterogeneousWithoutLocalPD()
noLocalTiDB := ref != nil && ref.Name != "" && tc.Spec.TiDB == nil

// common
{
// storage
// check "path" to be compatible with old version
if common.Get("path") == nil && common.Get("storage") == nil {
paths := []string{}
for i := range tc.Spec.TiFlash.StorageClaims {
paths = append(paths, fmt.Sprintf("/data%d/db", i))
}
if len(paths) == 0 {
paths = []string{"/data0/db"}
}
common.Set("storage.main.dir", paths)
}
// check "raft.kvstore_path" to be compatible with old version
if common.Get("raft.kvstore_path") == nil {
common.SetIfNil("storage.raft.dir", "/data0/kvstore")
}

// port
common.SetIfNil("tcp_port", int64(9000))
common.SetIfNil("http_port", int64(8123))

// flash
tidbStatusAddr := fmt.Sprintf("%s.%s.svc:10080", controller.TiDBMemberName(name), ns)
if noLocalTiDB {
tidbStatusAddr = fmt.Sprintf("%s.%s.svc%s:10080",
controller.TiDBMemberName(ref.Name), ref.Namespace, controller.FormatClusterDomain(ref.ClusterDomain))
}
common.SetIfNil("flash.tidb_status_addr", tidbStatusAddr)
common.SetIfNil("flash.service_addr", "0.0.0.0:3930")
common.SetIfNil("flash.flash_cluster.log", defaultClusterLog)
common.SetIfNil("flash.proxy.addr", "0.0.0.0:20170")
common.SetIfNil("flash.proxy.advertise-addr", fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:20170", controller.TiFlashMemberName(name), controller.TiFlashPeerMemberName(name), ns, controller.FormatClusterDomain(clusterDomain)))
common.SetIfNil("flash.proxy.data-dir", "/data0/proxy")
common.SetIfNil("flash.proxy.config", "/data0/proxy.toml")

// logger
common.SetIfNil("logger.errorlog", defaultErrorLog)
common.SetIfNil("logger.log", defaultServerLog)

// raft
pdAddr := fmt.Sprintf("%s.%s.svc:2379", controller.PDMemberName(name), ns)
if len(clusterDomain) > 0 {
pdAddr = "PD_ADDR"
} else if noLocalPD {
pdAddr = fmt.Sprintf("%s.%s.svc%s:2379", controller.PDMemberName(ref.Name), ref.Namespace, controller.FormatClusterDomain(ref.ClusterDomain))
}
common.SetIfNil("raft.pd_addr", pdAddr)
}

// proxy
{
proxy.SetIfNil("log-level", "info")
proxy.SetIfNil("server.engine-addr", fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:3930", controller.TiFlashMemberName(name), controller.TiFlashPeerMemberName(name), ns, controller.FormatClusterDomain(clusterDomain)))
proxy.SetIfNil("server.status-addr", "0.0.0.0:20292")
proxy.SetIfNil("server.advertise-status-addr", fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:20292", controller.TiFlashMemberName(name), controller.TiFlashPeerMemberName(name), ns, controller.FormatClusterDomain(clusterDomain)))
}

// Note the config of tiflash use "_" by convention, others(proxy) use "-".
if tc.IsTLSClusterEnabled() {
common.Set("security.ca_path", path.Join(tiflashCertPath, corev1.ServiceAccountRootCAKey))
common.Set("security.cert_path", path.Join(tiflashCertPath, corev1.TLSCertKey))
common.Set("security.key_path", path.Join(tiflashCertPath, corev1.TLSPrivateKeyKey))
common.SetIfNil("tcp_port_secure", int64(9000))
common.SetIfNil("https_port", int64(8123))
common.Del("http_port")
common.Del("tcp_port")

proxy.Set("security.ca-path", path.Join(tiflashCertPath, corev1.ServiceAccountRootCAKey))
proxy.Set("security.cert-path", path.Join(tiflashCertPath, corev1.TLSCertKey))
proxy.Set("security.key-path", path.Join(tiflashCertPath, corev1.TLSPrivateKeyKey))

if commonVal, proxyVal := common.Get("security.cert_allowed_cn"), proxy.Get("security.cert-allowed-cn"); commonVal != nil && proxyVal == nil {
proxy.Set("security.cert-allowed-cn", commonVal.Interface())
}
}

return config
}

func getTiFlashConfig(tc *v1alpha1.TidbCluster) *v1alpha1.TiFlashConfigWraper {
config := tc.Spec.TiFlash.Config.DeepCopy()
if config == nil {
Expand Down
Loading

0 comments on commit 9eb0658

Please sign in to comment.