Skip to content

Commit 5def523

Browse files
fanfanyangyangiSecloud
fanfanyangyang
authored and
iSecloud
committed
feat: 客户端克隆支持模糊匹配_权限克隆兼容scr的conn_log授权_查询权限补充域名视角 #6001 
1 parent d776511 commit 5def523

31 files changed

+440
-160
lines changed

‎dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil/hide_passowrd.go renamed to ‎dbm-services/common/go-pubpkg/mysqlcomm/hide_passowrd.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
* specific language governing permissions and limitations under the License.
99
*/
1010

11-
package mysqlutil
11+
package mysqlcomm
1212

1313
import (
1414
"regexp"

‎dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil/hide_password_test.go renamed to ‎dbm-services/common/go-pubpkg/mysqlcomm/hide_password_test.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,14 @@
88
* specific language governing permissions and limitations under the License.
99
*/
1010

11-
package mysqlutil_test
11+
package mysqlcomm
1212

1313
import (
1414
"testing"
15-
16-
"dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil"
1715
)
1816

1917
func TestClearSensitiveInformatio(t *testing.T) {
2018
t.Log("start ...")
2119
textString := "mysqldump \t\t-h1.1.1.1 \t\t-P26000 \t\t-uxasasda \t\t-p{xasx} \t\t--skip-opt \t\t--create-options \t"
22-
t.Log(mysqlutil.ClearSensitiveInformation(textString))
20+
t.Log(ClearSensitiveInformation(textString))
2321
}

‎dbm-services/mysql/db-partition/main.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package main
22

33
import (
4+
"dbm-services/mysql/db-partition/monitor"
45
"dbm-services/mysql/db-partition/service"
56
"net/http"
67
"os"
@@ -16,7 +17,6 @@ import (
1617
"dbm-services/common/go-pubpkg/apm/trace"
1718
"dbm-services/mysql/db-partition/assests"
1819
"dbm-services/mysql/db-partition/model"
19-
"dbm-services/mysql/db-partition/monitor"
2020
"dbm-services/mysql/db-partition/router"
2121
)
2222

@@ -33,7 +33,7 @@ func main() {
3333
}
3434
}
3535

36-
// 获取监控配置，多次尝试，获取监控配置失败，如果服务异常无法上报监控，所以让服务退出，可触发服务故障的告警。
36+
// 获取监控配置，多次尝试，获取监控配置失败
3737
monitor.InitMonitor()
3838

3939
// 注册定时任务

‎dbm-services/mysql/db-partition/model/init_env.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ func InitEnv() {
2828
viper.BindEnv("dbm_ticket_service", "DBM_TICKET_SERVICE")
2929
viper.BindEnv("bk_app_code", "BK_APP_CODE")
3030
viper.BindEnv("bk_app_secret", "BK_APP_SECRET")
31+
// 开启分区服务的蓝鲸监控
32+
viper.BindEnv("monitor", "MONITOR")
3133

3234
// pt-osc参数
3335
viper.BindEnv("pt.max_load.threads_running", "PT_MAX_LOAD_THREADS_RUNNING")

‎dbm-services/mysql/db-partition/monitor/monitor.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,9 @@ func GetMonitorSetting() (Setting, error) {
128128

129129
// InitMonitor 多次尝试获取监控配置，更新配置；获取失败退出
130130
func InitMonitor() {
131+
if !viper.GetBool("monitor") {
132+
return
133+
}
131134
i := 1
132135
for ; i <= 10; i++ {
133136
setting, err := GetMonitorSetting()

‎dbm-services/mysql/db-partition/service/check_partition.go

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -122,11 +122,15 @@ func CheckPartitionConfigs(configs []*PartitionConfig, dbtype string, splitCnt i
122122
checkFailSet := ConfigIdLogSet{}
123123
wg := sync.WaitGroup{}
124124
limit := rate.Every(time.Millisecond * 200) // QPS：5
125-
burst := 10 // 桶容量 10
125+
burst := 5 // 桶容量 5
126126
limiter := rate.NewLimiter(limit, burst)
127+
tokenBucket := make(chan int, 10) // 最大并行度
128+
127129
for _, config := range configs {
128130
wg.Add(1)
129-
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
131+
tokenBucket <- 0
132+
133+
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
130134
go func(config *PartitionConfig) {
131135
err := limiter.Wait(context.Background())
132136
if err != nil {
@@ -136,7 +140,7 @@ func CheckPartitionConfigs(configs []*PartitionConfig, dbtype string, splitCnt i
136140
return
137141
}
138142
CheckOnePartitionConfig(ctx, cancel, *config, &wg, &sqlSet, &nothingToDoSet, &checkFailSet, dbtype, splitCnt,
139-
fromCron, host)
143+
fromCron, host, &tokenBucket)
140144
}(config)
141145
}
142146
wg.Wait()
@@ -154,16 +158,16 @@ func CheckPartitionConfigs(configs []*PartitionConfig, dbtype string, splitCnt i
154158
// CheckOnePartitionConfig 检查一个分区规则是否需要执行，生成分区语句
155159
func CheckOnePartitionConfig(ctx context.Context, cancel context.CancelFunc, config PartitionConfig,
156160
wg *sync.WaitGroup, sqlSet *PartitionSqlSet, nothingToDoSet *ConfigIdLogSet, checkFailSet *ConfigIdLogSet,
157-
dbtype string, splitCnt int, fromCron bool, host Host) {
161+
dbtype string, splitCnt int, fromCron bool, host Host, tokenBucket *chan int) {
158162
fmt.Printf("do CheckOnePartitionConfig")
159163
var addSql, dropSql []string
160164
var err error
161165
var initSql []InitSql
162166
defer func() {
167+
<-*tokenBucket
163168
wg.Done()
164169
cancel()
165170
}()
166-
167171
finish := make(chan int, 1)
168172
go func() {
169173
defer func() {

‎dbm-services/mysql/db-partition/service/cron_basic_func.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -475,7 +475,7 @@ func DownLoadFilesCreateTicketByMachine(cloudMachineList map[int][]string, machi
475475
slog.Error("msg", msg, err)
476476
return
477477
}
478-
time.Sleep(60 * time.Second)
478+
time.Sleep(120 * time.Second)
479479
// 创建执行分区单据
480480
err = CreatePartitionTicket(files, clusterType, "mixed", vdate)
481481
if err != nil {
@@ -542,7 +542,7 @@ func DownLoadFilesCreateTicketByCluster(clusterIps map[string][]string, machineF
542542
}
543543
clusterFiles = append(clusterFiles, files...)
544544
}
545-
time.Sleep(60 * time.Second)
545+
time.Sleep(120 * time.Second)
546546
// 创建执行分区单据
547547
err = CreatePartitionTicket(clusterFiles, clusterType, domain, vdate)
548548
if err != nil {

‎dbm-services/mysql/db-priv/handler/add_priv.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -109,19 +109,20 @@ func (m *PrivService) GetPriv(c *gin.Context) {
109109
SendResponse(c, errno.ErrBind, err)
110110
return
111111
}
112-
formatted, count, download, hasPriv, noPriv, err := input.GetPriv()
112+
formattedForIp, formattedForCluster, count, download, hasPriv, noPriv, err := input.GetPriv()
113113
if err != nil {
114114
slog.Error(err.Error())
115115
SendResponse(c, err, nil)
116116
return
117117
}
118118
data := struct {
119-
Formatted []service.RelatedIp `json:"privs"`
120-
Count int `json:"count"`
121-
Download []service.GrantInfo `json:"download"`
122-
HasPriv []string `json:"has_priv"`
123-
NoPriv []string `json:"no_priv"`
124-
}{formatted, count, download, hasPriv, noPriv}
119+
FormattedForIp []service.RelatedIp `json:"privs_for_ip"`
120+
FormattedForCluster []service.RelatedDomain2 `json:"privs_for_cluster"`
121+
Count int `json:"count"`
122+
Download []service.GrantInfo `json:"download"`
123+
HasPriv []string `json:"has_priv"`
124+
NoPriv []string `json:"no_priv"`
125+
}{formattedForIp, formattedForCluster, count, download, hasPriv, noPriv}
125126
SendResponse(c, err, data)
126127
return
127128
}

‎dbm-services/mysql/db-priv/handler/clone_client_priv.go

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,14 @@ func (m *PrivService) CloneClientPriv(c *gin.Context) {
5353
return
5454
}
5555

56-
err = input.CloneClientPriv(string(body), ticket)
57-
SendResponse(c, err, nil)
56+
privs, err := input.CloneClientPriv(string(body), ticket)
57+
if err != nil {
58+
SendResponse(c, err, nil)
59+
return
60+
}
61+
SendResponse(c, err, ListResponse{
62+
Count: len(privs),
63+
Items: privs,
64+
})
5865
return
5966
}

‎dbm-services/mysql/db-priv/service/account_rule_mongodb.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,13 @@ func (m *AccountRulePara) MongoDBAddAccountRule(jsonPara string, ticket string)
5252
userPriv = strings.Trim(userPriv, ",")
5353
managerPriv = strings.Trim(managerPriv, ",")
5454
allTypePriv = strings.Trim(allTypePriv, ",")
55+
vtime := time.Now()
5556

5657
tx := DB.Self.Begin()
5758
for _, db := range dbs {
5859
accountRule = TbAccountRules{BkBizId: m.BkBizId, ClusterType: *m.ClusterType, AccountId: m.AccountId, Dbname: db,
5960
Priv: allTypePriv,
60-
DmlDdlPriv: userPriv, GlobalPriv: managerPriv, Creator: m.Operator, CreateTime: time.Now()}
61+
DmlDdlPriv: userPriv, GlobalPriv: managerPriv, Creator: m.Operator, CreateTime: vtime, UpdateTime: vtime}
6162
err = tx.Debug().Model(&TbAccountRules{}).Create(&accountRule).Error
6263
if err != nil {
6364
tx.Rollback()
@@ -68,7 +69,7 @@ func (m *AccountRulePara) MongoDBAddAccountRule(jsonPara string, ticket string)
6869
if err != nil {
6970
return err
7071
}
71-
log := PrivLog{BkBizId: m.BkBizId, Ticket: ticket, Operator: m.Operator, Para: jsonPara, Time: time.Now()}
72+
log := PrivLog{BkBizId: m.BkBizId, Ticket: ticket, Operator: m.Operator, Para: jsonPara, Time: vtime}
7273
AddPrivLog(log)
7374

7475
return nil

‎dbm-services/mysql/db-priv/service/clone_client_priv.go

Lines changed: 72 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ package service
22

33
import (
44
"context"
5+
"dbm-services/common/go-pubpkg/mysqlcomm"
56
"fmt"
7+
"log/slog"
68
"strings"
79
"sync"
810
"time"
@@ -48,18 +50,19 @@ func (m *CloneClientPrivParaList) CloneClientPrivDryRun() error {
4850
}
4951

5052
// CloneClientPriv 克隆客户端权限
51-
func (m *CloneClientPrivPara) CloneClientPriv(jsonPara string, ticket string) error {
53+
func (m *CloneClientPrivPara) CloneClientPriv(jsonPara string, ticket string) ([]ClusterGrantSql, error) {
5254
var errMsg Err
55+
var sqls ClusterGrants
5356
wg := sync.WaitGroup{}
5457
limit := rate.Every(time.Millisecond * 200) // QPS：5
5558
burst := 10 // 桶容量 10
5659
limiter := rate.NewLimiter(limit, burst)
5760

5861
if m.BkBizId == 0 {
59-
return errno.BkBizIdIsEmpty
62+
return nil, errno.BkBizIdIsEmpty
6063
}
6164
if m.BkCloudId == nil {
62-
return errno.CloudIdRequired
65+
return nil, errno.CloudIdRequired
6366
}
6467
if m.ClusterType == nil {
6568
ct := mysql
@@ -71,7 +74,7 @@ func (m *CloneClientPrivPara) CloneClientPriv(jsonPara string, ticket string) er
7174
client := util.NewClientByHosts(viper.GetString("dbmeta"))
7275
resp, errOuter := GetAllClustersInfo(client, BkBizIdPara{m.BkBizId})
7376
if errOuter != nil {
74-
return errOuter
77+
return nil, errOuter
7578
}
7679
var tempClusters []Cluster
7780
var clusters []Cluster
@@ -104,7 +107,7 @@ func (m *CloneClientPrivPara) CloneClientPriv(jsonPara string, ticket string) er
104107
}
105108
}
106109
if len(notExists) > 0 {
107-
return errno.DomainNotExists.AddBefore(strings.Join(notExists, ","))
110+
return nil, errno.DomainNotExists.AddBefore(strings.Join(notExists, ","))
108111
}
109112
} else {
110113
clusters = make([]Cluster, len(tempClusters))
@@ -113,7 +116,7 @@ func (m *CloneClientPrivPara) CloneClientPriv(jsonPara string, ticket string) er
113116

114117
errMsg.errs = validateIP(m.SourceIp, m.TargetIp, m.BkCloudId)
115118
if len(errMsg.errs) > 0 {
116-
return errno.ClonePrivilegesFail.Add("\n" + strings.Join(errMsg.errs, "\n"))
119+
return nil, errno.ClonePrivilegesFail.Add("\n" + strings.Join(errMsg.errs, "\n"))
117120
}
118121
// 获取业务下所有的集群，并行获取对旧的client授权的语句，替换旧client的ip为新client，执行导入
119122
// 一个协程失败，其报错信息添加到errMsg.errs。主协程wg.Wait()，等待所有协程执行完成才会返回。
@@ -125,39 +128,68 @@ func (m *CloneClientPrivPara) CloneClientPriv(jsonPara string, ticket string) er
125128
defer func() {
126129
wg.Done()
127130
}()
128-
129-
err := limiter.Wait(context.Background())
130-
if err != nil {
131-
AddError(&errMsg, item.ImmuteDomain, err)
131+
errOuter = limiter.Wait(context.Background())
132+
if errOuter != nil {
133+
AddError(&errMsg, item.ImmuteDomain, errOuter)
132134
return
133135
}
134-
136+
clusterGrant := ClusterGrantSql{ImmuteDomain: item.ImmuteDomain}
135137
if item.ClusterType == tendbha || item.ClusterType == tendbsingle {
136138
for _, storage := range item.Storages {
137139
address := fmt.Sprintf("%s:%d", storage.IP, storage.Port)
138-
userGrants, err := GetRemotePrivilege(address, m.SourceIp, item.BkCloudId,
139-
machineTypeBackend, m.User, false)
140+
// 在后端mysql中获取匹配的user@host列表
141+
_, _, matchHosts, err := MysqlUserList(address, item.BkCloudId, []string{m.SourceIp}, nil, "")
142+
if err != nil {
143+
AddError(&errMsg, address, err)
144+
continue
145+
}
146+
slog.Info("msg", "matchHosts", matchHosts)
147+
userGrants, err := GetRemotePrivilege(address, matchHosts, item.BkCloudId,
148+
machineTypeBackend, m.User, true)
140149
if err != nil {
141150
AddError(&errMsg, address, err)
142151
continue
143152
}
144-
userGrants = ReplaceHostInMysqlGrants(userGrants, m.SourceIp, m.TargetIp)
145-
err = ImportMysqlPrivileges(userGrants, address, item.BkCloudId)
153+
if len(userGrants) == 0 {
154+
continue
155+
}
156+
userGrants = ReplaceHostInMysqlGrants(userGrants, m.TargetIp)
157+
var grants []string
158+
for _, sql := range userGrants {
159+
grants = append(grants, sql.Grants...)
160+
}
161+
clusterGrant.Sqls = append(clusterGrant.Sqls, InstanceGrantSql{address,
162+
mysqlcomm.ClearIdentifyByInSQLs(grants)})
163+
err = ImportMysqlPrivileges(grants, address, item.BkCloudId)
146164
if err != nil {
147165
AddError(&errMsg, address, err)
148166
}
149167
}
150168
} else {
151169
for _, spider := range item.Proxies {
152170
address := fmt.Sprintf("%s:%d", spider.IP, spider.Port)
153-
userGrants, err := GetRemotePrivilege(address, m.SourceIp, item.BkCloudId,
154-
machineTypeSpider, m.User, false)
171+
_, _, matchHosts, err := MysqlUserList(address, item.BkCloudId, []string{m.SourceIp}, nil, "")
172+
if err != nil {
173+
AddError(&errMsg, address, err)
174+
continue
175+
}
176+
userGrants, err := GetRemotePrivilege(address, matchHosts, item.BkCloudId,
177+
machineTypeSpider, m.User, true)
155178
if err != nil {
156179
AddError(&errMsg, address, err)
157180
continue
158181
}
159-
userGrants = ReplaceHostInMysqlGrants(userGrants, m.SourceIp, m.TargetIp)
160-
err = ImportMysqlPrivileges(userGrants, address, item.BkCloudId)
182+
if len(userGrants) == 0 {
183+
continue
184+
}
185+
userGrants = ReplaceHostInMysqlGrants(userGrants, m.TargetIp)
186+
var grants []string
187+
for _, sql := range userGrants {
188+
grants = append(grants, sql.Grants...)
189+
}
190+
clusterGrant.Sqls = append(clusterGrant.Sqls, InstanceGrantSql{address,
191+
mysqlcomm.ClearIdentifyByInSQLs(grants)})
192+
err = ImportMysqlPrivileges(grants, address, item.BkCloudId)
161193
if err != nil {
162194
AddError(&errMsg, address, err)
163195
}
@@ -166,22 +198,39 @@ func (m *CloneClientPrivPara) CloneClientPriv(jsonPara string, ticket string) er
166198
if item.ClusterType == tendbha {
167199
for _, proxy := range item.Proxies {
168200
address := fmt.Sprintf("%s:%d", proxy.IP, proxy.AdminPort)
169-
proxyGrants, err := GetProxyPrivilege(address, m.SourceIp, item.BkCloudId, m.User)
201+
_, _, matchHosts, err := ProxyWhiteList(address, item.BkCloudId, []string{m.SourceIp}, nil, "")
202+
if err != nil {
203+
slog.Error("msg", "ProxyWhiteList", err)
204+
AddError(&errMsg, address, err)
205+
}
206+
slog.Info("msg", "matchHosts", matchHosts)
207+
proxyGrants, err := GetProxyPrivilege(address, matchHosts, item.BkCloudId, m.User)
170208
if err != nil {
209+
slog.Error("msg", "GetProxyPrivilege", err)
171210
AddError(&errMsg, address, err)
172211
}
173-
proxyGrants = ReplaceHostInProxyGrants(proxyGrants, m.SourceIp, m.TargetIp)
212+
if len(proxyGrants) == 0 {
213+
continue
214+
}
215+
proxyGrants = ReplaceHostInProxyGrants(proxyGrants, m.TargetIp)
216+
clusterGrant.Sqls = append(clusterGrant.Sqls, InstanceGrantSql{address, proxyGrants})
174217
err = ImportProxyPrivileges(proxyGrants, address, item.BkCloudId)
175218
if err != nil {
176219
AddError(&errMsg, address, err)
177220
}
178221
}
179222
}
223+
if len(clusterGrant.Sqls) > 0 {
224+
sqls.mu.Lock()
225+
sqls.resources = append(sqls.resources, clusterGrant)
226+
sqls.mu.Unlock()
227+
}
180228
}(item)
181229
}
182230
wg.Wait()
231+
slog.Info("msg", "clusterGrant", sqls.resources)
183232
if len(errMsg.errs) > 0 {
184-
return errno.ClonePrivilegesFail.Add("\n" + strings.Join(errMsg.errs, "\n"))
233+
return nil, errno.ClonePrivilegesFail.Add("\n" + strings.Join(errMsg.errs, "\n"))
185234
}
186-
return nil
235+
return sqls.resources, nil
187236
}

0 commit comments

Comments
 (0)