Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: slow update tc #306

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: slow update tc #306

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.21
require (
github.com/coreos/go-iptables v0.6.0
github.com/digitalocean/go-openvswitch v0.0.20240920
github.com/fsnotify/fsnotify v1.5.4
github.com/fsnotify/fsnotify v1.8.0
github.com/golang/protobuf v1.5.3
github.com/mitchellh/go-homedir v1.1.0
github.com/pkg/errors v0.9.1
Expand All @@ -17,10 +17,10 @@ require (
golang.org/x/net v0.21.0
google.golang.org/grpc v1.62.0
google.golang.org/protobuf v1.32.0
yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401
yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
yunion.io/x/onecloud v0.0.0-20241009134947-8e0507d0029f
yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318
yunion.io/x/onecloud v0.0.0-20241217113931-4132f245acc4
yunion.io/x/pkg v1.10.2
)

require (
Expand Down Expand Up @@ -119,8 +119,8 @@ require (
moul.io/http2curl/v2 v2.3.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.0.1 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240929084351-30a36ccf2201 // indirect
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 // indirect
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241216075633-a8665686cf63 // indirect
yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 // indirect
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e // indirect
yunion.io/x/sqlchemy v1.1.3-0.20240926163039-d41512b264e1 // indirect
yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c // indirect
Expand Down
25 changes: 12 additions & 13 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,8 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gernest/wow v0.1.0/go.mod h1:dEPabJRi5BneI1Nev1VWo0ZlcTWibHWp43qxKms4elY=
github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
Expand Down Expand Up @@ -605,7 +605,6 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down Expand Up @@ -783,25 +782,25 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240929084351-30a36ccf2201 h1:Qe+41cwNdSangAUPCEMV4YQWB8Twdusn1hZfdciK3Ro=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240929084351-30a36ccf2201/go.mod h1:rj/pb3DitJlQaQD8UW1oxx/KD+PzDZqoywzqRJaFE9A=
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug=
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241216075633-a8665686cf63 h1:j5XE/WcpbSolZfeQueXmtxiVVxdQWC2Qr9RtNnzEwR0=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241216075633-a8665686cf63/go.mod h1:rj/pb3DitJlQaQD8UW1oxx/KD+PzDZqoywzqRJaFE9A=
yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 h1:kLKQ6zbgPDQflRwoHFAjxNChcbhXIFgsUVLkJwiXu/8=
yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/go-openvswitch v0.0.20240920 h1:imjK9g2EZ3IzBdNZ3UEYobJJYw6bQCvUvgoTOIyis1o=
yunion.io/x/go-openvswitch v0.0.20240920/go.mod h1:OAtI/pEmN/EvxlkixiYp2nMQQEtEqzHcpWeE2AW2Bb8=
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401 h1:4l6ELFSQ0MBVInscZ8/yOtSWF0cwH5BT1ATN6dCtAqc=
yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f h1:N7V0uLqX9mED1HEPeIKcYpkWglnHL5hyn4ENX+VBeqM=
yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 h1:inY5o3LDa/zgsIZuPN0HmpzKIsu/lLgsBmMttuDPGj4=
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
yunion.io/x/onecloud v0.0.0-20241009134947-8e0507d0029f h1:wTat6DgcTgPW362zaviuWr3novfUHw3KwldfhRUfdko=
yunion.io/x/onecloud v0.0.0-20241009134947-8e0507d0029f/go.mod h1:NxDO0eScgEYp5NJPV8BWURe2/copiNBFqDIBAEBoFZw=
yunion.io/x/onecloud v0.0.0-20241217113931-4132f245acc4 h1:y7VBqX89ZAZmIjYdAb6hvOZhoBdsNOzzPwUARl7wccU=
yunion.io/x/onecloud v0.0.0-20241217113931-4132f245acc4/go.mod h1:bMRZe2muTlMeYiX7yHJwz5pTZtGMHeQK/BIrT4IdXRY=
yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318 h1:Fm7I8ypXHxeObY4u/VUGz78NsambemzTZ9fECyGKNi8=
yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
yunion.io/x/pkg v1.10.2 h1:oZhedvlvDsebIWcOvrOMJQ31SxzjxxqUXUBYseBjh7w=
yunion.io/x/pkg v1.10.2/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
yunion.io/x/sqlchemy v1.1.3-0.20240926163039-d41512b264e1 h1:HWPqY1I5JSmM6Sks6FyK9hnq/MjL7FDghM6M8DXHob0=
Expand Down
64 changes: 30 additions & 34 deletions pkg/agent/server/flowman.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@
package server

import (
"bytes"
"context"
"fmt"
"reflect"
"sync"
"sync/atomic"
Expand Down Expand Up @@ -99,64 +97,60 @@ var (
}
)

func (fm *FlowMan) mergeFlows() *utils.FlowSet {
merge := utils.NewFlowSet()
for _, fs := range fm.flowSets {
merge.Merge(fs)
}
return merge
}

func (fm *FlowMan) doCheck() {
log.Infof("flowman %s: do check waitCount %d", fm.bridge, fm.waitCount)
if atomic.LoadInt32(&fm.waitCount) != 0 {
return
}
defer log.Infof("flowman %s: check done", fm.bridge)
start := time.Now()

defer func() {
log.Infof("flowman %s: check done %f", fm.bridge, time.Since(start).Seconds())
}()

log.Infof("flowman %s: start check", fm.bridge)
var err error
// fs0: current flows
fs0, err := fm.doDumpFlows(excludeOvsTables)
if err != nil {
log.Errorf("FlowMan doCheck doDumpFlows fail %s", err)
return
}
fsAdd := utils.NewFlowSet()
fsDel := utils.NewFlowSet()
// flows1: flows to install
// flows1 := []*ovs.Flow{}
for _, fs1 := range fm.flowSets {
for _, f1 := range fs1.Flows() {
// flows1 = append(flows1, f1)
if !fs0.Contains(f1) {
fsAdd.Add(f1)
}
}
}
for _, f0 := range fs0.Flows() {
found := false
for _, fs1 := range fm.flowSets {
if fs1.Contains(f0) {
found = true
break
}
}
if !found {
fsDel.Add(f0)
}
}
flowsAdd := fsAdd.Flows()
flowsDel := fsDel.Flows()
log.Infof("flowman %s: %d flows in table", fm.bridge, fs0.Len())

merged := fm.mergeFlows()
log.Infof("flowman %s: %d flows in table and %d flows in memory", fm.bridge, fs0.Len(), merged.Len())
flowsAdd, flowsDel := fs0.Diff(merged)
fm.doCommitChange(flowsAdd, flowsDel)
if len(flowsAdd) > 0 || len(flowsDel) > 0 {

/*if len(flowsAdd) > 0 || len(flowsDel) > 0 {
buf := &bytes.Buffer{}
buf.WriteString(fmt.Sprintf("flowman %s: commit:\n", fm.bridge))
//fm.bufWriteFlows(buf, "000-flow", fs0.Flows())
//fm.bufWriteFlows(buf, "111-flow", flows1)
fm.bufWriteFlows(buf, "add-flow", flowsAdd)
fm.bufWriteFlows(buf, "del-flow", flowsDel)
log.Infof("%s", buf.String())
}
}*/
}

func (fm *FlowMan) bufWriteFlows(buf *bytes.Buffer, prefix string, flows []*ovs.Flow) {
/*func (fm *FlowMan) bufWriteFlows(buf *bytes.Buffer, prefix string, flows []*ovs.Flow) {
for i, f := range flows {
txt, _ := f.MarshalText()
buf.WriteString(fmt.Sprintf("%s:%2d: %s\n", prefix, i, txt))
}
}
}*/

func (fm *FlowMan) doCommitChange(flowsAdd, flowsDel []*ovs.Flow) error {
log.Infof("FlowMan %s doCommitChange flowsAdd %d flowsDel %d", fm.bridge, len(flowsAdd), len(flowsDel))
ofCli := ovs.New(ovs.Strict(), ovs.Debug(false)).OpenFlow
err := ofCli.AddFlowBundle(fm.bridge, func(tx *ovs.FlowTransaction) error {
mfs := make([]*ovs.MatchFlow, len(flowsDel))
Expand Down Expand Up @@ -291,13 +285,15 @@ func (fm *FlowMan) DelFlow(ctx context.Context, of *ovs.Flow) {
}

func (fm *FlowMan) SyncFlows(ctx context.Context) {
log.Debugf("flowman %s: SyncFlows", fm.bridge)
cmd := &flowManCmd{
Type: flowManCmdSyncFlows,
}
fm.sendCmd(ctx, cmd)
}

func (fm *FlowMan) updateFlows(ctx context.Context, who string, ofs []*ovs.Flow) {
log.Debugf("flowman %s: updateFlows %s", fm.bridge, who)
{
v := ctx.Value("waitData")
// The caller is responsible for coordinating access
Expand All @@ -324,7 +320,7 @@ func (fm *FlowMan) updateFlows(ctx context.Context, who string, ofs []*ovs.Flow)
fm.sendCmd(ctx, cmd)
}

func (fm *FlowMan) waitDecr(ctx context.Context, n int32) {
func (fm *FlowMan) waitDecr(n int32) {
atomic.AddInt32(&fm.waitCount, -n)
}

Expand Down
13 changes: 9 additions & 4 deletions pkg/agent/server/guest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ func (g *Guest) clearClassicFlows(ctx context.Context) {
g.clearPending()
}

func (g *Guest) updateTc(ctx context.Context) {
func (g *Guest) updateTc(ctx context.Context, sync bool) {
if g.watcher.tcMan == nil {
return
}
Expand All @@ -201,7 +201,7 @@ func (g *Guest) updateTc(ctx context.Context) {
d := nic.TcData()
data = append(data, d)
}
g.watcher.tcMan.AddIfaces(ctx, g.Who(), data)
g.watcher.tcMan.AddIfaces(ctx, g.Who(), data, sync)
}

func (g *Guest) clearTc(ctx context.Context) {
Expand Down Expand Up @@ -238,13 +238,18 @@ func (g *Guest) clearOvn(ctx context.Context) {
ovnMdMan.SetGuestNICs(ctx, g.Id, nil)
}

func (g *Guest) UpdateSettings(ctx context.Context) {
func (g *Guest) UpdateSettings(ctx context.Context, sync bool) {
start := time.Now()
err := g.refresh(ctx)
log.Infof("guest UpdateSettings refresh %f", time.Since(start).Seconds())
switch err {
case nil:
g.updateClassicFlows(ctx)
g.updateTc(ctx)
log.Infof("guest UpdateSettings updateClassicFlows %f", time.Since(start).Seconds())
g.updateTc(ctx, sync)
log.Infof("guest UpdateSettings updateTc %f", time.Since(start).Seconds())
g.updateOvn(ctx)
log.Infof("guest UpdateSettings updateOvn %f", time.Since(start).Seconds())
if g.HostId != "" {
g.watcher.agent.HostId(g.HostId)
}
Expand Down
8 changes: 4 additions & 4 deletions pkg/agent/server/hostlocal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func (hl *HostLocal) updateFlows(ctx context.Context) {
}
}

func (hl *HostLocal) updateTc(ctx context.Context) {
func (hl *HostLocal) updateTc(ctx context.Context, sync bool) {
if hl.watcher.tcMan == nil {
return
}
Expand All @@ -81,10 +81,10 @@ func (hl *HostLocal) updateTc(ctx context.Context) {
}
data = append(data, td)
}
hl.watcher.tcMan.AddIfaces(ctx, "hostlocal", data)
hl.watcher.tcMan.AddIfaces(ctx, "hostlocal", data, sync)
}

func (hl *HostLocal) UpdateSettings(ctx context.Context) {
func (hl *HostLocal) UpdateSettings(ctx context.Context, sync bool) {
hl.updateFlows(ctx)
hl.updateTc(ctx)
hl.updateTc(ctx, sync)
}
19 changes: 17 additions & 2 deletions pkg/agent/server/tcman.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,12 +105,15 @@ type TcManCmdType int
const (
TcManCmdAdd = iota
TcManCmdDel
TcManCmdSync
)

type TcManCmd struct {
typ TcManCmdType
who string
section *TcManSection
// if the command is executed synchronizedly
sync bool
}

// TODO
Expand Down Expand Up @@ -216,9 +219,13 @@ func (tm *TcMan) doCmd(ctx context.Context, cmd *TcManCmd) {
} else {
section.Update(cmd.section)
}
tm.doCheckSection(ctx, section)
if cmd.sync {
tm.doCheckSection(ctx, section)
}
case TcManCmdDel:
delete(tm.book, cmd.who)
case TcManCmdSync:
tm.doIdleCheck(ctx)
}
}

Expand All @@ -231,7 +238,7 @@ func (tm *TcMan) sendCmd(ctx context.Context, cmd *TcManCmd) {
}
}

func (tm *TcMan) AddIfaces(ctx context.Context, who string, data []*utils.TcData) {
func (tm *TcMan) AddIfaces(ctx context.Context, who string, data []*utils.TcData, sync bool) {
section := &TcManSection{
pages: map[string]*TcManPage{},
}
Expand All @@ -253,6 +260,7 @@ func (tm *TcMan) AddIfaces(ctx context.Context, who string, data []*utils.TcData
typ: TcManCmdAdd,
who: who,
section: section,
sync: sync,
}
tm.sendCmd(ctx, cmd)
}
Expand All @@ -264,3 +272,10 @@ func (tm *TcMan) ClearIfaces(ctx context.Context, who string) {
}
tm.sendCmd(ctx, cmd)
}

func (tm *TcMan) SyncAll(ctx context.Context) {
cmd := &TcManCmd{
typ: TcManCmdSync,
}
tm.sendCmd(ctx, cmd)
}
Loading
Loading