Skip to content

Commit

Permalink
iptables: fix subnet metrics rules (#4979)
Browse files Browse the repository at this point in the history 
Signed-off-by: zhangzujian <[email protected]>
  • Loading branch information
@zhangzujian
zhangzujian authored Feb 13, 2025
1 parent d7714f9 commit 7a36465
Showing 1 changed file with 64 additions and 78 deletions.
142 changes: 64 additions & 78 deletions pkg/daemon/gateway_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/klog/v2"
"k8s.io/utils/set"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/ovs"
Expand Down Expand Up @@ -730,13 +731,16 @@ func (c *Controller) setIptables() error {
)
}
}

_, subnetCidrs, err := c.getDefaultVpcSubnetsCIDR(protocol)
if err != nil {
klog.Errorf("get subnets failed, %+v", err)
return err
}

subnetNames := set.New[string]()
for name, subnetCidr := range subnetCidrs {
subnetNames.Insert(name)
iptablesRules = append(iptablesRules,
util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(fmt.Sprintf(`-m comment --comment %s,%s -s %s`, util.OvnSubnetGatewayIptables, name, subnetCidr))},
util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(fmt.Sprintf(`-m comment --comment %s,%s -d %s`, util.OvnSubnetGatewayIptables, name, subnetCidr))},
Expand All @@ -749,27 +753,31 @@ func (c *Controller) setIptables() error {
return err
}

pattern := fmt.Sprintf(`-m comment --comment "%s,`, util.OvnSubnetGatewayIptables)
for _, rule := range rules {
if !strings.Contains(rule, util.OvnSubnetGatewayIptables) {
if !strings.Contains(rule, pattern) {
continue
}

var inUse bool
for name := range subnetCidrs {
if slices.Contains(util.DoubleQuotedFields(rule), fmt.Sprintf("%s,%s", util.OvnSubnetGatewayIptables, name)) {
inUse = true
break
}
fields := util.DoubleQuotedFields(rule)
// -A FORWARD -d 10.16.0.0/16 -m comment --comment "ovn-subnet-gateway,ovn-default"
if len(fields) != 8 || fields[6] != "--comment" {
continue
}
commentFields := strings.Split(fields[7], ",")
if len(commentFields) != 2 {
continue
}
if subnetNames.Has(commentFields[1]) {
continue
}

if !inUse {
// rule[11:] skip "-A FORWARD "
if err = deleteIptablesRule(ipt, util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: util.DoubleQuotedFields(rule[11:])}); err != nil {
klog.Error(err)
return err
}
// use fields[2:] to skip prefix "-A FORWARD"
if err = deleteIptablesRule(ipt, util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: fields[2:]}); err != nil {
klog.Error(err)
return err
}
}

var natPreroutingRules, natPostroutingRules, ovnMasqueradeRules, manglePostroutingRules []util.IPTableRule
for _, rule := range iptablesRules {
if rule.Table == NAT {
Expand Down Expand Up @@ -1257,95 +1265,73 @@ func (c *Controller) setOvnSubnetGatewayMetric() {
}

for _, rule := range rules {
if !strings.Contains(rule, util.OvnSubnetGatewayIptables) {
continue
}

items := util.DoubleQuotedFields(rule)
cidr := ""
direction := ""
subnetName := ""
var currentPackets, currentPacketBytes int
if len(items) <= 10 {
if len(items) != 11 {
continue
}
for _, item := range items {
if strings.Contains(item, util.OvnSubnetGatewayIptables) {
cidr = items[3]

switch items[2] {
case "-s":
direction = "egress"
case "-d":
direction = "ingress"
default:
break
}

comments := strings.Split(items[7], ",")
if len(comments) != 2 {
break
}
subnetName = comments[1][:len(comments[1])-1]
currentPackets, err = strconv.Atoi(items[9])
if err != nil {
break
}
currentPacketBytes, err = strconv.Atoi(items[10])
if err != nil {
break
}
}
comments := strings.Split(items[7], ",")
if len(comments) != 2 || comments[0] != util.OvnSubnetGatewayIptables {
continue
}
subnetName := comments[1]

var direction string
switch items[2] {
case "-s":
direction = "egress"
case "-d":
direction = "ingress"
default:
continue
}

cidr := items[3]
proto := util.CheckProtocol(cidr)

if cidr == "" || direction == "" || subnetName == "" && proto != "" {
if proto == "" {
klog.Errorf("failed to get protocol from cidr %q", cidr)
continue
}

lastPacketBytes := 0
lastPackets := 0
diffPacketBytes := 0
diffPackets := 0
currentPackets, err := strconv.Atoi(items[9])
if err != nil {
klog.Errorf("failed to parse packets %q: %v", items[9], err)
continue
}
currentPacketBytes, err := strconv.Atoi(items[10])
if err != nil {
klog.Errorf("failed to parse packet bytes %q: %v", items[10], err)
continue
}

key := strings.Join([]string{subnetName, direction, proto}, "/")
if ret, ok := c.gwCounters[key]; ok {
lastPackets = ret.Packets
lastPacketBytes = ret.PacketBytes
} else {
c.gwCounters[key] = &util.GwIPtableCounters{
Packets: lastPackets,
PacketBytes: lastPacketBytes,
}
if c.gwCounters[key] == nil {
c.gwCounters[key] = new(util.GwIPtableCounters)
}
lastPackets, lastPacketBytes := c.gwCounters[key].Packets, c.gwCounters[key].PacketBytes
c.gwCounters[key].Packets, c.gwCounters[key].PacketBytes = currentPackets, currentPacketBytes

if lastPacketBytes == 0 && lastPackets == 0 {
if lastPackets == 0 && lastPacketBytes == 0 {
// the gwCounters may just initialize don't cal the diff values,
// it may loss packets to calculate during a metric period
c.gwCounters[key].Packets = currentPackets
c.gwCounters[key].PacketBytes = currentPacketBytes
continue
}

if currentPackets >= lastPackets && currentPacketBytes >= lastPacketBytes {
diffPacketBytes = currentPacketBytes - lastPacketBytes
diffPackets = currentPackets - lastPackets
} else {
if currentPackets < lastPackets || currentPacketBytes < lastPacketBytes {
// if currentPacketBytes < lastPacketBytes, the reason is that iptables rule is reset ,
// it may loss packets to calculate during a metric period
c.gwCounters[key].Packets = currentPackets
c.gwCounters[key].PacketBytes = currentPacketBytes
continue
}

c.gwCounters[key].Packets = currentPackets
c.gwCounters[key].PacketBytes = currentPacketBytes

diffPackets := currentPackets - lastPackets
diffPacketBytes := currentPacketBytes - lastPacketBytes
klog.V(3).Infof(`hostname %s key %s cidr %s direction %s proto %s has diffPackets %d diffPacketBytes %d currentPackets %d currentPacketBytes %d lastPackets %d lastPacketBytes %d`,
hostname, key, cidr, direction, proto, diffPackets, diffPacketBytes, currentPackets, currentPacketBytes, lastPackets, lastPacketBytes)
if diffPackets > 0 {
metricOvnSubnetGatewayPackets.WithLabelValues(hostname, key, cidr, direction, proto).Add(float64(diffPackets))
}
if diffPacketBytes > 0 {
metricOvnSubnetGatewayPacketBytes.WithLabelValues(hostname, key, cidr, direction, proto).Add(float64(diffPacketBytes))
}
metricOvnSubnetGatewayPackets.WithLabelValues(hostname, key, cidr, direction, proto).Add(float64(diffPackets))
metricOvnSubnetGatewayPacketBytes.WithLabelValues(hostname, key, cidr, direction, proto).Add(float64(diffPacketBytes))
}
}
}
Expand Down

0 comments on commit 7a36465

Please sign in to comment.