Skip to content

Commit

Permalink
Fix underlay cni (#1006)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lou-lan
lou-lan authored Nov 24, 2023
1 parent b044126 commit 4756385
Show file tree
Hide file tree
Showing 70 changed files with 601 additions and 2,619 deletions.
8 changes: 3 additions & 5 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ require (
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
github.com/cilium/ipam v0.0.0-20220824141044-46ef3d556735
github.com/go-faker/faker/v4 v4.2.0
github.com/go-kit/log v0.2.1
github.com/go-logr/logr v1.3.0
github.com/go-swagger/go-swagger v0.30.4
github.com/google/gops v0.3.27
github.com/google/uuid v1.4.0
github.com/gorilla/websocket v1.5.1
github.com/grafana/pyroscope-go v1.0.4
github.com/mdlayher/arp v0.0.0-20220221190821-c37aaafac7f9
github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118
github.com/mdlayher/ndp v0.0.0-20200602162440-17ab9e3e5567
Expand All @@ -21,7 +21,6 @@ require (
github.com/onsi/gomega v1.30.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.17.0
github.com/pyroscope-io/client v0.7.1
github.com/sasha-s/go-deadlock v0.3.1
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.0
Expand Down Expand Up @@ -58,7 +57,6 @@ require (
github.com/fatih/color v1.15.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/go-logr/zapr v1.2.4 // indirect
github.com/go-openapi/analysis v0.21.4 // indirect
github.com/go-openapi/errors v0.20.4 // indirect
Expand All @@ -79,8 +77,9 @@ require (
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect
github.com/gorilla/handlers v1.5.1 // indirect
github.com/grafana/pyroscope-go/godeltaprof v0.1.4 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
Expand Down Expand Up @@ -113,7 +112,6 @@ require (
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/pyroscope-io/godeltaprof v0.1.0 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
Expand Down
16 changes: 6 additions & 10 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,10 +120,6 @@ github.com/go-faker/faker/v4 v4.2.0/go.mod h1:F/bBy8GH9NxOxMInug5Gx4WYeG6fHJZ8Ol
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
Expand Down Expand Up @@ -285,8 +281,8 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a h1:fEBsGL/sjAuJrgah5XqmmYsTLzJp/TO9Lhy39gkverk=
github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
Expand All @@ -303,6 +299,10 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
github.com/grafana/pyroscope-go v1.0.4 h1:oyQX0BOkL+iARXzHuCdIF5TQ7/sRSel1YFViMHC7Bm0=
github.com/grafana/pyroscope-go v1.0.4/go.mod h1:0d7ftwSMBV/Awm7CCiYmHQEG8Y44Ma3YSjt+nWcWztY=
github.com/grafana/pyroscope-go/godeltaprof v0.1.4 h1:mDsJ3ngul7UfrHibGQpV66PbZ3q1T8glz/tK3bQKKEk=
github.com/grafana/pyroscope-go/godeltaprof v0.1.4/go.mod h1:1HSPtjU8vLG0jE9JrTdzjgFqdJ/VgN7fvxBNq3luJko=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
Expand Down Expand Up @@ -452,10 +452,6 @@ github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdO
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
github.com/pyroscope-io/client v0.7.1 h1:yFRhj3vbgjBxehvxQmedmUWJQ4CAfCHhn+itPsuWsHw=
github.com/pyroscope-io/client v0.7.1/go.mod h1:4h21iOU4pUOq0prKyDlvYRL+SCKsBc5wKiEtV+rJGqU=
github.com/pyroscope-io/godeltaprof v0.1.0 h1:UBqtjt0yZi4jTxqZmLAs34XG6ycS3vUTlhEUSq4NHLE=
github.com/pyroscope-io/godeltaprof v0.1.0/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
Expand Down
44 changes: 1 addition & 43 deletions pkg/agent/eip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,7 @@ func (r *eip) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.R

// newEipCtrl return a new egress ip controller
func newEipCtrl(mgr manager.Manager, log logr.Logger, cfg *config.Config) error {
lw := logWrapper{log: log}
an, err := layer2.New(lw, cfg.FileConfig.AnnounceExcludeRegexp)
an, err := layer2.New(log, cfg.FileConfig.AnnounceExcludeRegexp)
if err != nil {
return err
}
Expand All @@ -95,44 +94,3 @@ func newEipCtrl(mgr manager.Manager, log logr.Logger, cfg *config.Config) error

return nil
}

type logWrapper struct {
log logr.Logger
}

func (lw logWrapper) Log(keyVals ...interface{}) error {
fields := make([]interface{}, 0, len(keyVals)/2)
var msgValue interface{}
var kind string
for i := 0; i < len(keyVals); i += 2 {
key, ok := keyVals[i].(string)
if !ok {
key = fmt.Sprintf("%v", keyVals[i])
}
if key == "msg" {
msgValue = keyVals[i+1]
} else if key == "level" {
kind = fmt.Sprintf("%v", keyVals[i+1])
} else {
fields = append(fields, key, keyVals[i+1])
}
}

var msg string
if msgValue != nil {
msg = fmt.Sprintf("%v", msgValue)
}

switch kind {
case "debug":
lw.log.V(1).Info(msg, fields...)
case "warn":
lw.log.Info(msg, fields...)
case "error":
lw.log.Error(fmt.Errorf(msg), "", fields...)
default:
lw.log.Info(msg, fields...)
}

return nil
}
138 changes: 84 additions & 54 deletions pkg/agent/police.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,26 +184,32 @@ func (r *policeReconciler) initApplyPolicy() error {
}

for _, table := range r.mangleTables {
table.UpdateChain(&iptables.Chain{Name: "EGRESSGATEWAY-REPLY-ROUTING"})
table.UpdateChain(&iptables.Chain{Name: "EGRESSGATEWAY-MARK-REQUEST"})
chainMapRules := buildMangleStaticRule(baseMark)
chainMapRules := buildMangleStaticRule(
baseMark,
isEgressNode,
r.cfg.FileConfig.EnableGatewayReplyRoute,
uint32(r.cfg.FileConfig.GatewayReplyRouteMark),
)
for chain, rules := range chainMapRules {
table.InsertOrAppendRules(chain, rules)
}
}

// add forward rules for replay packet on gateway node, which should be enabled for spiderpool
if isEgressNode && r.cfg.FileConfig.EnableGatewayReplyRoute {
gatewayReplyRouteMark := r.cfg.FileConfig.GatewayReplyRouteMark
dev := r.cfg.FileConfig.VXLAN.Name

for _, table := range r.mangleTables {
table.UpdateChain(&iptables.Chain{Name: "EGRESSGATEWAY-REPLY-ROUTING"})
chainMapRules := buildReplyRouteIptables(uint32(gatewayReplyRouteMark), dev)
for chain, rules := range chainMapRules {
table.InsertOrAppendRules(chain, rules)
}
}
}
//if isEgressNode && r.cfg.FileConfig.EnableGatewayReplyRoute {
// gatewayReplyRouteMark := r.cfg.FileConfig.GatewayReplyRouteMark
// dev := r.cfg.FileConfig.VXLAN.Name
//
// for _, table := range r.mangleTables {
// table.UpdateChain(&iptables.Chain{Name: "EGRESSGATEWAY-REPLY-ROUTING"})
// chainMapRules := buildReplyRouteIptables(uint32(gatewayReplyRouteMark), dev)
// for chain, rules := range chainMapRules {
// table.InsertOrAppendRules(chain, rules)
// }
// }
//}

for _, table := range r.mangleTables {
rules := make([]iptables.Rule, 0)
Expand Down Expand Up @@ -236,6 +242,11 @@ func (r *policeReconciler) initApplyPolicy() error {
Name: "EGRESSGATEWAY-MARK-REQUEST",
Rules: rules,
})
table.UpdateChain(&iptables.Chain{
Name: "EGRESSGATEWAY-REPLY-ROUTING",
Rules: buildPreroutingReplyRouting(r.cfg.FileConfig.VXLAN.Name,
uint32(r.cfg.FileConfig.GatewayReplyRouteMark)),
})
}

for _, table := range r.natTables {
Expand Down Expand Up @@ -741,79 +752,98 @@ func buildFilterStaticRule(base uint32) map[string][]iptables.Rule {
return res
}

func buildMangleStaticRule(base uint32) map[string][]iptables.Rule {
res := map[string][]iptables.Rule{
"FORWARD": {{
func buildMangleStaticRule(base uint32,
isEgressNode bool,
enableGatewayReplyRoute bool, replyMark uint32) map[string][]iptables.Rule {

forward := []iptables.Rule{
{
Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xff000000),
Action: iptables.SetMaskedMarkAction{Mark: base, Mask: 0xffffffff},
Comment: []string{
"Accept for egress traffic from pod going to EgressTunnel",
},
}},
"POSTROUTING": {{
Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xffffffff),
Action: iptables.AcceptAction{},
Comment: []string{
"Accept for egress traffic from pod going to EgressTunnel",
},
}},
"PREROUTING": {{
},
}

postrouting := []iptables.Rule{{
Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xffffffff),
Action: iptables.AcceptAction{},
Comment: []string{
"Accept for egress traffic from pod going to EgressTunnel",
},
}}

prerouting := []iptables.Rule{
{
Match: iptables.MatchCriteria{},
Action: iptables.JumpAction{Target: "EGRESSGATEWAY-MARK-REQUEST"},
Comment: []string{
"Checking for EgressPolicy matched traffic",
},
}},
},
}
return res
}

func buildReplyRouteIptables(base uint32, dev string) map[string][]iptables.Rule {
res := map[string][]iptables.Rule{
"PREROUTING": {
if isEgressNode && enableGatewayReplyRoute {
prerouting = []iptables.Rule{
{
Match: iptables.MatchCriteria{},
Action: iptables.JumpAction{Target: "EGRESSGATEWAY-REPLY-ROUTING"},
Comment: []string{
"egressGateway Reply datapath rule, rule is from the EgressGateway",
},
},
},
"EGRESSGATEWAY-REPLY-ROUTING": {
{
Match: iptables.MatchCriteria{}.InInterface(dev),
Action: iptables.SetMaskedMarkAction{Mark: base, Mask: 0xffffffff},
Comment: []string{
"mark the traffic from the EgressGateway tunnel, rule is from the EgressGateway",
},
},
{
Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xffffffff),
Action: iptables.SaveConnMarkAction{SaveMask: base},
Comment: []string{
"save mark to the connection, rule is from the EgressGateway",
},
},
{
Match: iptables.MatchCriteria{}.ConntrackState("ESTABLISHED"),
Action: iptables.RestoreConnMarkAction{RestoreMask: 0},
Match: iptables.MatchCriteria{},
Action: iptables.JumpAction{Target: "EGRESSGATEWAY-MARK-REQUEST"},
Comment: []string{
"label for restoring connections, rule is from the EgressGateway",
"Checking for EgressPolicy matched traffic",
},
},
},
"POSTROUTING": {{
Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xffffffff),
}
postrouting = append(postrouting, iptables.Rule{
Match: iptables.MatchCriteria{}.MarkMatchesWithMask(replyMark, 0xffffffff),
Action: iptables.SetMaskedMarkAction{Mark: 0x00000000, Mask: 0xffffffff},
Comment: []string{
"clear the Mark of the inner package, rule is from the EgressGateway",
},
}},
})
}

res := map[string][]iptables.Rule{
"FORWARD": forward,
"POSTROUTING": postrouting,
"PREROUTING": prerouting,
}
return res
}

func buildPreroutingReplyRouting(vxlanName string, replyMark uint32) []iptables.Rule {
return []iptables.Rule{
{
Match: iptables.MatchCriteria{}.InInterface(vxlanName),
Action: iptables.SetMaskedMarkAction{Mark: replyMark, Mask: 0xffffffff},
Comment: []string{
"mark the traffic from the EgressGateway tunnel, rule is from the EgressGateway",
},
},
{
Match: iptables.MatchCriteria{}.MarkMatchesWithMask(replyMark, 0xffffffff),
Action: iptables.SaveConnMarkAction{SaveMask: replyMark},
Comment: []string{
"save mark to the connection, rule is from the EgressGateway",
},
},
{
Match: iptables.MatchCriteria{}.ConntrackState("ESTABLISHED"),
Action: iptables.RestoreConnMarkAction{RestoreMask: 0},
Comment: []string{
"label for restoring connections, rule is from the EgressGateway",
},
},
}
}

// reconcilePolicy reconcile egress policy
// watch update/delete events
// - ipset
Expand Down
Loading

0 comments on commit 4756385

Please sign in to comment.