This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new 5d5160fb fix: preserve legacy CNI values in upgrade e2e (#1011)
5d5160fb is described below
commit 5d5160fbd7193e6549409e634aecc98103714cdf
Author: mfordjody <[email protected]>
AuthorDate: Tue Aug 11 08:50:22 2026 +0800
fix: preserve legacy CNI values in upgrade e2e (#1011)
* fix: preserve legacy CNI values in upgrade e2e
* test: collect pod events on e2e failure
* fix: fall back to direct iptables rules without ipset
* fix: preserve established CNI connections
---
cni/pkg/nodeagent/iptables.go | 153 ++++++++++++++++++++++++++++++++++++-
cni/pkg/nodeagent/iptables_test.go | 98 +++++++++++++++++++++++-
tests/e2e/run.sh | 14 +++-
3 files changed, 258 insertions(+), 7 deletions(-)
diff --git a/cni/pkg/nodeagent/iptables.go b/cni/pkg/nodeagent/iptables.go
index 41830e7b..200c3885 100644
--- a/cni/pkg/nodeagent/iptables.go
+++ b/cni/pkg/nodeagent/iptables.go
@@ -18,8 +18,10 @@ package nodeagent
import (
"bytes"
"context"
+ "errors"
"fmt"
"net"
+ "os"
"os/exec"
)
@@ -53,6 +55,10 @@ type IPTablesRuleManager struct {
grpcInboundPort int
dryRun bool
runner CommandRunner
+ // directRules is used only when the node does not provide ipset. It
keeps
+ // the same inbound fence with destination-IP rules instead of failing
every
+ // managed Pod's sandbox creation.
+ directRules bool
}
func NewIPTablesRuleManager(conf NetConf) *IPTablesRuleManager {
@@ -76,7 +82,13 @@ func (m *IPTablesRuleManager) AddPodRules(ctx
context.Context, podIP string, exc
if err != nil {
return err
}
+ if m.directRules {
+ return m.addDirectPodRules(ctx, ip, excludedPorts)
+ }
if err := m.ensureBase(ctx); err != nil {
+ if m.useDirectRules(err) {
+ return m.addDirectPodRules(ctx, ip, excludedPorts)
+ }
return err
}
if err := m.runIPSet(ctx, "add", meshPodIPSet, ip, "-exist"); err !=
nil {
@@ -90,12 +102,24 @@ func (m *IPTablesRuleManager) DeletePodRules(ctx
context.Context, podIP string,
if err != nil {
return err
}
+ if m.directRules {
+ return m.deleteDirectPodRules(ctx, ip, excludedPorts)
+ }
for _, port := range excludedPorts {
if err := m.runIPSet(ctx, "del", meshExcludeIPSet,
excludeEntry(ip, port), "-exist"); err != nil {
+ if m.useDirectRules(err) {
+ return m.deleteDirectPodRules(ctx, ip,
excludedPorts)
+ }
return err
}
}
- return m.runIPSet(ctx, "del", meshPodIPSet, ip, "-exist")
+ if err := m.runIPSet(ctx, "del", meshPodIPSet, ip, "-exist"); err !=
nil {
+ if m.useDirectRules(err) {
+ return m.deleteDirectPodRules(ctx, ip, excludedPorts)
+ }
+ return err
+ }
+ return nil
}
// Reconcile rebuilds the whole fence from the supplied pod states.
@@ -105,7 +129,13 @@ func (m *IPTablesRuleManager) DeletePodRules(ctx
context.Context, podIP string,
// Without this the fence silently disappears for every already-running pod,
// so the node agent replays it from its own persisted state.
func (m *IPTablesRuleManager) Reconcile(ctx context.Context, states
[]PodState) error {
+ if m.directRules {
+ return m.reconcileDirectRules(ctx, states)
+ }
if err := m.ensureBase(ctx); err != nil {
+ if m.useDirectRules(err) {
+ return m.reconcileDirectRules(ctx, states)
+ }
return err
}
if err := m.runIPSet(ctx, "flush", meshPodIPSet); err != nil {
@@ -130,6 +160,123 @@ func (m *IPTablesRuleManager) Reconcile(ctx
context.Context, states []PodState)
return nil
}
+func (m *IPTablesRuleManager) useDirectRules(err error) bool {
+ if !errors.Is(err, exec.ErrNotFound) {
+ return false
+ }
+ m.directRules = true
+ fmt.Fprintln(os.Stderr, "dubbo-cni: ipset is unavailable; using direct
iptables inbound rules")
+ return true
+}
+
+func (m *IPTablesRuleManager) addDirectPodRules(ctx context.Context, ip
string, excludedPorts []int) error {
+ if err := m.ensureDirectBase(ctx); err != nil {
+ return err
+ }
+ rules, err := m.directPodRules(ip, excludedPorts)
+ if err != nil {
+ return err
+ }
+ m.deleteDirectRules(ctx, rules)
+ for _, rule := range rules {
+ if err := m.appendRule(ctx, rule...); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (m *IPTablesRuleManager) deleteDirectPodRules(ctx context.Context, ip
string, excludedPorts []int) error {
+ rules, err := m.directPodRules(ip, excludedPorts)
+ if err != nil {
+ return err
+ }
+ m.deleteDirectRules(ctx, rules)
+ return nil
+}
+
+func (m *IPTablesRuleManager) reconcileDirectRules(ctx context.Context, states
[]PodState) error {
+ if err := m.ensureDirectBase(ctx); err != nil {
+ return err
+ }
+ if err := m.run(ctx, "-w", "-t", "filter", "-F", meshInboundChain); err
!= nil {
+ return err
+ }
+ for _, state := range states {
+ ip, err := normalizePodIP(state.IP)
+ if err != nil {
+ // A malformed entry must not stop the remaining pods
from being restored.
+ continue
+ }
+ rules, err := m.directPodRules(ip, state.ExcludedPorts)
+ if err != nil {
+ return err
+ }
+ for _, rule := range rules {
+ if err := m.appendRule(ctx, rule...); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (m *IPTablesRuleManager) ensureDirectBase(ctx context.Context) error {
+ if err := m.runIgnoreExists(ctx, "-w", "-t", "filter", "-N",
meshInboundChain); err != nil {
+ return err
+ }
+ for _, chain := range []string{"FORWARD", "OUTPUT"} {
+ if err := m.run(ctx, "-w", "-t", "filter", "-C", chain, "-j",
meshInboundChain); err == nil {
+ continue
+ }
+ if err := m.run(ctx, "-w", "-t", "filter", "-I", chain, "1",
"-j", meshInboundChain); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (m *IPTablesRuleManager) directPodRules(ip string, excludedPorts []int)
([][]string, error) {
+ ports := make([]int, 0, len(excludedPorts)+3)
+ seen := make(map[int]struct{}, len(excludedPorts)+3)
+ addPort := func(port int) error {
+ if port < 1 || port > 65535 {
+ return fmt.Errorf("excluded port %d is out of range",
port)
+ }
+ if _, found := seen[port]; found {
+ return nil
+ }
+ seen[port] = struct{}{}
+ ports = append(ports, port)
+ return nil
+ }
+ for _, port := range excludedPorts {
+ if err := addPort(port); err != nil {
+ return nil, err
+ }
+ }
+ for _, port := range []int{m.grpcInboundPort, dxgateAdminPort,
dxproxyAdminPort} {
+ if err := addPort(port); err != nil {
+ return nil, err
+ }
+ }
+ rules := make([][]string, 0, len(ports)+1)
+ for _, port := range ports {
+ rules = append(rules, []string{"-d", ip, "-p", "tcp",
"--dport", fmt.Sprint(port), "-j", "RETURN"})
+ }
+ // Fence only connection attempts. A reply to an outbound xDS/activation
+ // connection also has the Pod as its destination, but is ESTABLISHED
and
+ // must pass; rejecting it would leave injected gateways permanently
+ // unready.
+ return append(rules, []string{"-d", ip, "-p", "tcp", "-m", "conntrack",
"--ctstate", "NEW", "-j", "REJECT"}), nil
+}
+
+func (m *IPTablesRuleManager) deleteDirectRules(ctx context.Context, rules
[][]string) {
+ for _, rule := range rules {
+ m.deleteRepeated(ctx, rule...)
+ }
+}
+
func (m *IPTablesRuleManager) addExcludedPorts(ctx context.Context, ip string,
ports []int) error {
for _, port := range ports {
if port < 1 || port > 65535 {
@@ -170,7 +317,9 @@ func (m *IPTablesRuleManager) ensureBase(ctx
context.Context) error {
allowGRPCInbound := []string{"-m", "set", "--match-set", meshPodIPSet,
"dst", "-p", "tcp", "--dport", fmt.Sprint(m.grpcInboundPort), "-j", "RETURN"}
allowDxgateAdmin := []string{"-m", "set", "--match-set", meshPodIPSet,
"dst", "-p", "tcp", "--dport", fmt.Sprint(dxgateAdminPort), "-j", "RETURN"}
allowDxproxyAdmin := []string{"-m", "set", "--match-set", meshPodIPSet,
"dst", "-p", "tcp", "--dport", fmt.Sprint(dxproxyAdminPort), "-j", "RETURN"}
- rejectOtherTCP := []string{"-m", "set", "--match-set", meshPodIPSet,
"dst", "-p", "tcp", "-j", "REJECT"}
+ // Reject only new inbound connections. Established replies to outbound
+ // control-plane traffic still target a managed Pod and must not be
fenced.
+ rejectOtherTCP := []string{"-m", "set", "--match-set", meshPodIPSet,
"dst", "-p", "tcp", "-m", "conntrack", "--ctstate", "NEW", "-j", "REJECT"}
m.deleteRepeated(ctx, allowExcluded...)
m.deleteRepeated(ctx, allowGRPCInbound...)
m.deleteRepeated(ctx, allowDxgateAdmin...)
diff --git a/cni/pkg/nodeagent/iptables_test.go
b/cni/pkg/nodeagent/iptables_test.go
index e1121391..db002649 100644
--- a/cni/pkg/nodeagent/iptables_test.go
+++ b/cni/pkg/nodeagent/iptables_test.go
@@ -17,6 +17,7 @@ package nodeagent
import (
"context"
+ "os/exec"
"strings"
"testing"
)
@@ -44,7 +45,7 @@ func TestIPTablesRuleManagerAddsGRPCInboundBoundaryRules(t
*testing.T) {
"-A DUBBO-GRPC-INBOUND -m set --match-set
DUBBO-GRPC-INBOUND-PODS dst -p tcp --dport 15080 -j RETURN",
"-A DUBBO-GRPC-INBOUND -m set --match-set
DUBBO-GRPC-INBOUND-PODS dst -p tcp --dport 26021 -j RETURN",
"-A DUBBO-GRPC-INBOUND -m set --match-set
DUBBO-GRPC-INBOUND-PODS dst -p tcp --dport 15020 -j RETURN",
- "-A DUBBO-GRPC-INBOUND -m set --match-set
DUBBO-GRPC-INBOUND-PODS dst -p tcp -j REJECT",
+ "-A DUBBO-GRPC-INBOUND -m set --match-set
DUBBO-GRPC-INBOUND-PODS dst -p tcp -m conntrack --ctstate NEW -j REJECT",
"ipset add DUBBO-GRPC-INBOUND-PODS 10.244.0.12 -exist",
"ipset add DUBBO-GRPC-INBOUND-EXCLUDE 10.244.0.12,tcp:9090
-exist",
} {
@@ -55,7 +56,7 @@ func TestIPTablesRuleManagerAddsGRPCInboundBoundaryRules(t
*testing.T) {
// The exclusion must be evaluated before the catch-all REJECT.
excludeAt := strings.Index(joined, "--match-set
DUBBO-GRPC-INBOUND-EXCLUDE dst,dst -p tcp -j RETURN")
- rejectAt := strings.Index(joined, "DUBBO-GRPC-INBOUND-PODS dst -p tcp
-j REJECT")
+ rejectAt := strings.Index(joined, "DUBBO-GRPC-INBOUND-PODS dst -p tcp
-m conntrack --ctstate NEW -j REJECT")
if excludeAt < 0 || rejectAt < 0 || excludeAt > rejectAt {
t.Fatalf("exclusion rule is not appended before the REJECT
rule:\n%s", joined)
}
@@ -95,6 +96,87 @@ func TestIPTablesRuleManagerReconcileRebuildsFence(t
*testing.T) {
}
}
+func TestIPTablesRuleManagerFallsBackToDirectRulesWithoutIPSet(t *testing.T) {
+ runner := &ipsetMissingRunner{}
+ conf, err := ParseNetConf([]byte(`{"grpcInboundPort":15080}`))
+ if err != nil {
+ t.Fatalf("ParseNetConf() failed: %v", err)
+ }
+ manager := NewIPTablesRuleManagerWithRunner(conf, runner)
+
+ if err := manager.AddPodRules(context.Background(), "10.244.0.12",
[]int{9090}); err != nil {
+ t.Fatalf("AddPodRules() failed: %v", err)
+ }
+
+ joined := strings.Join(runner.commands, "\n")
+ for _, want := range []string{
+ "ipset create DUBBO-GRPC-INBOUND-PODS hash:ip -exist",
+ "-A DUBBO-GRPC-INBOUND -d 10.244.0.12 -p tcp --dport 9090 -j
RETURN",
+ "-A DUBBO-GRPC-INBOUND -d 10.244.0.12 -p tcp --dport 15080 -j
RETURN",
+ "-A DUBBO-GRPC-INBOUND -d 10.244.0.12 -p tcp --dport 26021 -j
RETURN",
+ "-A DUBBO-GRPC-INBOUND -d 10.244.0.12 -p tcp --dport 15020 -j
RETURN",
+ "-A DUBBO-GRPC-INBOUND -d 10.244.0.12 -p tcp -m conntrack
--ctstate NEW -j REJECT",
+ } {
+ if !strings.Contains(joined, want) {
+ t.Fatalf("commands missing %q:\n%s", want, joined)
+ }
+ }
+ if strings.Contains(joined, "--match-set") {
+ t.Fatalf("ipset rule leaked into direct fallback:\n%s", joined)
+ }
+}
+
+func TestIPTablesRuleManagerReconcilesDirectRulesWithoutIPSet(t *testing.T) {
+ runner := &ipsetMissingRunner{}
+ conf, err := ParseNetConf([]byte(`{"grpcInboundPort":15080}`))
+ if err != nil {
+ t.Fatalf("ParseNetConf() failed: %v", err)
+ }
+ manager := NewIPTablesRuleManagerWithRunner(conf, runner)
+
+ if err := manager.Reconcile(context.Background(), []PodState{{
+ IP: "10.244.0.12",
+ ExcludedPorts: []int{9090},
+ }}); err != nil {
+ t.Fatalf("Reconcile() failed: %v", err)
+ }
+
+ joined := strings.Join(runner.commands, "\n")
+ for _, want := range []string{
+ "-F DUBBO-GRPC-INBOUND",
+ "-A DUBBO-GRPC-INBOUND -d 10.244.0.12 -p tcp --dport 9090 -j
RETURN",
+ "-A DUBBO-GRPC-INBOUND -d 10.244.0.12 -p tcp -m conntrack
--ctstate NEW -j REJECT",
+ } {
+ if !strings.Contains(joined, want) {
+ t.Fatalf("commands missing %q:\n%s", want, joined)
+ }
+ }
+}
+
+func TestIPTablesRuleManagerDeletesDirectRulesWithoutIPSet(t *testing.T) {
+ runner := &ipsetMissingRunner{}
+ conf, err := ParseNetConf([]byte(`{"grpcInboundPort":15080}`))
+ if err != nil {
+ t.Fatalf("ParseNetConf() failed: %v", err)
+ }
+ manager := NewIPTablesRuleManagerWithRunner(conf, runner)
+
+ if err := manager.DeletePodRules(context.Background(), "10.244.0.12",
[]int{9090}); err != nil {
+ t.Fatalf("DeletePodRules() failed: %v", err)
+ }
+
+ joined := strings.Join(runner.commands, "\n")
+ for _, want := range []string{
+ "-D DUBBO-GRPC-INBOUND -d 10.244.0.12 -p tcp --dport 9090 -j
RETURN",
+ "-D DUBBO-GRPC-INBOUND -d 10.244.0.12 -p tcp --dport 15080 -j
RETURN",
+ "-D DUBBO-GRPC-INBOUND -d 10.244.0.12 -p tcp -m conntrack
--ctstate NEW -j REJECT",
+ } {
+ if !strings.Contains(joined, want) {
+ t.Fatalf("commands missing %q:\n%s", want, joined)
+ }
+ }
+}
+
type recordingRunner struct {
commands []string
}
@@ -109,6 +191,18 @@ func (r *recordingRunner) Run(_ context.Context, name
string, args ...string) ([
return nil, nil
}
+type ipsetMissingRunner struct {
+ recordingRunner
+}
+
+func (r *ipsetMissingRunner) Run(ctx context.Context, name string, args
...string) ([]byte, error) {
+ if name == "ipset" {
+ r.commands = append(r.commands, name+" "+strings.Join(args, "
"))
+ return nil, &exec.Error{Name: name, Err: exec.ErrNotFound}
+ }
+ return r.recordingRunner.Run(ctx, name, args...)
+}
+
var errCommandFailed = commandFailedError{}
type commandFailedError struct{}
diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh
index 6c6ed42f..0deaf702 100755
--- a/tests/e2e/run.sh
+++ b/tests/e2e/run.sh
@@ -66,6 +66,13 @@ fail() {
echo "FAIL: $*" >&2
echo "--- diagnostics: pods ---" >&2
"${KUBECTL[@]}" get pods -A -o wide >&2 || true
+ # ContainerCreating failures have no container logs. Events and describe
+ # expose the kubelet/CNI/image error before cleanup deletes the cluster.
+ echo "--- diagnostics: events ---" >&2
+ "${KUBECTL[@]}" get events -A --sort-by=.lastTimestamp >&2 || true
+ echo "--- diagnostics: managed gateway pod descriptions ---" >&2
+ "${KUBECTL[@]}" -n "${APP_NS}" describe pods \
+ -l gateway.networking.k8s.io/gateway-name >&2 || true
echo "--- diagnostics: dubbod logs ---" >&2
"${KUBECTL[@]}" -n "${SYSTEM_NS}" logs deploy/dubbod --tail=100 >&2 || true
echo "--- diagnostics: managed gateway logs ---" >&2
@@ -234,10 +241,11 @@ install_dubbod() {
--set-string "gateway.image=${DXGATE_IMAGE}"
)
else
- # The previous chart still exposes its legacy CNI configuration.
+ # 0.4.3 predates Inherent. These values target its immutable legacy
+ # chart schema; renaming them leaves its CNI enabled during the upgrade.
chart_values=(
- --set "global.inherent.cni.enabled=false"
- --set-string "global.inherent.cni.image=${image}"
+ --set "global.proxyless.cni.enabled=false"
+ --set-string "global.proxyless.cni.image=${image}"
--set-string "global.gateway.image=${DXGATE_IMAGE}"
)
fi
