Add IPv6/dual-stack support

## Flags Changes

This replaces the proxy-init flags `--firewall-bin-path` and `--firewall-save-bin-path` with the flag `--iptables-mode` (with possible values `legacy` and `nft`).
Also the `--ipv6` flag has been added (default `true`).
Proxy-init won't be relying just on the iptables commands family (iptables-legacy, iptables-legacy-save, iptables-nft, iptable-nft-save), but also on the ip6tables command family, so it's better to know the mode we're in (legacy or nft) and whether ipv6 is enabled, to determine all the commands that need to be used instead of directly passing them as arguments.

After the set of rules run via iptables are processed, if `--ipv6` is true (which is the default), the same set of rules will be run via ip6tables.

Analog changes were applied to linkerd-cni as well.

## Backwards-Compatibility

This is backwards-compatible with older control planes as long as the older flags are not used. If those flags are used, an explanatory error is thrown (better than showing a deprecation message and failing later when legacy/nft iptables don't work).
If `--ipv6` is not passed (and thus defaults to true), this doesn't impact operation even if the cluster doesn't support IPv6; the ip6tables rules are applied but they're innocuous.
OTOH if there's no kernel support for IPv6 then the ip6tables command will fail but we'll just log the failure and not fail the linkerd-init container (nor the `add` command for linkerd-cni). This avoids having to explicitly set `--ipv6=false`, but it can be set if the user is aware of such limitations and wants to get rid of the errors.

## Linkerd IPv6 Support

This allows routing IPv6 traffic to the proxy, but is just the first step towards IPv6/dual-stack support. Control plane and proxy changes will come up next.

cni-plugin/integration/manifests/calico/linkerd-cni.yaml

@@ -80,7 +80,9 @@ data:
         "ports-to-redirect": [],
         "inbound-ports-to-ignore": ["4191","4190"],
         "simulate": false,
-        "use-wait-flag": false
+        "use-wait-flag": false,
+        "iptables-mode": "legacy",
+        "ipv6": true
       }
     }
 ---

cni-plugin/integration/manifests/cilium/linkerd-cni.yaml

@@ -80,7 +80,9 @@ data:
         "ports-to-redirect": [],
         "inbound-ports-to-ignore": ["4191","4190"],
         "simulate": false,
-        "use-wait-flag": false
+        "use-wait-flag": false,
+        "iptables-mode": "legacy",
+        "ipv6": true
       }
     }
 ---

cni-plugin/integration/manifests/flannel/linkerd-cni.yaml

@@ -84,7 +84,9 @@ data:
         "ports-to-redirect": [],
         "inbound-ports-to-ignore": ["4191","4190"],
         "simulate": false,
-        "use-wait-flag": false
+        "use-wait-flag": false,
+        "iptables-mode": "legacy",
+        "ipv6": true
       }
     }
 ---

cni-plugin/integration/testutil/test_util.go

@@ -49,8 +49,11 @@ type ProxyInit struct {
 	PortsToRedirect       []int    `json:"ports-to-redirect"`
 	InboundPortsToIgnore  []string `json:"inbound-ports-to-ignore"`
 	OutboundPortsToIgnore []string `json:"outbound-ports-to-ignore"`
+	SubnetsToIgnore       []string `json:"subnets-to-ignore"`
 	Simulate              bool     `json:"simulate"`
 	UseWaitFlag           bool     `json:"use-wait-flag"`
+	IPTablesMode          string   `json:"iptables-mode"`
+	IPv6                  bool     `json:"ipv6"`
 }
 
 // LinkerdPlugin is what we use for CNI configuration in the plugins section

cni-plugin/main.go

@@ -52,6 +52,8 @@ type ProxyInit struct {
 	SubnetsToIgnore       []string `json:"subnets-to-ignore"`
 	Simulate              bool     `json:"simulate"`
 	UseWaitFlag           bool     `json:"use-wait-flag"`
+	IPTablesMode          string   `json:"iptables-mode"`
+	IPv6                  bool     `json:"ipv6"`
 }
 
 // Kubernetes a K8s specific struct to hold config
@@ -219,8 +221,8 @@ func cmdAdd(args *skel.CmdArgs) error {
 				SimulateOnly:          conf.ProxyInit.Simulate,
 				NetNs:                 args.Netns,
 				UseWaitFlag:           conf.ProxyInit.UseWaitFlag,
-				FirewallBinPath:       "iptables-legacy",
-				FirewallSaveBinPath:   "iptables-legacy-save",
+				IPTablesMode:          conf.ProxyInit.IPTablesMode,
+				IPv6:                  conf.ProxyInit.IPv6,
 			}
 
 			// Check if there are any overridden ports to be skipped
@@ -292,17 +294,19 @@ func cmdAdd(args *skel.CmdArgs) error {
 				options.OutboundPortsToIgnore = append(options.OutboundPortsToIgnore, skippedPorts...)
 			}
 
-			firewallConfiguration, err := cmd.BuildFirewallConfiguration(&options)
-			if err != nil {
-				logEntry.Errorf("linkerd-cni: could not create a Firewall Configuration from the options: %v", options)
-				return err
+			// This ensures BC against linkerd2-cni older versions not yet passing this flag
+			if options.IPTablesMode == "" {
+				options.IPTablesMode = cmd.IPTablesModeLegacy
 			}
 
-			err = iptables.ConfigureFirewall(*firewallConfiguration)
-			if err != nil {
-				logEntry.Errorf("linkerd-cni: could not configure firewall: %s", err)
+			if err := buildAndConfigure(logEntry, &options, false); err != nil {
 				return err
 			}
+			if options.IPv6 {
+				if err := buildAndConfigure(logEntry, &options, true); err != nil {
+					return err
+				}
+			}
 		} else {
 			if containsInitContainer {
 				logEntry.Debug("linkerd-cni: linkerd-init initContainer is present, skipping.")
@@ -353,6 +357,24 @@ func getAPIServerPorts(ctx context.Context, api *kubernetes.Clientset) ([]string
 	return ports, nil
 }
 
+func buildAndConfigure(logEntry *logrus.Entry, options *cmd.RootOptions, ipv6 bool) error {
+	firewallConfiguration, err := cmd.BuildFirewallConfiguration(options, ipv6)
+	if err != nil {
+		logEntry.Errorf("linkerd-cni: could not create a Firewall Configuration from the options: %v", options)
+		return err
+	}
+
+	err = iptables.ConfigureFirewall(*firewallConfiguration)
+	// We couldn't find a robust way of checking IPv6 support besides trying to just call ip6tables-save.
+	// If IPv4 rules worked but not IPv6, let's not fail the container (the actual problem will get logged).
+	if !ipv6 && err != nil {
+		logEntry.Errorf("linkerd-cni: could not configure firewall: %s", err)
+		return err
+	}
+
+	return nil
+}
+
 func getAnnotationOverride(ctx context.Context, api *kubernetes.Clientset, pod *v1.Pod, key string) (string, error) {
 	// Check if the annotation is present on the pod
 	if override := pod.GetObjectMeta().GetAnnotations()[key]; override != "" {

justfile

@@ -224,6 +224,7 @@ _cni-plugin-setup-cilium:
     echo "Mounted /sys/fs/bpf to cilium-test-server cluster"
     helm repo add cilium https://helm.cilium.io/
     helm install cilium cilium/cilium --version 1.13.0 \
+        --kube-context k3d-l5d-cilium-test \
         --namespace kube-system \
         --set kubeProxyReplacement=partial \
         --set hostServices.enabled=false \

proxy-init/cmd/root.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"net"
 	"os/exec"
@@ -13,6 +14,22 @@ import (
 	"github.com/linkerd/linkerd2-proxy-init/internal/util"
 )
 
+const (
+	// IPTablesModeLegacy signals the usage of the iptables-legacy commands
+	IPTablesModeLegacy = "legacy"
+	// ipTablesModeNFT signals the usage of the iptables-nft commands
+	ipTablesModeNFT = "nft"
+
+	cmdLegacy         = "iptables-legacy"
+	cmdLegacySave     = "iptables-legacy-save"
+	cmdLegacyIPv6     = "ip6tables-legacy"
+	cmdLegacyIPv6Save = "ip6tables-legacy-save"
+	cmdNFT            = "iptables-nft"
+	cmdNFTSave        = "iptables-nft-save"
+	cmdNFTIPv6        = "ip6tables-nft"
+	cmdNFTIPv6Save    = "ip6tables-nft-save"
+)
+
 // RootOptions provides the information that will be used to build a firewall configuration.
 type RootOptions struct {
 	IncomingProxyPort     int
@@ -28,8 +45,12 @@ type RootOptions struct {
 	TimeoutCloseWaitSecs  int
 	LogFormat             string
 	LogLevel              string
-	FirewallBinPath       string
-	FirewallSaveBinPath   string
+	IPTablesMode          string
+	IPv6                  bool
+
+	// No longer supported
+	FirewallBinPath     string
+	FirewallSaveBinPath string
 }
 
 func newRootOptions() *RootOptions {
@@ -47,8 +68,8 @@ func newRootOptions() *RootOptions {
 		TimeoutCloseWaitSecs:  0,
 		LogFormat:             "plain",
 		LogLevel:              "info",
-		FirewallBinPath:       "iptables-legacy",
-		FirewallSaveBinPath:   "iptables-legacy-save",
+		IPTablesMode:          IPTablesModeLegacy,
+		IPv6:                  true,
 	}
 }
 
@@ -61,7 +82,7 @@ func NewRootCmd() *cobra.Command {
 		Use:   "proxy-init",
 		Short: "proxy-init adds a Kubernetes pod to the Linkerd service mesh",
 		Long:  "proxy-init adds a Kubernetes pod to the Linkerd service mesh.",
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, _ []string) error {
 
 			if options.TimeoutCloseWaitSecs != 0 {
 				sysctl := exec.Command("sysctl", "-w",
@@ -75,16 +96,35 @@ func NewRootCmd() *cobra.Command {
 				log.Info(string(out))
 			}
 
-			config, err := BuildFirewallConfiguration(options)
+			log.SetFormatter(getFormatter(options.LogFormat))
+			err := setLogLevel(options.LogLevel)
 			if err != nil {
 				return err
 			}
-			log.SetFormatter(getFormatter(options.LogFormat))
-			err = setLogLevel(options.LogLevel)
+
+			config, err := BuildFirewallConfiguration(options, false)
 			if err != nil {
 				return err
 			}
-			return iptables.ConfigureFirewall(*config)
+
+			if err = iptables.ConfigureFirewall(*config); err != nil {
+				return err
+			}
+
+			if !options.IPv6 {
+				return nil
+			}
+
+			config, err = BuildFirewallConfiguration(options, true)
+			if err != nil {
+				return err
+			}
+
+			// We couldn't find a robust way of checking IPv6 support besides trying to just call ip6tables-save.
+			// If IPv4 rules worked but not IPv6, let's not fail the container (the actual problem will get logged).
+			_ = iptables.ConfigureFirewall(*config)
+
+			return nil
 		},
 	}
 
@@ -101,13 +141,31 @@ func NewRootCmd() *cobra.Command {
 	cmd.PersistentFlags().IntVar(&options.TimeoutCloseWaitSecs, "timeout-close-wait-secs", options.TimeoutCloseWaitSecs, "Sets nf_conntrack_tcp_timeout_close_wait")
 	cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", options.LogFormat, "Configure log format ('plain' or 'json')")
 	cmd.PersistentFlags().StringVar(&options.LogLevel, "log-level", options.LogLevel, "Configure log level")
+	cmd.PersistentFlags().StringVar(&options.IPTablesMode, "iptables-mode", options.IPTablesMode, "Variant of iptables command to use (\"legacy\" or \"nft\")")
+	cmd.PersistentFlags().BoolVar(&options.IPv6, "ipv6", options.IPv6, "Set rules both via iptables and ip6tables to support dual-stack networking")
 	cmd.PersistentFlags().StringVar(&options.FirewallBinPath, "firewall-bin-path", options.FirewallBinPath, "Path to iptables binary")
 	cmd.PersistentFlags().StringVar(&options.FirewallSaveBinPath, "firewall-save-bin-path", options.FirewallSaveBinPath, "Path to iptables-save binary")
+
+	if err := cmd.PersistentFlags().MarkHidden("firewall-bin-path"); err != nil {
+		log.Fatal(err)
+	}
+	if err := cmd.PersistentFlags().MarkHidden("firewall-save-bin-path"); err != nil {
+		log.Fatal(err)
+	}
+
 	return cmd
 }
 
 // BuildFirewallConfiguration returns an iptables FirewallConfiguration suitable to use to configure iptables.
-func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfiguration, error) {
+func BuildFirewallConfiguration(options *RootOptions, ipv6 bool) (*iptables.FirewallConfiguration, error) {
+	if options.FirewallBinPath != "" || options.FirewallSaveBinPath != "" {
+		return nil, errors.New("--firewal-bin-path and firewall-save-bin-path are no longer supported; please use --iptables-mode instead")
+	}
+
+	if options.IPTablesMode != IPTablesModeLegacy && options.IPTablesMode != ipTablesModeNFT {
+		return nil, errors.New("--iptables-mode valid values are only \"legacy\" and \"nft\"")
+	}
+
 	if !util.IsValidPort(options.IncomingProxyPort) {
 		return nil, fmt.Errorf("--incoming-proxy-port must be a valid TCP port number")
 	}
@@ -116,6 +174,8 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu
 		return nil, fmt.Errorf("--outgoing-proxy-port must be a valid TCP port number")
 	}
 
+	cmd, cmdSave := getCommands(options.IPTablesMode, ipv6)
+
 	sanitizedSubnets := []string{}
 	for _, subnet := range options.SubnetsToIgnore {
 		subnet := strings.TrimSpace(subnet)
@@ -138,8 +198,8 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu
 		SimulateOnly:           options.SimulateOnly,
 		NetNs:                  options.NetNs,
 		UseWaitFlag:            options.UseWaitFlag,
-		BinPath:                options.FirewallBinPath,
-		SaveBinPath:            options.FirewallSaveBinPath,
+		BinPath:                cmd,
+		SaveBinPath:            cmdSave,
 	}
 
 	if len(options.PortsToRedirect) > 0 {
@@ -160,6 +220,21 @@ func getFormatter(format string) log.Formatter {
 	}
 }
 
+func getCommands(mode string, ipv6 bool) (string, string) {
+	if mode == IPTablesModeLegacy {
+		if ipv6 {
+			return cmdLegacyIPv6, cmdLegacyIPv6Save
+		}
+		return cmdLegacy, cmdLegacySave
+	}
+
+	if ipv6 {
+		return cmdNFTIPv6, cmdNFTIPv6Save
+	}
+
+	return cmdNFT, cmdNFTSave
+}
+
 func setLogLevel(logLevel string) error {
 	level, err := log.ParseLevel(logLevel)
 	if err != nil {

proxy-init/cmd/root_test.go

@@ -32,7 +32,7 @@ func TestBuildFirewallConfiguration(t *testing.T) {
 		options.OutgoingProxyPort = expectedOutgoingProxyPort
 		options.ProxyUserID = expectedProxyUserID
 
-		config, err := BuildFirewallConfiguration(options)
+		config, err := BuildFirewallConfiguration(options, false)
 		if err != nil {
 			t.Fatalf("Unexpected error: %s", err)
 		}
@@ -51,38 +51,43 @@ func TestBuildFirewallConfiguration(t *testing.T) {
 				options: &RootOptions{
 					IncomingProxyPort: -1,
 					OutgoingProxyPort: 1234,
+					IPTablesMode:      IPTablesModeLegacy,
 				},
 				errorMessage: "--incoming-proxy-port must be a valid TCP port number",
 			},
 			{
 				options: &RootOptions{
 					IncomingProxyPort: 100000,
 					OutgoingProxyPort: 1234,
+					IPTablesMode:      IPTablesModeLegacy,
 				},
 				errorMessage: "--incoming-proxy-port must be a valid TCP port number",
 			},
 			{
 				options: &RootOptions{
 					IncomingProxyPort: 1234,
 					OutgoingProxyPort: -1,
+					IPTablesMode:      IPTablesModeLegacy,
 				},
 				errorMessage: "--outgoing-proxy-port must be a valid TCP port number",
 			},
 			{
 				options: &RootOptions{
 					IncomingProxyPort: 1234,
 					OutgoingProxyPort: 100000,
+					IPTablesMode:      IPTablesModeLegacy,
 				},
 				errorMessage: "--outgoing-proxy-port must be a valid TCP port number",
 			},
 			{
 				options: &RootOptions{
 					SubnetsToIgnore: []string{"1.1.1.1/24", "0.0.0.0"},
+					IPTablesMode:    IPTablesModeLegacy,
 				},
 				errorMessage: "0.0.0.0 is not a valid CIDR address",
 			},
 		} {
-			_, err := BuildFirewallConfiguration(tt.options)
+			_, err := BuildFirewallConfiguration(tt.options, false)
 			if err == nil {
 				t.Fatalf("Expected error for config [%v], got nil", tt.options)
 			}
@@ -102,11 +107,12 @@ func TestBuildFirewallConfiguration(t *testing.T) {
 				// Tests that subnets are parsed properly and trimmed of excess whitespace
 				options: &RootOptions{
 					SubnetsToIgnore: []string{"1.1.1.1/24 "},
+					IPTablesMode:    IPTablesModeLegacy,
 				},
 				errorMessage: "",
 			},
 		} {
-			_, err := BuildFirewallConfiguration(tt.options)
+			_, err := BuildFirewallConfiguration(tt.options, false)
 			if err != nil {
 				t.Fatalf("Got error error for config [%v]", tt.options)
 			}