feat: update iptables monitor with ipv6 and bpf map reading capabilities (#3948)

* update label

* add ip6tables

* update readme

* update logging

* add ability to read bpf map to iptables monitor

* modify log

* display iptables rules block count only when check map enabled

* update readme

* remove unused function (noop)

* address linter

* update azure-iptables-monitor/iptables_monitor.go

Co-authored-by: Copilot <[email protected]>
Signed-off-by: Alexander <[email protected]>

* adjust event description

* adjust pinned path

---------

Signed-off-by: Alexander <[email protected]>
Co-authored-by: Copilot <[email protected]>

Author: QxBytes

azure-iptables-monitor/README.md

@@ -25,14 +25,20 @@ Follow the steps below to build and run the program:
 
 4. Start the program with:
     ```bash
-    ./azure-iptables-monitor --input=/etc/config/ --interval=300
+    ./azure-iptables-monitor -input=/etc/config/ -interval=300
     ```
-    - The `--input` flag specifies the directory containing allowed regex pattern files. Default: `/etc/config/`
-    - The `--interval` flag specifies how often to check iptables rules in seconds. Default: `300`
-    - The `--events` flag enables Kubernetes event creation for rule violations. Default: `false`
+    - The `-input` flag specifies the directory containing allowed regex pattern files. Default: `/etc/config/`
+    - The `-input6` flag specifies the directory containing allowed regex pattern files for IPv6 ip6tables. Default: `/etc/config6/`
+    - The `-interval` flag specifies how often to check iptables rules and the bpf map in seconds. Default: `300`
+    - The `-events` flag enables Kubernetes event creation for rule violations. Default: `false`
+    - The `-ipv6` flag enables IPv6 ip6tables monitoring using the IPv6 allowlists. Default: `false`
+    - The `-checkMap` flag enables checking the pinned bpf map specified in mapPath for increases. Default: `false`
+    - The `-mapPath` flag specifies the pinned bpf map path to check. Default: `/azure-block-iptables/iptables_block_event_counter`
     - The program must be in a k8s environment and `NODE_NAME` must be a set environment variable with the current node.
 
-5. The program will set the `user-iptables-rules` label to `true` on the specified ciliumnode resource if unexpected rules are found, or `false` if all rules match expected patterns. Proper RBAC is required for patching (patch for ciliumnodes, create for events, get for nodes).
+5. The program will set the `kubernetes.azure.com/user-iptables-rules` label to `true` on the specified ciliumnode resource if unexpected rules are found, or `false` if all rules match expected patterns. Proper RBAC is required for patching (patch for ciliumnodes, create for events, get for nodes).
+
+6. The program will also send out an event if the bpf map value specified increases between checks
 
 
 ## Pattern File Format
@@ -48,6 +54,7 @@ Each pattern file should contain one regex pattern per line:
 - `nat`, `mangle`, `filter`, `raw`, `security`: Patterns specific to each iptables table
 - Empty lines are ignored
 - Each line should be a valid Go regex pattern
+- The ipv6 config directory uses files with same names, but will match against ipv6 iptables rules
 
 ## Debugging
 

azure-iptables-monitor/go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cilium/ebpf v0.19.0
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect

azure-iptables-monitor/go.sum

@@ -4,6 +4,8 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cilium/ebpf v0.19.0 h1:Ro/rE64RmFBeA9FGjcTc+KmCeY6jXmryu6FfnzPRIao=
+github.com/cilium/ebpf v0.19.0/go.mod h1:fLCgMo3l8tZmAdM3B2XqdFzXBpwkcSTroaVqN08OWVY=
 github.com/coreos/go-iptables v0.8.0 h1:MPc2P89IhuVpLI7ETL/2tx3XZ61VeICZjYqDEgNsPRc=
 github.com/coreos/go-iptables v0.8.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=

azure-iptables-monitor/iptables_monitor.go

@@ -10,6 +10,7 @@ import (
 	"regexp"
 	"time"
 
+	"github.com/cilium/ebpf"
 	goiptables "github.com/coreos/go-iptables/iptables"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -27,12 +28,16 @@ import (
 var version string
 
 var (
-	configPath    = flag.String("input", "/etc/config/", "Name of the directory with the allowed regex files")
-	checkInterval = flag.Int("interval", 300, "How often to check iptables rules (in seconds)")
+	configPath4   = flag.String("input", "/etc/config/", "Name of the directory with the ipv4 allowed regex files")
+	configPath6   = flag.String("input6", "/etc/config6/", "Name of directory with the ipv6 allowed regex files")
+	checkInterval = flag.Int("interval", 300, "How often to check for user iptables rules and bpf map increases (in seconds)")
 	sendEvents    = flag.Bool("events", false, "Whether to send node events if unexpected iptables rules are detected")
+	ipv6Enabled   = flag.Bool("ipv6", false, "Whether to check ip6tables using the ipv6 allowlists")
+	checkMap      = flag.Bool("checkMap", false, "Whether to check the bpf map at mapPath for increases")
+	pinPath       = flag.String("mapPath", "/azure-block-iptables/iptables_block_event_counter", "Path to pinned bpf map")
 )
 
-const label = "user-iptables-rules"
+const label = "kubernetes.azure.com/user-iptables-rules"
 
 type FileLineReader interface {
 	Read(filename string) ([]string, error)
@@ -197,17 +202,19 @@ func hasUnexpectedRules(currentRules, allowedPatterns []string) bool {
 // nodeHasUserIPTablesRules returns true if the node has iptables rules that do not match the regex
 // specified in the rule's respective table: nat, mangle, filter, raw, or security
 // The global file's regexes can match to a rule in any table
-func nodeHasUserIPTablesRules(fileReader FileLineReader, iptablesClient IPTablesClient) bool {
+func nodeHasUserIPTablesRules(fileReader FileLineReader, path string, iptablesClient IPTablesClient) bool {
 	tables := []string{"nat", "mangle", "filter", "raw", "security"}
 
-	globalPatterns, err := fileReader.Read(filepath.Join(*configPath, "global"))
+	globalPatterns, err := fileReader.Read(filepath.Join(path, "global"))
 	if err != nil {
 		globalPatterns = []string{}
 		klog.V(2).Infof("No global patterns file found, using empty patterns")
 	}
 
 	userIPTablesRules := false
 
+	klog.V(2).Infof("Using reference patterns files in %s", path)
+
 	for _, table := range tables {
 		rules, err := GetRules(iptablesClient, table)
 		if err != nil {
@@ -216,7 +223,7 @@ func nodeHasUserIPTablesRules(fileReader FileLineReader, iptablesClient IPTables
 		}
 
 		var referencePatterns []string
-		referencePatterns, err = fileReader.Read(filepath.Join(*configPath, table))
+		referencePatterns, err = fileReader.Read(filepath.Join(path, table))
 		if err != nil {
 			referencePatterns = []string{}
 			klog.V(2).Infof("No reference patterns file found for table %s", table)
@@ -234,6 +241,23 @@ func nodeHasUserIPTablesRules(fileReader FileLineReader, iptablesClient IPTables
 	return userIPTablesRules
 }
 
+func getBPFMapValue() (uint64, error) {
+	m, err := ebpf.LoadPinnedMap(*pinPath, nil)
+	if err != nil {
+		return 0, fmt.Errorf("failed to load pinned map %s: %w", *pinPath, err)
+	}
+	defer m.Close()
+	// 0 is the key for # of blocks
+	key := uint32(0)
+	value := uint64(0)
+
+	if err := m.Lookup(&key, &value); err != nil {
+		return 0, fmt.Errorf("failed to lookup key %d in bpf map: %w", key, err)
+	}
+
+	return value, nil
+}
+
 func main() {
 	klog.InitFlags(nil)
 	flag.Parse()
@@ -263,6 +287,15 @@ func main() {
 		klog.Fatalf("failed to create iptables client: %v", err)
 	}
 
+	var ip6tablesClient IPTablesClient
+	if *ipv6Enabled {
+		ip6tablesClient, err = goiptables.New(goiptables.IPFamily(goiptables.ProtocolIPv6))
+		if err != nil {
+			klog.Fatalf("failed to create ip6tables client: %v", err)
+		}
+	}
+	klog.Infof("IPv6: %v", *ipv6Enabled)
+
 	// get current node name from environment variable
 	currentNodeName := os.Getenv("NODE_NAME")
 	if currentNodeName == "" {
@@ -273,8 +306,22 @@ func main() {
 
 	var fileReader FileLineReader = OSFileLineReader{}
 
+	previousBlocks := uint64(0)
+
 	for {
-		userIPTablesRulesFound := nodeHasUserIPTablesRules(fileReader, iptablesClient)
+		userIPTablesRulesFound := nodeHasUserIPTablesRules(fileReader, *configPath4, iptablesClient)
+		if userIPTablesRulesFound {
+			klog.Info("Above user iptables rules detected in IPv4 iptables")
+		}
+
+		// check ip6tables rules if enabled
+		if *ipv6Enabled {
+			userIP6TablesRulesFound := nodeHasUserIPTablesRules(fileReader, *configPath6, ip6tablesClient)
+			if userIP6TablesRulesFound {
+				klog.Info("Above user iptables rules detected in IPv6 iptables")
+			}
+			userIPTablesRulesFound = userIPTablesRulesFound || userIP6TablesRulesFound
+		}
 
 		// update label based on whether user iptables rules were found
 		err = patchLabel(dynamicClient, userIPTablesRulesFound, currentNodeName)
@@ -291,6 +338,28 @@ func main() {
 			}
 		}
 
+		// if disabled the number of blocks never increases from zero
+		currentBlocks := uint64(0)
+		if *checkMap {
+			// read bpf map to check for number of blocked iptables rules
+			currentBlocks, err = getBPFMapValue()
+			if err != nil {
+				klog.Errorf("failed to get bpf map value: %v", err)
+			}
+			klog.V(2).Infof("IPTables rules blocks: Previous: %d Current: %d", previousBlocks, currentBlocks)
+		}
+		// if number of blocked rules increased since last time
+		blockedRulesIncreased := currentBlocks > previousBlocks
+		if *sendEvents && blockedRulesIncreased {
+			msg := "A process attempted to add iptables rules to the node but was blocked since last check. " +
+				"iptables rules blocked because EBPF Host Routing is enabled: aka.ms/acnsperformance"
+			err = createNodeEvent(clientset, currentNodeName, "BlockedIPTablesRule", msg, corev1.EventTypeWarning)
+			if err != nil {
+				klog.Errorf("failed to create iptables block event: %v", err)
+			}
+		}
+		previousBlocks = currentBlocks
+
 		time.Sleep(time.Duration(*checkInterval) * time.Second)
 	}
 }

azure-iptables-monitor/iptables_monitor_test.go

@@ -217,7 +217,7 @@ func TestNodeHasUserIPTablesRules(t *testing.T) {
 			fileReader.files = tc.files
 			iptablesClient.rules = tc.rules
 
-			result := nodeHasUserIPTablesRules(fileReader, iptablesClient)
+			result := nodeHasUserIPTablesRules(fileReader, "/etc/config/", iptablesClient)
 			require.Equal(t, tc.expected, result, tc.description)
 		})
 	}

bpf-prog/azure-block-iptables/pkg/bpfprogram/program.go

@@ -18,7 +18,7 @@ import (
 
 const (
 	// BPFMapPinPath is the directory where BPF maps are pinned
-	BPFMapPinPath = "/sys/fs/bpf/block-iptables"
+	BPFMapPinPath = "/sys/fs/bpf/azure-block-iptables"
 	// EventCounterMapName is the name used for pinning the event counter map
 	EventCounterMapName = "iptables_block_event_counter"
 	// IptablesLegacyBlockProgramName is the name used for pinning the legacy iptables block program