feature: exclude ips and cidr ranges (#83)

README.md

@@ -337,6 +337,44 @@ For example, to limit the speed to 1 packet per 5 seconds:
 cat arp.cache | sx tcp --rate 1/5s --json -p 22,80,443 192.168.0.171
 ```
 
+### Exclude subnets
+
+Sometimes you need to exclude some ip addresses and subnets from scanning. This can be done with 
+the `--exclude` option. It specifies a file with IPs or subnets in CIDR notation to exclude, one-per line.
+
+For instance, to exclude RFC 1918 addresses, create a file `ips.txt` with the following contents:
+
+```
+10.0.0.0/8
+172.16.0.0/16
+192.168.0.0/16
+```
+
+You can also insert comments and blank lines:
+
+```
+# exclude RFC 1918 addresses
+10.0.0.0/8 # comment 1
+172.16.0.0/12 # comment 2
+192.168.0.0/16 # comment 3
+
+0.0.0.0/8 # used in initialization procedures (RFC 6890)
+
+# exclude RFC 5735 addresses
+127.0.0.0/8 # loopback address
+192.0.0.0/24 # reserved block for IETF protocol assignments
+224.0.0.0/4 # allocated for use in IPv4 multicast address assignments
+240.0.0.0/4 # reserved for future use
+
+# exclude Amazon network
+3.0.0.0/8
+
+# ip addresses are valid as well
+1.1.1.1
+```
+
+and run a scan with `--exclude ips.txt` option.
+
 ### Live LAN TCP SYN scanner
 
 As an example of scan composition, you can combine ARP and TCP SYN scans to create live TCP port scanner that periodically scan whole LAN network.

command/arp.go

@@ -96,6 +96,9 @@ func (o *arpCmdOpts) getLogger() (logger log.Logger, err error) {
 
 func (o *arpCmdOpts) newARPScanMethod(ctx context.Context) *arp.ScanMethod {
 	var reqgen scan.RequestGenerator = scan.NewIPRequestGenerator(scan.NewIPGenerator())
+	if o.excludeIPs != nil {
+		reqgen = scan.NewFilterIPRequestGenerator(reqgen, o.excludeIPs)
+	}
 	if o.liveTimeout > 0 {
 		reqgen = scan.NewLiveRequestGenerator(reqgen, o.liveTimeout)
 	}

command/config.go

@@ -1,6 +1,7 @@
 package command
 
 import (
+	"bufio"
 	"context"
 	"errors"
 	"io"
@@ -17,6 +18,7 @@ import (
 	"github.com/v-byte-cpu/sx/pkg/ip"
 	"github.com/v-byte-cpu/sx/pkg/scan"
 	"github.com/v-byte-cpu/sx/pkg/scan/arp"
+	"github.com/yl2chen/cidranger"
 	"go.uber.org/ratelimit"
 )
 
@@ -48,17 +50,23 @@ type packetScanCmdOpts struct {
 	rateCount  int
 	rateWindow time.Duration
 	exitDelay  time.Duration
+	excludeIPs scan.IPContainer
 
-	rawInterface string
-	rawSrcMAC    string
-	rawRateLimit string
+	rawInterface   string
+	rawSrcMAC      string
+	rawRateLimit   string
+	rawExcludeFile string
 }
 
 func (o *packetScanCmdOpts) initCliFlags(cmd *cobra.Command) {
 	cmd.Flags().BoolVar(&o.json, "json", false, "enable JSON output")
 	cmd.Flags().StringVarP(&o.rawInterface, "iface", "i", "", "set interface to send/receive packets")
 	cmd.Flags().IPVar(&o.srcIP, "srcip", nil, "set source IP address for generated packets")
 	cmd.Flags().StringVar(&o.rawSrcMAC, "srcmac", "", "set source MAC address for generated packets")
+	cmd.Flags().StringVar(&o.rawExcludeFile, "exclude", "",
+		strings.Join([]string{
+			"set file with IPs or subnets in CIDR notation to exclude, one-per line.",
+			"It is useful to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special-purpose addresses."}, "\n"))
 	cmd.Flags().StringVarP(&o.rawRateLimit, "rate", "r", "",
 		strings.Join([]string{
 			"set rate limit for generated packets",
@@ -87,6 +95,13 @@ func (o *packetScanCmdOpts) parseRawOptions() (err error) {
 			return
 		}
 	}
+	if len(o.rawExcludeFile) > 0 {
+		if o.excludeIPs, err = parseExcludeFile(func() (io.ReadCloser, error) {
+			return os.Open(o.rawExcludeFile)
+		}); err != nil {
+			return
+		}
+	}
 	return
 }
 
@@ -327,6 +342,11 @@ func (o *ipPortScanCmdOpts) parseScanConfig(scanName string, args []string) (c *
 }
 
 func (o *ipPortScanCmdOpts) newIPPortGenerator() (reqgen scan.RequestGenerator) {
+	defer func() {
+		if o.excludeIPs != nil {
+			reqgen = scan.NewFilterIPRequestGenerator(reqgen, o.excludeIPs)
+		}
+	}()
 	if len(o.ipFile) == 0 {
 		return scan.NewIPPortGenerator(scan.NewIPGenerator(), scan.NewPortGenerator())
 	}
@@ -352,16 +372,22 @@ type genericScanCmdOpts struct {
 	rateCount  int
 	rateWindow time.Duration
 	exitDelay  time.Duration
+	excludeIPs scan.IPContainer
 
-	rawPortRanges string
-	rawRateLimit  string
+	rawPortRanges  string
+	rawRateLimit   string
+	rawExcludeFile string
 }
 
 func (o *genericScanCmdOpts) initCliFlags(cmd *cobra.Command) {
 	cmd.Flags().BoolVar(&o.json, "json", false, "enable JSON output")
 	cmd.Flags().StringVarP(&o.rawPortRanges, "ports", "p", "", "set ports to scan")
 	cmd.Flags().StringVarP(&o.ipFile, "file", "f", "", "set JSONL file with ip/port pairs to scan")
 	cmd.Flags().IntVarP(&o.workers, "workers", "w", defaultWorkerCount, "set workers count")
+	cmd.Flags().StringVar(&o.rawExcludeFile, "exclude", "",
+		strings.Join([]string{
+			"set file with IPs or subnets in CIDR notation to exclude, one-per line.",
+			"It is useful to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special-purpose addresses."}, "\n"))
 	cmd.Flags().StringVarP(&o.rawRateLimit, "rate", "r", "",
 		strings.Join([]string{
 			"set rate limit for generated scan requests",
@@ -385,6 +411,13 @@ func (o *genericScanCmdOpts) parseRawOptions() (err error) {
 			return
 		}
 	}
+	if len(o.rawExcludeFile) > 0 {
+		if o.excludeIPs, err = parseExcludeFile(func() (io.ReadCloser, error) {
+			return os.Open(o.rawExcludeFile)
+		}); err != nil {
+			return
+		}
+	}
 	if o.workers <= 0 {
 		return errors.New("invalid workers count")
 	}
@@ -429,6 +462,11 @@ func (o *genericScanCmdOpts) newScanEngine(ctx context.Context, scanner scan.Sca
 }
 
 func (o *genericScanCmdOpts) newIPPortGenerator() (reqgen scan.RequestGenerator) {
+	defer func() {
+		if o.excludeIPs != nil {
+			reqgen = scan.NewFilterIPRequestGenerator(reqgen, o.excludeIPs)
+		}
+	}()
 	if len(o.ipFile) == 0 {
 		return scan.NewIPPortGenerator(scan.NewIPGenerator(), scan.NewPortGenerator())
 	}
@@ -525,3 +563,35 @@ func parseIPFlags(inputFlags string) (result uint8, err error) {
 	}
 	return
 }
+
+type openFileFunc func() (io.ReadCloser, error)
+
+func parseExcludeFile(openFile openFileFunc) (excludeIPs scan.IPContainer, err error) {
+	input, err := openFile()
+	if err != nil {
+		return
+	}
+	defer input.Close()
+
+	ranger := cidranger.NewPCTrieRanger()
+	scanner := bufio.NewScanner(input)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if comment := strings.Index(line, "#"); comment != -1 {
+			line = line[:comment]
+		}
+		line = strings.Trim(line, " ")
+		if len(line) == 0 {
+			continue
+		}
+		var ipnet *net.IPNet
+		if ipnet, err = ip.ParseIPNet(line); err != nil {
+			return
+		}
+		if err = ranger.Insert(cidranger.NewBasicRangerEntry(*ipnet)); err != nil {
+			return
+		}
+	}
+	excludeIPs = ranger
+	return
+}