From c06d6a6a57d1680722db5927dba20e9e3dbc006b Mon Sep 17 00:00:00 2001
From: zy9ard3 <67743789+zy9ard3@users.noreply.github.com>
Date: Wed, 27 Mar 2024 20:57:47 +0530
Subject: [PATCH 1/3] Update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 9bef2334..2011262e 100644
--- a/README.md
+++ b/README.md
@@ -117,6 +117,7 @@ Flags:
 INPUT:
    -u, -list string[]  target url / list to crawl
    -resume string      resume scan using resume.cfg
+   -e, -exclude string[]  exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)
 
 CONFIGURATION:
    -r, -resolvers string[]       list of custom resolver (file or comma separated)

From 650b0e449818f1b41b2d3db3def3fa37242422fa Mon Sep 17 00:00:00 2001
From: Mikhail Gaile <michael2ga3@gmail.com>
Date: Sat, 27 Apr 2024 23:56:12 +0300
Subject: [PATCH 2/3] fix: add user agent handler

---
 pkg/engine/hybrid/crawl.go | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/pkg/engine/hybrid/crawl.go b/pkg/engine/hybrid/crawl.go
index f3a7d628..ee7fda6a 100644
--- a/pkg/engine/hybrid/crawl.go
+++ b/pkg/engine/hybrid/crawl.go
@@ -237,14 +237,29 @@ func (c *Crawler) addHeadersToPage(page *rod.Page) {
 	if len(c.Headers) == 0 {
 		return
 	}
+
 	var arr []string
+	var userAgentSet bool
+
 	for k, v := range c.Headers {
-		arr = append(arr, k, v)
+		if k == "User-Agent" {
+			userAgentParams := &proto.NetworkSetUserAgentOverride{
+				UserAgent: v,
+			}
+			if err := page.SetUserAgent(userAgentParams); err != nil {
+				gologger.Error().Msgf("headless: could not set user agent: %v", err)
+			}
+			userAgentSet = true
+		} else {
+			arr = append(arr, k, v)
+		}
 	}
-	// ignore cleanup callback
-	_, err := page.SetExtraHeaders(arr)
-	if err != nil {
-		gologger.Error().Msgf("headless: could not set extra headers: %v", err)
+
+	if !userAgentSet || len(arr) > 0 {
+		_, err := page.SetExtraHeaders(arr)
+		if err != nil {
+			gologger.Error().Msgf("headless: could not set extra headers: %v", err)
+		}
 	}
 }
 

From 67756207e298adca69dc3b57329a474094b62804 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Mon, 13 May 2024 09:22:44 +0200
Subject: [PATCH 3/3] small refactor

---
 pkg/engine/hybrid/crawl.go | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/pkg/engine/hybrid/crawl.go b/pkg/engine/hybrid/crawl.go
index ee7fda6a..c100d36d 100644
--- a/pkg/engine/hybrid/crawl.go
+++ b/pkg/engine/hybrid/crawl.go
@@ -239,23 +239,22 @@ func (c *Crawler) addHeadersToPage(page *rod.Page) {
 	}
 
 	var arr []string
-	var userAgentSet bool
 
 	for k, v := range c.Headers {
-		if k == "User-Agent" {
+		switch {
+		case stringsutil.EqualFoldAny(k, "User-Agent"):
 			userAgentParams := &proto.NetworkSetUserAgentOverride{
 				UserAgent: v,
 			}
 			if err := page.SetUserAgent(userAgentParams); err != nil {
 				gologger.Error().Msgf("headless: could not set user agent: %v", err)
 			}
-			userAgentSet = true
-		} else {
+		default:
 			arr = append(arr, k, v)
 		}
 	}
 
-	if !userAgentSet || len(arr) > 0 {
+	if len(arr) > 0 {
 		_, err := page.SetExtraHeaders(arr)
 		if err != nil {
 			gologger.Error().Msgf("headless: could not set extra headers: %v", err)