added the option to disable following redirects (#588)

cmd/katana/main.go

@@ -105,6 +105,7 @@ pipelines offering both headless and non-headless crawling.`)
 		flagSet.StringVarP(&options.Strategy, "strategy", "s", "depth-first", "Visit strategy (depth-first, breadth-first)"),
 		flagSet.BoolVarP(&options.IgnoreQueryParams, "ignore-query-params", "iqp", false, "Ignore crawling same path with different query-param values"),
 		flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
+		flagSet.BoolVarP(&options.DisableRedirects, "disable-redirects", "dr", false, "disable following redirects (default false)"),
 	)
 
 	flagSet.CreateGroup("debug", "Debug",

pkg/engine/common/http.go

@@ -54,6 +54,9 @@ func BuildHttpClient(dialer *fastdialer.Dialer, options *types.Options, redirect
 		Transport: transport,
 		Timeout:   time.Duration(options.Timeout) * time.Second,
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if options.DisableRedirects {
+				return http.ErrUseLastResponse
+			}
 			if len(via) == 10 {
 				return errorutil.New("stopped after 10 redirects")
 			}

pkg/types/options.go

@@ -144,6 +144,8 @@ type Options struct {
 	Debug bool
 	// TlsImpersonate enables experimental tls ClientHello randomization for standard crawler
 	TlsImpersonate bool
+	//DisableRedirects disables the following of redirects
+	DisableRedirects bool
 }
 
 func (options *Options) ParseCustomHeaders() map[string]string {