Add resolver in katana

.github/workflows/lint-test.yml

@@ -13,18 +13,18 @@ jobs:
     name: Lint Test
     runs-on: ubuntu-latest-16-cores
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
           go-version: 1.19
           cache: true
 
-      - name: Checkout code
-        uses: actions/checkout@v3
-
       - name: Run golangci-lint
         uses: golangci/[email protected]
         with:
           version: latest
           args: --timeout 5m
-          working-directory: .
+          working-directory: .

cmd/katana/main.go

@@ -69,6 +69,7 @@ pipelines offering both headless and non-headless crawling.`)
 	)
 
 	flagSet.CreateGroup("config", "Configuration",
+		flagSet.StringSliceVarP(&options.Resolvers, "resolvers", "r", nil, "list of custom resolver (file or comma separated)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.IntVarP(&options.MaxDepth, "depth", "d", 2, "maximum depth to crawl"),
 		flagSet.BoolVarP(&options.ScrapeJSResponses, "js-crawl", "jc", false, "enable endpoint parsing / crawling in javascript file"),
 		flagSet.IntVarP(&options.CrawlDuration, "crawl-duration", "ct", 0, "maximum duration to crawl the target for"),

pkg/types/crawler_options.go

@@ -39,7 +39,11 @@ type CrawlerOptions struct {
 func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) {
 	extensionsValidator := extensions.NewValidator(options.ExtensionsMatch, options.ExtensionFilter)
 
-	fastdialerInstance, err := fastdialer.NewDialer(fastdialer.DefaultOptions)
+	dialerOpts := fastdialer.DefaultOptions
+	if len(options.Resolvers) > 0 {
+		dialerOpts.BaseResolvers = options.Resolvers
+	}
+	fastdialerInstance, err := fastdialer.NewDialer(dialerOpts)
 	if err != nil {
 		return nil, err
 	}

pkg/types/options.go

@@ -106,6 +106,8 @@ type Options struct {
 	HealthCheck bool
 	// ErrorLogFile specifies a file to write with the errors of all requests
 	ErrorLogFile string
+	// Resolvers contains custom resolvers
+	Resolvers goflags.StringSlice
 	// OutputMatchRegex is the regex to match output url
 	OutputMatchRegex goflags.StringSlice
 	// OutputFilterRegex is the regex to filter output url