From 56b30a2000e468e6e7d58c167244450978614427 Mon Sep 17 00:00:00 2001 From: shubhamrasal Date: Mon, 6 Mar 2023 16:16:08 +0530 Subject: [PATCH 1/3] Add resolver in katana --- cmd/katana/main.go | 1 + pkg/types/crawler_options.go | 6 +++++- pkg/types/options.go | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/katana/main.go b/cmd/katana/main.go index 86d42cae..1875d0da 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -69,6 +69,7 @@ pipelines offering both headless and non-headless crawling.`) ) flagSet.CreateGroup("config", "Configuration", + flagSet.StringSliceVarP(&options.Resolvers, "resolvers", "r", nil, "list of custom resolver (file or comma separated)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.IntVarP(&options.MaxDepth, "depth", "d", 2, "maximum depth to crawl"), flagSet.BoolVarP(&options.ScrapeJSResponses, "js-crawl", "jc", false, "enable endpoint parsing / crawling in javascript file"), flagSet.IntVarP(&options.CrawlDuration, "crawl-duration", "ct", 0, "maximum duration to crawl the target for"), diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go index bbf11750..ae82f704 100644 --- a/pkg/types/crawler_options.go +++ b/pkg/types/crawler_options.go @@ -39,7 +39,11 @@ type CrawlerOptions struct { func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { extensionsValidator := extensions.NewValidator(options.ExtensionsMatch, options.ExtensionFilter) - fastdialerInstance, err := fastdialer.NewDialer(fastdialer.DefaultOptions) + dialerOpts := fastdialer.DefaultOptions + if len(options.Resolvers) > 0 { + dialerOpts.BaseResolvers = options.Resolvers + } + fastdialerInstance, err := fastdialer.NewDialer(dialerOpts) if err != nil { return nil, err } diff --git a/pkg/types/options.go b/pkg/types/options.go index 1b5d7e07..6d87f4e8 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -105,6 +105,8 @@ type Options struct { HealthCheck bool // ErrorLogFile specifies a file to write with the errors of all requests ErrorLogFile string + // Resolvers contains custom resolvers for the tlsx client + Resolvers goflags.StringSlice } func (options *Options) ParseCustomHeaders() map[string]string { From 4b42a55c4e70efa0c9f745737f1d397725c9df94 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 7 Mar 2023 11:53:37 +0100 Subject: [PATCH 2/3] updating comment --- pkg/types/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/types/options.go b/pkg/types/options.go index 6d87f4e8..5a16b164 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -105,7 +105,7 @@ type Options struct { HealthCheck bool // ErrorLogFile specifies a file to write with the errors of all requests ErrorLogFile string - // Resolvers contains custom resolvers for the tlsx client + // Resolvers contains custom resolvers Resolvers goflags.StringSlice } From 0a489e5aab0e361fc45b74caf950bc1b5cad4908 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Fri, 10 Mar 2023 14:52:16 +0530 Subject: [PATCH 3/3] workflow fix --- .github/workflows/lint-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 657059ea..052975c4 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -13,18 +13,18 @@ jobs: name: Lint Test runs-on: ubuntu-latest-16-cores steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Go uses: actions/setup-go@v3 with: go-version: 1.19 cache: true - - name: Checkout code - uses: actions/checkout@v3 - - name: Run golangci-lint uses: golangci/golangci-lint-action@v3.4.0 with: version: latest args: --timeout 5m - working-directory: . + working-directory: . \ No newline at end of file