From 5a91a24bbf25de1b690ac915d3fd1a940eeb9aec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Sat, 19 Aug 2023 14:08:31 +0000 Subject: [PATCH 1/4] fix headless options --- cmd/katana/main.go | 2 +- internal/runner/options.go | 2 +- pkg/engine/hybrid/hybrid.go | 7 ++++++- pkg/types/options.go | 34 ++++++++++++++++++++++++---------- pkg/types/options_test.go | 29 ++++++++++++----------------- 5 files changed, 44 insertions(+), 30 deletions(-) diff --git a/cmd/katana/main.go b/cmd/katana/main.go index 07191397..0e026d4d 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -97,7 +97,7 @@ pipelines offering both headless and non-headless crawling.`) flagSet.BoolVarP(&options.Headless, "headless", "hl", false, "enable headless hybrid crawling (experimental)"), flagSet.BoolVarP(&options.UseInstalledChrome, "system-chrome", "sc", false, "use local installed chrome browser instead of katana installed"), flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen with headless mode"), - flagSet.StringSliceVarP(&options.HeadlessOptionalArguments, "headless-options", "ho", nil, "start headless chrome with additional options", goflags.FileCommaSeparatedStringSliceOptions), + flagSet.StringVarP(&options.HeadlessOptionalArguments, "headless-options", "ho", "", "start headless chrome with additional options"), flagSet.BoolVarP(&options.HeadlessNoSandbox, "no-sandbox", "nos", false, "start headless chrome in --no-sandbox mode"), flagSet.StringVarP(&options.ChromeDataDir, "chrome-data-dir", "cdd", "", "path to store chrome browser data"), flagSet.StringVarP(&options.SystemChromePath, "system-chrome-path", "scp", "", "use specified chrome browser for headless crawling"), diff --git a/internal/runner/options.go b/internal/runner/options.go index c3612c24..3e63b45c 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,7 +25,7 @@ func validateOptions(options *types.Options) error { if len(options.URLs) == 0 && !fileutil.HasStdin() { return errorutil.New("no inputs specified for crawler") } - if (options.HeadlessOptionalArguments != nil || options.HeadlessNoSandbox || options.SystemChromePath != "") && !options.Headless { + if (options.HeadlessOptionalArguments != "" || options.HeadlessNoSandbox || options.SystemChromePath != "") && !options.Headless { return errorutil.New("headless mode (-hl) is required if -ho, -nos or -scp are set") } if options.SystemChromePath != "" { diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index 89392ad3..c724dd3a 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/katana/pkg/engine/common" "github.com/projectdiscovery/katana/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" + sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" ps "github.com/shirou/gopsutil/v3/process" @@ -168,7 +169,11 @@ func buildChromeLauncher(options *types.CrawlerOptions, dataStore string) (*laun } for k, v := range options.Options.ParseHeadlessOptionalArguments() { - chromeLauncher.Set(flags.Flag(k), v) + if sliceutil.IsEmpty(v) { + chromeLauncher.Set(flags.Flag(k)) + } else { + chromeLauncher.Set(flags.Flag(k), v...) + } } return chromeLauncher, nil diff --git a/pkg/types/options.go b/pkg/types/options.go index dd9711c9..794c4499 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/katana/pkg/output" + sliceutil "github.com/projectdiscovery/utils/slice" ) // OnResultCallback (output.Result) @@ -96,7 +97,7 @@ type Options struct { // ShowBrowser specifies whether the show the browser in headless mode ShowBrowser bool // HeadlessOptionalArguments specifies optional arguments to pass to Chrome - HeadlessOptionalArguments goflags.StringSlice + HeadlessOptionalArguments string // HeadlessNoSandbox specifies if chrome should be start in --no-sandbox mode HeadlessNoSandbox bool // SystemChromePath : Specify the chrome binary path for headless crawling @@ -153,15 +154,28 @@ func (options *Options) ParseCustomHeaders() map[string]string { return customHeaders } -func (options *Options) ParseHeadlessOptionalArguments() map[string]string { - optionalArguments := make(map[string]string) - for _, v := range options.HeadlessOptionalArguments { - if argParts := strings.SplitN(v, "=", 2); len(argParts) >= 2 { - key := strings.TrimSpace(argParts[0]) - value := strings.TrimSpace(argParts[1]) - if key != "" && value != "" { - optionalArguments[key] = value - } +func (options *Options) ParseHeadlessOptionalArguments() map[string][]string { + optionalArguments := make(map[string][]string) + argParts := strings.Split(options.HeadlessOptionalArguments, "--") + for _, part := range argParts { + if strings.TrimSpace(part) == "" { + continue + } + keyValue := strings.SplitN(strings.TrimSpace(part), "=", 2) + if sliceutil.IsEmpty(keyValue) || keyValue[0] == "" { + continue + } + + key := "--" + keyValue[0] + if len(keyValue) == 2 && keyValue[1] != "" { + values := sliceutil.PruneEmptyStrings(strings.Split(keyValue[1], ",")) + sliceutil.VisitSequential(values, func(i int, v string) error { + values[i] = strings.TrimSpace(v) + return nil + }) + optionalArguments[key] = values + } else { + optionalArguments[key] = []string{} } } return optionalArguments diff --git a/pkg/types/options_test.go b/pkg/types/options_test.go index 82ac4fb4..deab7a29 100644 --- a/pkg/types/options_test.go +++ b/pkg/types/options_test.go @@ -53,47 +53,42 @@ func TestParseHeadlessOptionalArguments(t *testing.T) { tests := []struct { name string input string - want map[string]string + want map[string][]string }{ { name: "single value", - input: "a=b", - want: map[string]string{"a": "b"}, + input: "--a=b", + want: map[string][]string{"--a": {"b"}}, }, { name: "empty string", input: "", - want: map[string]string{}, + want: map[string][]string{}, }, { name: "empty key", input: "=b", - want: map[string]string{}, + want: map[string][]string{}, }, { name: "empty value", - input: "a=", - want: map[string]string{}, + input: "--a=", + want: map[string][]string{"--a": {}}, }, { name: "double input", - input: "a=b,c=d", - want: map[string]string{"a": "b", "c": "d"}, + input: "--a=b,--c=d", + want: map[string][]string{"--a": {"b"}, "--c": {"d"}}, }, { name: "duplicated input", - input: "a=b,a=b", - want: map[string]string{"a": "b"}, + input: "--a=b,--a=b", + want: map[string][]string{"--a": {"b"}}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - strsl := goflags.StringSlice{} - for _, v := range strings.Split(tt.input, ",") { - //nolint - strsl.Set(v) - } - opt := Options{HeadlessOptionalArguments: strsl} + opt := Options{HeadlessOptionalArguments: tt.input} got := opt.ParseHeadlessOptionalArguments() require.Equal(t, tt.want, got) }) From 35dc425e92de272070990bea91a36ecaadecc4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 21 Aug 2023 10:59:36 +0300 Subject: [PATCH 2/4] Revert "fix headless options" This reverts commit 5a91a24bbf25de1b690ac915d3fd1a940eeb9aec. --- cmd/katana/main.go | 2 +- internal/runner/options.go | 2 +- pkg/engine/hybrid/hybrid.go | 7 +------ pkg/types/options.go | 34 ++++++++++------------------------ pkg/types/options_test.go | 29 +++++++++++++++++------------ 5 files changed, 30 insertions(+), 44 deletions(-) diff --git a/cmd/katana/main.go b/cmd/katana/main.go index 0e026d4d..07191397 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -97,7 +97,7 @@ pipelines offering both headless and non-headless crawling.`) flagSet.BoolVarP(&options.Headless, "headless", "hl", false, "enable headless hybrid crawling (experimental)"), flagSet.BoolVarP(&options.UseInstalledChrome, "system-chrome", "sc", false, "use local installed chrome browser instead of katana installed"), flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen with headless mode"), - flagSet.StringVarP(&options.HeadlessOptionalArguments, "headless-options", "ho", "", "start headless chrome with additional options"), + flagSet.StringSliceVarP(&options.HeadlessOptionalArguments, "headless-options", "ho", nil, "start headless chrome with additional options", goflags.FileCommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.HeadlessNoSandbox, "no-sandbox", "nos", false, "start headless chrome in --no-sandbox mode"), flagSet.StringVarP(&options.ChromeDataDir, "chrome-data-dir", "cdd", "", "path to store chrome browser data"), flagSet.StringVarP(&options.SystemChromePath, "system-chrome-path", "scp", "", "use specified chrome browser for headless crawling"), diff --git a/internal/runner/options.go b/internal/runner/options.go index 3e63b45c..c3612c24 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,7 +25,7 @@ func validateOptions(options *types.Options) error { if len(options.URLs) == 0 && !fileutil.HasStdin() { return errorutil.New("no inputs specified for crawler") } - if (options.HeadlessOptionalArguments != "" || options.HeadlessNoSandbox || options.SystemChromePath != "") && !options.Headless { + if (options.HeadlessOptionalArguments != nil || options.HeadlessNoSandbox || options.SystemChromePath != "") && !options.Headless { return errorutil.New("headless mode (-hl) is required if -ho, -nos or -scp are set") } if options.SystemChromePath != "" { diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index c724dd3a..89392ad3 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -11,7 +11,6 @@ import ( "github.com/projectdiscovery/katana/pkg/engine/common" "github.com/projectdiscovery/katana/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" - sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" ps "github.com/shirou/gopsutil/v3/process" @@ -169,11 +168,7 @@ func buildChromeLauncher(options *types.CrawlerOptions, dataStore string) (*laun } for k, v := range options.Options.ParseHeadlessOptionalArguments() { - if sliceutil.IsEmpty(v) { - chromeLauncher.Set(flags.Flag(k)) - } else { - chromeLauncher.Set(flags.Flag(k), v...) - } + chromeLauncher.Set(flags.Flag(k), v) } return chromeLauncher, nil diff --git a/pkg/types/options.go b/pkg/types/options.go index 794c4499..dd9711c9 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -7,7 +7,6 @@ import ( "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/katana/pkg/output" - sliceutil "github.com/projectdiscovery/utils/slice" ) // OnResultCallback (output.Result) @@ -97,7 +96,7 @@ type Options struct { // ShowBrowser specifies whether the show the browser in headless mode ShowBrowser bool // HeadlessOptionalArguments specifies optional arguments to pass to Chrome - HeadlessOptionalArguments string + HeadlessOptionalArguments goflags.StringSlice // HeadlessNoSandbox specifies if chrome should be start in --no-sandbox mode HeadlessNoSandbox bool // SystemChromePath : Specify the chrome binary path for headless crawling @@ -154,28 +153,15 @@ func (options *Options) ParseCustomHeaders() map[string]string { return customHeaders } -func (options *Options) ParseHeadlessOptionalArguments() map[string][]string { - optionalArguments := make(map[string][]string) - argParts := strings.Split(options.HeadlessOptionalArguments, "--") - for _, part := range argParts { - if strings.TrimSpace(part) == "" { - continue - } - keyValue := strings.SplitN(strings.TrimSpace(part), "=", 2) - if sliceutil.IsEmpty(keyValue) || keyValue[0] == "" { - continue - } - - key := "--" + keyValue[0] - if len(keyValue) == 2 && keyValue[1] != "" { - values := sliceutil.PruneEmptyStrings(strings.Split(keyValue[1], ",")) - sliceutil.VisitSequential(values, func(i int, v string) error { - values[i] = strings.TrimSpace(v) - return nil - }) - optionalArguments[key] = values - } else { - optionalArguments[key] = []string{} +func (options *Options) ParseHeadlessOptionalArguments() map[string]string { + optionalArguments := make(map[string]string) + for _, v := range options.HeadlessOptionalArguments { + if argParts := strings.SplitN(v, "=", 2); len(argParts) >= 2 { + key := strings.TrimSpace(argParts[0]) + value := strings.TrimSpace(argParts[1]) + if key != "" && value != "" { + optionalArguments[key] = value + } } } return optionalArguments diff --git a/pkg/types/options_test.go b/pkg/types/options_test.go index deab7a29..82ac4fb4 100644 --- a/pkg/types/options_test.go +++ b/pkg/types/options_test.go @@ -53,42 +53,47 @@ func TestParseHeadlessOptionalArguments(t *testing.T) { tests := []struct { name string input string - want map[string][]string + want map[string]string }{ { name: "single value", - input: "--a=b", - want: map[string][]string{"--a": {"b"}}, + input: "a=b", + want: map[string]string{"a": "b"}, }, { name: "empty string", input: "", - want: map[string][]string{}, + want: map[string]string{}, }, { name: "empty key", input: "=b", - want: map[string][]string{}, + want: map[string]string{}, }, { name: "empty value", - input: "--a=", - want: map[string][]string{"--a": {}}, + input: "a=", + want: map[string]string{}, }, { name: "double input", - input: "--a=b,--c=d", - want: map[string][]string{"--a": {"b"}, "--c": {"d"}}, + input: "a=b,c=d", + want: map[string]string{"a": "b", "c": "d"}, }, { name: "duplicated input", - input: "--a=b,--a=b", - want: map[string][]string{"--a": {"b"}}, + input: "a=b,a=b", + want: map[string]string{"a": "b"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - opt := Options{HeadlessOptionalArguments: tt.input} + strsl := goflags.StringSlice{} + for _, v := range strings.Split(tt.input, ",") { + //nolint + strsl.Set(v) + } + opt := Options{HeadlessOptionalArguments: strsl} got := opt.ParseHeadlessOptionalArguments() require.Equal(t, tt.want, got) }) From f2746263ec925fcf312a3b56ac68211b8d6c0851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 21 Aug 2023 08:17:29 +0000 Subject: [PATCH 3/4] use StringSliceVarP --- pkg/types/options.go | 13 ++++++++++++- pkg/types/options_test.go | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pkg/types/options.go b/pkg/types/options.go index dd9711c9..a572a80b 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -154,14 +154,25 @@ func (options *Options) ParseCustomHeaders() map[string]string { } func (options *Options) ParseHeadlessOptionalArguments() map[string]string { - optionalArguments := make(map[string]string) + var ( + lastKey string + optionalArguments = make(map[string]string) + ) for _, v := range options.HeadlessOptionalArguments { + if v == "" { + continue + } if argParts := strings.SplitN(v, "=", 2); len(argParts) >= 2 { key := strings.TrimSpace(argParts[0]) value := strings.TrimSpace(argParts[1]) if key != "" && value != "" { optionalArguments[key] = value + lastKey = key } + } else if !strings.HasPrefix(v, "--") { + optionalArguments[lastKey] += "," + v + } else { + optionalArguments[v] = "" } } return optionalArguments diff --git a/pkg/types/options_test.go b/pkg/types/options_test.go index 82ac4fb4..3f73bea3 100644 --- a/pkg/types/options_test.go +++ b/pkg/types/options_test.go @@ -85,6 +85,21 @@ func TestParseHeadlessOptionalArguments(t *testing.T) { input: "a=b,a=b", want: map[string]string{"a": "b"}, }, + { + name: "values with dash with boolean flag at the end", + input: "--a=a/b,c/d--z--n--m/a,--c=k,--h", + want: map[string]string{"--a": "a/b,c/d--z--n--m/a", "--c": "k", "--h": ""}, + }, + { + name: "values with dash boolean flag at the beginning", + input: "--h,--a=a/b,c/d--z--n--m/a,--c=k", + want: map[string]string{"--h": "", "--a": "a/b,c/d--z--n--m/a", "--c": "k"}, + }, + { + name: "values with dash boolean flag in the middle", + input: "--a=a/b,c/d--z--n--m/a,--h,--c=k", + want: map[string]string{"--a": "a/b,c/d--z--n--m/a", "--h": "", "--c": "k"}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 396f8313654645f751d20612373d93add302e818 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 25 Aug 2023 15:24:32 +0200 Subject: [PATCH 4/4] disabling default logger --- internal/runner/options.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/runner/options.go b/internal/runner/options.go index c3612c24..5f003e53 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/katana/pkg/utils" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" + logutil "github.com/projectdiscovery/utils/log" "gopkg.in/yaml.v3" ) @@ -112,7 +113,7 @@ func configureOutput(options *types.Options) { gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) } - // logutil.DisableDefaultLogger() + logutil.DisableDefaultLogger() } func initExampleFormFillConfig() error {