From fb52abf5adb1cefdd9353d61567dfdbe3b68c07d Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Wed, 3 May 2023 10:35:22 +0530 Subject: [PATCH 1/7] remove redundant c.browser.Connect() --- pkg/engine/hybrid/hybrid.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index 6c6b4396..ecd7427f 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -135,9 +135,6 @@ func (c *Crawler) Crawl(rootURL string) error { // create a new browser instance (default to incognito mode) if c.Options.Options.HeadlessNoIncognito { - if err := c.browser.Connect(); err != nil { - return err - } crawlSession.Browser = c.browser } else { var err error From e82c3eb26fbcc696705ea67ffdde3032fcc47f94 Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Thu, 4 May 2023 12:41:19 +0530 Subject: [PATCH 2/7] Add functional test --- .github/workflows/functional-test.yml | 30 ++++++++++++++ cmd/functional-test/main.go | 58 +++++++++++++++++++++++++++ cmd/functional-test/run.sh | 20 +++++++++ internal/testutils/helper.go | 13 ++++++ internal/testutils/integration.go | 27 +++++++++++++ internal/testutils/testutils.go | 30 ++++++++++++++ 6 files changed, 178 insertions(+) create mode 100644 .github/workflows/functional-test.yml create mode 100644 cmd/functional-test/main.go create mode 100644 cmd/functional-test/run.sh create mode 100644 internal/testutils/helper.go create mode 100644 internal/testutils/integration.go create mode 100644 internal/testutils/testutils.go diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml new file mode 100644 index 00000000..3f1fc504 --- /dev/null +++ b/.github/workflows/functional-test.yml @@ -0,0 +1,30 @@ +name: ๐Ÿงช Functional Test + +on: + pull_request: + paths: + - '**.go' + - '**.mod' + workflow_dispatch: + +jobs: + functional: + name: Functional Test + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + steps: + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.19 + + - name: Check out code + uses: actions/checkout@v3 + + - name: Functional Tests + run: | + chmod +x run.sh + bash run.sh ${{ matrix.os }} + working-directory: cmd/functional-test diff --git a/cmd/functional-test/main.go b/cmd/functional-test/main.go new file mode 100644 index 00000000..7e76e1e9 --- /dev/null +++ b/cmd/functional-test/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" + "github.com/projectdiscovery/katana/internal/testutils" +) + +var ( + debug = os.Getenv("DEBUG") == "true" + success = aurora.Green("[โœ“]").String() + failed = aurora.Red("[โœ˜]").String() + errored = false + devKatanaBinary = flag.String("dev", "", "Dev Branch Katana Binary") +) + +func main() { + flag.Parse() + if err := runFunctionalTests(); err != nil { + log.Fatalf("Could not run functional tests: %s\n", err) + } + if errored { + os.Exit(1) + } +} + +func runFunctionalTests() error { + for _, testcase := range testutils.TestCases { + if err := runIndividualTestCase(testcase); err != nil { + errored = true + fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, testcase.Name, err) + } else { + fmt.Printf("%s Test \"%s\" passed!\n", success, testcase.Name) + } + } + return nil +} + +func runIndividualTestCase(testcase testutils.TestCase) error { + argsParts := strings.Fields(testcase.Args) + devOutput, err := testutils.RunKatanaBinaryAndGetResults(testcase.Target, *devKatanaBinary, debug, argsParts) + if err != nil { + return errors.Wrap(err, "could not run Katana dev test") + } + if testcase.CompareFunc != nil { + return testcase.CompareFunc(testcase.Target, devOutput) + } + if !testutils.CompareOutput(devOutput, testcase.Expected) { + return errors.Errorf("expected output %s, got %s", testcase.Expected, devOutput) + } + return nil +} diff --git a/cmd/functional-test/run.sh b/cmd/functional-test/run.sh new file mode 100644 index 00000000..af44e4f5 --- /dev/null +++ b/cmd/functional-test/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# reading os type from arguments +CURRENT_OS=$1 + +if [ "${CURRENT_OS}" == "windows-latest" ];then + extension=.exe +fi + +echo "::group::Building functional-test binary" +go build -o functional-test$extension +echo "::endgroup::" + +echo "::group::Building katana binary from current branch" +go build -o katana_dev$extension ../katana +echo "::endgroup::" + + +echo 'Starting katana functional test' +./functional-test$extension -dev ./katana_dev$extension diff --git a/internal/testutils/helper.go b/internal/testutils/helper.go new file mode 100644 index 00000000..3f20b9d1 --- /dev/null +++ b/internal/testutils/helper.go @@ -0,0 +1,13 @@ +package testutils + +func CompareOutput(input, expected []string) bool { + if len(input) != len(expected) { + return false + } + for i, v := range input { + if v != expected[i] { + return false + } + } + return true +} diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go new file mode 100644 index 00000000..b77a22c8 --- /dev/null +++ b/internal/testutils/integration.go @@ -0,0 +1,27 @@ +package testutils + +import ( + "fmt" + "os/exec" + "strings" +) + +func RunKatanaBinaryAndGetResults(target string, katanaBinary string, debug bool, args []string) ([]string, error) { + cmd := exec.Command("bash", "-c") + cmdLine := fmt.Sprintf(`echo %s | %s `, target, katanaBinary) + cmdLine += strings.Join(args, " ") + + cmd.Args = append(cmd.Args, cmdLine) + data, err := cmd.Output() + if err != nil { + return nil, err + } + parts := []string{} + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go new file mode 100644 index 00000000..9cd01442 --- /dev/null +++ b/internal/testutils/testutils.go @@ -0,0 +1,30 @@ +package testutils + +import ( + "strings" + + errorutils "github.com/projectdiscovery/utils/errors" +) + +type TestCase struct { + Name string + Target string + Args string + Expected []string + CompareFunc func(target string, got []string) error +} + +var TestCases = []TestCase{ + { + Name: "Headless Browser Without Incognito", + Target: "https://www.hackerone.com/", + Expected: nil, + Args: "-headless -no-incognito -depth 2 -silent", + CompareFunc: func(target string, got []string) error { + if strings.Contains(got[0], target) && len(got) > 10 { + return nil + } + return errorutils.New("expected > 10 results, but got %v ", len(got)) + }, + }, +} From 738b9c523ca4fb3591472331009369da4f44843e Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Thu, 4 May 2023 17:03:59 +0530 Subject: [PATCH 3/7] create config folder if doesn't exist --- internal/runner/options.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/runner/options.go b/internal/runner/options.go index 432f1a53..24ad30d6 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -125,6 +125,9 @@ func initExampleFormFillConfig() error { if fileutil.FileExists(defaultConfig) { return nil } + if err := os.MkdirAll(filepath.Dir(defaultConfig), 0775); err != nil { + return err + } exampleConfig, err := os.Create(defaultConfig) if err != nil { return errorutil.NewWithErr(err).Msgf("could not get home directory") From 75851afd855090c8f934659f882e1dabb13a813a Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Thu, 4 May 2023 17:35:28 +0530 Subject: [PATCH 4/7] change test compare logic --- internal/testutils/testutils.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index 9cd01442..142d8dad 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -16,15 +16,17 @@ type TestCase struct { var TestCases = []TestCase{ { - Name: "Headless Browser Without Incognito", + Name: "Headless Browser Without Incognito", Target: "https://www.hackerone.com/", Expected: nil, Args: "-headless -no-incognito -depth 2 -silent", CompareFunc: func(target string, got []string) error { - if strings.Contains(got[0], target) && len(got) > 10 { - return nil + for _, res := range got { + if strings.Contains(res, target) { + return nil + } } - return errorutils.New("expected > 10 results, but got %v ", len(got)) + return errorutils.New("expected %v target in output, but got %v ", strings.Join(got, "\n")) }, }, } From 420398de1f47b1c30dd162689de111344addab1a Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Thu, 4 May 2023 17:43:04 +0530 Subject: [PATCH 5/7] fix lint error --- internal/testutils/testutils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index 142d8dad..8c85fbdc 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -26,7 +26,7 @@ var TestCases = []TestCase{ return nil } } - return errorutils.New("expected %v target in output, but got %v ", strings.Join(got, "\n")) + return errorutils.New("expected %v target in output, but got %v ", target, strings.Join(got, "\n")) }, }, } From 8f97690c94e1cb6fbe091b056bdf49effafb135e Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 4 May 2023 18:18:29 +0530 Subject: [PATCH 6/7] create incognito browser in New --- pkg/engine/hybrid/hybrid.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index ecd7427f..62ca90bd 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -97,6 +97,16 @@ func New(options *types.CrawlerOptions) (*Crawler, error) { return nil, browserErr } + // create a new browser instance (default to incognito mode) + if !options.Options.HeadlessNoIncognito { + incognito, err := browser.Incognito() + if err != nil { + chromeLauncher.Kill() + return nil, errorutil.NewWithErr(err).Msgf("failed to create incognito browser") + } + browser = incognito + } + shared, err := common.NewShared(options) if err != nil { return nil, errorutil.NewWithErr(err).WithTag("hybrid") @@ -133,17 +143,6 @@ func (c *Crawler) Crawl(rootURL string) error { } defer crawlSession.CancelFunc() - // create a new browser instance (default to incognito mode) - if c.Options.Options.HeadlessNoIncognito { - crawlSession.Browser = c.browser - } else { - var err error - crawlSession.Browser, err = c.browser.Incognito() - if err != nil { - return err - } - } - gologger.Info().Msgf("Started headless crawling for => %v", rootURL) if err := c.Do(crawlSession, c.navigateRequest); err != nil { return errorutil.NewWithErr(err).WithTag("standard") From a6de137ef9e1ca653e3daf265c01201ed2c44c03 Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Thu, 4 May 2023 20:01:48 +0530 Subject: [PATCH 7/7] fix nil pointer err --- pkg/engine/hybrid/hybrid.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index 62ca90bd..d9073311 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -138,6 +138,7 @@ func (c *Crawler) Close() error { // Crawl crawls a URL with the specified options func (c *Crawler) Crawl(rootURL string) error { crawlSession, err := c.NewCrawlSessionWithURL(rootURL) + crawlSession.Browser = c.browser if err != nil { return errorutil.NewWithErr(err).WithTag("hybrid") }