remove redundant c.browser.Connect() (#428)

* remove redundant c.browser.Connect()

* Add functional test

* create config folder if doesn't exist

* change test compare logic

* fix lint error

* create incognito browser in New

* fix nil pointer err

---------

Co-authored-by: Tarun Koyalwar <[email protected]>

.github/workflows/functional-test.yml

@@ -0,0 +1,30 @@
+name: 🧪 Functional Test
+
+on:
+  pull_request:
+    paths:
+      - '**.go'
+      - '**.mod'
+  workflow_dispatch:
+
+jobs:  
+  functional:
+    name: Functional Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.19
+
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Functional Tests
+        run: |
+          chmod +x run.sh
+          bash run.sh ${{ matrix.os }}
+        working-directory: cmd/functional-test

cmd/functional-test/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/katana/internal/testutils"
+)
+
+var (
+	debug           = os.Getenv("DEBUG") == "true"
+	success         = aurora.Green("[✓]").String()
+	failed          = aurora.Red("[✘]").String()
+	errored         = false
+	devKatanaBinary = flag.String("dev", "", "Dev Branch Katana Binary")
+)
+
+func main() {
+	flag.Parse()
+	if err := runFunctionalTests(); err != nil {
+		log.Fatalf("Could not run functional tests: %s\n", err)
+	}
+	if errored {
+		os.Exit(1)
+	}
+}
+
+func runFunctionalTests() error {
+	for _, testcase := range testutils.TestCases {
+		if err := runIndividualTestCase(testcase); err != nil {
+			errored = true
+			fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, testcase.Name, err)
+		} else {
+			fmt.Printf("%s Test \"%s\" passed!\n", success, testcase.Name)
+		}
+	}
+	return nil
+}
+
+func runIndividualTestCase(testcase testutils.TestCase) error {
+	argsParts := strings.Fields(testcase.Args)
+	devOutput, err := testutils.RunKatanaBinaryAndGetResults(testcase.Target, *devKatanaBinary, debug, argsParts)
+	if err != nil {
+		return errors.Wrap(err, "could not run Katana dev test")
+	}
+	if testcase.CompareFunc != nil {
+		return testcase.CompareFunc(testcase.Target, devOutput)
+	}
+	if !testutils.CompareOutput(devOutput, testcase.Expected) {
+		return errors.Errorf("expected output %s, got %s", testcase.Expected, devOutput)
+	}
+	return nil
+}

cmd/functional-test/run.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# reading os type from arguments
+CURRENT_OS=$1
+
+if [ "${CURRENT_OS}" == "windows-latest" ];then
+    extension=.exe
+fi
+
+echo "::group::Building functional-test binary"
+go build -o functional-test$extension
+echo "::endgroup::"
+
+echo "::group::Building katana binary from current branch"
+go build -o katana_dev$extension ../katana
+echo "::endgroup::"
+
+
+echo 'Starting katana functional test'
+./functional-test$extension -dev ./katana_dev$extension

internal/runner/options.go

@@ -125,6 +125,9 @@ func initExampleFormFillConfig() error {
 	if fileutil.FileExists(defaultConfig) {
 		return nil
 	}
+	if err := os.MkdirAll(filepath.Dir(defaultConfig), 0775); err != nil {
+		return err
+	}
 	exampleConfig, err := os.Create(defaultConfig)
 	if err != nil {
 		return errorutil.NewWithErr(err).Msgf("could not get home directory")

internal/testutils/helper.go

@@ -0,0 +1,13 @@
+package testutils
+
+func CompareOutput(input, expected []string) bool {
+	if len(input) != len(expected) {
+		return false
+	}
+	for i, v := range input {
+		if v != expected[i] {
+			return false
+		}
+	}
+	return true
+}

internal/testutils/integration.go

@@ -0,0 +1,27 @@
+package testutils
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+func RunKatanaBinaryAndGetResults(target string, katanaBinary string, debug bool, args []string) ([]string, error) {
+	cmd := exec.Command("bash", "-c")
+	cmdLine := fmt.Sprintf(`echo %s | %s `, target, katanaBinary)
+	cmdLine += strings.Join(args, " ")
+
+	cmd.Args = append(cmd.Args, cmdLine)
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}

internal/testutils/testutils.go

@@ -0,0 +1,32 @@
+package testutils
+
+import (
+	"strings"
+
+	errorutils "github.com/projectdiscovery/utils/errors"
+)
+
+type TestCase struct {
+	Name        string
+	Target      string
+	Args        string
+	Expected    []string
+	CompareFunc func(target string, got []string) error
+}
+
+var TestCases = []TestCase{
+	{
+		Name:     "Headless Browser Without Incognito",
+		Target:   "https://www.hackerone.com/",
+		Expected: nil,
+		Args:     "-headless -no-incognito -depth 2 -silent",
+		CompareFunc: func(target string, got []string) error {
+			for _, res := range got {
+				if strings.Contains(res, target) {
+					return nil
+				}
+			}
+			return errorutils.New("expected %v target in output, but got %v ", target, strings.Join(got, "\n"))
+		},
+	},
+}

pkg/engine/hybrid/hybrid.go

@@ -97,6 +97,16 @@ func New(options *types.CrawlerOptions) (*Crawler, error) {
 		return nil, browserErr
 	}
 
+	// create a new browser instance (default to incognito mode)
+	if !options.Options.HeadlessNoIncognito {
+		incognito, err := browser.Incognito()
+		if err != nil {
+			chromeLauncher.Kill()
+			return nil, errorutil.NewWithErr(err).Msgf("failed to create incognito browser")
+		}
+		browser = incognito
+	}
+
 	shared, err := common.NewShared(options)
 	if err != nil {
 		return nil, errorutil.NewWithErr(err).WithTag("hybrid")
@@ -128,25 +138,12 @@ func (c *Crawler) Close() error {
 // Crawl crawls a URL with the specified options
 func (c *Crawler) Crawl(rootURL string) error {
 	crawlSession, err := c.NewCrawlSessionWithURL(rootURL)
+	crawlSession.Browser = c.browser
 	if err != nil {
 		return errorutil.NewWithErr(err).WithTag("hybrid")
 	}
 	defer crawlSession.CancelFunc()
 
-	// create a new browser instance (default to incognito mode)
-	if c.Options.Options.HeadlessNoIncognito {
-		if err := c.browser.Connect(); err != nil {
-			return err
-		}
-		crawlSession.Browser = c.browser
-	} else {
-		var err error
-		crawlSession.Browser, err = c.browser.Incognito()
-		if err != nil {
-			return err
-		}
-	}
-
 	gologger.Info().Msgf("Started headless crawling for => %v", rootURL)
 	if err := c.Do(crawlSession, c.navigateRequest); err != nil {
 		return errorutil.NewWithErr(err).WithTag("standard")