diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml new file mode 100644 index 00000000..3f1fc504 --- /dev/null +++ b/.github/workflows/functional-test.yml @@ -0,0 +1,30 @@ +name: ๐Ÿงช Functional Test + +on: + pull_request: + paths: + - '**.go' + - '**.mod' + workflow_dispatch: + +jobs: + functional: + name: Functional Test + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + steps: + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.19 + + - name: Check out code + uses: actions/checkout@v3 + + - name: Functional Tests + run: | + chmod +x run.sh + bash run.sh ${{ matrix.os }} + working-directory: cmd/functional-test diff --git a/cmd/functional-test/main.go b/cmd/functional-test/main.go new file mode 100644 index 00000000..7e76e1e9 --- /dev/null +++ b/cmd/functional-test/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" + "github.com/projectdiscovery/katana/internal/testutils" +) + +var ( + debug = os.Getenv("DEBUG") == "true" + success = aurora.Green("[โœ“]").String() + failed = aurora.Red("[โœ˜]").String() + errored = false + devKatanaBinary = flag.String("dev", "", "Dev Branch Katana Binary") +) + +func main() { + flag.Parse() + if err := runFunctionalTests(); err != nil { + log.Fatalf("Could not run functional tests: %s\n", err) + } + if errored { + os.Exit(1) + } +} + +func runFunctionalTests() error { + for _, testcase := range testutils.TestCases { + if err := runIndividualTestCase(testcase); err != nil { + errored = true + fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, testcase.Name, err) + } else { + fmt.Printf("%s Test \"%s\" passed!\n", success, testcase.Name) + } + } + return nil +} + +func runIndividualTestCase(testcase testutils.TestCase) error { + argsParts := strings.Fields(testcase.Args) + devOutput, err := testutils.RunKatanaBinaryAndGetResults(testcase.Target, *devKatanaBinary, debug, argsParts) + if err != nil { + return errors.Wrap(err, "could not run Katana dev test") + } + if testcase.CompareFunc != nil { + return testcase.CompareFunc(testcase.Target, devOutput) + } + if !testutils.CompareOutput(devOutput, testcase.Expected) { + return errors.Errorf("expected output %s, got %s", testcase.Expected, devOutput) + } + return nil +} diff --git a/cmd/functional-test/run.sh b/cmd/functional-test/run.sh new file mode 100644 index 00000000..af44e4f5 --- /dev/null +++ b/cmd/functional-test/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# reading os type from arguments +CURRENT_OS=$1 + +if [ "${CURRENT_OS}" == "windows-latest" ];then + extension=.exe +fi + +echo "::group::Building functional-test binary" +go build -o functional-test$extension +echo "::endgroup::" + +echo "::group::Building katana binary from current branch" +go build -o katana_dev$extension ../katana +echo "::endgroup::" + + +echo 'Starting katana functional test' +./functional-test$extension -dev ./katana_dev$extension diff --git a/internal/runner/options.go b/internal/runner/options.go index 432f1a53..24ad30d6 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -125,6 +125,9 @@ func initExampleFormFillConfig() error { if fileutil.FileExists(defaultConfig) { return nil } + if err := os.MkdirAll(filepath.Dir(defaultConfig), 0775); err != nil { + return err + } exampleConfig, err := os.Create(defaultConfig) if err != nil { return errorutil.NewWithErr(err).Msgf("could not get home directory") diff --git a/internal/testutils/helper.go b/internal/testutils/helper.go new file mode 100644 index 00000000..3f20b9d1 --- /dev/null +++ b/internal/testutils/helper.go @@ -0,0 +1,13 @@ +package testutils + +func CompareOutput(input, expected []string) bool { + if len(input) != len(expected) { + return false + } + for i, v := range input { + if v != expected[i] { + return false + } + } + return true +} diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go new file mode 100644 index 00000000..b77a22c8 --- /dev/null +++ b/internal/testutils/integration.go @@ -0,0 +1,27 @@ +package testutils + +import ( + "fmt" + "os/exec" + "strings" +) + +func RunKatanaBinaryAndGetResults(target string, katanaBinary string, debug bool, args []string) ([]string, error) { + cmd := exec.Command("bash", "-c") + cmdLine := fmt.Sprintf(`echo %s | %s `, target, katanaBinary) + cmdLine += strings.Join(args, " ") + + cmd.Args = append(cmd.Args, cmdLine) + data, err := cmd.Output() + if err != nil { + return nil, err + } + parts := []string{} + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go new file mode 100644 index 00000000..8c85fbdc --- /dev/null +++ b/internal/testutils/testutils.go @@ -0,0 +1,32 @@ +package testutils + +import ( + "strings" + + errorutils "github.com/projectdiscovery/utils/errors" +) + +type TestCase struct { + Name string + Target string + Args string + Expected []string + CompareFunc func(target string, got []string) error +} + +var TestCases = []TestCase{ + { + Name: "Headless Browser Without Incognito", + Target: "https://www.hackerone.com/", + Expected: nil, + Args: "-headless -no-incognito -depth 2 -silent", + CompareFunc: func(target string, got []string) error { + for _, res := range got { + if strings.Contains(res, target) { + return nil + } + } + return errorutils.New("expected %v target in output, but got %v ", target, strings.Join(got, "\n")) + }, + }, +} diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index 6c6b4396..d9073311 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -97,6 +97,16 @@ func New(options *types.CrawlerOptions) (*Crawler, error) { return nil, browserErr } + // create a new browser instance (default to incognito mode) + if !options.Options.HeadlessNoIncognito { + incognito, err := browser.Incognito() + if err != nil { + chromeLauncher.Kill() + return nil, errorutil.NewWithErr(err).Msgf("failed to create incognito browser") + } + browser = incognito + } + shared, err := common.NewShared(options) if err != nil { return nil, errorutil.NewWithErr(err).WithTag("hybrid") @@ -128,25 +138,12 @@ func (c *Crawler) Close() error { // Crawl crawls a URL with the specified options func (c *Crawler) Crawl(rootURL string) error { crawlSession, err := c.NewCrawlSessionWithURL(rootURL) + crawlSession.Browser = c.browser if err != nil { return errorutil.NewWithErr(err).WithTag("hybrid") } defer crawlSession.CancelFunc() - // create a new browser instance (default to incognito mode) - if c.Options.Options.HeadlessNoIncognito { - if err := c.browser.Connect(); err != nil { - return err - } - crawlSession.Browser = c.browser - } else { - var err error - crawlSession.Browser, err = c.browser.Incognito() - if err != nil { - return err - } - } - gologger.Info().Msgf("Started headless crawling for => %v", rootURL) if err := c.Do(crawlSession, c.navigateRequest); err != nil { return errorutil.NewWithErr(err).WithTag("standard")