`))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
for _, navigationRequest := range bodyAudioTagParser(resp) {
gotURL = append(gotURL, navigationRequest.URL)
}
@@ -106,25 +106,25 @@ func TestBodyParsers(t *testing.T) {
t.Run("img", func(t *testing.T) {
t.Run("dynsrc", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(`![]()
`))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyImgTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/img/dynsrc.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("longdesc", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(`
`))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyImgTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/img/longdesc.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("lowsrc", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(`![]()
`))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyImgTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/img/lowsrc.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("src", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(`
`))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyImgTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/img/src.found", navigationRequests[0].URL, "could not get correct url")
})
@@ -132,7 +132,7 @@ func TestBodyParsers(t *testing.T) {
var gotURL []string
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(`![]()
`))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
for _, navigationResponse := range bodyImgTagParser(resp) {
gotURL = append(gotURL, navigationResponse.URL)
}
@@ -147,7 +147,7 @@ func TestBodyParsers(t *testing.T) {
//
// parsed, _ = url.Parse("https://security-crawl-maze.app/html/body/frameset/frame/src.html")
// var gotURL []string
- // resp := navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Body: []byte(`
+ // resp := navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Body: []byte(`
// The test contains an inline string with known extension - /string-known-extension.pdf
// The test contains an inline string - ./test/html/misc/string/dot-slash-prefix.found
// The test contains an inline string - ../test/html/misc/string/dot-dot-slash-prefix.found
@@ -165,19 +165,19 @@ func TestBodyParsers(t *testing.T) {
})
t.Run("object", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(``))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyObjectTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/object/data.found", navigationRequests[0].URL, "could not get correct url")
documentReader, _ = goquery.NewDocumentFromReader(strings.NewReader(``))
- resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests = bodyObjectTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/object/codebase.found", navigationRequests[0].URL, "could not get correct url")
documentReader, _ = goquery.NewDocumentFromReader(strings.NewReader(``))
- resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests = bodyObjectTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/object/param/value.found", navigationRequests[0].URL, "could not get correct url")
})
@@ -185,20 +185,20 @@ func TestBodyParsers(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(``))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodySvgTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/svg/image/xlink.found", navigationRequests[0].URL, "could not get correct url")
documentReader, _ = goquery.NewDocumentFromReader(strings.NewReader(``))
- resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests = bodySvgTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/svg/script/xlink.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("table", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(`
`))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyTableTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/table/background.found", navigationRequests[0].URL, "could not get correct url")
@@ -207,80 +207,80 @@ func TestBodyParsers(t *testing.T) {
|
`))
- resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests = bodyTableTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/table/td/background.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("video", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(``))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyVideoTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/video/poster.found", navigationRequests[0].URL, "could not get correct url")
documentReader, _ = goquery.NewDocumentFromReader(strings.NewReader(``))
- resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests = bodyVideoTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/video/src.found", navigationRequests[0].URL, "could not get correct url")
documentReader, _ = goquery.NewDocumentFromReader(strings.NewReader(``))
- resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests = bodyVideoTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/video/track/src.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("applet", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(``))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyAppletTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/applet/archive.found", navigationRequests[0].URL, "could not get correct url")
documentReader, _ = goquery.NewDocumentFromReader(strings.NewReader(``))
- resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests = bodyAppletTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/applet/codebase.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("link", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyLinkHrefTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/css/font-face.css", navigationRequests[0].URL, "could not get correct url")
documentReader, _ = goquery.NewDocumentFromReader(strings.NewReader(``))
- resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests = bodyLinkHrefTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/head/link/href.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("base", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(``))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyBaseHrefTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/head/base/href.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("manifest", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(``))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyHtmlManifestTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/manifest.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("doctype", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(`
`))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyHtmlDoctypeTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/doctype.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("import", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(``))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyImportImplementationTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/head/import/implementation.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("embed", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyEmbedTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/embed/src.found", navigationRequests[0].URL, "could not get correct url")
})
@@ -295,7 +295,7 @@ func TestBodyParsers(t *testing.T) {
// `))
- // resp := navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ // resp := navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
// bodyFrameTagParser(resp, func(resp navigation.Request) {
// gotURL = resp.URL
// })
@@ -304,14 +304,14 @@ func TestBodyParsers(t *testing.T) {
t.Run("iframe", func(t *testing.T) {
t.Run("src", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyIframeTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/iframe/src.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("srcdoc", func(t *testing.T) {
//var gotURL string
//documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- //resp := navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ //resp := navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
//bodyIframeTagParser(resp, func(resp navigation.Request) {
// gotURL = resp.URL
//})
@@ -320,38 +320,38 @@ func TestBodyParsers(t *testing.T) {
})
t.Run("input", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyInputSrcTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/input/src.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("isindex", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyIsindexActionTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/isindex/action.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("script", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyScriptSrcTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/script/src.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("button", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyButtonFormactionTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/form/button/formaction.found", navigationRequests[0].URL, "could not get correct url")
})
t.Run("form", func(t *testing.T) {
t.Run("get", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyFormTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/form/action-get.found?test1=test&test2=test", navigationRequests[0].URL, "could not get correct url")
})
t.Run("post", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := bodyFormTagParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/body/form/action-post.found", navigationRequests[0].URL, "could not get correct url")
require.Equal(t, "POST", navigationRequests[0].Method, "could not get correct method")
@@ -361,28 +361,28 @@ func TestBodyParsers(t *testing.T) {
t.Run("meta", func(t *testing.T) {
// var gotURL string
// documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- // resp := navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ // resp := navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
// bodyMetaContentTagParser(resp, func(resp navigation.Request) {
// gotURL = resp.URL
// })
// require.Equal(t, "https://security-crawl-maze.app/test/html/head/meta/content-redirect.found", gotURL, "could not get correct url")
//
// documentReader, _ = goquery.NewDocumentFromReader(strings.NewReader(``))
- // resp = navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ // resp = navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
// bodyMetaContentTagParser(resp, func(resp navigation.Request) {
// gotURL = resp.URL
// })
// require.Equal(t, "https://security-crawl-maze.app/test/html/head/meta/content-csp.found", gotURL, "could not get correct url")
//
// documentReader, _ = goquery.NewDocumentFromReader(strings.NewReader(``))
- // resp = navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ // resp = navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
// bodyMetaContentTagParser(resp, func(resp navigation.Request) {
// gotURL = resp.URL
// })
// require.Equal(t, "https://security-crawl-maze.app/test/html/head/meta/content-pinned-websites.found", gotURL, "could not get correct url")
//
// documentReader, _ = goquery.NewDocumentFromReader(strings.NewReader(``))
- // resp = navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ // resp = navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
// bodyMetaContentTagParser(resp, func(resp navigation.Request) {
// gotURL = resp.URL
// })
@@ -391,23 +391,23 @@ func TestBodyParsers(t *testing.T) {
}
func TestScriptParsers(t *testing.T) {
- parsed, _ := url.Parse("https://security-crawl-maze.app/html/script/xyz/")
+ parsed, _ := urlutil.Parse("https://security-crawl-maze.app/html/script/xyz/")
t.Run("content", func(t *testing.T) {
documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(""))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
navigationRequests := scriptContentRegexParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/script/content.do", navigationRequests[0].URL, "could not get correct url")
})
t.Run("js", func(t *testing.T) {
- parsed, _ = url.Parse("https://security-crawl-maze.app/html/script/xyz/data.js")
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Body: "var endpoint='/test/html/script/body.do';"}
+ parsed, _ = urlutil.Parse("https://security-crawl-maze.app/html/script/xyz/data.js")
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Body: "var endpoint='/test/html/script/body.do';"}
navigationRequests := scriptJSFileRegexParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/script/body.do", navigationRequests[0].URL, "could not get correct url")
- parsed, _ = url.Parse("https://security-crawl-maze.app/html/script/xyz/")
- resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}, Header: http.Header{"Content-Type": []string{"application/javascript"}}}, Body: "var endpoint='/test/html/script/body-content-type.do';"}
+ parsed, _ = urlutil.Parse("https://security-crawl-maze.app/html/script/xyz/")
+ resp = &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}, Header: http.Header{"Content-Type": []string{"application/javascript"}}}, Body: "var endpoint='/test/html/script/body-content-type.do';"}
navigationRequests = scriptJSFileRegexParser(resp)
require.Equal(t, "https://security-crawl-maze.app/test/html/script/body-content-type.do", navigationRequests[0].URL, "could not get correct url")
@@ -415,11 +415,11 @@ func TestScriptParsers(t *testing.T) {
}
func TestRegexBodyParsers(t *testing.T) {
- parsed, _ := url.Parse("https://security-crawl-maze.app/contact")
+ parsed, _ := urlutil.Parse("https://security-crawl-maze.app/contact")
t.Run("regexbody", func(t *testing.T) {
output.CustomFieldsMap = make(map[string]output.CustomFieldConfig)
resp := &navigation.Response{
- Resp: &http.Response{Request: &http.Request{URL: parsed}},
+ Resp: &http.Response{Request: &http.Request{URL: parsed.URL}},
Depth: 0,
Body: "some content contact@example.com",
}
@@ -439,7 +439,7 @@ func TestRegexBodyParsers(t *testing.T) {
t.Run("regexheader", func(t *testing.T) {
output.CustomFieldsMap = make(map[string]output.CustomFieldConfig)
resp := &navigation.Response{
- Resp: &http.Response{Request: &http.Request{URL: parsed},
+ Resp: &http.Response{Request: &http.Request{URL: parsed.URL},
Header: http.Header{
"server": []string{"ECS (dcb/7F84)"},
},
@@ -462,7 +462,7 @@ func TestRegexBodyParsers(t *testing.T) {
t.Run("regexresponse", func(t *testing.T) {
output.CustomFieldsMap = make(map[string]output.CustomFieldConfig)
resp := &navigation.Response{
- Resp: &http.Response{Request: &http.Request{URL: parsed},
+ Resp: &http.Response{Request: &http.Request{URL: parsed.URL},
Header: http.Header{
"server": []string{"ECS (dcb/7F84)"},
},
diff --git a/pkg/engine/standard/standard.go b/pkg/engine/standard/standard.go
index 55240462..b90429fb 100644
--- a/pkg/engine/standard/standard.go
+++ b/pkg/engine/standard/standard.go
@@ -1,6 +1,7 @@
package standard
import (
+ "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/katana/pkg/engine/common"
"github.com/projectdiscovery/katana/pkg/types"
errorutil "github.com/projectdiscovery/utils/errors"
@@ -32,7 +33,7 @@ func (c *Crawler) Crawl(rootURL string) error {
return errorutil.NewWithErr(err).WithTag("standard")
}
defer crawlSession.CancelFunc()
-
+ gologger.Info().Msgf("Started standard crawling for => %v", rootURL)
if err := c.Do(crawlSession, c.makeRequest); err != nil {
return errorutil.NewWithErr(err).WithTag("standard")
}
diff --git a/pkg/output/fields.go b/pkg/output/fields.go
index 824625aa..e22dc7b4 100644
--- a/pkg/output/fields.go
+++ b/pkg/output/fields.go
@@ -7,8 +7,10 @@ import (
"path"
"strings"
+ "github.com/projectdiscovery/gologger"
errorutil "github.com/projectdiscovery/utils/errors"
stringsutil "github.com/projectdiscovery/utils/strings"
+ urlutil "github.com/projectdiscovery/utils/url"
"golang.org/x/net/publicsuffix"
)
@@ -59,8 +61,9 @@ func validateFieldNames(names string) error {
// storeFields stores fields for a result into individual files
// based on name.
func storeFields(output *Result, storeFields []string) {
- parsed, err := url.Parse(output.Request.URL)
+ parsed, err := urlutil.Parse(output.Request.URL)
if err != nil {
+ gologger.Warning().Msgf("storeFields: failed to parse url %v got %v", output.Request.URL, err)
return
}
@@ -68,13 +71,13 @@ func storeFields(output *Result, storeFields []string) {
etld, _ := publicsuffix.EffectiveTLDPlusOne(hostname)
rootURL := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
for _, field := range storeFields {
- if result := getValueForField(output, parsed, hostname, etld, rootURL, field); result != "" {
- appendToFileField(parsed, field, result)
+ if result := getValueForField(output, parsed.URL, hostname, etld, rootURL, field); result != "" {
+ appendToFileField(parsed.URL, field, result)
}
if _, ok := CustomFieldsMap[field]; ok {
results := getValueForCustomField(output)
for _, result := range results {
- appendToFileField(parsed, result.field, result.value)
+ appendToFileField(parsed.URL, result.field, result.value)
}
}
}
@@ -94,7 +97,7 @@ func appendToFileField(parsed *url.URL, field, data string) {
// formatField formats output results based on fields from fieldNames
func formatField(output *Result, fields string) []fieldOutput {
var svalue []fieldOutput
- parsed, _ := url.Parse(output.Request.URL)
+ parsed, _ := urlutil.Parse(output.Request.URL)
if parsed == nil {
return svalue
}
diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go
index 9ff7e200..90a0fca5 100644
--- a/pkg/types/crawler_options.go
+++ b/pkg/types/crawler_options.go
@@ -2,7 +2,6 @@ package types
import (
"context"
- "net/url"
"time"
"github.com/projectdiscovery/fastdialer/fastdialer"
@@ -12,6 +11,7 @@ import (
"github.com/projectdiscovery/katana/pkg/utils/scope"
"github.com/projectdiscovery/ratelimit"
errorutil "github.com/projectdiscovery/utils/errors"
+ urlutil "github.com/projectdiscovery/utils/url"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
)
@@ -115,12 +115,12 @@ func (c *CrawlerOptions) ValidatePath(path string) bool {
// ValidateScope validates scope for an AbsURL
func (c *CrawlerOptions) ValidateScope(absURL, rootHostname string) (bool, error) {
- parsed, err := url.Parse(absURL)
+ parsed, err := urlutil.Parse(absURL)
if err != nil {
return false, err
}
if c.ScopeManager != nil {
- return c.ScopeManager.Validate(parsed, rootHostname)
+ return c.ScopeManager.Validate(parsed.URL, rootHostname)
}
return true, nil
}
diff --git a/pkg/types/options.go b/pkg/types/options.go
index 6dbaa96c..7b55ba8e 100644
--- a/pkg/types/options.go
+++ b/pkg/types/options.go
@@ -120,6 +120,8 @@ type Options struct {
DisableUpdateCheck bool
//IgnoreQueryParams ignore crawling same path with different query-param values
IgnoreQueryParams bool
+ // Debug
+ Debug bool
}
func (options *Options) ParseCustomHeaders() map[string]string {
diff --git a/pkg/utils/extensions/extensions.go b/pkg/utils/extensions/extensions.go
index e0efb760..87c61316 100644
--- a/pkg/utils/extensions/extensions.go
+++ b/pkg/utils/extensions/extensions.go
@@ -1,13 +1,15 @@
package extensions
import (
- "net/url"
"path"
"strings"
+
+ "github.com/projectdiscovery/gologger"
+ urlutil "github.com/projectdiscovery/utils/url"
)
// defaultDenylist is the default list of extensions to be denied
-var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".txt", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"}
+var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".txt", ".vob", ".wav", ".webm", ".webp", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"}
// Validator is a validator for file extension
type Validator struct {
@@ -37,8 +39,11 @@ func NewValidator(extensionsMatch, extensionsFilter []string) *Validator {
// ValidatePath returns true if an extension is allowed by the validator
func (e *Validator) ValidatePath(item string) bool {
var extension string
- u, _ := url.Parse(item)
- if u != nil {
+ u, err := urlutil.Parse(item)
+ if err != nil {
+ gologger.Warning().Msgf("validatepath: failed to parse url %v got %v", item, err)
+ }
+ if u.Path != "" {
extension = strings.ToLower(path.Ext(u.Path))
} else {
extension = strings.ToLower(path.Ext(item))
diff --git a/pkg/utils/scope/scope_test.go b/pkg/utils/scope/scope_test.go
index 2a53a8c3..6f8e794a 100644
--- a/pkg/utils/scope/scope_test.go
+++ b/pkg/utils/scope/scope_test.go
@@ -1,9 +1,9 @@
package scope
import (
- "net/url"
"testing"
+ urlutil "github.com/projectdiscovery/utils/url"
"github.com/stretchr/testify/require"
)
@@ -12,13 +12,13 @@ func TestManagerValidate(t *testing.T) {
manager, err := NewManager([]string{`example`}, []string{`logout\.php`}, "dn", false)
require.NoError(t, err, "could not create scope manager")
- parsed, _ := url.Parse("https://test.com/index.php/example")
- validated, err := manager.Validate(parsed, "test.com")
+ parsed, _ := urlutil.Parse("https://test.com/index.php/example")
+ validated, err := manager.Validate(parsed.URL, "test.com")
require.NoError(t, err, "could not validate url")
require.True(t, validated, "could not get correct in-scope validation")
- parsed, _ = url.Parse("https://test.com/logout.php")
- validated, err = manager.Validate(parsed, "another.com")
+ parsed, _ = urlutil.Parse("https://test.com/logout.php")
+ validated, err = manager.Validate(parsed.URL, "another.com")
require.NoError(t, err, "could not validate url")
require.False(t, validated, "could not get correct out-scope validation")
})
@@ -27,8 +27,8 @@ func TestManagerValidate(t *testing.T) {
manager, err := NewManager(nil, nil, "dn", false)
require.NoError(t, err, "could not create scope manager")
- parsed, _ := url.Parse("https://testanother.com/index.php")
- validated, err := manager.Validate(parsed, "test.com")
+ parsed, _ := urlutil.Parse("https://testanother.com/index.php")
+ validated, err := manager.Validate(parsed.URL, "test.com")
require.NoError(t, err, "could not validate host")
require.True(t, validated, "could not get correct in-scope validation")
})
@@ -36,8 +36,8 @@ func TestManagerValidate(t *testing.T) {
manager, err := NewManager(nil, nil, "rdn", false)
require.NoError(t, err, "could not create scope manager")
- parsed, _ := url.Parse("https://subdomain.example.com/logout.php")
- validated, err := manager.Validate(parsed, "example.com")
+ parsed, _ := urlutil.Parse("https://subdomain.example.com/logout.php")
+ validated, err := manager.Validate(parsed.URL, "example.com")
require.NoError(t, err, "could not validate host")
require.True(t, validated, "could not get correct in-scope validation")
})
@@ -45,8 +45,8 @@ func TestManagerValidate(t *testing.T) {
manager, err := NewManager(nil, nil, "rdn", false)
require.NoError(t, err, "could not create scope manager")
- parsed, _ := url.Parse("http://localhost:8082/logout.php")
- validated, err := manager.Validate(parsed, "localhost")
+ parsed, _ := urlutil.Parse("http://localhost:8082/logout.php")
+ validated, err := manager.Validate(parsed.URL, "localhost")
require.NoError(t, err, "could not validate host")
require.True(t, validated, "could not get correct in-scope validation")
})
@@ -54,18 +54,18 @@ func TestManagerValidate(t *testing.T) {
manager, err := NewManager(nil, nil, "fqdn", false)
require.NoError(t, err, "could not create scope manager")
- parsed, _ := url.Parse("https://test.com/index.php")
- validated, err := manager.Validate(parsed, "test.com")
+ parsed, _ := urlutil.Parse("https://test.com/index.php")
+ validated, err := manager.Validate(parsed.URL, "test.com")
require.NoError(t, err, "could not validate host")
require.True(t, validated, "could not get correct in-scope validation")
- parsed, _ = url.Parse("https://subdomain.example.com/logout.php")
- validated, err = manager.Validate(parsed, "example.com")
+ parsed, _ = urlutil.Parse("https://subdomain.example.com/logout.php")
+ validated, err = manager.Validate(parsed.URL, "example.com")
require.NoError(t, err, "could not validate host")
require.False(t, validated, "could not get correct out-scope validation")
- parsed, _ = url.Parse("https://example.com/logout.php")
- validated, err = manager.Validate(parsed, "another.com")
+ parsed, _ = urlutil.Parse("https://example.com/logout.php")
+ validated, err = manager.Validate(parsed.URL, "another.com")
require.NoError(t, err, "could not validate host")
require.False(t, validated, "could not get correct out-scope validation")
})
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
index ec8b3a89..8b6ec0e3 100644
--- a/pkg/utils/utils.go
+++ b/pkg/utils/utils.go
@@ -1,16 +1,21 @@
package utils
import (
- "net/url"
"strings"
"github.com/lukasbob/srcset"
- stringsutil "github.com/projectdiscovery/utils/strings"
+ "github.com/projectdiscovery/gologger"
+ urlutil "github.com/projectdiscovery/utils/url"
)
// IsURL returns true if a provided string is URL
func IsURL(url string) bool {
- return stringsutil.HasPrefixAny(url, "http://", "https://")
+ if value, err := urlutil.Parse(url); err == nil {
+ return value.Hostname() != ""
+ } else {
+ gologger.Debug().Msgf("IsURL: failed to parse url %v got %v", url, err)
+ }
+ return false
}
// ParseSRCSetTag parses srcset tag returning found URLs
@@ -73,7 +78,7 @@ func FlattenHeaders(headers map[string][]string) map[string]string {
// ReplaceAllQueryParam replaces all the query param with the given value
func ReplaceAllQueryParam(reqUrl, val string) string {
- u, err := url.Parse(reqUrl)
+ u, err := urlutil.Parse(reqUrl)
if err != nil {
return reqUrl
}
`))
- resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Reader: documentReader}
+ resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader}
for _, navigationResponse := range bodyImgTagParser(resp) {
gotURL = append(gotURL, navigationResponse.URL)
}
@@ -147,7 +147,7 @@ func TestBodyParsers(t *testing.T) {
//
// parsed, _ = url.Parse("https://security-crawl-maze.app/html/body/frameset/frame/src.html")
// var gotURL []string
- // resp := navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed}}, Body: []byte(`