diff --git a/.gitignore b/.gitignore index d04dfe5a..f9e3efb5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ security-crawl-maze/ cmd/katana/katana katana -*.exe \ No newline at end of file +*.exe +katana_output/ +katana_responses/ \ No newline at end of file diff --git a/cmd/katana/main.go b/cmd/katana/main.go index 5ff39030..5d051521 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -111,6 +111,8 @@ pipelines offering both headless and non-headless crawling.`) flagSet.CreateGroup("output", "Output", flagSet.StringVarP(&options.OutputFile, "output", "o", "", "file to write output to"), + flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"), + flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"), flagSet.BoolVarP(&options.JSON, "json", "j", false, "write output in JSONL(ines) format"), flagSet.BoolVarP(&options.NoColors, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"), flagSet.BoolVar(&options.Silent, "silent", false, "display output only"), diff --git a/go.mod b/go.mod index 22f7c1d5..34be2ece 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.2 github.com/projectdiscovery/retryablehttp-go v1.0.5 github.com/projectdiscovery/stringsutil v0.0.2 + github.com/projectdiscovery/utils v0.0.4-0.20221201124851-f8524345b6d3 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/shirou/gopsutil/v3 v3.22.11 @@ -53,7 +54,6 @@ require ( github.com/projectdiscovery/networkpolicy v0.0.3 // indirect github.com/projectdiscovery/retryabledns v1.0.17 // indirect github.com/projectdiscovery/sliceutil v0.0.1 // indirect - github.com/projectdiscovery/utils v0.0.4-0.20221201124851-f8524345b6d3 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect diff --git a/internal/runner/options.go b/internal/runner/options.go index a856c37c..4427fdf9 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -32,6 +32,13 @@ func validateOptions(options *types.Options) error { if (options.HeadlessOptionalArguments != nil || options.HeadlessNoSandbox) && !options.Headless { return errors.New("headless mode (-hl) is required if -ho or -nos are set") } + if options.StoreResponseDir != "" && !options.StoreResponse { + gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n") + options.StoreResponse = true + } + if options.Headless && options.StoreResponse { + return errors.New("Impossible to store responses in headless mode") + } gologger.DefaultLogger.SetFormatter(formatter.NewCLI(options.NoColors)) return nil } diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index 187a3a6d..daf24046 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -254,7 +254,7 @@ func (c *Crawler) makeParseResponseCallback(queue *queue.VarietyQueue) func(nr n return } if scopeValidated || c.options.Options.DisplayOutScope { - _ = c.options.OutputWriter.Write(result) + _ = c.options.OutputWriter.Write(result, nil) } if c.options.Options.OnResult != nil { c.options.Options.OnResult(*result) diff --git a/pkg/engine/standard/crawl.go b/pkg/engine/standard/crawl.go index cc585d21..3cfcc176 100644 --- a/pkg/engine/standard/crawl.go +++ b/pkg/engine/standard/crawl.go @@ -67,6 +67,9 @@ func (c *Crawler) makeRequest(ctx context.Context, request navigation.Request, r return navigation.Response{}, nil } + resp.Body = io.NopCloser(strings.NewReader(string(data))) + _ = c.options.OutputWriter.Write(nil, resp) + response.Body = data response.Resp = resp response.Reader, err = goquery.NewDocumentFromReader(bytes.NewReader(data)) diff --git a/pkg/engine/standard/standard.go b/pkg/engine/standard/standard.go index 407b0e12..b4631b9f 100644 --- a/pkg/engine/standard/standard.go +++ b/pkg/engine/standard/standard.go @@ -164,7 +164,7 @@ func (c *Crawler) makeParseResponseCallback(queue *queue.VarietyQueue) func(nr n return } if scopeValidated || c.options.Options.DisplayOutScope { - _ = c.options.OutputWriter.Write(result) + _ = c.options.OutputWriter.Write(result, nil) } if c.options.Options.OnResult != nil { c.options.Options.OnResult(*result) diff --git a/pkg/output/output.go b/pkg/output/output.go index d3762eb1..3001cc0f 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -1,7 +1,9 @@ package output import ( + "net/http" "os" + "path/filepath" "regexp" "strings" "sync" @@ -17,20 +19,24 @@ type Writer interface { // Close closes the output writer interface Close() error // Write writes the event to file and/or screen. - Write(*Result) error + Write(*Result, *http.Response) error } -var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) +var ( + decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) +) // StandardWriter is an standard output writer structure type StandardWriter struct { - storeFields []string - fields string - json bool - verbose bool - aurora aurora.Aurora - outputFile *fileWriter - outputMutex *sync.Mutex + storeFields []string + fields string + json bool + verbose bool + aurora aurora.Aurora + outputFile *fileWriter + outputMutex *sync.Mutex + storeResponse bool + storeResponseDir string } // Options contains the configuration options for output writer @@ -61,16 +67,22 @@ type Result struct { Attribute string `json:"attribute,omitempty"` } -const storeFieldsDirectory = "katana_output" +const ( + storeFieldsDirectory = "katana_output" + indexFile = "index.txt" + DefaultResponseDir = "katana_responses" +) // New returns a new output writer instance -func New(colors, json, verbose bool, file, fields, storeFields string) (Writer, error) { +func New(colors, json, verbose, storeResponse bool, file, fields, storeFields, storeResponseDir string) (Writer, error) { writer := &StandardWriter{ - fields: fields, - json: json, - verbose: verbose, - aurora: aurora.NewAurora(colors), - outputMutex: &sync.Mutex{}, + fields: fields, + json: json, + verbose: verbose, + aurora: aurora.NewAurora(colors), + outputMutex: &sync.Mutex{}, + storeResponse: storeResponse, + storeResponseDir: storeResponseDir, } // Perform validations for fields and store-fields if fields != "" { @@ -92,40 +104,71 @@ func New(colors, json, verbose bool, file, fields, storeFields string) (Writer, } writer.outputFile = output } + if storeResponse { + writer.storeResponseDir = DefaultResponseDir + if storeResponseDir != DefaultResponseDir && storeResponseDir != "" { + writer.storeResponseDir = storeResponseDir + } + _ = os.RemoveAll(writer.storeResponseDir) + _ = os.MkdirAll(writer.storeResponseDir, os.ModePerm) + _, err := newFileOutputWriter(filepath.Join(writer.storeResponseDir, indexFile)) + if err != nil { + return nil, errors.Wrap(err, "could not create index file") + } + } return writer, nil } // Write writes the event to file and/or screen. -func (w *StandardWriter) Write(event *Result) error { - if len(w.storeFields) > 0 { - storeFields(event, w.storeFields) - } - var data []byte - var err error +func (w *StandardWriter) Write(event *Result, resp *http.Response) error { + if event != nil { + if len(w.storeFields) > 0 { + storeFields(event, w.storeFields) + } + var data []byte + var err error - if w.json { - data, err = w.formatJSON(event) - } else { - data, err = w.formatScreen(event) - } - if err != nil { - return errors.Wrap(err, "could not format output") - } - if len(data) == 0 { - return nil - } - w.outputMutex.Lock() - defer w.outputMutex.Unlock() + if w.json { + data, err = w.formatJSON(event) + } else { + data, err = w.formatScreen(event) + } + if err != nil { + return errors.Wrap(err, "could not format output") + } + if len(data) == 0 { + return nil + } + w.outputMutex.Lock() + defer w.outputMutex.Unlock() - gologger.Silent().Msgf("%s", string(data)) - if w.outputFile != nil { - if !w.json { - data = decolorizerRegex.ReplaceAll(data, []byte("")) + gologger.Silent().Msgf("%s", string(data)) + if w.outputFile != nil { + if !w.json { + data = decolorizerRegex.ReplaceAll(data, []byte("")) + } + if writeErr := w.outputFile.Write(data); writeErr != nil { + return errors.Wrap(err, "could not write to output") + } } - if writeErr := w.outputFile.Write(data); writeErr != nil { - return errors.Wrap(err, "could not write to output") + } + + if w.storeResponse && resp != nil { + if file, err := getResponseFile(w.storeResponseDir, resp.Request.URL.String()); err == nil { + data, err := w.formatResponse(resp) + if err != nil { + return errors.Wrap(err, "could not store response") + } + if err := updateIndex(w.storeResponseDir, resp); err != nil { + return errors.Wrap(err, "could not store response") + } + if writeErr := file.Write(data); writeErr != nil { + return errors.Wrap(err, "could not store response") + } + file.Close() } } + return nil } diff --git a/pkg/output/responses.go b/pkg/output/responses.go new file mode 100644 index 00000000..e18391e0 --- /dev/null +++ b/pkg/output/responses.go @@ -0,0 +1,127 @@ +package output + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + urlutil "github.com/projectdiscovery/utils/url" +) + +func getResponseHash(URL string) string { + hash := sha1.Sum([]byte(URL)) + return hex.EncodeToString(hash[:]) +} + +func (w *StandardWriter) formatResponse(resp *http.Response) ([]byte, error) { + builder := &bytes.Buffer{} + + builder.WriteString(resp.Request.URL.String()) + builder.WriteString("\n\n\n") + + builder.WriteString(resp.Request.Method) + builder.WriteString(" ") + path := resp.Request.URL.Path + if resp.Request.URL.Fragment != "" { + path = path + "#" + resp.Request.URL.Fragment + } + builder.WriteString(path) + builder.WriteString(" ") + builder.WriteString(resp.Request.Proto) + builder.WriteString("\n") + builder.WriteString("Host: " + resp.Request.Host) + builder.WriteRune('\n') + for k, v := range resp.Request.Header { + builder.WriteString(k + ": " + strings.Join(v, "; ") + "\n") + } + + if resp.Request.Body != nil { + bodyResp, _ := io.ReadAll(resp.Request.Body) + if string(bodyResp) != "" { + builder.WriteString("\n") + builder.WriteString(string(bodyResp)) + } + } + builder.WriteString("\n\n") + + builder.WriteString(resp.Proto) + builder.WriteString(" ") + builder.WriteString(resp.Status) + builder.WriteString("\n") + for k, v := range resp.Header { + builder.WriteString(k + ": " + strings.Join(v, "; ") + "\n") + } + builder.WriteString("\n") + body, _ := io.ReadAll(resp.Body) + builder.WriteString(string(body)) + + return builder.Bytes(), nil +} + +func getResponseHost(URL string) (string, error) { + u, err := urlutil.ParseWithScheme(URL) + if err != nil { + return "", err + } + + return u.Host, nil +} + +func createHostDir(storeResponseFolder, domain string) string { + _ = os.MkdirAll(filepath.Join(storeResponseFolder, domain), os.ModePerm) + return filepath.Join(storeResponseFolder, domain) +} + +func getResponseFile(storeResponseFolder, URL string) (*fileWriter, error) { + domain, err := getResponseHost(URL) + if err != nil { + return nil, err + } + output, err := newFileOutputWriter(getResponseFileName(storeResponseFolder, domain, URL)) + if err != nil { + return nil, errors.Wrap(err, "could not create output file") + } + + return output, nil +} + +func getResponseFileName(storeResponseFolder, domain, URL string) string { + folder := createHostDir(storeResponseFolder, domain) + file := getResponseHash(URL) + ".txt" + return filepath.Join(folder, file) +} + +func updateIndex(storeResponseFolder string, resp *http.Response) error { + index, err := os.OpenFile(filepath.Join(storeResponseFolder, indexFile), os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + + defer index.Close() + + builder := &bytes.Buffer{} + + domain, err := getResponseHost(resp.Request.URL.String()) + if err != nil { + return err + } + + builder.WriteString(getResponseFileName(storeResponseFolder, domain, resp.Request.URL.String())) + builder.WriteRune(' ') + builder.WriteString(resp.Request.URL.String()) + builder.WriteRune(' ') + builder.WriteString("(" + resp.Status + ")") + builder.WriteRune('\n') + + if _, writeErr := index.Write(builder.Bytes()); writeErr != nil { + return errors.Wrap(err, "could not update index") + } + + return nil +} diff --git a/pkg/output/responses_test.go b/pkg/output/responses_test.go new file mode 100644 index 00000000..6f9529e7 --- /dev/null +++ b/pkg/output/responses_test.go @@ -0,0 +1,62 @@ +package output + +import ( + "bytes" + "io" + "net/http" + "net/url" + "testing" + + "github.com/stretchr/testify/require" +) + +var ( + resp = http.Response{ + StatusCode: 200, + Status: "200 OK", + Proto: "HTTP/1.1", + Header: http.Header{}, + Body: io.NopCloser(bytes.NewReader([]byte("test body"))), + Request: &http.Request{ + Method: http.MethodGet, + URL: &url.URL{ + Scheme: "https", + Host: "projectdiscovery.io", + Path: "/", + }, + Host: "projectdiscovery.io", + Proto: "HTTP/1.1", + Header: http.Header{}, + }, + } + out = `https://projectdiscovery.io/ + + +GET / HTTP/1.1 +Host: projectdiscovery.io +Test: test + + +HTTP/1.1 200 OK +Test: test + +test body` +) + +func TestFormatResponses(t *testing.T) { + tests := []struct { + Resp *http.Response + Result string + }{ + {Resp: &resp, Result: out}, + } + + w := StandardWriter{} + for _, test := range tests { + test.Resp.Request.Header.Add("test", "test") + test.Resp.Header.Add("test", "test") + result, err := w.formatResponse(test.Resp) + require.Nil(t, err) + require.Equal(t, test.Result, string(result), "could not equal value") + } +} diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go index 30f48eaa..b243eb29 100644 --- a/pkg/types/crawler_options.go +++ b/pkg/types/crawler_options.go @@ -49,7 +49,7 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { return nil, errors.Wrap(err, "could not create filter") } - outputWriter, err := output.New(!options.NoColors, options.JSON, options.Verbose, options.OutputFile, options.Fields, options.StoreFields) + outputWriter, err := output.New(!options.NoColors, options.JSON, options.Verbose, options.StoreResponse, options.OutputFile, options.Fields, options.StoreFields, options.StoreResponseDir) if err != nil { return nil, errors.Wrap(err, "could not create output writer") } diff --git a/pkg/types/options.go b/pkg/types/options.go index 2271e9dc..758dbec3 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -89,6 +89,10 @@ type Options struct { HeadlessNoSandbox bool // OnResult allows callback function on a result OnResult OnResultCallback + // StoreResponse specifies if katana should store http requests/responses + StoreResponse bool + // StoreResponseDir specifies if katana should use a custom directory to store http requests/responses + StoreResponseDir string } func (options *Options) ParseCustomHeaders() map[string]string {