Option to store http responses to file

.gitignore

@@ -1,4 +1,6 @@
 security-crawl-maze/
 cmd/katana/katana
 katana
-*.exe
+*.exe
+katana_output/
+katana_responses/

cmd/katana/main.go

@@ -111,6 +111,8 @@ pipelines offering both headless and non-headless crawling.`)
 
 	flagSet.CreateGroup("output", "Output",
 		flagSet.StringVarP(&options.OutputFile, "output", "o", "", "file to write output to"),
+		flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"),
+		flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"),
 		flagSet.BoolVarP(&options.JSON, "json", "j", false, "write output in JSONL(ines) format"),
 		flagSet.BoolVarP(&options.NoColors, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
 		flagSet.BoolVar(&options.Silent, "silent", false, "display output only"),

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/projectdiscovery/ratelimit v0.0.2
 	github.com/projectdiscovery/retryablehttp-go v1.0.5
 	github.com/projectdiscovery/stringsutil v0.0.2
+	github.com/projectdiscovery/utils v0.0.4-0.20221201124851-f8524345b6d3
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.4.0
 	github.com/shirou/gopsutil/v3 v3.22.11
@@ -53,7 +54,6 @@ require (
 	github.com/projectdiscovery/networkpolicy v0.0.3 // indirect
 	github.com/projectdiscovery/retryabledns v1.0.17 // indirect
 	github.com/projectdiscovery/sliceutil v0.0.1 // indirect
-	github.com/projectdiscovery/utils v0.0.4-0.20221201124851-f8524345b6d3 // indirect
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect

internal/runner/options.go

@@ -32,6 +32,13 @@ func validateOptions(options *types.Options) error {
 	if (options.HeadlessOptionalArguments != nil || options.HeadlessNoSandbox) && !options.Headless {
 		return errors.New("headless mode (-hl) is required if -ho or -nos are set")
 	}
+	if options.StoreResponseDir != "" && !options.StoreResponse {
+		gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n")
+		options.StoreResponse = true
+	}
+	if options.Headless && options.StoreResponse {
+		return errors.New("Impossible to store responses in headless mode")
+	}
 	gologger.DefaultLogger.SetFormatter(formatter.NewCLI(options.NoColors))
 	return nil
 }

pkg/engine/hybrid/hybrid.go

@@ -254,7 +254,7 @@ func (c *Crawler) makeParseResponseCallback(queue *queue.VarietyQueue) func(nr n
 			return
 		}
 		if scopeValidated || c.options.Options.DisplayOutScope {
-			_ = c.options.OutputWriter.Write(result)
+			_ = c.options.OutputWriter.Write(result, nil)
 		}
 		if c.options.Options.OnResult != nil {
 			c.options.Options.OnResult(*result)

pkg/engine/standard/crawl.go

@@ -67,6 +67,9 @@ func (c *Crawler) makeRequest(ctx context.Context, request navigation.Request, r
 		return navigation.Response{}, nil
 	}
 
+	resp.Body = io.NopCloser(strings.NewReader(string(data)))
+	_ = c.options.OutputWriter.Write(nil, resp)
+
 	response.Body = data
 	response.Resp = resp
 	response.Reader, err = goquery.NewDocumentFromReader(bytes.NewReader(data))

pkg/engine/standard/standard.go

@@ -164,7 +164,7 @@ func (c *Crawler) makeParseResponseCallback(queue *queue.VarietyQueue) func(nr n
 			return
 		}
 		if scopeValidated || c.options.Options.DisplayOutScope {
-			_ = c.options.OutputWriter.Write(result)
+			_ = c.options.OutputWriter.Write(result, nil)
 		}
 		if c.options.Options.OnResult != nil {
 			c.options.Options.OnResult(*result)

pkg/output/output.go

@@ -1,7 +1,9 @@
 package output
 
 import (
+	"net/http"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"sync"
@@ -17,20 +19,24 @@ type Writer interface {
 	// Close closes the output writer interface
 	Close() error
 	// Write writes the event to file and/or screen.
-	Write(*Result) error
+	Write(*Result, *http.Response) error
 }
 
-var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
+var (
+	decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
+)
 
 // StandardWriter is an standard output writer structure
 type StandardWriter struct {
-	storeFields []string
-	fields      string
-	json        bool
-	verbose     bool
-	aurora      aurora.Aurora
-	outputFile  *fileWriter
-	outputMutex *sync.Mutex
+	storeFields      []string
+	fields           string
+	json             bool
+	verbose          bool
+	aurora           aurora.Aurora
+	outputFile       *fileWriter
+	outputMutex      *sync.Mutex
+	storeResponse    bool
+	storeResponseDir string
 }
 
 // Options contains the configuration options for output writer
@@ -61,16 +67,22 @@ type Result struct {
 	Attribute string `json:"attribute,omitempty"`
 }
 
-const storeFieldsDirectory = "katana_output"
+const (
+	storeFieldsDirectory = "katana_output"
+	indexFile            = "index.txt"
+	DefaultResponseDir   = "katana_responses"
+)
 
 // New returns a new output writer instance
-func New(colors, json, verbose bool, file, fields, storeFields string) (Writer, error) {
+func New(colors, json, verbose, storeResponse bool, file, fields, storeFields, storeResponseDir string) (Writer, error) {
 	writer := &StandardWriter{
-		fields:      fields,
-		json:        json,
-		verbose:     verbose,
-		aurora:      aurora.NewAurora(colors),
-		outputMutex: &sync.Mutex{},
+		fields:           fields,
+		json:             json,
+		verbose:          verbose,
+		aurora:           aurora.NewAurora(colors),
+		outputMutex:      &sync.Mutex{},
+		storeResponse:    storeResponse,
+		storeResponseDir: storeResponseDir,
 	}
 	// Perform validations for fields and store-fields
 	if fields != "" {
@@ -92,40 +104,71 @@ func New(colors, json, verbose bool, file, fields, storeFields string) (Writer,
 		}
 		writer.outputFile = output
 	}
+	if storeResponse {
+		writer.storeResponseDir = DefaultResponseDir
+		if storeResponseDir != DefaultResponseDir && storeResponseDir != "" {
+			writer.storeResponseDir = storeResponseDir
+		}
+		_ = os.RemoveAll(writer.storeResponseDir)
+		_ = os.MkdirAll(writer.storeResponseDir, os.ModePerm)
+		_, err := newFileOutputWriter(filepath.Join(writer.storeResponseDir, indexFile))
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create index file")
+		}
+	}
 	return writer, nil
 }
 
 // Write writes the event to file and/or screen.
-func (w *StandardWriter) Write(event *Result) error {
-	if len(w.storeFields) > 0 {
-		storeFields(event, w.storeFields)
-	}
-	var data []byte
-	var err error
+func (w *StandardWriter) Write(event *Result, resp *http.Response) error {
+	if event != nil {
+		if len(w.storeFields) > 0 {
+			storeFields(event, w.storeFields)
+		}
+		var data []byte
+		var err error
 
-	if w.json {
-		data, err = w.formatJSON(event)
-	} else {
-		data, err = w.formatScreen(event)
-	}
-	if err != nil {
-		return errors.Wrap(err, "could not format output")
-	}
-	if len(data) == 0 {
-		return nil
-	}
-	w.outputMutex.Lock()
-	defer w.outputMutex.Unlock()
+		if w.json {
+			data, err = w.formatJSON(event)
+		} else {
+			data, err = w.formatScreen(event)
+		}
+		if err != nil {
+			return errors.Wrap(err, "could not format output")
+		}
+		if len(data) == 0 {
+			return nil
+		}
+		w.outputMutex.Lock()
+		defer w.outputMutex.Unlock()
 
-	gologger.Silent().Msgf("%s", string(data))
-	if w.outputFile != nil {
-		if !w.json {
-			data = decolorizerRegex.ReplaceAll(data, []byte(""))
+		gologger.Silent().Msgf("%s", string(data))
+		if w.outputFile != nil {
+			if !w.json {
+				data = decolorizerRegex.ReplaceAll(data, []byte(""))
+			}
+			if writeErr := w.outputFile.Write(data); writeErr != nil {
+				return errors.Wrap(err, "could not write to output")
+			}
 		}
-		if writeErr := w.outputFile.Write(data); writeErr != nil {
-			return errors.Wrap(err, "could not write to output")
+	}
+
+	if w.storeResponse && resp != nil {
+		if file, err := getResponseFile(w.storeResponseDir, resp.Request.URL.String()); err == nil {
+			data, err := w.formatResponse(resp)
+			if err != nil {
+				return errors.Wrap(err, "could not store response")
+			}
+			if err := updateIndex(w.storeResponseDir, resp); err != nil {
+				return errors.Wrap(err, "could not store response")
+			}
+			if writeErr := file.Write(data); writeErr != nil {
+				return errors.Wrap(err, "could not store response")
+			}
+			file.Close()
 		}
 	}
+
 	return nil
 }
 

pkg/output/responses.go

@@ -0,0 +1,127 @@
+package output
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/hex"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	urlutil "github.com/projectdiscovery/utils/url"
+)
+
+func getResponseHash(URL string) string {
+	hash := sha1.Sum([]byte(URL))
+	return hex.EncodeToString(hash[:])
+}
+
+func (w *StandardWriter) formatResponse(resp *http.Response) ([]byte, error) {
+	builder := &bytes.Buffer{}
+
+	builder.WriteString(resp.Request.URL.String())
+	builder.WriteString("\n\n\n")
+
+	builder.WriteString(resp.Request.Method)
+	builder.WriteString(" ")
+	path := resp.Request.URL.Path
+	if resp.Request.URL.Fragment != "" {
+		path = path + "#" + resp.Request.URL.Fragment
+	}
+	builder.WriteString(path)
+	builder.WriteString(" ")
+	builder.WriteString(resp.Request.Proto)
+	builder.WriteString("\n")
+	builder.WriteString("Host: " + resp.Request.Host)
+	builder.WriteRune('\n')
+	for k, v := range resp.Request.Header {
+		builder.WriteString(k + ": " + strings.Join(v, "; ") + "\n")
+	}
+
+	if resp.Request.Body != nil {
+		bodyResp, _ := io.ReadAll(resp.Request.Body)
+		if string(bodyResp) != "" {
+			builder.WriteString("\n")
+			builder.WriteString(string(bodyResp))
+		}
+	}
+	builder.WriteString("\n\n")
+
+	builder.WriteString(resp.Proto)
+	builder.WriteString(" ")
+	builder.WriteString(resp.Status)
+	builder.WriteString("\n")
+	for k, v := range resp.Header {
+		builder.WriteString(k + ": " + strings.Join(v, "; ") + "\n")
+	}
+	builder.WriteString("\n")
+	body, _ := io.ReadAll(resp.Body)
+	builder.WriteString(string(body))
+
+	return builder.Bytes(), nil
+}
+
+func getResponseHost(URL string) (string, error) {
+	u, err := urlutil.ParseWithScheme(URL)
+	if err != nil {
+		return "", err
+	}
+
+	return u.Host, nil
+}
+
+func createHostDir(storeResponseFolder, domain string) string {
+	_ = os.MkdirAll(filepath.Join(storeResponseFolder, domain), os.ModePerm)
+	return filepath.Join(storeResponseFolder, domain)
+}
+
+func getResponseFile(storeResponseFolder, URL string) (*fileWriter, error) {
+	domain, err := getResponseHost(URL)
+	if err != nil {
+		return nil, err
+	}
+	output, err := newFileOutputWriter(getResponseFileName(storeResponseFolder, domain, URL))
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create output file")
+	}
+
+	return output, nil
+}
+
+func getResponseFileName(storeResponseFolder, domain, URL string) string {
+	folder := createHostDir(storeResponseFolder, domain)
+	file := getResponseHash(URL) + ".txt"
+	return filepath.Join(folder, file)
+}
+
+func updateIndex(storeResponseFolder string, resp *http.Response) error {
+	index, err := os.OpenFile(filepath.Join(storeResponseFolder, indexFile), os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+
+	defer index.Close()
+
+	builder := &bytes.Buffer{}
+
+	domain, err := getResponseHost(resp.Request.URL.String())
+	if err != nil {
+		return err
+	}
+
+	builder.WriteString(getResponseFileName(storeResponseFolder, domain, resp.Request.URL.String()))
+	builder.WriteRune(' ')
+	builder.WriteString(resp.Request.URL.String())
+	builder.WriteRune(' ')
+	builder.WriteString("(" + resp.Status + ")")
+	builder.WriteRune('\n')
+
+	if _, writeErr := index.Write(builder.Bytes()); writeErr != nil {
+		return errors.Wrap(err, "could not update index")
+	}
+
+	return nil
+}