Added implementation for issue 447 (#460)

* Added implementation for issue #447 Two new command line options are introduced: -or, -omit-raw omit raw requests/responses from output -ob, -omit-body omit response body from output * misc update --------- Co-authored-by: sandeep <[email protected]>
projectdiscovery · Jun 4, 2023 · 4e4124c · 4e4124c
1 parent d40ecdb
commit 4e4124c
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -178,7 +178,9 @@ OUTPUT:
    -o, -output string                file to write output to
    -sr, -store-response              store http requests/responses
    -srd, -store-response-dir string  store http requests/responses to custom directory
-   -j, -json                         write output in JSONL(ines) format
+   -or, -omit-raw                    omit raw requests/responses from jsonl output
+   -ob, -omit-body                   omit response body from jsonl output
+   -j, -jsonl                        write output in jsonl format
    -nc, -no-color                    disable output content coloring (ANSI escape codes)
    -silent                           display output only
    -v, -verbose                      display verbose output

diff --git a/cmd/katana/main.go b/cmd/katana/main.go
@@ -136,7 +136,9 @@ pipelines offering both headless and non-headless crawling.`)
 		flagSet.StringVarP(&options.OutputFile, "output", "o", "", "file to write output to"),
 		flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"),
 		flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"),
-		flagSet.BoolVarP(&options.JSON, "json", "j", false, "write output in JSONL(ines) format"),
+		flagSet.BoolVarP(&options.OmitRaw, "omit-raw", "or", false, "omit raw requests/responses from jsonl output"),
+		flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body from jsonl output"),
+		flagSet.BoolVarP(&options.JSON, "jsonl", "j", false, "write output in jsonl format"),
 		flagSet.BoolVarP(&options.NoColors, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
 		flagSet.BoolVar(&options.Silent, "silent", false, "display output only"),
 		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "display verbose output"),

diff --git a/pkg/output/options.go b/pkg/output/options.go
@@ -8,6 +8,8 @@ type Options struct {
 	JSON             bool
 	Verbose          bool
 	StoreResponse    bool
+	OmitRaw          bool
+	OmitBody         bool
 	OutputFile       string
 	Fields           string
 	StoreFields      string

diff --git a/pkg/output/output.go b/pkg/output/output.go
@@ -44,6 +44,8 @@ type StandardWriter struct {
 	outputMutex      *sync.Mutex
 	storeResponse    bool
 	storeResponseDir string
+	omitRaw          bool
+	omitBody         bool
 	errorFile        *fileWriter
 	matchRegex       []*regexp.Regexp
 	filterRegex      []*regexp.Regexp
@@ -59,6 +61,8 @@ func New(options Options) (Writer, error) {
 		outputMutex:      &sync.Mutex{},
 		storeResponse:    options.StoreResponse,
 		storeResponseDir: options.StoreResponseDir,
+		omitRaw:          options.OmitRaw,
+		omitBody:         options.OmitBody,
 		matchRegex:       options.MatchRegex,
 		filterRegex:      options.FilterRegex,
 	}
@@ -137,6 +141,14 @@ func (w *StandardWriter) Write(result *Result) error {
 		var data []byte
 		var err error
 
+		if w.omitRaw {
+			result.Request.Raw = ""
+			result.Response.Raw = ""
+		}
+		if w.omitBody {
+			result.Response.Body = ""
+		}
+
 		if w.json {
 			data, err = w.formatJSON(result)
 		} else {

diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go
@@ -66,6 +66,8 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) {
 		Fields:           options.Fields,
 		StoreFields:      options.StoreFields,
 		StoreResponseDir: options.StoreResponseDir,
+		OmitRaw:          options.OmitRaw,
+		OmitBody:         options.OmitBody,
 		FieldConfig:      options.FieldConfig,
 		ErrorLogFile:     options.ErrorLogFile,
 		MatchRegex:       options.MatchRegex,

diff --git a/pkg/types/options.go b/pkg/types/options.go
@@ -98,6 +98,10 @@ type Options struct {
 	StoreResponse bool
 	// StoreResponseDir specifies if katana should use a custom directory to store http requests/responses
 	StoreResponseDir string
+	// OmitRaw omits raw requests/responses from the output
+	OmitRaw bool
+	// OmitBody omits the response body from the output
+	OmitBody bool
 	// ChromeDataDir : 	Specify the --user-data-dir to chrome binary to preserve sessions
 	ChromeDataDir string
 	// HeadlessNoIncognito specifies if chrome should be started without incognito mode