ExecuteWithCallback: Callback does not fire if UseOutputWriter is configured

[Xenov-X]

Nuclei version:

github.com/projectdiscovery/nuclei/v3/lib:latest

Current Behavior:

When NewNucleiEngine is configured with UseOutputWriter the callback function within ExecuteWithCallback never fires.

Expected Behavior:

On detection of a result, the callback function fires, irrespective of OutputWriter configuration

Steps To Reproduce:

Example: Configured with OutputWriter - Callback function FoundResult does not execute

package main

import (
	"fmt"
	"strconv"
	"time"

	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
	"github.com/projectdiscovery/nuclei/v3/pkg/output"
	"github.com/projectdiscovery/nuclei/v3/pkg/types"
)

func main() {

	filters := nuclei.TemplateFilters{
		IDs: []string{"tls-version"},
	}

	targets := []string{"https://scanme.nmap.org", "https://scanme.sh", "https://honey.scanme.sh"}
	NucleiScan(filters, targets)
	fmt.Println("Done!")
}

func NucleiScan(filters nuclei.TemplateFilters, targets []string) {

	timestamp := time.Now().Unix()
	OutFile := "/tmp/nuclei_data_log_" + strconv.FormatInt(timestamp, 10) + ".json"
	ErrFile := "/tmp/nuclei_err_log_" + strconv.FormatInt(timestamp, 10) + ".json"

	outputWriter, err := output.NewStandardWriter(&types.Options{
		ResponseReadSize: 10 * 1024 * 1024,
		ResponseSaveSize: 10 * 1024 * 1024,
		JSONL:            true,
		Output:           OutFile,
		ErrorLogFile:     ErrFile,
	})

	ne, err := nuclei.NewNucleiEngine(
		nuclei.UseOutputWriter(outputWriter),
		nuclei.WithTemplateFilters(filters),
		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064, Interval: 1}), // optionally enable metrics server for better observability,
		nuclei.WithConcurrency(nuclei.Concurrency{
			TemplateConcurrency: 125,
			HostConcurrency:     25,
		}),
	)
	if err != nil {
		panic(err)
	}

	ne.LoadTargets(targets, true)

	err = ne.ExecuteWithCallback(FoundResult)
	if err != nil {
		panic(err)
	}
	defer ne.Close()

	fmt.Println(OutFile)
	fmt.Println(ErrFile)
}

func FoundResult(event *output.ResultEvent) {
	fmt.Println("BEEP FOUND RESULT")
}

Example: No OutputWriter - Callback function FoundResult executes on each result

package main

import (
	"fmt"
	"strconv"
	"time"

	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
	"github.com/projectdiscovery/nuclei/v3/pkg/output"
)

func main() {

	filters := nuclei.TemplateFilters{
		IDs: []string{"tls-version"},
	}

	targets := []string{"https://scanme.nmap.org", "https://scanme.sh", "https://honey.scanme.sh"}
	NucleiScan(filters, targets)
	fmt.Println("Done!")
}

func NucleiScan(filters nuclei.TemplateFilters, targets []string) {

	timestamp := time.Now().Unix()
	OutFile := "/tmp/nuclei_data_log_" + strconv.FormatInt(timestamp, 10) + ".json"
	ErrFile := "/tmp/nuclei_err_log_" + strconv.FormatInt(timestamp, 10) + ".json"

	// outputWriter, err := output.NewStandardWriter(&types.Options{
	// 	ResponseReadSize: 10 * 1024 * 1024,
	// 	ResponseSaveSize: 10 * 1024 * 1024,
	// 	JSONL:            true,
	// 	Output:           OutFile,
	// 	ErrorLogFile:     ErrFile,
	// })

	ne, err := nuclei.NewNucleiEngine(
		//nuclei.UseOutputWriter(outputWriter),
		nuclei.WithTemplateFilters(filters),
		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064, Interval: 1}), // optionally enable metrics server for better observability,
		nuclei.WithConcurrency(nuclei.Concurrency{
			TemplateConcurrency: 125,
			HostConcurrency:     25,
		}),
	)
	if err != nil {
		panic(err)
	}

	ne.LoadTargets(targets, true)

	err = ne.ExecuteWithCallback(FoundResult)
	if err != nil {
		panic(err)
	}
	defer ne.Close()

	fmt.Println(OutFile)
	fmt.Println(ErrFile)
}

func FoundResult(event *output.ResultEvent) {
	fmt.Println("BEEP FOUND RESULT")
}

Workaround

Potential workaround by not defining NewStandardWriter and reimplementing the conversion to JSONL to save to disk - However, this would then not log errors to the ErrorLogFile available in NewStandardWriter options

[tarunKoyalwar]

thanks for creating issue @Xenov-X , yeah ideally callback's present in ExecuteWithResult should be called regardless if output.Writer is implemented or not .
In recent PR we have introduced MultiWriter(https://github.com/projectdiscovery/nuclei/blob/25cc17318ef474f07786250b908cbe08ac025e21/pkg/output/multi_writer.go) to wrap multiple output.Writer's i will implement this functionality in that PR .

we are experimenting with new design without nested callbacks #4373 . this issue will be fixed with new executer function available in that PR and existing callback option will be removed