use std config directories (projectdiscovery#4228)

* use std config directories

* use NUCLEI_CONFIG_DIR env for config dir

* add template sources option in sdk

* add cloud.projectdiscovery.io to trusted domain

* fix failing test

integration_tests/protocols/code/py-env-var.yaml

@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input baz"
-# digest: 4a0a004730450221009b73a0d173beca99aff61fcfe366d929760411f0b9facfe5502a0f2e677342fa02201327b024e52a89891bb90ecdd2f7521370fe40ac89b3f93e9e5f5072f0192de7
+# digest: 4a0a004730450221008132561626bc3ef36822cb33518b731d96056a898165966920163c60088aca8a022030f7ca08e18d24f031d511fdb89dd8fd1e83a681bdc67dd062bd47039132f911

integration_tests/protocols/code/py-file.yaml

@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 490a00463044022037b485132a0ff1f52bde1ce71908fd692d93d9cda15d5b70275c7fc89a0aa257022043f36ca1c7062d4d74dde377e5a0065c1507bfcad2a8389772eebfa99fd16af2
+# digest: 4b0a00483046022100dd46d2316163dd5f073bdf84d038958114c6a00914e737f0daa12827994eaa7a022100c8a6bbdedd0c6dc315e6c61f98dc3add9121b8ab340d333401dc58962284fc9a

integration_tests/protocols/code/py-interactsh.yaml

@@ -26,4 +26,4 @@ code:
         part: interactsh_protocol
         words:
           - "http"
-# digest: 4a0a004730450221008d0e317bd37ab7dfdbaf3d26ed2072521f0ea00a61d45d0e6d592886faeac1840220410070cd29ffa05113b387c5afcd13fd65dab2b67f21583a1aeddb352ab5aa88
+# digest: 4b0a00483046022100b5084304ca60c6c7d89e0a5f23ed82a26f59cb2c8ccb3a90535792d4d77cd80d022100eea2c5a3164f83a9b0bcf60e637e7a710358cef7a96c0fc016185cce3f23d6a4

integration_tests/protocols/code/py-snippet.yaml

@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4b0a00483046022100b7fef4d710fa7e3b8916dc990a381551deb42c558d92a043358325f3994078f5022100db47c9f752185ad42af1a5268788567f88fcd8e41053674fdb637f01f02bbbaa
+# digest: 4a0a00473045022030a0b1fddd6c5ac0c5d217eef447c7ea54e69a044eb12376d06d5c5aa8171f67022100bb150ff1bf3b3ee0dead7ffec6cc038c860f0f660df1fc5e61eed871a439d6f4

v2/cmd/integration-test/custom-dir.go

@@ -19,7 +19,7 @@ func (h *customConfigDirTest) Execute(filePath string) error {
 		return err
 	}
 	defer os.RemoveAll(customTempDirectory)
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "8x8exch02.8x8.com", debug, "-config-directory", customTempDirectory)
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, []string{"NUCLEI_CONFIG_DIR=" + customTempDirectory}, "-t", filePath, "-u", "8x8exch02.8x8.com")
 	if err != nil {
 		return err
 	}

v2/cmd/integration-test/loader.go

@@ -55,7 +55,7 @@ func (h *remoteTemplateList) Execute(templateList string) error {
 	}
 	defer os.Remove("test-config.yaml")
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml")
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@@ -78,7 +78,7 @@ func (h *excludedTemplate) Execute(templateList string) error {
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-t", templateList, "-include-templates", templateList)
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-t", templateList, "-include-templates", templateList)
 	if err != nil {
 		return err
 	}
@@ -112,7 +112,7 @@ func (h *remoteTemplateListNotAllowed) Execute(templateList string) error {
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/template_list")
 	if err == nil {
 		return fmt.Errorf("expected error for not allowed remote template list url")
 	}
@@ -154,7 +154,7 @@ func (h *remoteWorkflowList) Execute(workflowList string) error {
 	}
 	defer os.Remove("test-config.yaml")
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml")
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@@ -170,7 +170,7 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/404")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}
@@ -186,7 +186,7 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/404")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}

v2/cmd/nuclei/main.go

@@ -262,7 +262,6 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.Interface, "interface", "i", "", "network interface to use for network scan"),
 		flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"),
 		flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"),
-		flagSet.StringVar(&options.CustomConfigDir, "config-directory", "", "override the default config path ($home/.config)"),
 		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 10*1024*1024, "max response size to read in bytes"),
 		flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"),
 		flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
@@ -388,7 +387,11 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&options.NoTables, "no-tables", false, "do not display pretty-printed tables"),
 		flagSet.IntVar(&options.OutputLimit, "limit", 100, "limit the number of output to display"),
 	)
-
+	// nuclei has multiple migrations
+	// ex: resume.cfg moved to platform standard cache dir from config dir
+	// ex: config.yaml moved to platform standard config dir from linux specific config dir
+	// and hence it will be attempted in config package during init
+	goflags.DisableAutoConfigMigration = true
 	_ = flagSet.Parse()
 
 	gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug)
@@ -401,8 +404,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 	if options.LeaveDefaultPorts {
 		http.LeaveDefaultPorts = true
 	}
-	if options.CustomConfigDir != "" {
-		config.DefaultConfig.SetConfigDir(options.CustomConfigDir)
+	if customConfigDir := os.Getenv(config.NucleiConfigDirEnv); customConfigDir != "" {
+		config.DefaultConfig.SetConfigDir(customConfigDir)
 		readFlagsConfig(flagSet)
 	}
 	if cfgFile != "" {
@@ -424,7 +427,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 // cleanupOldResumeFiles cleans up resume files older than 10 days.
 func cleanupOldResumeFiles() {
-	root := config.DefaultConfig.GetConfigDir()
+	root := config.DefaultConfig.GetCacheDir()
 	filter := fileutil.FileFilters{
 		OlderThan: 24 * time.Hour * 10, // cleanup on the 10th day
 		Prefix:    "resume-",
@@ -469,6 +472,8 @@ func disableUpdatesCallback() {
 // printVersion prints the nuclei version and exits.
 func printVersion() {
 	gologger.Info().Msgf("Nuclei Engine Version: %s", config.Version)
+	gologger.Info().Msgf("Nuclei Config Directory: %s", config.DefaultConfig.GetConfigDir())
+	gologger.Info().Msgf("Nuclei Cache Directory: %s", config.DefaultConfig.GetCacheDir()) // cache dir contains resume files
 	os.Exit(0)
 }
 

v2/go.mod

@@ -39,7 +39,7 @@ require (
 	github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
 	github.com/xanzy/go-gitlab v0.84.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/net v0.15.0
+	golang.org/x/net v0.16.0
 	golang.org/x/oauth2 v0.11.0
 	golang.org/x/text v0.13.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -79,7 +79,7 @@ require (
 	github.com/praetorian-inc/fingerprintx v1.1.9
 	github.com/projectdiscovery/dsl v0.0.22-0.20230911020052-7ab80c9abba8
 	github.com/projectdiscovery/fasttemplate v0.0.2
-	github.com/projectdiscovery/goflags v0.1.20
+	github.com/projectdiscovery/goflags v0.1.24-0.20231009194911-044c556377a1
 	github.com/projectdiscovery/gologger v1.1.11
 	github.com/projectdiscovery/gostruct v0.0.1
 	github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6
@@ -91,7 +91,7 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.4
 	github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1
-	github.com/projectdiscovery/utils v0.0.54
+	github.com/projectdiscovery/utils v0.0.58-0.20231009161115-60268dca6e8f
 	github.com/projectdiscovery/wappalyzergo v0.0.107
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02
@@ -128,6 +128,7 @@ require (
 	github.com/cloudflare/cfssl v1.6.4 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/containerd/continuity v0.4.2 // indirect
+	github.com/denisbrodbeck/machineid v1.0.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/docker/cli v24.0.5+incompatible // indirect
@@ -276,12 +277,12 @@ require (
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	goftp.io/server/v2 v2.0.1 // indirect
-	golang.org/x/crypto v0.13.0
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/crypto v0.14.0
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect