From ab0f0ea92bc38ce0bab72d11e04d23b8641cc329 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:34:52 +0530 Subject: [PATCH] Printing bug fixes (#4238) * fix printing issue of unsigned workflows * add NUCLEI_LOG_ALL env variable * enable tlsx -dns and update print msg * misc update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- DEBUG.md | 1 + v2/go.mod | 16 ++++++------ v2/go.sum | 36 +++++++++++++-------------- v2/internal/runner/runner.go | 7 +++++- v2/pkg/catalog/config/nucleiconfig.go | 10 ++++++++ v2/pkg/catalog/disk/find.go | 3 +++ v2/pkg/catalog/loader/loader.go | 18 +++++++++++--- v2/pkg/parsers/parser.go | 12 ++++++--- v2/pkg/protocols/ssl/ssl.go | 1 + v2/pkg/templates/compile.go | 15 ++++++++--- v2/pkg/templates/log.go | 3 ++- v2/pkg/utils/stats/stats.go | 19 ++++++++++++++ 12 files changed, 103 insertions(+), 38 deletions(-) diff --git a/DEBUG.md b/DEBUG.md index 2fbf6b56d2..8d94b51f1b 100644 --- a/DEBUG.md +++ b/DEBUG.md @@ -39,6 +39,7 @@ Nuclei was built with some environment variables in mind to help with debugging. | `DEBUG=true` | Enables Printing Stack Traces for all errors | | `SHOW_DSL_ERRORS=true` | Enables Printing DSL Errors (that are hidden by default) | | `HIDE_TEMPLATE_SIG_WARNING=true` | Hides Template Signature Verification Warnings | +| `NUCLEI_LOG_ALL=true` | Log All Events that were skipped in verbose mode | diff --git a/v2/go.mod b/v2/go.mod index e1e9a0d6d9..36482c79cc 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -22,11 +22,11 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.19 github.com/projectdiscovery/fastdialer v0.0.37 - github.com/projectdiscovery/hmap v0.0.17 + github.com/projectdiscovery/hmap v0.0.20 github.com/projectdiscovery/interactsh v1.1.6 github.com/projectdiscovery/rawhttp v0.1.18 - github.com/projectdiscovery/retryabledns v1.0.35 - github.com/projectdiscovery/retryablehttp-go v1.0.25 + github.com/projectdiscovery/retryabledns v1.0.36 + github.com/projectdiscovery/retryablehttp-go v1.0.28 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -39,7 +39,7 @@ require ( github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.16.0 + golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.11.0 golang.org/x/text v0.13.0 gopkg.in/yaml.v2 v2.4.0 @@ -84,14 +84,14 @@ require ( github.com/projectdiscovery/gostruct v0.0.1 github.com/projectdiscovery/gozero v0.0.1 github.com/projectdiscovery/httpx v1.3.4 - github.com/projectdiscovery/mapcidr v1.1.2 + github.com/projectdiscovery/mapcidr v1.1.9 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.1.4 + github.com/projectdiscovery/tlsx v1.1.5 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.58-0.20231009161115-60268dca6e8f + github.com/projectdiscovery/utils v0.0.58-0.20231011230949-78c6b5a8ff28 github.com/projectdiscovery/wappalyzergo v0.0.107 github.com/redis/go-redis/v9 v9.1.0 github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02 @@ -178,7 +178,7 @@ require ( github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/projectdiscovery/asnmap v1.0.4 // indirect + github.com/projectdiscovery/asnmap v1.0.5 // indirect github.com/projectdiscovery/cdncheck v1.0.9 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/quic-go/quic-go v0.38.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index 280fa87549..c64db5345b 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -783,8 +783,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/praetorian-inc/fingerprintx v1.1.9 h1:zWbG/Fdan0s/dvXkeaHb/CdFTz/yEEzrAF4iCzok3r8= github.com/praetorian-inc/fingerprintx v1.1.9/go.mod h1:k6EJIHe/Da4DH5e4JuoZHe+qSGq/KPUmXGaK+xW74OI= -github.com/projectdiscovery/asnmap v1.0.4 h1:dmXrls7Y0Sdeb6cLlKGwdYX8h1K9q2iYOGXioD4U2AY= -github.com/projectdiscovery/asnmap v1.0.4/go.mod h1:iTLDyYsblEwYHcLiKZCRx8Et+xV7NlvgGLusANpgegc= +github.com/projectdiscovery/asnmap v1.0.5 h1:euWZuyLUkWhRMi8x3b4JXvKBDU9qRIcrh6p9BtYe4LM= +github.com/projectdiscovery/asnmap v1.0.5/go.mod h1:7YUiCMshTEKzPlV1kgVp7l6Z7ZrAvxH6ufYq5mNnurY= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg= @@ -807,14 +807,14 @@ github.com/projectdiscovery/gostruct v0.0.1 h1:1KvR6Pn4mDbQqoLEQzhRfHpbreLno2R9x github.com/projectdiscovery/gostruct v0.0.1/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.1 h1:f08ZnYlbDZV/TNGDvIXV9s/oB/sAI+HWaSbW4em4aKM= github.com/projectdiscovery/gozero v0.0.1/go.mod h1:/dHwbly+1lhOX9UreVure4lEe7K4hIHeu/c/wZGNTDo= -github.com/projectdiscovery/hmap v0.0.17 h1:QpVMjuLEwVkioAOhAFcn409ATB4rK3DkAEmqXghJcpI= -github.com/projectdiscovery/hmap v0.0.17/go.mod h1:d5kXPXHfQWZZzm5TFAZ88a+vjOjcMCRMnTj4XXkyhxk= +github.com/projectdiscovery/hmap v0.0.20 h1:2W0TLRWNx3ACZo9Q60JsuAntQ8OprGAoe/4Fi5QdUHI= +github.com/projectdiscovery/hmap v0.0.20/go.mod h1:XI17aljoGOQhzcLq5iw8GKtSi5SmDTh0r5vRzq6dsJ0= github.com/projectdiscovery/httpx v1.3.4 h1:1tCP7YRngCDi2a8PvvcYqmpR1H9X7Qgn89uazKL65eg= github.com/projectdiscovery/httpx v1.3.4/go.mod h1:5JlNJcEHPF9ByFFNEcaXEAs8yZYsUC6E9Q3VGfDpPeY= github.com/projectdiscovery/interactsh v1.1.6 h1:Jm09jXtV/3zPWIkf1+KpbPR6TnjXI/4SJQE2tMvVZQ8= github.com/projectdiscovery/interactsh v1.1.6/go.mod h1:mJm+f7qfxgtiCJ+FclViaw5oqNMCkuMu8l+MjxHDmY0= -github.com/projectdiscovery/mapcidr v1.1.2 h1:Mmq/nPqvVc7fjvH/kJVK0IBOny/LrJIxZ4tQsLPCrsA= -github.com/projectdiscovery/mapcidr v1.1.2/go.mod h1:Aoq0x/wJl6KDbtQ8OcPkjIDCqx2iEyx5ty1nzso8wXM= +github.com/projectdiscovery/mapcidr v1.1.9 h1:m8Dn6C3J+WpO4Myzi1IL5e4fvvTfTeRDc5miI5kL08I= +github.com/projectdiscovery/mapcidr v1.1.9/go.mod h1:TZGwVbVuRAMfHj5+9JT3zjL7WjTXVdAbmt/Ixt7M91M= github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8= github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc= github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= @@ -825,20 +825,20 @@ github.com/projectdiscovery/rawhttp v0.1.18 h1:wTs6CePrjcIz5/SrxkluOrCGOk3F9Ddt3 github.com/projectdiscovery/rawhttp v0.1.18/go.mod h1:nwTySMnfI7qFMQEC9PHdklXGWED8FDcEOnA8DGZqu/A= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.35 h1:lPX8f7exDaiNJc/4Rc44xQfFK9BpA8ZLtpQ+te2ymLU= -github.com/projectdiscovery/retryabledns v1.0.35/go.mod h1:V4nRoHJzK2UmlGgKMRduLBkgNNMXJXmJchB5Wui8s4c= -github.com/projectdiscovery/retryablehttp-go v1.0.25 h1:IhNSwWSnWYorp1Dcsh6whqy5Lm9QX738PXvWasTbVRo= -github.com/projectdiscovery/retryablehttp-go v1.0.25/go.mod h1:0oqaVWsBVMpZ1P9Dk1lkoNXFXD4B9MHKidbtD+VmEPU= +github.com/projectdiscovery/retryabledns v1.0.36 h1:AJKfbNEEZQW9X/26sf7AAWIsQ7xwfc110GqWO6lpr3w= +github.com/projectdiscovery/retryabledns v1.0.36/go.mod h1:E8tz9NUFW8zaZtjyw9KOb3oRUOKFwgEAcHuTme6QNNg= +github.com/projectdiscovery/retryablehttp-go v1.0.28 h1:vxohVncGHRce0Zfr+HzqRznXCkuolJKsEvYYYUyv/8Q= +github.com/projectdiscovery/retryablehttp-go v1.0.28/go.mod h1:GBhpuxhXFxgykmmafj0IK+7g2E0gS1WiitDiwAUAx0E= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.1.4 h1:jXRvichO/ZfhYERch1CbNS1PRbS2KgSBj7JoWslEpIw= -github.com/projectdiscovery/tlsx v1.1.4/go.mod h1:crzMlxOokVQDwGVm51JPZi1ZAgzxhNl1KVRmbff6pkI= +github.com/projectdiscovery/tlsx v1.1.5 h1:S8KV2ckcjW3hDBa/REmDdsZfHwYJ9eKoZ7rtgETkwkM= +github.com/projectdiscovery/tlsx v1.1.5/go.mod h1:0a0TdWb3fYeVpuPsJuf5AGtwZIKwkY0kxdO9lojU6S4= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.58-0.20231009161115-60268dca6e8f h1:5GMMQ6d7vqLMvjfibclgWgptj7vm9iDAz8xgRCYd+iI= -github.com/projectdiscovery/utils v0.0.58-0.20231009161115-60268dca6e8f/go.mod h1:5ub86JF91NnI3nTMIzEpL/pfsNb0jtHznzKi9hv03X4= +github.com/projectdiscovery/utils v0.0.58-0.20231011230949-78c6b5a8ff28 h1:qNPijCB3yli8veOf5huqs6NT7RMXwFdtf2xKpiKurK8= +github.com/projectdiscovery/utils v0.0.58-0.20231011230949-78c6b5a8ff28/go.mod h1:rsR5Kzjrb+/Yp7JSnEblLk4LfU4zH5Z7wQn8RzaGSdY= github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8= github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -891,8 +891,8 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02 h1:Nk74A6E84pynxLN74hIrQ7Q3cS0/0L5I7coOLNSFAMs= github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02/go.mod h1:OGEfzIZJs5m/VgAb1BvWR8fH17RTQWx84HTB1koGf9s= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -1226,8 +1226,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= -golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 893bf9ab26..566327a68c 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -699,7 +699,12 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { stats.Display(parsers.SyntaxWarningStats) stats.Display(parsers.SyntaxErrorStats) stats.Display(parsers.RuntimeWarningsStats) - stats.Display(parsers.UnsignedWarning) + if r.options.Verbose { + // only print these stats in verbose mode + stats.DisplayAsWarning(parsers.HeadlessFlagWarningStats) + stats.DisplayAsWarning(parsers.TemplatesExecutedStats) + } + stats.DisplayAsWarning(parsers.UnsignedWarning) cfg := config.DefaultConfig diff --git a/v2/pkg/catalog/config/nucleiconfig.go b/v2/pkg/catalog/config/nucleiconfig.go index 788abe9ee0..fb4be387e8 100644 --- a/v2/pkg/catalog/config/nucleiconfig.go +++ b/v2/pkg/catalog/config/nucleiconfig.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/utils/env" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" @@ -33,6 +34,7 @@ type Config struct { TemplateVersion string `json:"nuclei-templates-version,omitempty"` NucleiIgnoreHash string `json:"nuclei-ignore-hash,omitempty"` + LogAllEvents bool `json:"-"` // when enabled logs all events (more than verbose) // LatestXXX are not meant to be used directly and is used as // local cache of nuclei version check endpoint @@ -324,6 +326,14 @@ func init() { homeDir: folderutil.HomeDirOrDefault(""), configDir: ConfigDir, } + + // when enabled will log events in more verbosity than -v or -debug + // ex: N templates are excluded + // with this switch enabled nuclei will print details of above N templates + if value := env.GetEnvOrDefault("NUCLEI_LOG_ALL", false); value { + DefaultConfig.LogAllEvents = true + } + // try to read config from file if err := DefaultConfig.ReadTemplatesConfig(); err != nil { gologger.Verbose().Msgf("config file not found, creating new config file at %s", DefaultConfig.getTemplatesConfigFilePath()) diff --git a/v2/pkg/catalog/disk/find.go b/v2/pkg/catalog/disk/find.go index 98b8cf5be5..85abe32874 100644 --- a/v2/pkg/catalog/disk/find.go +++ b/v2/pkg/catalog/disk/find.go @@ -81,6 +81,9 @@ func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) { // try to handle deprecated template paths absPath := BackwardsCompatiblePaths(c.templatesDirectory, target) if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target { + if config.DefaultConfig.LogAllEvents { + gologger.DefaultLogger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath) + } deprecatedPathsCounter++ } diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 67e6589078..e1967fa0aa 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -8,9 +8,11 @@ import ( "sort" "strings" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" cfg "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" @@ -395,12 +397,15 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ } else if parsed != nil { if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless { // donot include headless template in final list if headless flag is not set - gologger.Warning().Msgf("Headless flag is required for headless template '%s'\n", templatePath) + stats.Increment(parsers.HeadlessFlagWarningStats) + if config.DefaultConfig.LogAllEvents { + gologger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) + } } else if len(parsed.RequestsCode) > 0 && !parsed.Verified && len(parsed.Workflows) == 0 { // donot include unverified 'Code' protocol custom template in final list stats.Increment(parsers.UnsignedWarning) - if store.config.ExecutorOptions.Options.VerboseVerbose { // only shown in -vv - gologger.Verbose().Msgf("Skipping Unverified custom template %s", templatePath) + if config.DefaultConfig.LogAllEvents { + gologger.Print().Msgf("[%v] Tampered/Unsigned template at %v.\n", aurora.Yellow("WRN").String(), templatePath) } } else { loadedTemplates = append(loadedTemplates, parsed) @@ -408,6 +413,13 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ } } if err != nil { + if strings.Contains(err.Error(), filter.ErrExcluded.Error()) { + stats.Increment(parsers.TemplatesExecutedStats) + if config.DefaultConfig.LogAllEvents { + gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error()) + } + continue + } gologger.Warning().Msg(err.Error()) } } diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index f321213eea..0056bde50b 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -139,10 +139,12 @@ var ( ) const ( - SyntaxWarningStats = "syntax-warnings" - SyntaxErrorStats = "syntax-errors" - RuntimeWarningsStats = "runtime-warnings" - UnsignedWarning = "unsigned-warnings" + SyntaxWarningStats = "syntax-warnings" + SyntaxErrorStats = "syntax-errors" + RuntimeWarningsStats = "runtime-warnings" + UnsignedWarning = "unsigned-warnings" + HeadlessFlagWarningStats = "headless-flag-missing-warnings" + TemplatesExecutedStats = "templates-executed" ) func init() { @@ -152,6 +154,8 @@ func init() { stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") stats.NewEntry(RuntimeWarningsStats, "Found %d templates with runtime error (use -validate flag for further examination)") stats.NewEntry(UnsignedWarning, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)") + stats.NewEntry(HeadlessFlagWarningStats, "Excluded %d headless templates (disabled as default), use -headless option to run headless templates.") + stats.NewEntry(TemplatesExecutedStats, "Excluded %d templates with known weak matchers / tags excluded from default run using .nuclei-ignore") } // ParseTemplate parses a template and returns a *templates.Template structure diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 7a9f83b171..a32e849195 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -132,6 +132,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { Fastdialer: client, ClientHello: true, ServerHello: true, + DisplayDns: true, } tlsxService, err := tlsx.New(tlsxOptions) diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index e67228a07e..3fd4aa412a 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -8,9 +8,11 @@ import ( "sync" "sync/atomic" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" "gopkg.in/yaml.v2" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/js/compiler" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -42,7 +44,7 @@ func init() { for _, verifier := range signer.DefaultTemplateVerifiers { SignatureStats[verifier.Identifier()] = &atomic.Uint64{} } - SignatureStats["unsigned"] = &atomic.Uint64{} + SignatureStats[Unsigned] = &atomic.Uint64{} } // Parse parses a yaml request template file @@ -256,7 +258,10 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option if err != nil { return nil, err } - if !template.Verified { + if !template.Verified && len(template.Workflows) == 0 { + if config.DefaultConfig.LogAllEvents { + gologger.DefaultLogger.Print().Msgf("[%v] Template %s is not signed or tampered\n", aurora.Yellow("WRN").String(), template.ID) + } SignatureStats[Unsigned].Add(1) } return template, nil @@ -273,7 +278,11 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option return nil, err } isVerified := template.Verified - if !template.Verified { + if !template.Verified && len(template.Workflows) == 0 { + // workflows are not signed by default + if config.DefaultConfig.LogAllEvents { + gologger.DefaultLogger.Print().Msgf("[%v] Template %s is not signed or tampered\n", aurora.Yellow("WRN").String(), template.ID) + } SignatureStats[Unsigned].Add(1) } diff --git a/v2/pkg/templates/log.go b/v2/pkg/templates/log.go index 7d82591593..d705a86ec2 100644 --- a/v2/pkg/templates/log.go +++ b/v2/pkg/templates/log.go @@ -6,6 +6,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -57,7 +58,7 @@ func PrintDeprecatedProtocolNameMsgIfApplicable(isSilent bool, verbose bool) { if count > 0 && !isSilent { gologger.Print().Msgf("[%v] Found %v templates loaded with deprecated protocol syntax, update before v3 for continued support.\n", aurora.Yellow("WRN").String(), count) } - if verbose { + if config.DefaultConfig.LogAllEvents { _ = deprecatedProtocolNameTemplates.Iterate(func(k string, v bool) error { gologger.Print().Msgf(" - %s\n", k) return nil diff --git a/v2/pkg/utils/stats/stats.go b/v2/pkg/utils/stats/stats.go index 9b2b7f2f0f..a001d98980 100644 --- a/v2/pkg/utils/stats/stats.go +++ b/v2/pkg/utils/stats/stats.go @@ -40,6 +40,10 @@ func Display(name string) { Default.Display(name) } +func DisplayAsWarning(name string) { + Default.DisplayAsWarning(name) +} + // GetValue returns the value for a set variable func GetValue(name string) int64 { return Default.GetValue(name) @@ -85,6 +89,21 @@ func (s *Storage) Display(name string) { gologger.Error().Label("WRN").Msgf(data.description, dataValue) } +func (s *Storage) DisplayAsWarning(name string) { + s.mutex.RLock() + data, ok := s.data[name] + s.mutex.RUnlock() + if !ok { + return + } + + dataValue := atomic.LoadInt64(&data.value) + if dataValue == 0 { + return // don't show for nil stats + } + gologger.Warning().Label("WRN").Msgf(data.description, dataValue) +} + // GetValue returns the value for a set variable func (s *Storage) GetValue(name string) int64 { s.mutex.RLock()