From 3064788d35a8ec8623eb3a54eca920ea289f1259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 19 Aug 2024 23:02:27 +0300 Subject: [PATCH 1/6] fix race condition --- .../http/httpclientpool/clientpool.go | 20 +++++++++++++++++++ pkg/protocols/http/request.go | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 2c244556ae..da7d5467e2 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -32,6 +32,7 @@ var ( forceMaxRedirects int normalClient *retryablehttp.Client clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] + rawHttpClientMu sync.Mutex ) // Init initializes the clientpool implementation @@ -102,6 +103,22 @@ type Configuration struct { ResponseHeaderTimeout time.Duration } +func (c *Configuration) Clone() *Configuration { + clone := *c + if c.Connection != nil { + cloneConnection := &ConnectionConfiguration{ + DisableKeepAlive: c.Connection.DisableKeepAlive, + } + if c.Connection.HasCookieJar() { + cookiejar := *c.Connection.GetCookieJar() + cloneConnection.SetCookieJar(&cookiejar) + } + clone.Connection = cloneConnection + } + + return &clone +} + // Hash returns the hash of the configuration to allow client pooling func (c *Configuration) Hash() string { builder := &strings.Builder{} @@ -131,6 +148,9 @@ func (c *Configuration) HasStandardOptions() bool { // GetRawHTTP returns the rawhttp request client func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client { + rawHttpClientMu.Lock() + defer rawHttpClientMu.Unlock() + if rawHttpClient == nil { rawHttpOptions := rawhttp.DefaultOptions if types.ProxyURL != "" { diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 98e1932477..3955325599 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -770,7 +770,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // check for cookie related configuration if input.CookieJar != nil { - connConfiguration := request.connConfiguration + connConfiguration := request.connConfiguration.Clone() connConfiguration.Connection.SetCookieJar(input.CookieJar) modifiedConfig = connConfiguration } @@ -778,7 +778,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ updatedTimeout, ok := generatedRequest.request.Context().Value(httpclientpool.WithCustomTimeout{}).(httpclientpool.WithCustomTimeout) if ok { if modifiedConfig == nil { - modifiedConfig = request.connConfiguration + connConfiguration := request.connConfiguration.Clone() + modifiedConfig = connConfiguration } modifiedConfig.ResponseHeaderTimeout = updatedTimeout.Timeout } From 50023428d0ae90f5ed09ebe4392364e06129ecb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 19 Aug 2024 23:09:07 +0300 Subject: [PATCH 2/6] fix lint --- lib/sdk.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/sdk.go b/lib/sdk.go index 04b084f244..daeb68c14b 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -178,7 +178,8 @@ func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []by if err != nil { return data, err } - buff := bytes.NewBuffer(signer.RemoveSignatureFromData(data)) + _, content := signer.ExtractSignatureAndContent(data) + buff := bytes.NewBuffer(content) buff.WriteString("\n" + signatureData) return buff.Bytes(), err } From 46782ff90c29d2bc04ca130763ffbf95e95c5b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 21 Aug 2024 11:26:17 +0300 Subject: [PATCH 3/6] use sync.Once --- pkg/protocols/http/httpclientpool/clientpool.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index da7d5467e2..5750d5aead 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -29,10 +29,10 @@ import ( var ( rawHttpClient *rawhttp.Client + rawHttpClientOnce sync.Once forceMaxRedirects int normalClient *retryablehttp.Client clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] - rawHttpClientMu sync.Mutex ) // Init initializes the clientpool implementation @@ -148,10 +148,7 @@ func (c *Configuration) HasStandardOptions() bool { // GetRawHTTP returns the rawhttp request client func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client { - rawHttpClientMu.Lock() - defer rawHttpClientMu.Unlock() - - if rawHttpClient == nil { + rawHttpClientOnce.Do(func() { rawHttpOptions := rawhttp.DefaultOptions if types.ProxyURL != "" { rawHttpOptions.Proxy = types.ProxyURL @@ -162,7 +159,7 @@ func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client { } rawHttpOptions.Timeout = options.Options.GetTimeouts().HttpTimeout rawHttpClient = rawhttp.NewClient(rawHttpOptions) - } + }) return rawHttpClient } From 35a0d673ad8e12b11e90e8e0090feb26ea042b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 21 Aug 2024 11:36:33 +0300 Subject: [PATCH 4/6] remove redundant code --- pkg/protocols/common/protocolstate/state.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index b7c7796fa8..b21a1be9e5 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -214,6 +214,5 @@ func Close() { Dialer.Close() Dialer = nil } - Dialer = nil StopActiveMemGuardian() } From d1f4c98cd76f82e556b4a9c9406c45bcec32ae7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 21 Aug 2024 15:03:41 +0300 Subject: [PATCH 5/6] Revert "remove redundant code" This reverts commit 35a0d673ad8e12b11e90e8e0090feb26ea042b46. --- pkg/protocols/common/protocolstate/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index b21a1be9e5..b7c7796fa8 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -214,5 +214,6 @@ func Close() { Dialer.Close() Dialer = nil } + Dialer = nil StopActiveMemGuardian() } From 5e102b782ba28386288460f7e6f1fec08ea514fa Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Wed, 21 Aug 2024 16:09:47 +0200 Subject: [PATCH 6/6] fixing race + nil crash --- pkg/protocols/common/protocolstate/state.go | 15 +++++++++++++-- pkg/protocols/http/httpclientpool/clientpool.go | 4 ++-- pkg/protocols/http/request.go | 6 +++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index b7c7796fa8..5c56fdf77e 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "net/url" + "sync" "github.com/go-sql-driver/mysql" "github.com/pkg/errors" @@ -19,9 +20,17 @@ import ( // Dialer is a shared fastdialer instance for host DNS resolution var ( - Dialer *fastdialer.Dialer + muDialer sync.RWMutex + Dialer *fastdialer.Dialer ) +func GetDialer() *fastdialer.Dialer { + muDialer.RLock() + defer muDialer.RUnlock() + + return Dialer +} + func ShouldInit() bool { return Dialer == nil } @@ -210,10 +219,12 @@ func interfaceAddresses(interfaceName string) ([]net.Addr, error) { // Close closes the global shared fastdialer func Close() { + muDialer.Lock() + defer muDialer.Unlock() + if Dialer != nil { Dialer.Close() Dialer = nil } - Dialer = nil StopActiveMemGuardian() } diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 5750d5aead..65879818f6 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -250,7 +250,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl transport := &http.Transport{ ForceAttemptHTTP2: options.ForceAttemptHTTP2, - DialContext: protocolstate.Dialer.Dial, + DialContext: protocolstate.GetDialer().Dial, DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { if options.TlsImpersonate { return protocolstate.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil) @@ -258,7 +258,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl if options.HasClientCertificates() || options.ForceAttemptHTTP2 { return protocolstate.Dialer.DialTLSWithConfig(ctx, network, addr, tlsConfig) } - return protocolstate.Dialer.DialTLS(ctx, network, addr) + return protocolstate.GetDialer().DialTLS(ctx, network, addr) }, MaxIdleConns: maxIdleConns, MaxIdleConnsPerHost: maxIdleConnsPerHost, diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 3955325599..5a1219b83f 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -942,7 +942,11 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if input.MetaInput.CustomIP != "" { outputEvent["ip"] = input.MetaInput.CustomIP } else { - outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname) + dialer := protocolstate.GetDialer() + if dialer != nil { + outputEvent["ip"] = dialer.GetDialedIP(hostname) + } + // try getting cname request.addCNameIfAvailable(hostname, outputEvent) }