diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 559d7879cc..252203d0d5 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -12,7 +12,7 @@ jobs: name: Test Builds strategy: matrix: - go-version: [1.20.x] + go-version: [1.21.x] os: [ubuntu-latest, windows-latest, macOS-latest] runs-on: ${{ matrix.os }} @@ -22,6 +22,11 @@ jobs: with: go-version: ${{ matrix.go-version }} + - name: Set up Python # required for running python code in py-snippet.yaml integration test + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 79b154af8b..a6bafd84fd 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.20.x + go-version: 1.21.x - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 9c93c08847..24ba452e40 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.20.x + go-version: 1.21.x - name: Checkout code uses: actions/checkout@v3 diff --git a/.github/workflows/performance-test.yaml b/.github/workflows/performance-test.yaml index 09c3f2765a..2e59772307 100644 --- a/.github/workflows/performance-test.yaml +++ b/.github/workflows/performance-test.yaml @@ -11,7 +11,7 @@ jobs: name: Test Performance strategy: matrix: - go-version: [1.20.x] + go-version: [1.21.x] os: [ubuntu-latest, macOS-latest] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml index c7d0f38638..2fca548ace 100644 --- a/.github/workflows/publish-docs.yaml +++ b/.github/workflows/publish-docs.yaml @@ -18,7 +18,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v4 with: - go-version: 1.20.x + go-version: 1.21.x - name: Generate YAML Syntax Documentation id: generate-docs diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 4a40bf9edc..314d7b06ed 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: 1.20.x + go-version: 1.21.x - uses: goreleaser/goreleaser-action@v4 with: diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 26917f0efe..df645d6e8f 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.20.x + go-version: 1.21.x - name: release test uses: goreleaser/goreleaser-action@v4 diff --git a/.github/workflows/template-validate.yml b/.github/workflows/template-validate.yml index 5ebb74280b..566e5ed337 100644 --- a/.github/workflows/template-validate.yml +++ b/.github/workflows/template-validate.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.20.x + go-version: 1.21.x - name: Template Validation run: | diff --git a/Dockerfile b/Dockerfile index 23a40a61e7..1c6090822a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.20.6-alpine AS build-env +FROM golang:1.21-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app diff --git a/README.md b/README.md index da7601a44c..49f6dc0e83 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp # Install Nuclei -Nuclei requires **go1.20** to install successfully. Run the following command to install the latest version - +Nuclei requires **go1.21** to install successfully. Run the following command to install the latest version - ```sh go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest diff --git a/README_CN.md b/README_CN.md index d39738c06f..79c4aaa238 100644 --- a/README_CN.md +++ b/README_CN.md @@ -52,7 +52,7 @@ Nuclei使用零误报的定制模板向目标发送请求，同时可以对主 # 安装Nuclei -Nuclei需要**go1.20**才能安装成功。执行下列命令安装最新版本的Nuclei +Nuclei需要**go1**才能安装成功。执行下列命令安装最新版本的Nuclei ```sh go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest diff --git a/README_ID.md b/README_ID.md index 04fcd3e8c5..55bc95849e 100644 --- a/README_ID.md +++ b/README_ID.md @@ -52,7 +52,7 @@ Kami memiliki [repositori khusus](https://github.com/projectdiscovery/nuclei-tem # Instalasi Nuclei -Nuclei membutuhkan **go1.20** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru - +Nuclei membutuhkan **go1.21** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru - ```sh go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest diff --git a/README_KR.md b/README_KR.md index 9608083d6c..49e874ee94 100644 --- a/README_KR.md +++ b/README_KR.md @@ -50,7 +50,7 @@ Nuclei는 템플릿을 기반으로 대상 간에 요청을 보내기 위해 사 # 설치 -Nuclei를 성공적으로 설치하기 위해서 **go1.20**가 필요합니다. 다음 명령을 실행하여 최신 버전을 설치합니다. +Nuclei를 성공적으로 설치하기 위해서 **go1.21**가 필요합니다. 다음 명령을 실행하여 최신 버전을 설치합니다. ```sh go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest diff --git a/docs/mint.json b/docs/mint.json index 22990e2bc9..00c6505c95 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -72,6 +72,7 @@ "template-guide/dns", "template-guide/file", "template-guide/javascript", + "template-guide/code", { "group": "Operators", "pages": [ diff --git a/docs/template-guide/code.mdx b/docs/template-guide/code.mdx new file mode 100644 index 0000000000..01ed003b56 --- /dev/null +++ b/docs/template-guide/code.mdx @@ -0,0 +1,342 @@ +--- +title: "Code" +--- + +## Code Requests (beta) + +Nuclei enables the execution of external code on the host operating system. This feature allows security researchers, pentesters, and developers to extend the capabilities of Nuclei and perform complex actions beyond the scope of regular supported protocol-based testing. + +By leveraging this capability, Nuclei can interact with the underlying operating system and execute custom scripts or commands, opening up a wide range of possibilities. It enables users to perform tasks such as system-level configurations, file operations, network interactions, and more. This level of control and flexibility empowers users to tailor their security testing workflows according to their specific requirements. + +However, it's important to exercise caution while utilizing this feature, as executing external code on the host operating system carries inherent risks. It is crucial to ensure that the executed code is secure, thoroughly tested, and does not pose any unintended consequences or security risks to the target system. + +## Template Signing (beta) + +Template signing via the private-public key mechanism is a crucial aspect of ensuring the integrity and authenticity of templates. This mechanism involves the use of asymmetric cryptography, specifically ECDSA algorithm, to create a secure and verifiable signature. + +In this process, a template author generates a private key that remains confidential and securely stored. The corresponding public key is then shared with the template consumers. When a template is created or modified, the author signs it using their private key, generating a unique signature that is attached to the template. + +Template consumers can verify the authenticity and integrity of a signed template by using the author's public key. By applying the appropriate cryptographic algorithm (ECDSA), they can validate the signature and ensure that the template has not been tampered with since it was signed. This provides a level of trust, as any modifications or unauthorized changes to the template would result in a failed verification process. + +By employing the private-public key mechanism, template signing adds an additional layer of security and trust to the template ecosystem. It helps establish the identity of the template author and ensures that the templates used in various systems are genuine and have not been altered maliciously. + +### What does signing a template mean ? + +Template signing is a mechanism to ensure the integrity and authenticity of templates. The primary goal is to provide template writers/consumers a way to trust crowdsource/custom templates ensuring that they are not tampered + +All Official nuclei templates include a digital signature in them and are verified by nuclei while loading templates using ProjectDiscovery's public key shipped with nuclei binary itself. + +Individuals / Organizations running nuclei in their work environment can generate their own key-pair with `nuclei` and sign their custom templates with their private key, thus ensuring that only authorized templates are being used in their environment. + +This also allows entities to fully utilize the power of new protocols like `code` and `javascript` without worrying about malicious custom templates being used in their environment. + +**Points to note** + +- Template signing is optional for all protocols except `code`. + +- Code File References (ex: `source: protocols/code/pyfile.py`) are allowed and content of these files is included in the template digest + +- Payload File References (ex: `payloads: protocols/http/params.txt`) are not included in the template digest as it is treated as a payload/helper and not actual code that is being executed + +- Template Signing is deterministic while both signing and verifying a template i.e if a code file is referenced in a template that is present outside of templates directory with `-lfa` flag then verification will fail if same template is used without `-lfa` flag. (Note this only applies to `-lfa` i.e local file access flag only) + +## How to sign custom templates + +Simplest and recommended way to generate key-pair and signing/verfifying templates is to use `nuclei` itself. + +#### When Signing a template if key-pair does not exist then nuclei will prompt user to generate a new key-pair with options + +```console +$ ./nuclei -t my-template.yaml -sign -v +[INF] Generating new key-pair for signing templates +[*] Enter User/Organization Name (exit to abort) : projectdiscovery/nuclei-templates +[*] Enter passphrase (exit to abort): +[*] Enter same passphrase again: +[INF] Successfully generated new key-pair for signing templates + +``` + +> Note: Passphrase is optional and can be left blank when used private key is encrypted with passphrase using PEMCipherAES256 Algo + +#### Signing a template with existing key-pair + +```console +$ ./nuclei -t ~/nuclei-templates/http -sign -v +[INF] All templates signatures were elaborated success=6464 failed=0 +``` + +### Template Digest + +When a template is signed, a digest is generated and added to the template. This digest is a hash of the template content and is used to verify the integrity of the template. If the template is modified after signing, the digest will change, and the signature verification will fail which happens during template loading. + +```yaml +# digest: 4a0a00473045022100eb01da6b97893e7868c584f330a0cd52df9bddac005860bb8595ba5b8aed58c9022050043feac68d69045cf320cba9298a2eb2e792ea4720d045d01e803de1943e7d:4a3eb6b4988d95847d4203be25ed1d46 +``` + +It is in the format of `signature:fragment` where signature is digital signature of template which is used to verify integrity of template +and fragment is a metadata generated by md5 hashing public key to disable re-signing of code templates not written by you. + +fragment is meant to act like a speed bump to prevent mass-signing of code protocol templates to prevent any unintended misuse. + +### Where are keys stored ? + +key-pair generated by nuclei are stored in 2 files in `$config/nuclei/keys` directory where `$config` is system specific config directory + +``` +$ la ~/.config/nuclei/keys +total 16 +-rw------- 1 tarun staff 251B Oct 4 21:45 nuclei-user-private-key.pem # encrypted private key with passphrase +-rw------- 1 tarun staff 572B Oct 4 21:45 nuclei-user.crt # self signed certificate which includes public key and identifier (i.e user/org name) +``` + +### Sharing and Using Public Key + +Public key is stored in $config/nuclei/keys/nuclei-user.crt and can be shared with other users / organizations to verify templates signed by you. + +#### Using Public Key + +- A simple way to use public key is to copy it to $config/nuclei/keys directory of other user's machine + +- Another way is to use environment variable `NUCLEI_USER_CERTIFICATE=xxx` to specify path of public key or content of public key directly + +```console +$ export NUCLEI_USER_CERTIFICATE=path/to/nuclei-user.crt +``` + +or + +```console +$ export NUCLEI_USER_CERTIFICATE=$(cat path/to/nuclei-user.crt) +``` + +#### Verifying Templates + +Everytime `nuclei` is run, it loads user certificate (aka public key) from above locations and uses it to verify templates. + +`nuclei` also prints identifier of public key being used and warns user of unsigned custom templates + +``` +[INF] Executing 6219 signed templates from projectdiscovery/nuclei-templates +[WRN] Executing 687 unsigned templates. Use with caution. +``` + +### Managing Private Key + +Private key is stored in $config/nuclei/keys/nuclei-user-private-key.pem and is encrypted with passphrase if provided while generating key-pair. + +It is not used/loaded by default by nuclei and is only used on demand i.e when signing templates using `-sign` flag + +Some Users might want to store / backup or move private key to different location or machine and `nuclei` doesn't enforce any restrictions on that. + +#### Using Private Key + +- A simple way to use private key is to copy it to $config/nuclei/keys directory of other user's machine + +- Another way is to use environment variable `NUCLEI_USER_PRIVATE_KEY=xxx` to specify path of private key or content of private key directly + +```console +$ export NUCLEI_USER_PRIVATE_KEY=path/to/nuclei-user-private-key.pem +``` + +or + +```console +$ export NUCLEI_USER_PRIVATE_KEY=$(cat path/to/nuclei-user-private-key.pem) +``` + +> Note: You are responsible for securing and managing private key and nuclei has no accountability for any loss of private key + + +## Code + +In the context of template creation, a code block is used to indicate the start of the requests for the template. This block marks the beginning of the code-related instructions. + +```yaml +# Start the requests for the template right here +code: +``` + +To execute the code, a list of engines is specified, which are searched sequentially until a valid one is found on the system. The engine names must match the corresponding binary names on the system. + +```yaml +- engine: + - py + - python3 +``` + +The code to be executed can be provided either as an external file or as a code snippet directly within the template. + +For an external file: + +```yaml +source: protocols/code/pyfile.py +``` + +For a code snippet: +```yaml +source: | + import sys + print("hello from " + sys.stdin.read()) +``` + +The target is passed to the template via stdin, and the output of the executed code is available for further processing in matchers and extractors. In the case of the Code protocol, the body part represents all data printed to stdout during the execution of the code. + +#### Matchers / Extractor Parts + +Valid `part` values supported by **Code** protocol for Matchers / Extractor are - + +| Value | Description | +| -------- | ---------------------------------------------------- | +| response | execution output (trailing whitespaces are filtered) | +| stderr | Raw Stderr Output(if any) | + + +#### **Example Code Template** + +The provided example demonstrates the execution of a Python script within the template. The specified engines are searched in the given order, and the code snippet is executed accordingly. Additionally, a matcher is included to check if the code's stdout contains the phrase "hello from input." (input must be passed as target with nuclei) + +```yaml +id: py-code-snippet + +info: + name: py-code-snippet + author: pdteam + severity: info + tags: code + description: | + py-code-snippet + +code: + - engine: + - py + - python3 + source: | + import sys + print("hello from " + sys.stdin.read()) + + matchers: + - type: word + words: + - "hello from input" +# digest: 4a0a00473045022067a69eb337ffa56d1c8e2cc57b7f74a5eb3294e6f366c9074778b2da3f1d795d02210096d6acda6acd2fe0ff005b08a9c0b72b63f599532ec6493f44b8518265d0e5fd:4a3eb6b4988d95847d4203be25ed1d46 +``` + +### Optional Fields for Code Protocol + +Apart from required fields mentioned above, Code protocol also supports following optional fields to further customize the execution of code. + +#### Args + +Args are arguments that are sent to engine while executing the code. For example if we want to bypass execution policy in powershell for specific template this can be done by adding following args to the template. + +```yaml + - engine: + - powershell + - powershell.exe + args: + - -ExecutionPolicy + - Bypass + - -File +``` + +#### Pattern + +Pattern field can be used to customize name / extension of temporary file while executing a code snippet in a template + +```yaml + pattern: "*.ps1" +``` + +adding `pattern: "*.ps1"` will make sure that name of temporary file given pattern + + +### Example Code Template with Args and Pattern + +Below is a example code template where we are executing a powershell script while customizing behaviour of execution policy and setting pattern to `*.ps1` + +```yaml +id: ps1-code-snippet + +info: + name: ps1-code-snippet + author: pdteam + severity: info + tags: code + description: | + ps1-code-snippet + +code: + - engine: + - powershell + - powershell.exe + args: + - -ExecutionPolicy + - Bypass + - -File + pattern: "*.ps1" + source: | + $stdin = [Console]::In + $line = $stdin.ReadLine() + Write-Host "hello from $line" + + matchers: + - type: word + words: + - "hello from input" +# digest: 4a0a00473045022100eb01da6b97893e7868c584f330a0cd52df9bddac005860bb8595ba5b8aed58c9022050043feac68d69045cf320cba9298a2eb2e792ea4720d045d01e803de1943e7d:4a3eb6b4988d95847d4203be25ed1d46 +``` + +For more examples, please refer to example [code-templates](https://github.com/projectdiscovery/nuclei/blob/3a5f9d626ea7b632ccca601b658acd9758f8f01b/integration_tests/protocols/code) in integration tests. + + +## FAQ + +### I got this error when running a template . What does it mean ? + +``` +./nuclei -u scanme.sh -t simple-code.yaml + + __ _ + ____ __ _______/ /__ (_) + / __ \/ / / / ___/ / _ \/ / + / / / / /_/ / /__/ / __/ / +/_/ /_/\__,_/\___/_/\___/_/ v3.0.0-dev + + projectdiscovery.io + +[WRN] Found 1 unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them) +[INF] Current nuclei version: v3.0.0-dev (development) +[INF] Current nuclei-templates version: v9.6.4 (latest) +[WRN] Executing 1 unsigned templates. Use with caution. +[INF] Targets loaded for current scan: 1 +[INF] No results found. Better luck next time! +[FTL] Could not run nuclei: no templates provided for scan +``` + +Here `simple-code.yaml` is a code protocol template which is not signed or content of template has been modified after signing which indicates loss of integrity of template. +If you are template writer then you can go ahead and sign the template using `-sign` flag and if you are template consumer then you should carefully examine the template before signing it. + +### What does `re-signing code templates are not allowed for security reasons` error mean? + +```bash +nuclei -u scanme.sh -t simple-code.yaml -sign + +[ERR] could not sign 'simple-code.yaml': [signer:RUNTIME] re-signing code templates are not allowed for security reasons. +[INF] All templates signatures were elaborated success=0 failed=1 +``` + +The error message `re-signing code templates are not allowed for security reasons` comes from the Nuclei engine. This error indicates that a code template initially signed by another user and someone is trying to re-sign it. + +This measure was implemented to prevent running untrusted templates unknowingly, which might lead to potential security issues. +When you encounter this error, it suggests that you're dealing with a template that has been signed by another user Likely, the original signer is not you or the team from projectdiscovery. + +By default, Nuclei disallows executing code templates that are signed by anyone other than you or from the public templates provided by projectdiscovery/nuclei-templates. + +This is done to prevent potential security abuse using code templates. + +To resolve this error: + 1. Open and thoroughly examine the code template for any modifications. + 2. Manually remove the existing digest signature from the template. + 3. Sign the template again. + +This way, you can ensure that only templates verified and trusted by you (or projectdiscovery) are run, thus maintaining a secure environment. \ No newline at end of file diff --git a/integration_tests/flow/iterate-values-flow.yaml b/integration_tests/flow/iterate-values-flow.yaml index 3abea9e4cb..b92dee4a45 100644 --- a/integration_tests/flow/iterate-values-flow.yaml +++ b/integration_tests/flow/iterate-values-flow.yaml @@ -7,10 +7,10 @@ info: flow: | - http(0) + http(1) for(let email of template["emails"]) { set("email",email); - http(1); + http(2); } http: diff --git a/integration_tests/protocols/code/ecdsa-priv-key.pem b/integration_tests/protocols/code/ecdsa-priv-key.pem deleted file mode 100644 index f6e5984fc0..0000000000 --- a/integration_tests/protocols/code/ecdsa-priv-key.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIEjTOyV8a3ZbhEM1Ti58cQrZNvKEmig+Yw8NKtePvcZ1oAoGCCqGSM49 -AwEHoUQDQgAErRysbgMYhazyMIfpkpvlrtzzCFhqc6zr0aLhXtmtHcJQ8YVhexSx -nbnzC//84yryOKkBRHOfH+xwrQvZzPbiRw== ------END EC PRIVATE KEY----- diff --git a/integration_tests/protocols/code/ecdsa-pub-key.pem b/integration_tests/protocols/code/ecdsa-pub-key.pem deleted file mode 100644 index e4cf331ae7..0000000000 --- a/integration_tests/protocols/code/ecdsa-pub-key.pem +++ /dev/null @@ -1,4 +0,0 @@ ------BEGIN PUBLIC KEY----- -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAErRysbgMYhazyMIfpkpvlrtzzCFhq -c6zr0aLhXtmtHcJQ8YVhexSxnbnzC//84yryOKkBRHOfH+xwrQvZzPbiRw== ------END PUBLIC KEY----- diff --git a/integration_tests/protocols/code/ps1-snippet.yaml b/integration_tests/protocols/code/ps1-snippet.yaml index a0116a2299..9d6c91f06d 100644 --- a/integration_tests/protocols/code/ps1-snippet.yaml +++ b/integration_tests/protocols/code/ps1-snippet.yaml @@ -25,5 +25,4 @@ code: matchers: - type: word words: - - "hello from input" -# digest: 4a0a00473045022023beecb1c4ef5b3b3a4d936a689d0fa5fea35524d23bbc12001fa0b21ca2500b02210082484d006ee0663ba1c8450ff0d10eb053308137af25cde223406c3423c4e5d1 \ No newline at end of file + - "hello from input" \ No newline at end of file diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index 00760e954c..d76ca02ecc 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4a0a004730450221008132561626bc3ef36822cb33518b731d96056a898165966920163c60088aca8a022030f7ca08e18d24f031d511fdb89dd8fd1e83a681bdc67dd062bd47039132f911 \ No newline at end of file +# digest: 4a0a00473045022100eb01da6b97893e7868c584f330a0cd52df9bddac005860bb8595ba5b8aed58c9022050043feac68d69045cf320cba9298a2eb2e792ea4720d045d01e803de1943e7d:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index 45882f57a6..790daeb0fe 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4b0a00483046022100dd46d2316163dd5f073bdf84d038958114c6a00914e737f0daa12827994eaa7a022100c8a6bbdedd0c6dc315e6c61f98dc3add9121b8ab340d333401dc58962284fc9a \ No newline at end of file +# digest: 4a0a00473045022100863e07e45c5fa8d808022dfd60679145e17b4ad6c97b493ef28adaf586407dc3022001f2b2d6e565123c0ef51921862352b0b5499b4adfbf5a92af20eb77107c4920:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index 02f1d125f4..173c3db55d 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 4b0a00483046022100b5084304ca60c6c7d89e0a5f23ed82a26f59cb2c8ccb3a90535792d4d77cd80d022100eea2c5a3164f83a9b0bcf60e637e7a710358cef7a96c0fc016185cce3f23d6a4 \ No newline at end of file +# digest: 4a0a004730450220785cbdcb0925c922fb34055b3b9277dec165e2f3ba938f5fd7488d400b11a1f5022100dc67027e9e8d6f249c8fc68d61866d636b137bd28e6870a716fbbe969f8b672b:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-nosig.yaml b/integration_tests/protocols/code/py-nosig.yaml new file mode 100644 index 0000000000..d8bd0ac6ac --- /dev/null +++ b/integration_tests/protocols/code/py-nosig.yaml @@ -0,0 +1,21 @@ +id: py-nosig + +info: + name: py-nosig + author: pdteam + severity: info + tags: code + description: | + Python code without signature + +code: + - engine: + - py + - python3 + source: | + print("py unsigned code") + + matchers: + - type: word + words: + - "py unsigned code" \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index b3822837a5..b0e0971e6b 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -12,6 +12,7 @@ code: - engine: - py - python3 + - python source: | import sys print("hello from " + sys.stdin.read()) @@ -20,4 +21,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a00473045022030a0b1fddd6c5ac0c5d217eef447c7ea54e69a044eb12376d06d5c5aa8171f67022100bb150ff1bf3b3ee0dead7ffec6cc038c860f0f660df1fc5e61eed871a439d6f4 \ No newline at end of file +# digest: 4a0a00473045022067a69eb337ffa56d1c8e2cc57b7f74a5eb3294e6f366c9074778b2da3f1d795d02210096d6acda6acd2fe0ff005b08a9c0b72b63f599532ec6493f44b8518265d0e5fd:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/rsa-priv-key b/integration_tests/protocols/code/rsa-priv-key deleted file mode 100644 index 35ac058c32..0000000000 --- a/integration_tests/protocols/code/rsa-priv-key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn -NhAAAAAwEAAQAAAQEAx8mFIi6bhD/oZZFmFziBuadPZ2mdmI1w+yFLC701SZxDFfHum2Fk -EJPmArA0uTWf8cebwo1QCJPfGmap6APqerxYfOZzquKcZj3nulwjcn+QBSaYgEomucCELK -llmlme6mSklOJ7tddgAAWUim13dR/yHsqRb9T2Ew0W2tK9s8E6mQ5B3Q4+4pJikgsyaoEO -3817NrXDYxM5dmtv2Z3FC9tu+aBOgQDiXPb8rIGszFqYexRgubaQvVpCTSA3K+lojC87r/ -GCzhGnVw9PhekcGDjxZwYy1kASRpCSxe5vER/ST6nY521sYTMlskXhozaFeaXQf1hgJoQt -N+BWuOv6iwAAA9CaQMkzmkDJMwAAAAdzc2gtcnNhAAABAQDHyYUiLpuEP+hlkWYXOIG5p0 -9naZ2YjXD7IUsLvTVJnEMV8e6bYWQQk+YCsDS5NZ/xx5vCjVAIk98aZqnoA+p6vFh85nOq -4pxmPee6XCNyf5AFJpiASia5wIQsqWWaWZ7qZKSU4nu112AABZSKbXd1H/IeypFv1PYTDR -ba0r2zwTqZDkHdDj7ikmKSCzJqgQ7fzXs2tcNjEzl2a2/ZncUL2275oE6BAOJc9vysgazM -Wph7FGC5tpC9WkJNIDcr6WiMLzuv8YLOEadXD0+F6RwYOPFnBjLWQBJGkJLF7m8RH9JPqd -jnbWxhMyWyReGjNoV5pdB/WGAmhC034Fa46/qLAAAAAwEAAQAAAQEAhHfL/JQmrZOqRd4Y -cQEYkr6q2Yif5Ay0gu7aUZhNAtfHa3+UlBYJQSyvb8zhyIQT3z5YurD0Bhv17A1yTtJ54J -ONoJM00nOa+/fD9D8vibWnCqNCrp17++H4QZy4L5WI1yWQMt/Q4wtBLgKFMRvP/ysFYQEz -WZE87/jX1JOzEMG8J+RmyvRyLWsm1dERfSl7e2Fub42bn8lSy4fH1rnlsX6M3w01g1YCNa -GqGGHncVtw/xPt3y/c7LMCMnz1esN4JCSEoLkUoL7yckjGMU90UzxaYO8xKk8uk69RkQ+o -jRau3nKodMgdHqYQJZ23F3dlVzeBMDoJGEpW5t0+qqaEEQAAAIEApStSDLRjd2K9bBa3u8 -Gkoc3W4A3PeFyVlKQbtGnpE0EBQvn1X6susq4fGZUam6H7aoI32f9pGp/At2e/Xr5TcFSL -YyIpAuBHjPemhsduH6PNMPk8I6tqeK/5ZgSWKRyrEaTkXS4KT9CHLBzNtJ1O0bH1eCO0xF -PQlU2+WAX7VZkAAACBAOwbtK/wh3Wu8o4Y6Dwps0pSVQ6pe0RrhdFdkg+dfgsy2ZM73Cs9 -THkEtKvenkQz+gi+eITFnWaN6GTgGBseb3QN2yGs3LnRF8L/H15R6p7dZ+q2R9HoyuSnzK -U6vrvuZrPxd/Nu3ttUc019bVWEvHSQw+lGCoQd+JCJL6zqjGCvAAAAgQDYnnSkcCp5fkth -/OzcRW25ZrZOpEZjsaQJm4m9wAsOjmMD7BU+5DKwIveanCp1YjgiJNsIIa/H4UeSbsrRGv -Y7eVcyJbA2fxf3QlIip3EcufVRmK+pXqmkCq68R0y49q8yrKhpdm7EGWWb4axq3JX2Yi3W -6xiSJ5uI+Y5CzbHi5QAAABVtYXJjb0BERVNLVE9QLUVMVDA4NVIBAgME ------END OPENSSH PRIVATE KEY----- diff --git a/integration_tests/protocols/code/rsa-pub-key b/integration_tests/protocols/code/rsa-pub-key deleted file mode 100644 index 96b55daf10..0000000000 --- a/integration_tests/protocols/code/rsa-pub-key +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHyYUiLpuEP+hlkWYXOIG5p09naZ2YjXD7IUsLvTVJnEMV8e6bYWQQk+YCsDS5NZ/xx5vCjVAIk98aZqnoA+p6vFh85nOq4pxmPee6XCNyf5AFJpiASia5wIQsqWWaWZ7qZKSU4nu112AABZSKbXd1H/IeypFv1PYTDRba0r2zwTqZDkHdDj7ikmKSCzJqgQ7fzXs2tcNjEzl2a2/ZncUL2275oE6BAOJc9vysgazMWph7FGC5tpC9WkJNIDcr6WiMLzuv8YLOEadXD0+F6RwYOPFnBjLWQBJGkJLF7m8RH9JPqdjnbWxhMyWyReGjNoV5pdB/WGAmhC034Fa46/qL pd@test diff --git a/integration_tests/protocols/code/rsa-signed.yaml b/integration_tests/protocols/code/rsa-signed.yaml deleted file mode 100644 index f983bf5295..0000000000 --- a/integration_tests/protocols/code/rsa-signed.yaml +++ /dev/null @@ -1,22 +0,0 @@ -id: rsa-signed-code-snippet - -info: - name: rsa-signed-code-snippet - author: pdteam - severity: info - tags: code - description: | - rsa-signed-code-snippet - -code: - - engine: - - py - - python3 - source: | - print("rsa signed code") - - matchers: - - type: word - words: - - "rsa signed code" -# digest: 34ff81030101095369676e617475726501ff820001030106466f726d6174010c000104426c6f62010a00010452657374010a000000fe0110ff8201077373682d72736101fe0100000d6ef7e5efcb8f8d543805c44ae59bc23d22f5302537dd20efa298aab362552ffc77633dc0898b6718c3e939d1972d906f7e3f8276a8062a3ca587df5cb9a3d4542f335d38d3d8804c83172ddd47d1b58c4dee389377786a7be181021c838c4e90402d19ec445a8cd322fa5ec4606f444f753d7d500b9767001dae17ed4c5640ff4c7655f88742cbd46fe5ba502c8cf3725aff6f5de4e0998821522c847648797897319c4d4c611fda2b22093c85b1f66ba121b9f46bab6f1468ca6a662ed6e9029aff1853d2753e5b17cc4ff776fc5c15b70abae35ab0ca82b58dc0bf5339b98c34e3316393f229ea460cb134787442a958b0c9802a0ea6c79f43d13c934900 \ No newline at end of file diff --git a/integration_tests/protocols/keys/ci-private-key.pem b/integration_tests/protocols/keys/ci-private-key.pem new file mode 100644 index 0000000000..6890d5f72a --- /dev/null +++ b/integration_tests/protocols/keys/ci-private-key.pem @@ -0,0 +1,5 @@ +-----BEGIN PD NUCLEI USER PRIVATE KEY----- +MHcCAQEEIEywlBGZ94ARrBT+1fTu/Ii7HGfJc4y7kK4aGYvDMYm5oAoGCCqGSM49 +AwEHoUQDQgAEnyVUkFKJx92/8doQ//VAPCrzB4dqvNgwLRZPC/oAieVpNG8HDGNw +PJ7qB7ovIfGwDOW98vQwsRG4TmgFlZr0rQ== +-----END PD NUCLEI USER PRIVATE KEY----- diff --git a/integration_tests/protocols/keys/ci.crt b/integration_tests/protocols/keys/ci.crt new file mode 100644 index 0000000000..bef501ad4b --- /dev/null +++ b/integration_tests/protocols/keys/ci.crt @@ -0,0 +1,9 @@ +-----BEGIN PD NUCLEI USER CERTIFICATE----- +MIIBPzCB56ADAgECAgRlHGgmMAoGCCqGSM49BAMCMA0xCzAJBgNVBAMTAkNJMB4X +DTIzMTAwMzE5MTQ0NloXDTI3MTAwMjE5MTQ0NlowDTELMAkGA1UEAxMCQ0kwWTAT +BgcqhkjOPQIBBggqhkjOPQMBBwNCAASfJVSQUonH3b/x2hD/9UA8KvMHh2q82DAt +Fk8L+gCJ5Wk0bwcMY3A8nuoHui8h8bAM5b3y9DCxEbhOaAWVmvStozUwMzAOBgNV +HQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAK +BggqhkjOPQQDAgNHADBEAiBgUdbAcSbDpkNNQscZog/pAuaRV4sk7fbOlTRcjZTL +qQIgdtvG1w7l9VAtk6gx+HJa3BP9IFhSfT+a3UCuJy2p2iA= +-----END PD NUCLEI USER CERTIFICATE----- diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index bd03245fa3..63362fbca4 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -3,14 +3,13 @@ package main import ( "errors" "log" - "os" "path/filepath" osutils "github.com/projectdiscovery/utils/os" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" - "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) var codeTestCases = []TestCaseInfo{ @@ -18,53 +17,29 @@ var codeTestCases = []TestCaseInfo{ {Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}}, {Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}}, {Path: "protocols/code/unsigned.yaml", TestCase: &unsignedCode{}}, - {Path: "protocols/code/rsa-signed.yaml", TestCase: &rsaSignedCode{}}, + {Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}}, {Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}}, {Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() }}, } -var ( - ecdsaPrivateKeyAbsPath string - ecdsaPublicKeyAbsPath string - - // rsaPrivateKeyAbsPath string - rsaPublicKeyAbsPath string +const ( + testCertFile = "protocols/keys/ci.crt" + testKeyFile = "protocols/keys/ci-private-key.pem" ) +var testcertpath = "" + func init() { - var err error - ecdsaPrivateKeyAbsPath, err = filepath.Abs("protocols/code/ecdsa-priv-key.pem") - if err != nil { - panic(err) - } - ecdsaPublicKeyAbsPath, err = filepath.Abs("protocols/code/ecdsa-pub-key.pem") - if err != nil { - panic(err) - } + // allow local file access to load content of file references in template + // in order to sign them for testing purposes + templates.TemplateSignerLFA() - // rsaPrivateKeyAbsPath, err = filepath.Abs("protocols/code/rsa-priv-key.pem") - // if err != nil { - // panic(err) - // } - rsaPublicKeyAbsPath, err = filepath.Abs("protocols/code/rsa-pub-key.pem") + tsigner, err := signer.NewTemplateSignerFromFiles(testCertFile, testKeyFile) if err != nil { panic(err) } - signTemplates() -} - -// signTemplates tests the signing procedure on various platforms -func signTemplates() { - signerOptions := &signer.Options{ - PrivateKeyName: ecdsaPrivateKeyAbsPath, - PublicKeyName: ecdsaPublicKeyAbsPath, - Algorithm: signer.ECDSA, - } - sign, err := signer.New(signerOptions) - if err != nil { - log.Fatalf("couldn't create crypto engine: %s\n", err) - } + testcertpath, _ = filepath.Abs(testCertFile) for _, v := range codeTestCases { templatePath := v.Path @@ -81,39 +56,31 @@ func signTemplates() { } // skip - // - unsigned test case + // - unsigned test cases if _, ok := testCase.(*unsignedCode); ok { continue } - // - already rsa signed - if _, ok := testCase.(*rsaSignedCode); ok { + if _, ok := testCase.(*codePyNoSig); ok { continue } - - if err := utils.ProcessFile(sign, templatePath); err != nil { - log.Fatalf("Could not walk directory: %s\n", err) + if err := templates.SignTemplate(tsigner, templatePath); err != nil { + log.Fatalf("Could not sign template %v got: %s\n", templatePath, err) } } -} -func prepareEnv(keypath string) { - os.Setenv("NUCLEI_SIGNATURE_PUBLIC_KEY", keypath) - os.Setenv("NUCLEI_SIGNATURE_ALGORITHM", "ecdsa") } -func tearDownEnv() { - os.Unsetenv("NUCLEI_SIGNATURE_PUBLIC_KEY") - os.Unsetenv("NUCLEI_SIGNATURE_ALGORITHM") +func getEnvValues() []string { + return []string{ + signer.CertEnvVarName + "=" + testcertpath, + } } type codeSnippet struct{} // Execute executes a test case and returns an error if occurred func (h *codeSnippet) Execute(filePath string) error { - prepareEnv(ecdsaPublicKeyAbsPath) - defer tearDownEnv() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "input", debug) + results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input") if err != nil { return err } @@ -124,10 +91,7 @@ type codeFile struct{} // Execute executes a test case and returns an error if occurred func (h *codeFile) Execute(filePath string) error { - prepareEnv(ecdsaPublicKeyAbsPath) - defer tearDownEnv() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "input", debug) + results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input") if err != nil { return err } @@ -138,10 +102,7 @@ type codeEnvVar struct{} // Execute executes a test case and returns an error if occurred func (h *codeEnvVar) Execute(filePath string) error { - prepareEnv(ecdsaPublicKeyAbsPath) - defer tearDownEnv() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "input", debug, "-V", "baz=baz") + results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-V", "baz=baz") if err != nil { return err } @@ -152,10 +113,7 @@ type unsignedCode struct{} // Execute executes a test case and returns an error if occurred func (h *unsignedCode) Execute(filePath string) error { - prepareEnv(ecdsaPublicKeyAbsPath) - defer tearDownEnv() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "input", debug) + results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input") // should error out if err != nil { @@ -166,14 +124,11 @@ func (h *unsignedCode) Execute(filePath string) error { return errors.Join(expectResultsCount(results, 1), errors.New("unsigned template was executed")) } -type rsaSignedCode struct{} +type codePyNoSig struct{} // Execute executes a test case and returns an error if occurred -func (h *rsaSignedCode) Execute(filePath string) error { - prepareEnv(rsaPublicKeyAbsPath) - defer tearDownEnv() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "input", debug) +func (h *codePyNoSig) Execute(filePath string) error { + results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input") // should error out if err != nil { diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index d9d0c681ac..61f82daf4c 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -23,6 +23,7 @@ var ( debug = os.Getenv("DEBUG") == "true" githubAction = os.Getenv("GH_ACTION") == "true" customTests = os.Getenv("TESTS") + protocol = os.Getenv("PROTO") success = aurora.Green("[✓]").String() failed = aurora.Red("[✘]").String() @@ -131,6 +132,11 @@ func runTests(customTemplatePaths []string) []string { var failedTestTemplatePaths []string for proto, testCaseInfos := range protocolTests { + if protocol != "" { + if !strings.EqualFold(proto, protocol) { + continue + } + } if len(customTemplatePaths) == 0 { fmt.Printf("Running test cases for %q protocol\n", aurora.Blue(proto)) } @@ -170,7 +176,7 @@ func execute(testCase testutils.TestCase, templatePath string) (string, error) { func expectResultsCount(results []string, expectedNumbers ...int) error { match := sliceutil.Contains(expectedNumbers, len(results)) if !match { - return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected) \nResults:\n\t%s\n", len(results), expectedNumbers, strings.Join(results, "\n\t")) + return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected) \nResults:\n\t%s\n", len(results), expectedNumbers, strings.Join(results, "\n\t")) // nolint:all } return nil } diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index e06077c966..1148c95661 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -23,11 +23,12 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/types/scanstrategy" - "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/monitor" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" @@ -53,40 +54,28 @@ func main() { // sign the templates if requested - only glob syntax is supported if options.SignTemplates { - privKey := os.Getenv(signer.PrivateKeyEnvVarName) - if privKey == "" { - gologger.Fatal().Msgf("private key '%s' not defined ", signer.PrivateKeyEnvVarName) - } - pubKey := os.Getenv(signer.PublicKeyEnvVarName) - if pubKey == "" { - gologger.Fatal().Msgf("public key '%s' not defined ", signer.PublicKeyEnvVarName) - } - signerOptions := &signer.Options{ - Algorithm: signer.RSA, - } - if fileutil.FileExists(privKey) { - signerOptions.PrivateKeyName = privKey - } else { - signerOptions.PrivateKeyData = []byte(privKey) - } - if fileutil.FileExists(pubKey) { - signerOptions.PublicKeyName = pubKey - } else { - signerOptions.PublicKeyData = []byte(pubKey) - } - sign, err := signer.New(signerOptions) + tsigner, err := signer.NewTemplateSigner(nil, nil) // will read from env , config or generate new keys if err != nil { gologger.Fatal().Msgf("couldn't initialize signer crypto engine: %s\n", err) } + successCounter := 0 + errorCounter := 0 for _, item := range options.Templates { err := filepath.WalkDir(item, func(iterItem string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { + if err != nil || d.IsDir() || !strings.HasSuffix(iterItem, extensions.YAML) { + // skip non yaml files return nil } - if err := utils.ProcessFile(sign, iterItem); err != nil { - gologger.Warning().Msgf("could not sign '%s': %s\n", iterItem, err) + if err := templates.SignTemplate(tsigner, iterItem); err != nil { + if err != templates.ErrNotATemplate { + // skip warnings and errors as given items are not templates + errorCounter++ + gologger.Error().Msgf("could not sign '%s': %s\n", iterItem, err) + } + } else { + successCounter++ } return nil @@ -94,8 +83,8 @@ func main() { if err != nil { gologger.Error().Msgf("%s\n", err) } - gologger.Info().Msgf("All templates signatures were elaborated\n") } + gologger.Info().Msgf("All templates signatures were elaborated success=%d failed=%d\n", successCounter, errorCounter) return } diff --git a/v2/cmd/sign-templates/main.go b/v2/cmd/sign-templates/main.go deleted file mode 100644 index a2eb6f90c3..0000000000 --- a/v2/cmd/sign-templates/main.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "io/fs" - "log" - "path/filepath" - - "github.com/projectdiscovery/goflags" - "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" - "github.com/projectdiscovery/nuclei/v2/pkg/utils" -) - -type options struct { - Templates goflags.StringSlice - Algorithm string - PrivateKeyName string - PrivateKeyPassPhrase string - PublicKeyName string -} - -func ParseOptions() (*options, error) { - opts := &options{} - flagSet := goflags.NewFlagSet() - flagSet.SetDescription(`sign-templates is a utility to perform template signature`) - - flagSet.CreateGroup("sign", "sign", - flagSet.StringSliceVarP(&opts.Templates, "templates", "t", nil, "templates files/folders to sign", goflags.CommaSeparatedStringSliceOptions), - flagSet.StringVarP(&opts.Algorithm, "algorithm", "a", "rsa", "signature algorithm (rsa, ecdsa)"), - flagSet.StringVarP(&opts.PrivateKeyName, "private-key", "prk", "", "private key env var name or file location"), - flagSet.StringVarP(&opts.PrivateKeyPassPhrase, "private-key-pass", "prkp", "", "private key passphrase env var name or file location"), - flagSet.StringVarP(&opts.PublicKeyName, "public-key", "puk", "", "public key env var name or file location"), - ) - - if err := flagSet.Parse(); err != nil { - return nil, err - } - - return opts, nil -} - -func main() { - opts, err := ParseOptions() - if err != nil { - log.Fatalf("couldn't parse options: %s\n", err) - } - - algo, err := signer.ParseAlgorithm(opts.Algorithm) - if err != nil { - log.Fatal("unknown algorithm type") - } - - signerOptions := &signer.Options{ - PrivateKeyName: opts.PrivateKeyName, - PassphraseName: opts.PrivateKeyPassPhrase, - PublicKeyName: opts.PublicKeyName, - Algorithm: algo, - } - sign, err := signer.New(signerOptions) - if err != nil { - log.Fatalf("couldn't create crypto engine: %s\n", err) - } - - for _, templateItem := range opts.Templates { - if err := processItem(sign, templateItem); err != nil { - log.Fatalf("Could not walk directory: %s\n", err) - } - } -} - -func processItem(sign *signer.Signer, item string) error { - return filepath.WalkDir(item, func(iterItem string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return nil - } - - if err := utils.ProcessFile(sign, iterItem); err != nil { - return err - } - - return nil - }) -} diff --git a/v2/detect-ssl-issuer.yaml b/v2/detect-ssl-issuer.yaml new file mode 100644 index 0000000000..0b4c50273c --- /dev/null +++ b/v2/detect-ssl-issuer.yaml @@ -0,0 +1,20 @@ +id: ssl-issuer + +info: + name: Detect SSL Certificate Issuer + author: Lingtren + severity: info + description: | + Extract the issuer's organization from the target's certificate. Issuers are entities which sign and distribute certificates. + tags: ssl + metadata: + max-request: 1 + +ssl: + - address: "{{Host}}:{{Port}}" + + extractors: + - type: json + json: + - " .issuer_org[]" +# digest: 4b0a00483046022100bd4c4049c78917614a2b671e1221dcbe381ce1815e59b2417440e4a8eff70e13022100856bfb849ee53d189f9cdd14940de44c326f1a20837bf65bb5b4a3595fa33138 \ No newline at end of file diff --git a/v2/go.mod b/v2/go.mod index e020bba34d..e1e9a0d6d9 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/nuclei/v2 -go 1.20 +go 1.21 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible @@ -22,11 +22,11 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.19 github.com/projectdiscovery/fastdialer v0.0.37 - github.com/projectdiscovery/hmap v0.0.18 + github.com/projectdiscovery/hmap v0.0.17 github.com/projectdiscovery/interactsh v1.1.6 github.com/projectdiscovery/rawhttp v0.1.18 github.com/projectdiscovery/retryabledns v1.0.35 - github.com/projectdiscovery/retryablehttp-go v1.0.26 + github.com/projectdiscovery/retryablehttp-go v1.0.25 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -77,12 +77,12 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/ory/dockertest/v3 v3.10.0 github.com/praetorian-inc/fingerprintx v1.1.9 - github.com/projectdiscovery/dsl v0.0.22-0.20230911020052-7ab80c9abba8 + github.com/projectdiscovery/dsl v0.0.20 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.24-0.20231009194911-044c556377a1 github.com/projectdiscovery/gologger v1.1.11 github.com/projectdiscovery/gostruct v0.0.1 - github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 + github.com/projectdiscovery/gozero v0.0.1 github.com/projectdiscovery/httpx v1.3.4 github.com/projectdiscovery/mapcidr v1.1.2 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 @@ -98,6 +98,7 @@ require ( github.com/sashabaranov/go-openai v1.15.3 github.com/stretchr/testify v1.8.4 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 + golang.org/x/term v0.13.0 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -277,7 +278,7 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/zap v1.25.0 // indirect goftp.io/server/v2 v2.0.1 // indirect - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/mod v0.13.0 // indirect golang.org/x/sys v0.13.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 9d705898e7..280fa87549 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -46,6 +46,7 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= @@ -114,6 +115,7 @@ github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8Tw= @@ -177,6 +179,7 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -188,7 +191,9 @@ github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7Std github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bsm/ginkgo/v2 v2.9.5 h1:rtVBYPs3+TC5iLUVOis1B9tjLTup7Cj5IfzosKtvTJ0= +github.com/bsm/ginkgo/v2 v2.9.5/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= +github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.8.0 h1:ea0Xadu+sHlu7x5O3gKhRpQ1IKiMrSiHttPF0ybECuA= @@ -240,6 +245,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -287,6 +293,7 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= +github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= @@ -304,10 +311,12 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b h1:XMw3j+4AEXLeL/uyiZ7/qYE1X7Ul05RTwWBhzxCLi+0= github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b/go.mod h1:l2Jrml4vojDomW5jdDJhIS60KdbrE9uPYhyAq/7OnF4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= @@ -321,6 +330,7 @@ github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A= github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -328,6 +338,7 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE= github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -346,11 +357,13 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-pg/pg v8.0.7+incompatible h1:ty/sXL1OZLo+47KK9N8llRcmbA9tZasqbQ/OO4ld53g= github.com/go-pg/pg v8.0.7+incompatible/go.mod h1:a2oXow+aFOrvwcKs3eIA0lNFmMilrxK2sOkB5NWe0vA= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -366,6 +379,7 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -603,6 +617,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -708,6 +723,7 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -716,9 +732,12 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -772,8 +791,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= -github.com/projectdiscovery/dsl v0.0.22-0.20230911020052-7ab80c9abba8 h1:llDw01bhwrcR9HTijzaCSbXp6Vs/urGRihx75NNJknE= -github.com/projectdiscovery/dsl v0.0.22-0.20230911020052-7ab80c9abba8/go.mod h1:k39cUvYjFWRJUa7ayOBRJ5EzAawJKo7XGibPub3DZEA= +github.com/projectdiscovery/dsl v0.0.20 h1:CKgstMXRMFe+R8NKaQbI0W2XSSlApXlC+Uw3MvwmHgY= +github.com/projectdiscovery/dsl v0.0.20/go.mod h1:dYgXhuJCqQtdezpBH8zFiwaAjohuIg9Au3vvamMzY5U= github.com/projectdiscovery/fastdialer v0.0.37 h1:GEn0VYD/Q+KWiUlQDPP5stvIauN8+gE/eooPzrwidic= github.com/projectdiscovery/fastdialer v0.0.37/go.mod h1:e4Rg9mQ5mNCDFV37njgGCognM0PdLq5f8CcljBqTkRw= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= @@ -786,10 +805,10 @@ github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqV github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/gostruct v0.0.1 h1:1KvR6Pn4mDbQqoLEQzhRfHpbreLno2R9xqRCCt5tgmU= github.com/projectdiscovery/gostruct v0.0.1/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 h1:M74WAoZ99q/LJPHC8aIWIt8+FLh699KqLm2CUSHoytA= -github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6/go.mod h1:jCpXNvLUCPMzm5AhJv8wtnUt/7rz0TY2SsqvKQ8tn2E= -github.com/projectdiscovery/hmap v0.0.18 h1:L9+55rpXYXdPvTWBlXPsXM2xtivZa+CzRz6z3nfZyX8= -github.com/projectdiscovery/hmap v0.0.18/go.mod h1:kTyoFd6dyhIkBRtaLOqpVZeVLBf94FFhiLFIu+Z0g/8= +github.com/projectdiscovery/gozero v0.0.1 h1:f08ZnYlbDZV/TNGDvIXV9s/oB/sAI+HWaSbW4em4aKM= +github.com/projectdiscovery/gozero v0.0.1/go.mod h1:/dHwbly+1lhOX9UreVure4lEe7K4hIHeu/c/wZGNTDo= +github.com/projectdiscovery/hmap v0.0.17 h1:QpVMjuLEwVkioAOhAFcn409ATB4rK3DkAEmqXghJcpI= +github.com/projectdiscovery/hmap v0.0.17/go.mod h1:d5kXPXHfQWZZzm5TFAZ88a+vjOjcMCRMnTj4XXkyhxk= github.com/projectdiscovery/httpx v1.3.4 h1:1tCP7YRngCDi2a8PvvcYqmpR1H9X7Qgn89uazKL65eg= github.com/projectdiscovery/httpx v1.3.4/go.mod h1:5JlNJcEHPF9ByFFNEcaXEAs8yZYsUC6E9Q3VGfDpPeY= github.com/projectdiscovery/interactsh v1.1.6 h1:Jm09jXtV/3zPWIkf1+KpbPR6TnjXI/4SJQE2tMvVZQ8= @@ -808,17 +827,16 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.35 h1:lPX8f7exDaiNJc/4Rc44xQfFK9BpA8ZLtpQ+te2ymLU= github.com/projectdiscovery/retryabledns v1.0.35/go.mod h1:V4nRoHJzK2UmlGgKMRduLBkgNNMXJXmJchB5Wui8s4c= -github.com/projectdiscovery/retryablehttp-go v1.0.26 h1:eZYNRRvj7lv05+XrsQU61o1sYTcPwKbmSfiOJfUOArg= -github.com/projectdiscovery/retryablehttp-go v1.0.26/go.mod h1:lCvCUZs1MK5gLi2yUT6Lw/ciVj8Wr2SnNeIJGXxWKHo= +github.com/projectdiscovery/retryablehttp-go v1.0.25 h1:IhNSwWSnWYorp1Dcsh6whqy5Lm9QX738PXvWasTbVRo= +github.com/projectdiscovery/retryablehttp-go v1.0.25/go.mod h1:0oqaVWsBVMpZ1P9Dk1lkoNXFXD4B9MHKidbtD+VmEPU= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= +github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.4 h1:jXRvichO/ZfhYERch1CbNS1PRbS2KgSBj7JoWslEpIw= github.com/projectdiscovery/tlsx v1.1.4/go.mod h1:crzMlxOokVQDwGVm51JPZi1ZAgzxhNl1KVRmbff6pkI= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.58-0.20231009151631-3681bca54127 h1:GNsAR3SPNcWE3wpW+HJJsPOGWnmdMdECXZN3k3zqkTI= -github.com/projectdiscovery/utils v0.0.58-0.20231009151631-3681bca54127/go.mod h1:5ub86JF91NnI3nTMIzEpL/pfsNb0jtHznzKi9hv03X4= github.com/projectdiscovery/utils v0.0.58-0.20231009161115-60268dca6e8f h1:5GMMQ6d7vqLMvjfibclgWgptj7vm9iDAz8xgRCYd+iI= github.com/projectdiscovery/utils v0.0.58-0.20231009161115-60268dca6e8f/go.mod h1:5ub86JF91NnI3nTMIzEpL/pfsNb0jtHznzKi9hv03X4= github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8= @@ -874,6 +892,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02 h1:Nk74A6E84pynxLN74hIrQ7Q3cS0/0L5I7coOLNSFAMs= github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02/go.mod h1:OGEfzIZJs5m/VgAb1BvWR8fH17RTQWx84HTB1koGf9s= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -946,6 +965,7 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= +github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= @@ -956,6 +976,7 @@ github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vl github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= +github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -1076,6 +1097,7 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -1231,6 +1253,7 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1316,6 +1339,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1528,6 +1552,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo= +gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 51ecace493..fb81b0ba58 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -20,7 +20,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" - "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" protocoltypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" fileutil "github.com/projectdiscovery/utils/file" @@ -78,10 +77,6 @@ func ParseOptions(options *types.Options) { // Load the resolvers if user asked for them loadResolvers(options) - if err := loadTemplateSignaturesKeys(options); err != nil && !getBoolEnvValue("HIDE_TEMPLATE_SIG_WARNING") { - gologger.Warning().Msgf("Could not initialize code template verifier: %s\n", err) - } - err := protocolinit.Init(options) if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) @@ -434,35 +429,6 @@ func readEnvInputVars(options *types.Options) { } } -func loadTemplateSignaturesKeys(options *types.Options) error { - if options.CodeTemplateSignaturePublicKey == "" { - return errors.New("public key not defined") - } - - if options.CodeTemplateSignatureAlgorithm == "" { - return errors.New("signature algorithm not defined") - } - - signatureAlgo, err := signer.ParseAlgorithm(options.CodeTemplateSignatureAlgorithm) - if err != nil { - return err - } - - signerOptions := &signer.Options{Algorithm: signatureAlgo} - if fileutil.FileExists(options.CodeTemplateSignaturePublicKey) { - signerOptions.PublicKeyName = options.CodeTemplateSignaturePublicKey - } else { - signerOptions.PublicKeyData = []byte(options.CodeTemplateSignaturePublicKey) - } - - verifier, err := signer.NewVerifier(signerOptions) - if err != nil { - return err - } - - return signer.AddToDefault(verifier) -} - func getBoolEnvValue(key string) bool { value := os.Getenv(key) return strings.EqualFold(value, "true") diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 2d797f67e0..893bf9ab26 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -699,6 +699,7 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { stats.Display(parsers.SyntaxWarningStats) stats.Display(parsers.SyntaxErrorStats) stats.Display(parsers.RuntimeWarningsStats) + stats.Display(parsers.UnsignedWarning) cfg := config.DefaultConfig @@ -712,6 +713,15 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { if len(store.Workflows()) > 0 { gologger.Info().Msgf("Workflows loaded for current scan: %d", len(store.Workflows())) } + for k, v := range templates.SignatureStats { + if v.Load() > 0 { + if k != templates.Unsigned { + gologger.Info().Msgf("Executing %d signed templates from %s", v.Load(), k) + } else if !r.options.Silent { + gologger.DefaultLogger.Print().Msgf("[%v] Executing %d unsigned templates. Use with caution.", aurora.BrightYellow("WRN"), v.Load()) + } + } + } if r.hmapInputProvider.Count() > 0 { gologger.Info().Msgf("Targets loaded for current scan: %d", r.hmapInputProvider.Count()) } diff --git a/v2/key.go b/v2/key.go new file mode 100644 index 0000000000..cc5adad565 --- /dev/null +++ b/v2/key.go @@ -0,0 +1,6 @@ +package v2 + +import _ "embed" + +//go:embed nuclei.crt +var NucleiCert []byte diff --git a/v2/nuclei.crt b/v2/nuclei.crt new file mode 100644 index 0000000000..f553377c97 --- /dev/null +++ b/v2/nuclei.crt @@ -0,0 +1,11 @@ +-----BEGIN PD NUCLEI USER CERTIFICATE----- +MIIBgDCCASWgAwIBAgIEZSUZ3jAKBggqhkjOPQQDAjAsMSowKAYDVQQDEyFwcm9q +ZWN0ZGlzY292ZXJ5L251Y2xlaS10ZW1wbGF0ZXMwHhcNMjMxMDEwMDkzMTEwWhcN +MjcxMDA5MDkzMTEwWjAsMSowKAYDVQQDEyFwcm9qZWN0ZGlzY292ZXJ5L251Y2xl +aS10ZW1wbGF0ZXMwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASTaiE41H7LWudF +SMCfnqguQMwEte7dz/FRfK2lmezE02w+I2VwcS3j5cPwNaqYRAJkQhk6+7li0GpG +9fb11Fs2ozUwMzAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwEw +DAYDVR0TAQH/BAIwADAKBggqhkjOPQQDAgNJADBGAiEAhFsWwLDcWks3RUv3ujCs +4V1reu6KL+kELrCCQWu5FiUCIQDZbtqL30GPGYaPSpVmd6BKrZDBOfUVBsoCS7pS +q3JLHQ== +-----END PD NUCLEI USER CERTIFICATE----- \ No newline at end of file diff --git a/v2/pkg/catalog/config/nucleiconfig.go b/v2/pkg/catalog/config/nucleiconfig.go index 0c5e6963d0..788abe9ee0 100644 --- a/v2/pkg/catalog/config/nucleiconfig.go +++ b/v2/pkg/catalog/config/nucleiconfig.go @@ -70,6 +70,12 @@ func (c *Config) WriteVersionCheckData(ignorehash, nucleiVersion, templatesVersi return nil } +// GetTemplateDir returns the nuclei templates directory absolute path +func (c *Config) GetTemplateDir() string { + val, _ := filepath.Abs(c.TemplatesDirectory) + return val +} + // DisableUpdateCheck disables update check and template updates func (c *Config) DisableUpdateCheck() { c.disableUpdates = true @@ -111,6 +117,11 @@ func (c *Config) GetConfigDir() string { return c.configDir } +// GetKeysDir returns the nuclei signer keys directory +func (c *Config) GetKeysDir() string { + return filepath.Join(c.configDir, "keys") +} + // GetAllCustomTemplateDirs returns all custom template directories func (c *Config) GetAllCustomTemplateDirs() []string { return []string{c.CustomS3TemplatesDirectory, c.CustomGitHubTemplatesDirectory, c.CustomGitLabTemplatesDirectory, c.CustomAzureTemplatesDirectory} diff --git a/v2/pkg/catalog/disk/path.go b/v2/pkg/catalog/disk/path.go index 18673db293..0280641c43 100644 --- a/v2/pkg/catalog/disk/path.go +++ b/v2/pkg/catalog/disk/path.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" fileutil "github.com/projectdiscovery/utils/file" urlutil "github.com/projectdiscovery/utils/url" ) @@ -37,12 +38,11 @@ func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) { return potentialPath, nil } - if c.templatesDirectory != "" { - templatePath := filepath.Join(c.templatesDirectory, templateName) - if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination { - return potentialPath, nil - } + templatePath = filepath.Join(config.DefaultConfig.GetTemplateDir(), templateName) + if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination { + return potentialPath, nil } + return "", fmt.Errorf("no such path found: %s", templateName) } @@ -50,7 +50,7 @@ var errNoValidCombination = errors.New("no valid combination found") // tryResolve attempts to load locate the target by iterating across all the folders tree func (c *DiskCatalog) tryResolve(fullPath string) (string, error) { - if _, err := os.Stat(fullPath); !os.IsNotExist(err) { + if fileutil.FileExists(fullPath) { return fullPath, nil } return "", errNoValidCombination diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index def4038ff9..67e6589078 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -393,11 +393,15 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ } gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err) } else if parsed != nil { - if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless { - gologger.Warning().Msgf("Headless flag is required for headless template %s\n", templatePath) - } else if len(parsed.RequestsCode) > 0 && !parsed.Verified { - gologger.Warning().Msgf("The template is not verified: '%s'\n", templatePath) + // donot include headless template in final list if headless flag is not set + gologger.Warning().Msgf("Headless flag is required for headless template '%s'\n", templatePath) + } else if len(parsed.RequestsCode) > 0 && !parsed.Verified && len(parsed.Workflows) == 0 { + // donot include unverified 'Code' protocol custom template in final list + stats.Increment(parsers.UnsignedWarning) + if store.config.ExecutorOptions.Options.VerboseVerbose { // only shown in -vv + gologger.Verbose().Msgf("Skipping Unverified custom template %s", templatePath) + } } else { loadedTemplates = append(loadedTemplates, parsed) } diff --git a/v2/pkg/js/compiler/compiler.go b/v2/pkg/js/compiler/compiler.go index 5d83556c2c..9e0e5cac13 100644 --- a/v2/pkg/js/compiler/compiler.go +++ b/v2/pkg/js/compiler/compiler.go @@ -5,6 +5,7 @@ import ( "runtime/debug" "github.com/dop251/goja" + "github.com/dop251/goja/parser" "github.com/dop251/goja_nodejs/console" "github.com/dop251/goja_nodejs/require" jsoniter "github.com/json-iterator/go" @@ -34,6 +35,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/js/global" "github.com/projectdiscovery/nuclei/v2/pkg/js/libs/goconsole" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" ) // Compiler provides a runtime to execute goja runtime @@ -109,6 +111,7 @@ func (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error // VM returns a new goja runtime for the compiler. func (c *Compiler) VM() *goja.Runtime { runtime := c.newRuntime(false) + runtime.SetParserOptions(parser.WithDisableSourceMaps) c.registerHelpersForVM(runtime) return runtime } @@ -199,7 +202,7 @@ func convertOutputToResult(output interface{}) (ExecuteResult, error) { // newRuntime creates a new goja runtime // TODO: Add support for runtime reuse for helper functions func (c *Compiler) newRuntime(reuse bool) *goja.Runtime { - return goja.New() + return protocolstate.NewJSRuntime() } // registerHelpersForVM registers all the helper functions for the goja runtime. diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 8d3938ca18..f321213eea 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -11,7 +11,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache" - "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" @@ -143,6 +142,7 @@ const ( SyntaxWarningStats = "syntax-warnings" SyntaxErrorStats = "syntax-errors" RuntimeWarningsStats = "runtime-warnings" + UnsignedWarning = "unsigned-warnings" ) func init() { @@ -151,6 +151,7 @@ func init() { stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") stats.NewEntry(RuntimeWarningsStats, "Found %d templates with runtime error (use -validate flag for further examination)") + stats.NewEntry(UnsignedWarning, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)") } // ParseTemplate parses a template and returns a *templates.Template structure @@ -165,14 +166,6 @@ func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Tem template := &templates.Template{} - // check if the template is verified - for _, verifier := range signer.DefaultVerifiers { - if template.Verified { - break - } - template.Verified, _ = signer.Verify(verifier, data) - } - switch config.GetTemplateFormatFromExt(templatePath) { case config.JSON: err = json.Unmarshal(data, template) @@ -186,7 +179,6 @@ func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Tem err = fmt.Errorf("failed to identify template format expected JSON or YAML but got %v", templatePath) } if err != nil { - stats.Increment(SyntaxErrorStats) return nil, err } diff --git a/v2/pkg/protocols/code/code.go b/v2/pkg/protocols/code/code.go index a4d18803c7..c048b2da88 100644 --- a/v2/pkg/protocols/code/code.go +++ b/v2/pkg/protocols/code/code.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gozero" + gozerotypes "github.com/projectdiscovery/gozero/types" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" @@ -20,10 +21,11 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" - fileutil "github.com/projectdiscovery/utils/file" + errorutil "github.com/projectdiscovery/utils/errors" ) // Request is a request for the SSL protocol @@ -63,17 +65,13 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } engine, err := gozero.New(gozeroOptions) if err != nil { - return err + return errorutil.NewWithErr(err).Msgf("[%s] engines '%s' not available on host", options.TemplateID, strings.Join(request.Engine, ",")) } request.gozero = engine var src *gozero.Source - if fileutil.FileExists(request.Source) { - src, err = gozero.NewSourceWithFile(request.Source) - } else { - src, err = gozero.NewSourceWithString(request.Source, request.Pattern) - } + src, err = gozero.NewSourceWithString(request.Source, request.Pattern) if err != nil { return err } @@ -86,6 +84,18 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } + for _, matcher := range compiled.Matchers { + // default matcher part for code protocol is response + if matcher.Part == "" || matcher.Part == "body" { + matcher.Part = "response" + } + } + for _, extractor := range compiled.Extractors { + // default extractor part for code protocol is response + if extractor.Part == "" || extractor.Part == "body" { + extractor.Part = "response" + } + } request.CompiledOperators = compiled } return nil @@ -126,34 +136,35 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa for name, value := range variables { v := fmt.Sprint(value) v, interactshURLs = request.options.Interactsh.Replace(v, interactshURLs) - metaSrc.AddVariable(gozero.Variable{Name: name, Value: v}) + metaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v}) } gOutput, err := request.gozero.Eval(context.Background(), request.src, metaSrc) if err != nil { return err } - defer func() { - if err := gOutput.Cleanup(); err != nil { - gologger.Warning().Msgf("%s\n", err) - } - }() + gologger.Verbose().Msgf("[%s] Executed code on local machine %v", request.options.TemplateID, input.MetaInput.Input) - dataOutput, err := gOutput.ReadAll() - if err != nil { - return err + if vardump.EnableVarDump { + gologger.Debug().Msgf("Code Protocol request variables: \n%s\n", vardump.DumpVariables(variables)) } - dataOutputString := fmtStdout(string(dataOutput)) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Debug().Msgf("[%s] Dumped Executed Source Code for %v\n\n%v\n", request.options.TemplateID, input.MetaInput.Input, request.Source) + } + + dataOutputString := fmtStdout(gOutput.Stdout.String()) data := make(output.InternalEvent) data["type"] = request.Type().String() - data["response"] = string(dataOutput) - data["body"] = dataOutputString + data["response"] = dataOutputString // response contains filtered output (eg without trailing \n) data["input"] = input.MetaInput.Input data["template-path"] = request.options.TemplatePath data["template-id"] = request.options.TemplateID data["template-info"] = request.options.TemplateInfo + if gOutput.Stderr.Len() > 0 { + data["stderr"] = fmtStdout(gOutput.Stderr.String()) + } // expose response variables in proto_var format // this is no-op if the template is not a multi protocol template @@ -181,10 +192,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } if request.options.Options.Debug || request.options.Options.DebugResponse || request.options.Options.StoreResponse { - msg := fmt.Sprintf("[%s] Dumped Code Execution for %s", request.options.TemplateID, input.MetaInput.Input) + msg := fmt.Sprintf("[%s] Dumped Code Execution for %s\n\n", request.options.TemplateID, input.MetaInput.Input) if request.options.Options.Debug || request.options.Options.DebugResponse { gologger.Debug().Msg(msg) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataOutputString, request.options.Options.NoColor, false)) + gologger.Print().Msgf("%s\n\n", responsehighlighter.Highlight(event.OperatorsResult, dataOutputString, request.options.Options.NoColor, false)) } if request.options.Options.StoreResponse { request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dataOutputString)) diff --git a/v2/pkg/protocols/common/generators/load_test.go b/v2/pkg/protocols/common/generators/load_test.go index 49c00ef2f9..eed974e1dd 100644 --- a/v2/pkg/protocols/common/generators/load_test.go +++ b/v2/pkg/protocols/common/generators/load_test.go @@ -66,7 +66,7 @@ func TestLoadPayloads(t *testing.T) { values, err := generator.loadPayloads(map[string]interface{}{ "new": "/etc/passwd", }, "/random") - require.Error(t, err, "could load payloads") + require.Error(t, err, "could load payloads got %v", values) require.Equal(t, 0, len(values), "could get values") // testcase when loading file from template directory and template file is at root i.e / diff --git a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go index 371de99260..1480de5c57 100644 --- a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go +++ b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go @@ -6,6 +6,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) // CreateEvent wraps the outputEvent with the result of the operators defined on the request @@ -21,7 +23,8 @@ func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent out // Dump response variables if ran in debug mode if vardump.EnableVarDump { - gologger.Debug().Msgf("Protocol response variables: \n%s\n", vardump.DumpVariables(outputEvent)) + protoName := cases.Title(language.English).String(request.Type().String()) + gologger.Debug().Msgf("%v Protocol response variables: \n%s\n", protoName, vardump.DumpVariables(outputEvent)) } for _, compiledOperator := range request.GetCompiledOperators() { if compiledOperator != nil { diff --git a/v2/pkg/protocols/common/protocolstate/file.go b/v2/pkg/protocols/common/protocolstate/file.go index 0c120dcf14..cc837a684a 100644 --- a/v2/pkg/protocols/common/protocolstate/file.go +++ b/v2/pkg/protocols/common/protocolstate/file.go @@ -1,7 +1,7 @@ package protocolstate import ( - "path/filepath" + "strings" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" errorutil "github.com/projectdiscovery/utils/errors" @@ -18,25 +18,17 @@ var ( // this respects the sandbox rules and only loads files from // allowed directories func NormalizePath(filePath string) (string, error) { - filePath = filepath.Clean(filePath) - templateDirectory := config.DefaultConfig.TemplatesDirectory - - tmpPath := filepath.Join(templateDirectory, filePath) - var err error - tmpPath, err = filepath.Abs(tmpPath) - if err != nil { - return "", errorutil.NewWithErr(err).Msgf("could not get absolute path of %v", tmpPath) + if lfaAllowed { + return filePath, nil } - // first try to resolve this path with 'nuclei-templates' directory as base - if fileutil.FileOrFolderExists(tmpPath) { - // this is a valid and allowed path - return tmpPath, nil + cleaned, err := fileutil.ResolveNClean(filePath, config.DefaultConfig.GetTemplateDir()) + if err != nil { + return "", errorutil.NewWithErr(err).Msgf("could not resolve and clean path %v", filePath) } - // for security reasons , access to files outside nuclei-templates directory is not allowed + // only allow files inside nuclei-templates directory // even current working directory is not allowed - // when lfa is allowed any path is allowed - if lfaAllowed { - return filePath, nil + if strings.HasPrefix(cleaned, config.DefaultConfig.GetTemplateDir()) { + return cleaned, nil } return "", errorutil.New("path %v is outside nuclei-template directory and -lfa is not enabled", filePath) } diff --git a/v2/pkg/protocols/common/protocolstate/js.go b/v2/pkg/protocols/common/protocolstate/js.go new file mode 100644 index 0000000000..9e522db476 --- /dev/null +++ b/v2/pkg/protocols/common/protocolstate/js.go @@ -0,0 +1,20 @@ +package protocolstate + +import ( + "github.com/dop251/goja" + "github.com/dop251/goja/parser" + "github.com/projectdiscovery/gologger" +) + +// NewJSRuntime returns a new javascript runtime +// with defaults set +// i.e sourcemap parsing is disabled by default +func NewJSRuntime() *goja.Runtime { + vm := goja.New() + vm.SetParserOptions(parser.WithDisableSourceMaps) + // disable eval by default + if err := vm.Set("eval", "undefined"); err != nil { + gologger.Error().Msgf("could not set eval to undefined: %s", err) + } + return vm +} diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index c2e1fcdbed..a132ad3e68 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -80,7 +80,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error { if vardump.EnableVarDump { - gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(vars)) + gologger.Debug().Msgf("DNS Protocol request variables: \n%s\n", vardump.DumpVariables(vars)) } // Compile each request for the template based on the URL diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 1b3cee291c..3cd9cfd091 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -265,7 +265,7 @@ func (p *Page) NavigateURL(action *Action, out map[string]string, allvars map[st allvars = generators.MergeMaps(allvars, defaultReqVars) if vardump.EnableVarDump { - gologger.Debug().Msgf("Final Protocol request variables: \n%s\n", vardump.DumpVariables(allvars)) + gologger.Debug().Msgf("Headless Protocol request variables: \n%s\n", vardump.DumpVariables(allvars)) } // Evaluate the target url with all variables diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 3955d6da48..59728f3a98 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -94,7 +94,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p defer instance.Close() if vardump.EnableVarDump { - gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(payloads)) + gologger.Debug().Msgf("Headless Protocol request variables: \n%s\n", vardump.DumpVariables(payloads)) } instance.SetInteractsh(request.options.Interactsh) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 726baece90..4e5ab3d98c 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -132,7 +132,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, finalVars := generators.MergeMaps(allVars, payloads) if vardump.EnableVarDump { - gologger.Debug().Msgf("Final Protocol request variables: \n%s\n", vardump.DumpVariables(finalVars)) + gologger.Debug().Msgf("HTTP Protocol request variables: \n%s\n", vardump.DumpVariables(finalVars)) } // Note: If possible any changes to current logic (i.e evaluate -> then parse URL) diff --git a/v2/pkg/protocols/javascript/js.go b/v2/pkg/protocols/javascript/js.go index ab4cdd2593..73320b4922 100644 --- a/v2/pkg/protocols/javascript/js.go +++ b/v2/pkg/protocols/javascript/js.go @@ -112,6 +112,11 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { matcher.Part = "response" } } + for _, extractor := range compiled.Extractors { + if extractor.Part == "" { + extractor.Part = "response" + } + } if err := compiled.Compile(); err != nil { return errorutil.NewWithTag(request.TemplateID, "could not compile operators got %v", err) } @@ -275,7 +280,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV templateCtx.Merge(payloadValues) if vardump.EnableVarDump { - gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues)) + gologger.Debug().Msgf("Javascript Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues)) } if request.PreCondition != "" { diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index d51071bcc1..5f720bebd6 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -152,7 +152,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac interimValues := generators.MergeMaps(variables, payloads) if vardump.EnableVarDump { - gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(interimValues)) + gologger.Debug().Msgf("Network Protocol request variables: \n%s\n", vardump.DumpVariables(interimValues)) } inputEvents := make(map[string]interface{}) diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 26f0818054..7a9f83b171 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -192,7 +192,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants) if vardump.EnableVarDump { - gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues)) + gologger.Debug().Msgf("SSL Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues)) } finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) diff --git a/v2/pkg/protocols/utils/variables.go b/v2/pkg/protocols/utils/variables.go index 92e562ce12..230f1f15de 100644 --- a/v2/pkg/protocols/utils/variables.go +++ b/v2/pkg/protocols/utils/variables.go @@ -149,7 +149,11 @@ func generateVariables(inputURL *urlutil.URL, removeTrailingSlash bool) map[stri case BaseURL: knownVariables[v] = parsed.String() case RootURL: - knownVariables[v] = fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host) + if parsed.Scheme != "" { + knownVariables[v] = fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host) + } else { + knownVariables[v] = parsed.Host + } case Hostname: knownVariables[v] = parsed.Host case Host: diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 4e8481a007..ed2e7bf9e1 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -208,7 +208,7 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context, } if vardump.EnableVarDump { - gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues)) + gologger.Debug().Msgf("Websocket Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues)) } finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index ac9ae7828b..6ab686ff06 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -99,7 +99,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants) if vardump.EnableVarDump { - gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables)) + gologger.Debug().Msgf("Whois Protocol request variables: \n%s\n", vardump.DumpVariables(variables)) } // and replace placeholders diff --git a/v2/pkg/reporting/dedupe/dedupe.go b/v2/pkg/reporting/dedupe/dedupe.go index a30de782aa..85a9200a10 100644 --- a/v2/pkg/reporting/dedupe/dedupe.go +++ b/v2/pkg/reporting/dedupe/dedupe.go @@ -7,7 +7,6 @@ package dedupe import ( "crypto/sha1" "os" - "reflect" "unsafe" "github.com/syndtr/goleveldb/leveldb" @@ -119,7 +118,6 @@ func (s *Storage) Index(result *output.ResultEvent) (bool, error) { // // Reference - https://stackoverflow.com/questions/59209493/how-to-use-unsafe-get-a-byte-slice-from-a-string-without-memory-copy func unsafeToBytes(data string) []byte { - var buf = *(*[]byte)(unsafe.Pointer(&data)) - (*reflect.SliceHeader)(unsafe.Pointer(&buf)).Cap = len(data) - return buf + var buf = (*[]byte)(unsafe.Pointer(&data)) + return *buf } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 2a95527590..e67228a07e 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -1,15 +1,17 @@ package templates import ( + "encoding/json" "fmt" "io" - "path/filepath" "reflect" - "strings" + "sync" + "sync/atomic" "github.com/pkg/errors" "gopkg.in/yaml.v2" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/js/compiler" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -20,19 +22,27 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/retryablehttp-go" errorutil "github.com/projectdiscovery/utils/errors" - fileutil "github.com/projectdiscovery/utils/file" stringsutil "github.com/projectdiscovery/utils/strings" ) var ( ErrCreateTemplateExecutor = errors.New("cannot create template executer") ErrIncompatibleWithOfflineMatching = errors.New("template can't be used for offline matching") + parsedTemplatesCache *cache.Templates + // track how many templates are verfied and by which signer + SignatureStats = map[string]*atomic.Uint64{} ) -var parsedTemplatesCache *cache.Templates +const ( + Unsigned = "unsigned" +) func init() { parsedTemplatesCache = cache.New() + for _, verifier := range signer.DefaultTemplateVerifiers { + SignatureStats[verifier.Identifier()] = &atomic.Uint64{} + } + SignatureStats["unsigned"] = &atomic.Uint64{} } // Parse parses a yaml request template file @@ -223,20 +233,80 @@ mainLoop: // ParseTemplateFromReader reads the template from reader // returns the parsed template func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, options protocols.ExecutorOptions) (*Template, error) { - template := &Template{} data, err := io.ReadAll(reader) if err != nil { return nil, err } - data = template.expandPreprocessors(data) - if preprocessor != nil { - data = preprocessor.Process(data) + // a preprocessor is a variable like + // {{randstr}} which is replaced before unmarshalling + // as it is known to be a random static value per template + hasPreprocessor := false + allPreprocessors := getPreprocessors(preprocessor) + for _, preprocessor := range allPreprocessors { + if preprocessor.Exists(data) { + hasPreprocessor = true + break + } + } + + if !hasPreprocessor { + // if no preprocessors exists parse template and exit + template, err := parseTemplate(data, options) + if err != nil { + return nil, err + } + if !template.Verified { + SignatureStats[Unsigned].Add(1) + } + return template, nil + } + + // if preprocessor is required / exists in this template + // first unmarshal it and check if its verified + // persist verified status value and then + // expand all preprocessor and reparse template + + // === signature verification befoer preprocessors === + template, err := parseTemplate(data, options) + if err != nil { + return nil, err + } + isVerified := template.Verified + if !template.Verified { + SignatureStats[Unsigned].Add(1) } - if err := yaml.Unmarshal(data, template); err != nil { + // ==== execute preprocessors ====== + for _, v := range allPreprocessors { + data = v.Process(data) + } + reParsed, err := parseTemplate(data, options) + if err != nil { return nil, err } + reParsed.Verified = isVerified + return reParsed, nil +} + +// this method does not include any kind of preprocessing +func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, error) { + template := &Template{} + var err error + switch config.GetTemplateFormatFromExt(template.Path) { + case config.JSON: + err = json.Unmarshal(data, template) + case config.YAML: + err = yaml.Unmarshal(data, template) + default: + // assume its yaml + if err = yaml.Unmarshal(data, template); err != nil { + return nil, err + } + } + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("failed to parse %s", template.Path) + } if utils.IsBlank(template.Info.Name) { return nil, errors.New("no template name field provided") @@ -259,25 +329,6 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option // if request id is not present template.validateAllRequestIDs() - // TODO: we should add a syntax check here or somehow use a javascript linter - // simplest option for now seems to compile using goja and see if it fails - if strings.TrimSpace(template.Flow) != "" { - if len(template.Flow) > 0 && filepath.Ext(template.Flow) == ".js" && fileutil.FileExists(template.Flow) { - // load file respecting sandbox - file, err := options.Options.LoadHelperFile(template.Flow, options.TemplatePath, options.Catalog) - if err != nil { - return nil, errorutil.NewWithErr(err).Msgf("loading flow file from %v denied", template.Flow) - } - defer file.Close() - if bin, err := io.ReadAll(file); err == nil { - template.Flow = string(bin) - } else { - return nil, errorutil.NewWithErr(err).Msgf("something went wrong failed to read file") - } - } - options.Flow = template.Flow - } - // create empty context args for template scope options.CreateTemplateCtxStore() options.ProtocolType = template.Type() @@ -285,7 +336,7 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option // initialize the js compiler if missing if options.JsCompiler == nil { - options.JsCompiler = compiler.New() + options.JsCompiler = GetJsCompiler() } template.Options = &options @@ -294,6 +345,12 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option return nil, fmt.Errorf("no requests defined for %s", template.ID) } + // load `flow` and `source` in code protocol from file + // if file is referenced instead of actual source code + if err := template.ImportFileRefs(template.Options); err != nil { + return nil, errorutil.NewWithErr(err).Msgf("failed to load file refs for %s", template.ID) + } + if err := template.compileProtocolRequests(options); err != nil { return nil, err } @@ -310,12 +367,25 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option template.parseSelfContainedRequests() // check if the template is verified - for _, verifier := range signer.DefaultVerifiers { + // only valid templates can be verified or signed + for _, verifier := range signer.DefaultTemplateVerifiers { + template.Verified, _ = verifier.Verify(data, template) if template.Verified { + SignatureStats[verifier.Identifier()].Add(1) break } - template.Verified, _ = signer.Verify(verifier, data) } - return template, nil } + +var ( + jsCompiler *compiler.Compiler + jsCompilerOnce = sync.OnceFunc(func() { + jsCompiler = compiler.New() + }) +) + +func GetJsCompiler() *compiler.Compiler { + jsCompilerOnce() + return jsCompiler +} diff --git a/v2/pkg/templates/preprocessors.go b/v2/pkg/templates/preprocessors.go index cc76ddc7d5..f730ec0983 100644 --- a/v2/pkg/templates/preprocessors.go +++ b/v2/pkg/templates/preprocessors.go @@ -9,13 +9,36 @@ import ( ) type Preprocessor interface { + // Process processes the data and returns the processed data. Process(data []byte) []byte + // Exists check if the preprocessor exists in the data. + Exists(data []byte) bool } -var preprocessorRegex = regexp.MustCompile(`{{([a-z0-9_]+)}}`) +var ( + preprocessorRegex = regexp.MustCompile(`{{([a-z0-9_]+)}}`) + defaultPreprocessors = []Preprocessor{} +) + +func getPreprocessors(preprocessor Preprocessor) []Preprocessor { + if preprocessor != nil { + // append() function adds the elements to existing slice if space is available + // else it creates a new slice and copies the elements to new slice + // this may cause race-conditions hence we do it explicitly + tmp := make([]Preprocessor, 0, len(defaultPreprocessors)+1) + tmp = append(tmp, preprocessor) + tmp = append(tmp, defaultPreprocessors...) + return tmp + } + return defaultPreprocessors +} -// expandPreprocessors expands the pre-processors if any for a template data. -func (template *Template) expandPreprocessors(data []byte) []byte { +var _ Preprocessor = &randStrPreprocessor{} + +type randStrPreprocessor struct{} + +// Process processes the data and returns the processed data. +func (r *randStrPreprocessor) Process(data []byte) []byte { foundMap := make(map[string]struct{}) for _, expression := range preprocessorRegex.FindAllStringSubmatch(string(data), -1) { @@ -37,3 +60,8 @@ func (template *Template) expandPreprocessors(data []byte) []byte { } return data } + +// Exists check if the preprocessor exists in the data. +func (r *randStrPreprocessor) Exists(data []byte) bool { + return bytes.Contains(data, []byte("randstr")) +} diff --git a/v2/pkg/templates/signer/default.go b/v2/pkg/templates/signer/default.go index 56378330bb..c05eda94b8 100644 --- a/v2/pkg/templates/signer/default.go +++ b/v2/pkg/templates/signer/default.go @@ -1,29 +1,41 @@ package signer import ( - "errors" + "github.com/projectdiscovery/gologger" + v2 "github.com/projectdiscovery/nuclei/v2" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + errorutil "github.com/projectdiscovery/utils/errors" ) -const ( - PrivateKeyEnvVarName = "NUCLEI_SIGNATURE_PRIVATE_KEY" - PublicKeyEnvVarName = "NUCLEI_SIGNATURE_PUBLIC_KEY" - AlgorithmEnvVarName = "NUCLEI_SIGNATURE_ALGORITHM" -) - -var DefaultVerifiers []*Signer +// DefaultTemplateVerifiers contains the default template verifiers +var DefaultTemplateVerifiers []*TemplateSigner func init() { - // add default pd verifier - if verifier, err := NewVerifier(&Options{PublicKeyData: pdPublicKey, Algorithm: RSA}); err == nil { - DefaultVerifiers = append(DefaultVerifiers, verifier) + h := &KeyHandler{ + UserCert: v2.NucleiCert, + } + if err := h.ParseUserCert(); err != nil { + gologger.Error().Msgf("Could not parse pd nuclei certificate: %s\n", err) + return + } + DefaultTemplateVerifiers = append(DefaultTemplateVerifiers, &TemplateSigner{handler: h}) + + // try to load default user cert + usr := &KeyHandler{} + if err := usr.ReadCert(CertEnvVarName, config.DefaultConfig.GetKeysDir()); err == nil { + if err := usr.ParseUserCert(); err != nil { + gologger.Error().Msgf("malformed user cert found: %s\n", err) + return + } + DefaultTemplateVerifiers = append(DefaultTemplateVerifiers, &TemplateSigner{handler: usr}) } } -func AddToDefault(s *Signer) error { +// AddSignerToDefault adds a signer to the default list of signers +func AddSignerToDefault(s *TemplateSigner) error { if s == nil { - return errors.New("signer is nil") + return errorutil.New("signer is nil") } - - DefaultVerifiers = append(DefaultVerifiers, s) + DefaultTemplateVerifiers = append(DefaultTemplateVerifiers, s) return nil } diff --git a/v2/pkg/templates/signer/handler.go b/v2/pkg/templates/signer/handler.go new file mode 100644 index 0000000000..860719143b --- /dev/null +++ b/v2/pkg/templates/signer/handler.go @@ -0,0 +1,292 @@ +package signer + +import ( + "bytes" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "os" + "path/filepath" + "time" + + "github.com/projectdiscovery/gologger" + fileutil "github.com/projectdiscovery/utils/file" + "github.com/rs/xid" + "golang.org/x/term" +) + +const ( + CertType = "PD NUCLEI USER CERTIFICATE" + PrivateKeyType = "PD NUCLEI USER PRIVATE KEY" + CertFilename = "nuclei-user.crt" + PrivateKeyFilename = "nuclei-user-private-key.pem" + CertEnvVarName = "NUCLEI_USER_CERTIFICATE" + PrivateKeyEnvName = "NUCLEI_USER_PRIVATE_KEY" +) + +var ( + ErrNoCertificate = fmt.Errorf("nuclei user certificate not found") + ErrNoPrivateKey = fmt.Errorf("nuclei user private key not found") + SkipGeneratingKeys = false + noUserPassphrase = false +) + +// KeyHandler handles the key generation and management +// of signer public and private keys +type KeyHandler struct { + UserCert []byte + PrivateKey []byte + cert *x509.Certificate + ecdsaPubKey *ecdsa.PublicKey + ecdsaKey *ecdsa.PrivateKey +} + +// ReadUserCert reads the user certificate from environment variable or given directory +func (k *KeyHandler) ReadCert(envName, dir string) error { + // read from env + if cert := k.getEnvContent(envName); cert != nil { + k.UserCert = cert + return nil + } + // read from disk + if cert, err := os.ReadFile(filepath.Join(dir, CertFilename)); err == nil { + k.UserCert = cert + return nil + } + return ErrNoCertificate +} + +// ReadPrivateKey reads the private key from environment variable or given directory +func (k *KeyHandler) ReadPrivateKey(envName, dir string) error { + // read from env + if privateKey := k.getEnvContent(envName); privateKey != nil { + k.PrivateKey = privateKey + return nil + } + // read from disk + if privateKey, err := os.ReadFile(filepath.Join(dir, PrivateKeyFilename)); err == nil { + k.PrivateKey = privateKey + return nil + } + return ErrNoPrivateKey +} + +// ParseUserCert parses the user certificate and returns the public key +func (k *KeyHandler) ParseUserCert() error { + block, _ := pem.Decode(k.UserCert) + if block == nil { + return fmt.Errorf("failed to parse PEM block containing the certificate") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return err + } + if cert.Subject.CommonName == "" { + return fmt.Errorf("invalid certificate: expected common name to be set") + } + k.cert = cert + var ok bool + k.ecdsaPubKey, ok = cert.PublicKey.(*ecdsa.PublicKey) + if !ok { + return fmt.Errorf("failed to parse ecdsa public key from cert") + } + return nil +} + +// ParsePrivateKey parses the private key and returns the private key +func (k *KeyHandler) ParsePrivateKey() error { + block, _ := pem.Decode(k.PrivateKey) + if block == nil { + return fmt.Errorf("failed to parse PEM block containing the private key") + } + // if pem block is encrypted , decrypt it + if x509.IsEncryptedPEMBlock(block) { // nolint: all + gologger.Info().Msgf("Private Key is encrypted with passphrase") + fmt.Printf("[*] Enter passphrase (exit to abort): ") + bin, err := term.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + return err + } + fmt.Println() + if string(bin) == "exit" { + return fmt.Errorf("private key requires passphrase, but none was provided") + } + block.Bytes, err = x509.DecryptPEMBlock(block, bin) // nolint: all + if err != nil { + return err + } + } + var err error + k.ecdsaKey, err = x509.ParseECPrivateKey(block.Bytes) + if err != nil { + return err + } + return nil +} + +// GenerateKeyPair generates a new key-pair for signing code templates +func (k *KeyHandler) GenerateKeyPair() { + + gologger.Info().Msgf("Generating new key-pair for signing templates") + fmt.Printf("[*] Enter User/Organization Name (exit to abort) : ") + + // get user/organization name + identifier := "" + _, err := fmt.Scanln(&identifier) + if err != nil { + gologger.Fatal().Msgf("failed to read user/organization name: %s", err) + } + if identifier == "exit" { + gologger.Fatal().Msgf("exiting key-pair generation") + } + if identifier == "" { + gologger.Fatal().Msgf("user/organization name cannot be empty") + } + + // generate new key-pair + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + gologger.Fatal().Msgf("failed to generate ecdsa key-pair: %s", err) + } + + // create x509 certificate with user/organization name and public key + // self-signed certificate with generated private key + k.UserCert, err = k.generateCertWithKey(identifier, privateKey) + if err != nil { + gologger.Fatal().Msgf("failed to create certificate: %s", err) + } + + // marshal private key + k.PrivateKey, err = k.marshalPrivateKey(privateKey) + if err != nil { + gologger.Fatal().Msgf("failed to marshal ecdsa private key: %s", err) + } + gologger.Info().Msgf("Successfully generated new key-pair for signing templates") +} + +// SaveToDisk saves the generated key-pair to the given directory +func (k *KeyHandler) SaveToDisk(dir string) error { + _ = fileutil.FixMissingDirs(filepath.Join(dir, CertFilename)) // not required but just in case will take care of missing dirs in path + if err := os.WriteFile(filepath.Join(dir, CertFilename), k.UserCert, 0600); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, PrivateKeyFilename), k.PrivateKey, 0600); err != nil { + return err + } + return nil +} + +// getEnvContent returns the content of the environment variable +// if it is a file then it loads its content +func (k *KeyHandler) getEnvContent(name string) []byte { + val := os.Getenv(name) + if val == "" { + return nil + } + if fileutil.FileExists(val) { + data, err := os.ReadFile(val) + if err != nil { + gologger.Fatal().Msgf("failed to read file: %s", err) + } + return data + } + return []byte(val) +} + +// generateCertWithKey creates a self-signed certificate with the given identifier and private key +func (k *KeyHandler) generateCertWithKey(identifier string, privateKey *ecdsa.PrivateKey) ([]byte, error) { + // Setting up the certificate + notBefore := time.Now() + notAfter := notBefore.Add(4 * 365 * 24 * time.Hour) + + serialNumber := big.NewInt(xid.New().Time().Unix()) + // create certificate template + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: identifier, + }, + SignatureAlgorithm: x509.ECDSAWithSHA256, + NotBefore: notBefore, + NotAfter: notAfter, + PublicKey: &privateKey.PublicKey, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageServerAuth, + }, + IsCA: false, + BasicConstraintsValid: true, + } + // Create the certificate + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + if err != nil { + return nil, err + } + + var certOut bytes.Buffer + if err := pem.Encode(&certOut, &pem.Block{Type: CertType, Bytes: derBytes}); err != nil { + return nil, err + } + return certOut.Bytes(), nil +} + +// marshalPrivateKey marshals the private key and encrypts it with the given passphrase +func (k *KeyHandler) marshalPrivateKey(privateKey *ecdsa.PrivateKey) ([]byte, error) { + + var passphrase []byte + // get passphrase to encrypt private key before saving to disk + if !noUserPassphrase { + fmt.Printf("[*] Enter passphrase (exit to abort): ") + passphrase = getPassphrase() + } + + // marshal private key + privateKeyData, err := x509.MarshalECPrivateKey(privateKey) + if err != nil { + gologger.Fatal().Msgf("failed to marshal ecdsa private key: %s", err) + } + // pem encode keys + pemBlock := &pem.Block{ + Type: PrivateKeyType, Bytes: privateKeyData, + } + // encrypt private key if passphrase is provided + if len(passphrase) > 0 { + // encode it with passphrase + // this function is deprecated since go 1.16 but go stdlib does not want to provide any alternative + // see: https://github.com/golang/go/issues/8860 + encBlock, err := x509.EncryptPEMBlock(rand.Reader, pemBlock.Type, pemBlock.Bytes, passphrase, x509.PEMCipherAES256) // nolint: all + if err != nil { + gologger.Fatal().Msgf("failed to encrypt private key: %s", err) + } + pemBlock = encBlock + } + return pem.EncodeToMemory(pemBlock), nil +} + +func getPassphrase() []byte { + bin, err := term.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + gologger.Fatal().Msgf("could not read passphrase: %s", err) + } + fmt.Println() + if string(bin) == "exit" { + gologger.Fatal().Msgf("exiting") + } + fmt.Printf("[*] Enter same passphrase again: ") + bin2, err := term.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + gologger.Fatal().Msgf("could not read passphrase: %s", err) + } + fmt.Println() + // review: should we allow empty passphrase? + // we currently allow empty passphrase + if string(bin) != string(bin2) { + gologger.Fatal().Msgf("passphrase did not match try again") + } + return bin +} diff --git a/v2/pkg/templates/signer/handler_test.go b/v2/pkg/templates/signer/handler_test.go new file mode 100644 index 0000000000..449e2aeead --- /dev/null +++ b/v2/pkg/templates/signer/handler_test.go @@ -0,0 +1,60 @@ +package signer + +import ( + "bytes" + "os" + "os/exec" + "strings" + "testing" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" +) + +// This Unit Test generates a new key pair and parses it +// to ensure that the key handler works as expected. +func TestKeyHandler(t *testing.T) { + if val := os.Getenv("KEY_HANDLER_CI"); val != "1" { + cmd := exec.Command(os.Args[0], "-test.run=^TestKeyHandler$", "-test.v") + cmd.Env = append(cmd.Env, "KEY_HANDLER_CI=1") + var buff bytes.Buffer + cmd.Stdin = &buff + buff.WriteString("CIUSER\n") + buff.WriteString("\n") + out, err := cmd.CombinedOutput() + if !strings.Contains(string(out), "PASS\n") || err != nil { + t.Fatalf("%s\n(exit status %v)", string(out), err) + } + return + } + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) + h := &KeyHandler{} + noUserPassphrase = true + h.GenerateKeyPair() + if h.UserCert == nil { + t.Fatal("no user cert found") + } + if h.PrivateKey == nil { + t.Fatal("no private key found") + } + + // now parse the cert and private key + if err := h.ParseUserCert(); err != nil { + t.Fatal(err) + } + if err := h.ParsePrivateKey(); err != nil { + t.Fatal(err) + } + if h.ecdsaKey == nil { + t.Fatal("no ecdsa key found") + } + if h.ecdsaPubKey == nil { + t.Fatal("no ecdsa public key found") + } + if h.cert == nil { + t.Fatal("no certificate found") + } + if h.cert.Subject.CommonName != "CIUSER" { + t.Fatal("invalid user name found") + } +} diff --git a/v2/pkg/templates/signer/options.go b/v2/pkg/templates/signer/options.go deleted file mode 100644 index 27c2a4f037..0000000000 --- a/v2/pkg/templates/signer/options.go +++ /dev/null @@ -1,42 +0,0 @@ -package signer - -import ( - "errors" - "regexp" - "strings" -) - -type AlgorithmType uint8 - -const ( - RSA AlgorithmType = iota - ECDSA - Undefined -) - -func ParseAlgorithm(algorithm string) (AlgorithmType, error) { - algorithm = strings.ToLower(strings.TrimSpace(algorithm)) - switch algorithm { - case "ecdsa": - return ECDSA, nil - case "rsa": - return RSA, nil - default: - return Undefined, nil - } -} - -type Options struct { - PrivateKeyName string - PrivateKeyData []byte - PassphraseName string - PassphraseData []byte - PublicKeyName string - PublicKeyData []byte - Algorithm AlgorithmType -} - -var ( - ReDigest = regexp.MustCompile(`(?m)^#\sdigest:\s.+$`) - ErrUnknownAlgorithm = errors.New("unknown algorithm") -) diff --git a/v2/pkg/templates/signer/rsa_public_key b/v2/pkg/templates/signer/rsa_public_key deleted file mode 100644 index 9e969ba6b5..0000000000 --- a/v2/pkg/templates/signer/rsa_public_key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN PUBLIC KEY----- -xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -xxxxxxxxxxxxRSA PUBLIC KEY PLACEHOLDERxxxxxxxxxxxxxxxxxxxxxx -xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ------END PUBLIC KEY----- diff --git a/v2/pkg/templates/signer/rsa_public_key.go b/v2/pkg/templates/signer/rsa_public_key.go deleted file mode 100644 index 2c1e1f3386..0000000000 --- a/v2/pkg/templates/signer/rsa_public_key.go +++ /dev/null @@ -1,8 +0,0 @@ -package signer - -import ( - _ "embed" -) - -//go:embed rsa_public_key -var pdPublicKey []byte diff --git a/v2/pkg/templates/signer/signer.go b/v2/pkg/templates/signer/signer.go deleted file mode 100644 index 52e94295d7..0000000000 --- a/v2/pkg/templates/signer/signer.go +++ /dev/null @@ -1,236 +0,0 @@ -package signer - -import ( - "bytes" - "crypto/ecdsa" - "crypto/rand" - "crypto/sha256" - "crypto/x509" - "encoding/gob" - "encoding/pem" - "errors" - "fmt" - "os" - - fileutil "github.com/projectdiscovery/utils/file" - "golang.org/x/crypto/ssh" -) - -type Signer struct { - options *Options - sshSigner ssh.Signer - sshVerifier ssh.PublicKey - ecdsaSigner *ecdsa.PrivateKey - ecdsaVerifier *ecdsa.PublicKey -} - -func New(options *Options) (*Signer, error) { - var ( - privateKeyData, passphraseData, publicKeyData []byte - err error - ) - if options.PrivateKeyName != "" { - privateKeyData, err = readKeyFromFileOrEnv(options.PrivateKeyName) - if err != nil { - return nil, err - } - } else { - privateKeyData = options.PrivateKeyData - } - - if options.PassphraseName != "" { - passphraseData = readKeyFromFileOrEnvWithDefault(options.PassphraseName, []byte{}) - } else { - passphraseData = options.PassphraseData - } - - if options.PublicKeyName != "" { - publicKeyData, err = readKeyFromFileOrEnv(options.PublicKeyName) - if err != nil { - return nil, err - } - } else { - publicKeyData = options.PublicKeyData - } - - signer := &Signer{options: options} - - switch signer.options.Algorithm { - case RSA: - signer.sshSigner, signer.sshVerifier, err = parseRsa(privateKeyData, passphraseData, publicKeyData) - case ECDSA: - signer.ecdsaSigner, signer.ecdsaVerifier, err = parseECDSA(privateKeyData, publicKeyData) - default: - return nil, ErrUnknownAlgorithm - } - - if err != nil { - return nil, err - } - - return signer, nil -} - -func NewVerifier(options *Options) (*Signer, error) { - var ( - publicKeyData []byte - err error - ) - if options.PublicKeyName != "" { - publicKeyData, err = readKeyFromFileOrEnv(options.PublicKeyName) - if err != nil { - return nil, err - } - } else { - publicKeyData = options.PublicKeyData - } - - signer := &Signer{options: options} - - switch signer.options.Algorithm { - case RSA: - signer.sshVerifier, err = parseRsaPublicKey(publicKeyData) - case ECDSA: - signer.ecdsaVerifier, err = parseECDSAPublicKey(publicKeyData) - default: - return nil, ErrUnknownAlgorithm - } - - if err != nil { - return nil, err - } - - return signer, nil -} - -func (s *Signer) Sign(data []byte) ([]byte, error) { - dataHash := sha256.Sum256(data) - switch s.options.Algorithm { - case RSA: - sshSignature, err := s.sshSigner.Sign(rand.Reader, dataHash[:]) - if err != nil { - return nil, err - } - var signatureData bytes.Buffer - if err := gob.NewEncoder(&signatureData).Encode(sshSignature); err != nil { - return nil, err - } - return signatureData.Bytes(), nil - case ECDSA: - ecdsaSignature, err := ecdsa.SignASN1(rand.Reader, s.ecdsaSigner, dataHash[:]) - if err != nil { - return nil, err - } - var signatureData bytes.Buffer - if err := gob.NewEncoder(&signatureData).Encode(ecdsaSignature); err != nil { - return nil, err - } - return signatureData.Bytes(), nil - default: - return nil, ErrUnknownAlgorithm - } -} - -func (s *Signer) Verify(data, signatureData []byte) (bool, error) { - dataHash := sha256.Sum256(data) - switch s.options.Algorithm { - case RSA: - signature := &ssh.Signature{} - if err := gob.NewDecoder(bytes.NewReader(signatureData)).Decode(&signature); err != nil { - return false, err - } - if err := s.sshVerifier.Verify(dataHash[:], signature); err != nil { - return false, err - } - return true, nil - case ECDSA: - var signature []byte - if err := gob.NewDecoder(bytes.NewReader(signatureData)).Decode(&signature); err != nil { - return false, err - } - return ecdsa.VerifyASN1(s.ecdsaVerifier, dataHash[:], signature), nil - default: - return false, ErrUnknownAlgorithm - } -} - -func parseRsa(privateKeyData, passphraseData, publicKeyData []byte) (ssh.Signer, ssh.PublicKey, error) { - privateKey, err := parseRsaPrivateKey(privateKeyData, passphraseData) - if err != nil { - return nil, nil, err - } - - publicKey, err := parseRsaPublicKey(publicKeyData) - if err != nil { - return nil, nil, err - } - - return privateKey, publicKey, nil -} - -func parseRsaPrivateKey(privateKeyData, passphraseData []byte) (ssh.Signer, error) { - if len(passphraseData) > 0 { - return ssh.ParsePrivateKeyWithPassphrase(privateKeyData, passphraseData) - } - return ssh.ParsePrivateKey(privateKeyData) -} - -func parseRsaPublicKey(publicKeyData []byte) (ssh.PublicKey, error) { - publicKey, _, _, _, err := ssh.ParseAuthorizedKey(publicKeyData) - if err != nil { - return nil, err - } - - return publicKey, nil -} - -func parseECDSA(privateKeyData, publicKeyData []byte) (*ecdsa.PrivateKey, *ecdsa.PublicKey, error) { - privateKey, err := parseECDSAPrivateKey(privateKeyData) - if err != nil { - return nil, nil, err - } - publicKey, err := parseECDSAPublicKey(publicKeyData) - if err != nil { - return nil, nil, err - } - return privateKey, publicKey, nil -} - -func parseECDSAPrivateKey(privateKeyData []byte) (*ecdsa.PrivateKey, error) { - blockPriv, _ := pem.Decode(privateKeyData) - return x509.ParseECPrivateKey(blockPriv.Bytes) -} - -func parseECDSAPublicKey(publicKeyData []byte) (*ecdsa.PublicKey, error) { - blockPub, _ := pem.Decode(publicKeyData) - if blockPub == nil { - return nil, errors.New("failed to parse PEM block containing the public key") - } - genericPublicKey, err := x509.ParsePKIXPublicKey(blockPub.Bytes) - if err != nil { - return nil, err - } - if publicKey, ok := genericPublicKey.(*ecdsa.PublicKey); ok { - return publicKey, nil - } - - return nil, errors.New("couldn't parse ecdsa public key") -} - -func readKeyFromFileOrEnvWithDefault(keypath string, defaultValue []byte) []byte { - keyValue, err := readKeyFromFileOrEnv(keypath) - if err != nil { - return defaultValue - } - return keyValue -} - -func readKeyFromFileOrEnv(keypath string) ([]byte, error) { - if fileutil.FileExists(keypath) { - return os.ReadFile(keypath) - } - if keydata := os.Getenv(keypath); keydata != "" { - return []byte(keydata), nil - } - return nil, fmt.Errorf("Key not found in file or environment variable: %s", keypath) -} diff --git a/v2/pkg/templates/signer/tmpl_signer.go b/v2/pkg/templates/signer/tmpl_signer.go new file mode 100644 index 0000000000..0c1ae49ee6 --- /dev/null +++ b/v2/pkg/templates/signer/tmpl_signer.go @@ -0,0 +1,234 @@ +package signer + +import ( + "bytes" + "crypto/ecdsa" + "crypto/md5" + "crypto/rand" + "crypto/sha256" + "encoding/gob" + "encoding/hex" + "errors" + "fmt" + "os" + "regexp" + "strings" + "sync" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + errorutil "github.com/projectdiscovery/utils/errors" +) + +var ( + ReDigest = regexp.MustCompile(`(?m)^#\sdigest:\s.+$`) + ErrUnknownAlgorithm = errors.New("unknown algorithm") + SignaturePattern = "# digest: " + SignatureFmt = SignaturePattern + "%x" + ":%v" // `#digest: :` +) + +func RemoveSignatureFromData(data []byte) []byte { + return bytes.Trim(ReDigest.ReplaceAll(data, []byte("")), "\n") +} + +func GetSignatureFromData(data []byte) []byte { + return ReDigest.Find(data) +} + +// SignableTemplate is a template that can be signed +type SignableTemplate interface { + // GetFileImports returns a list of files that are imported by the template + GetFileImports() []string + // HasCodeProtocol returns true if the template has a code protocol section + HasCodeProtocol() bool +} + +type TemplateSigner struct { + sync.Once + handler *KeyHandler + fragment string +} + +// Identifier returns the identifier for the template signer +func (t *TemplateSigner) Identifier() string { + return t.handler.cert.Subject.CommonName +} + +// fragment is optional part of signature that is used to identify the user +// who signed the template via md5 hash of public key +func (t *TemplateSigner) GetUserFragment() string { + // wrap with sync.Once to reduce unnecessary md5 hashing + t.Do(func() { + if t.handler.ecdsaPubKey != nil { + hashed := md5.Sum(t.handler.ecdsaPubKey.X.Bytes()) + t.fragment = fmt.Sprintf("%x", hashed) + } + }) + return t.fragment +} + +// Sign signs the given template with the template signer and returns the signature +func (t *TemplateSigner) Sign(data []byte, tmpl SignableTemplate) (string, error) { + // while re-signing template check if it has a code protocol + // if it does then verify that it is signed by current signer + // if not then return error + if tmpl.HasCodeProtocol() { + sig := GetSignatureFromData(data) + arr := strings.SplitN(string(sig), ":", 3) + if len(arr) == 2 { + // signature has no fragment + return "", errorutil.NewWithTag("signer", "re-signing code templates are not allowed for security reasons.") + } + if len(arr) == 3 { + // signature has fragment verify if it is equal to current fragment + fragment := t.GetUserFragment() + if fragment != arr[2] { + return "", errorutil.NewWithTag("signer", "re-signing code templates are not allowed for security reasons.") + } + } + } + + buff := bytes.NewBuffer(RemoveSignatureFromData(data)) + // if file has any imports process them + for _, file := range tmpl.GetFileImports() { + bin, err := os.ReadFile(file) + if err != nil { + return "", err + } + buff.WriteRune('\n') + buff.Write(bin) + } + signatureData, err := t.sign(buff.Bytes()) + if err != nil { + return "", err + } + return signatureData, nil +} + +// Signs given data with the template signer +// Note: this should not be used for signing templates as file references +// in templates are not processed use template.SignTemplate() instead +func (t *TemplateSigner) sign(data []byte) (string, error) { + dataHash := sha256.Sum256(data) + ecdsaSignature, err := ecdsa.SignASN1(rand.Reader, t.handler.ecdsaKey, dataHash[:]) + if err != nil { + return "", err + } + var signatureData bytes.Buffer + if err := gob.NewEncoder(&signatureData).Encode(ecdsaSignature); err != nil { + return "", err + } + return fmt.Sprintf(SignatureFmt, signatureData.Bytes(), t.GetUserFragment()), nil +} + +// Verify verifies the given template with the template signer +func (t *TemplateSigner) Verify(data []byte, tmpl SignableTemplate) (bool, error) { + digestData := ReDigest.Find(data) + if len(digestData) == 0 { + return false, errors.New("digest not found") + } + + digestData = bytes.TrimSpace(bytes.TrimPrefix(digestData, []byte(SignaturePattern))) + // remove fragment from digest as it is used for re-signing purposes only + digestString := strings.TrimSuffix(string(digestData), ":"+t.GetUserFragment()) + digest, err := hex.DecodeString(digestString) + if err != nil { + return false, err + } + + buff := bytes.NewBuffer(RemoveSignatureFromData(data)) + // if file has any imports process them + for _, file := range tmpl.GetFileImports() { + bin, err := os.ReadFile(file) + if err != nil { + return false, err + } + buff.WriteRune('\n') + buff.Write(bin) + } + + return t.verify(buff.Bytes(), digest) +} + +// Verify verifies the given data with the template signer +// Note: this should not be used for verifying templates as file references +// in templates are not processed +func (t *TemplateSigner) verify(data, signatureData []byte) (bool, error) { + dataHash := sha256.Sum256(data) + + var signature []byte + if err := gob.NewDecoder(bytes.NewReader(signatureData)).Decode(&signature); err != nil { + return false, err + } + return ecdsa.VerifyASN1(t.handler.ecdsaPubKey, dataHash[:], signature), nil +} + +// NewTemplateSigner creates a new signer for signing templates +func NewTemplateSigner(cert, privateKey []byte) (*TemplateSigner, error) { + handler := &KeyHandler{} + var err error + if cert != nil || privateKey != nil { + handler.UserCert = cert + handler.PrivateKey = privateKey + } else { + err = handler.ReadCert(CertEnvVarName, config.DefaultConfig.GetKeysDir()) + if err == nil { + err = handler.ReadPrivateKey(PrivateKeyEnvName, config.DefaultConfig.GetKeysDir()) + } + } + if err != nil && !SkipGeneratingKeys { + if err != ErrNoCertificate && err != ErrNoPrivateKey { + gologger.Info().Msgf("Invalid user cert found : %s\n", err) + } + // generating new keys + handler.GenerateKeyPair() + if err := handler.SaveToDisk(config.DefaultConfig.GetKeysDir()); err != nil { + gologger.Fatal().Msgf("could not save generated keys to disk: %s\n", err) + } + // do not continue further let user re-run the command + os.Exit(0) + } else if err != nil && SkipGeneratingKeys { + return nil, err + } + + if err := handler.ParseUserCert(); err != nil { + return nil, err + } + if err := handler.ParsePrivateKey(); err != nil { + return nil, err + } + return &TemplateSigner{ + handler: handler, + }, nil +} + +// NewTemplateSignerFromFiles creates a new signer for signing templates +func NewTemplateSignerFromFiles(cert, privKey string) (*TemplateSigner, error) { + certData, err := os.ReadFile(cert) + if err != nil { + return nil, err + } + privKeyData, err := os.ReadFile(privKey) + if err != nil { + return nil, err + } + return NewTemplateSigner(certData, privKeyData) +} + +// NewTemplateSigVerifier creates a new signer for verifying templates +func NewTemplateSigVerifier(cert []byte) (*TemplateSigner, error) { + handler := &KeyHandler{} + if cert != nil { + handler.UserCert = cert + } else { + if err := handler.ReadCert(CertEnvVarName, config.DefaultConfig.GetKeysDir()); err != nil { + return nil, err + } + } + if err := handler.ParseUserCert(); err != nil { + return nil, err + } + return &TemplateSigner{ + handler: handler, + }, nil +} diff --git a/v2/pkg/templates/signer/util.go b/v2/pkg/templates/signer/util.go deleted file mode 100644 index 70f4308c76..0000000000 --- a/v2/pkg/templates/signer/util.go +++ /dev/null @@ -1,50 +0,0 @@ -package signer - -import ( - "bytes" - "encoding/hex" - "errors" - "fmt" -) - -const ( - SignaturePattern = "# digest: " - SignatureFmt = SignaturePattern + "%x" -) - -func RemoveSignatureFromData(data []byte) []byte { - return bytes.Trim(ReDigest.ReplaceAll(data, []byte("")), "\n") -} - -func Sign(sign *Signer, data []byte) (string, error) { - if sign == nil { - return "", errors.New("invalid nil signer") - } - cleanedData := RemoveSignatureFromData(data) - signatureData, err := sign.Sign(cleanedData) - if err != nil { - return "", err - } - - return fmt.Sprintf(SignatureFmt, signatureData), nil -} - -func Verify(sign *Signer, data []byte) (bool, error) { - if sign == nil { - return false, errors.New("invalid nil verifier") - } - digestData := ReDigest.Find(data) - if len(digestData) == 0 { - return false, errors.New("digest not found") - } - - digestData = bytes.TrimSpace(bytes.TrimPrefix(digestData, []byte(SignaturePattern))) - digest, err := hex.DecodeString(string(digestData)) - if err != nil { - return false, err - } - - cleanedData := RemoveSignatureFromData(data) - - return sign.Verify(cleanedData, digest) -} diff --git a/v2/pkg/templates/template_sign.go b/v2/pkg/templates/template_sign.go new file mode 100644 index 0000000000..58e60fdbcb --- /dev/null +++ b/v2/pkg/templates/template_sign.go @@ -0,0 +1,99 @@ +package templates + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + errorutil "github.com/projectdiscovery/utils/errors" +) + +// Due to file references in sensitive fields of template +// ex: javascript code in flow or bash command in code.Source etc +// signing / verifying template is only possible after loading the template +// with these fields resolved + +var ( + defaultOpts *types.Options = types.DefaultOptions() + initOnce = sync.OnceFunc(func() { + _ = protocolstate.Init(defaultOpts) + _ = protocolinit.Init(defaultOpts) + }) + ErrNotATemplate = errorutil.NewWithTag("signer", "given filePath is not a template") +) + +// New Signer/Verification logic requires it to load content of file references +// and this is done respecting sandbox restrictions to avoid any security issues +// AllowLocalFileAccess is a function that allows local file access by disabling sandbox restrictions +// and **MUST** be called before signing / verifying any templates for intialization +func TemplateSignerLFA() { + defaultOpts.AllowLocalFileAccess = true +} + +// VerifyTemplateSignature verifies the signature of the template +// using default signers +func VerifyTemplateSignature(templatePath string) (bool, error) { + template, _, err := getTemplate(templatePath) + if err != nil { + return false, err + } + return template.Verified, nil +} + +// SignTemplate signs the tempalate using custom signer +func SignTemplate(templateSigner *signer.TemplateSigner, templatePath string) error { + // sign templates requires code files such as javsacript bash command to be included + // in template hence we first load template and append such resolved file references to content + initOnce() + + // signing is only supported on yaml nuclei templates + if !strings.HasSuffix(templatePath, extensions.YAML) { + return ErrNotATemplate + } + + template, bin, err := getTemplate(templatePath) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to get template from disk") + } + if len(template.Workflows) > 0 { + // signing workflows is not supported at least yet + return ErrNotATemplate + } + if !template.Verified { + signatureData, err := templateSigner.Sign(bin, template) + if err != nil { + return err + } + buff := bytes.NewBuffer(signer.RemoveSignatureFromData(bin)) + buff.WriteString("\n" + signatureData) + return os.WriteFile(templatePath, buff.Bytes(), 0644) + } + return nil +} + +func getTemplate(templatePath string) (*Template, []byte, error) { + catalog := disk.NewCatalog(filepath.Dir(templatePath)) + executerOpts := protocols.ExecutorOptions{ + Catalog: catalog, + Options: defaultOpts, + TemplatePath: templatePath, + } + bin, err := os.ReadFile(templatePath) + if err != nil { + return nil, bin, err + } + template, err := ParseTemplateFromReader(bytes.NewReader(bin), nil, executerOpts) + if err != nil { + return nil, bin, errorutil.NewWithErr(err).Msgf("failed to parse template") + } + return template, bin, nil +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 8aea97b52e..b1b11967c8 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -3,7 +3,10 @@ package templates import ( "encoding/json" + "io" + "path/filepath" "strconv" + "strings" validate "github.com/go-playground/validator/v10" "github.com/projectdiscovery/nuclei/v2/pkg/model" @@ -22,6 +25,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" "go.uber.org/multierr" "gopkg.in/yaml.v2" ) @@ -152,6 +156,9 @@ type Template struct { // RequestsQueue contains all template requests in order (both protocol & request order) RequestsQueue []protocols.Request `yaml:"-" json:"-"` + + // ImportedFiles contains list of files whose contents are imported after template was compiled + ImportedFiles []string `yaml:"-" json:"-"` } // Type returns the type of the template @@ -184,6 +191,11 @@ func (template *Template) Type() types.ProtocolType { } } +// HasCodeProtocol returns true if the template has a code protocol section +func (template *Template) HasCodeProtocol() bool { + return len(template.RequestsCode) > 0 +} + // validateAllRequestIDs check if that protocol already has given id if not // then is is manually set to proto_index func (template *Template) validateAllRequestIDs() { @@ -193,35 +205,36 @@ func (template *Template) validateAllRequestIDs() { if len(template.RequestsCode) > 1 { for i, req := range template.RequestsCode { if req.ID == "" { - req.ID = req.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i+1) } } } + if len(template.RequestsDNS) > 1 { for i, req := range template.RequestsDNS { if req.ID == "" { - req.ID = req.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i+1) } } } if len(template.RequestsFile) > 1 { for i, req := range template.RequestsFile { if req.ID == "" { - req.ID = req.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i+1) } } } if len(template.RequestsHTTP) > 1 { for i, req := range template.RequestsHTTP { if req.ID == "" { - req.ID = req.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i+1) } } } if len(template.RequestsHeadless) > 1 { for i, req := range template.RequestsHeadless { if req.ID == "" { - req.ID = req.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i+1) } } @@ -229,35 +242,35 @@ func (template *Template) validateAllRequestIDs() { if len(template.RequestsNetwork) > 1 { for i, req := range template.RequestsNetwork { if req.ID == "" { - req.ID = req.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i+1) } } } if len(template.RequestsSSL) > 1 { for i, req := range template.RequestsSSL { if req.ID == "" { - req.ID = req.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i+1) } } } if len(template.RequestsWebsocket) > 1 { for i, req := range template.RequestsWebsocket { if req.ID == "" { - req.ID = req.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i+1) } } } if len(template.RequestsWHOIS) > 1 { for i, req := range template.RequestsWHOIS { if req.ID == "" { - req.ID = req.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i+1) } } } if len(template.RequestsJavascript) > 1 { for i, req := range template.RequestsJavascript { if req.ID == "" { - req.ID = req.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i+1) } } } @@ -325,6 +338,76 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error return nil } +// ImportFileRefs checks if sensitive fields like `flow` , `source` in code protocol are referencing files +// instead of actual javascript / engine code if so it loads the file contents and replaces the reference +func (template *Template) ImportFileRefs(options *protocols.ExecutorOptions) error { + var errs []error + + loadFile := func(source string) (string, bool) { + // load file respecting sandbox + data, err := options.Options.LoadHelperFile(source, options.TemplatePath, options.Catalog) + if err == nil { + defer data.Close() + bin, err := io.ReadAll(data) + if err == nil { + return string(bin), true + } else { + errs = append(errs, err) + } + } else { + errs = append(errs, err) + } + return "", false + } + + // for code protocol requests + for _, request := range template.RequestsCode { + // simple test to check if source is a file or a snippet + if len(strings.Split(request.Source, "\n")) == 1 && fileutil.FileExists(request.Source) { + if val, ok := loadFile(request.Source); ok { + template.ImportedFiles = append(template.ImportedFiles, request.Source) + request.Source = val + } + } + } + + // flow code references + if template.Flow != "" { + if len(template.Flow) > 0 && filepath.Ext(template.Flow) == ".js" && fileutil.FileExists(template.Flow) { + if val, ok := loadFile(template.Flow); ok { + template.ImportedFiles = append(template.ImportedFiles, template.Flow) + template.Flow = val + } + } + options.Flow = template.Flow + } + + // for multiprotocol requests + // mutually exclusive with flow + if len(template.RequestsQueue) > 0 && template.Flow == "" { + // this is most likely a multiprotocol template + for _, req := range template.RequestsQueue { + if req.Type() == types.CodeProtocol { + request := req.(*code.Request) + // simple test to check if source is a file or a snippet + if len(strings.Split(request.Source, "\n")) == 1 && fileutil.FileExists(request.Source) { + if val, ok := loadFile(request.Source); ok { + template.ImportedFiles = append(template.ImportedFiles, request.Source) + request.Source = val + } + } + } + } + } + + return multierr.Combine(errs...) +} + +// GetFileImports returns a list of files that are imported by the template +func (template *Template) GetFileImports() []string { + return template.ImportedFiles +} + // addProtocolsToQueue adds protocol requests to the queue and preserves order of the protocols and requests func (template *Template) addRequestsToQueue(keys ...string) { for _, key := range keys { diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index 1fdffa97be..5156291cee 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -137,6 +137,40 @@ func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]str return results, err } +// RunNucleiArgsWithEnvAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL) +func RunNucleiArgsWithEnvAndGetResults(debug bool, env []string, extra ...string) ([]string, error) { + cmd := exec.Command("./nuclei") + extra = append(extra, ExtraDebugArgs...) + cmd.Env = append(os.Environ(), env...) + cmd.Args = append(cmd.Args, extra...) + cmd.Args = append(cmd.Args, "-duc") // disable auto updates + cmd.Args = append(cmd.Args, "-interactions-poll-duration", "1") + cmd.Args = append(cmd.Args, "-interactions-cooldown-period", "10") + cmd.Args = append(cmd.Args, "-allow-local-file-access") + if debug { + cmd.Args = append(cmd.Args, "-debug") + cmd.Stderr = os.Stderr + fmt.Println(cmd.String()) + } else { + cmd.Args = append(cmd.Args, "-silent") + } + data, err := cmd.Output() + if debug { + fmt.Println(string(data)) + } + if len(data) < 1 && err != nil { + return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) + } + var parts []string + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} + var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\d+)`) // RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template diff --git a/v2/pkg/tmplexec/exec.go b/v2/pkg/tmplexec/exec.go index a20ca84560..08234e1b1b 100644 --- a/v2/pkg/tmplexec/exec.go +++ b/v2/pkg/tmplexec/exec.go @@ -155,7 +155,6 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (e *TemplateExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { - gologger.Info().Msgf("[%s] Running on %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint()) userCallback := func(event *output.InternalWrappedEvent) { if event != nil { callback(event) diff --git a/v2/pkg/tmplexec/flow/flow_executor.go b/v2/pkg/tmplexec/flow/flow_executor.go index 06abdc3243..8cff7d9776 100644 --- a/v2/pkg/tmplexec/flow/flow_executor.go +++ b/v2/pkg/tmplexec/flow/flow_executor.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -25,7 +26,7 @@ import ( var ( // ErrInvalidRequestID is a request id error - ErrInvalidRequestID = errorutil.NewWithFmt("invalid request id '%s' provided") + ErrInvalidRequestID = errorutil.NewWithFmt("[%s] invalid request id '%s' provided") ) // FlowExecutor is a flow executor for executing a flow @@ -84,7 +85,7 @@ func NewFlowExecutor(requests []protocols.Request, input *contextargs.Context, o }, protoFunctions: map[string]func(call goja.FunctionCall) goja.Value{}, results: results, - jsVM: goja.New(), + jsVM: protocolstate.NewJSRuntime(), input: input, } return f @@ -124,6 +125,7 @@ func (f *FlowExecutor) Compile() error { counter := 0 proto := strings.ToLower(p) // donot use loop variables in callback functions directly for index := range requests { + counter++ // start index from 1 request := f.allProtocols[proto][index] if request.GetID() != "" { // if id is present use it @@ -132,7 +134,6 @@ func (f *FlowExecutor) Compile() error { // fallback to using index as id // always allow index as id as a fallback reqMap[strconv.Itoa(counter)] = request - counter++ } // ---define hook that allows protocol/request execution from js----- // --- this is the actual callback that is executed when function is invoked in js---- diff --git a/v2/pkg/tmplexec/flow/flow_internal.go b/v2/pkg/tmplexec/flow/flow_internal.go index 15765a1476..3e9beb7b06 100644 --- a/v2/pkg/tmplexec/flow/flow_internal.go +++ b/v2/pkg/tmplexec/flow/flow_internal.go @@ -83,9 +83,9 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req for _, id := range opts.reqIDS { req, ok := reqMap[id] if !ok { - gologger.Error().Msgf("invalid request id '%s' provided", id) + gologger.Error().Msgf("[%v] invalid request id '%s' provided", f.options.TemplateID, id) // compile error - if err := f.allErrs.Set(opts.protoName+":"+id, ErrInvalidRequestID.Msgf(id)); err != nil { + if err := f.allErrs.Set(opts.protoName+":"+id, ErrInvalidRequestID.Msgf(f.options.TemplateID, id)); err != nil { gologger.Error().Msgf("failed to store flow runtime errors got %v", err) } return matcherStatus.Load() diff --git a/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml index 6c040ff7b7..8b5987e305 100644 --- a/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml +++ b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml @@ -8,7 +8,7 @@ info: reference: https://example-reference-link flow: | - dns("0"); + dns(1); template["nameservers"].forEach(nameserver => { set("nameserver",nameserver); dns("probe-ns"); diff --git a/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml index 5be54ef27d..8d5f7906c6 100644 --- a/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml +++ b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml @@ -8,10 +8,10 @@ info: reference: https://example-reference-link flow: | - dns("0"); + dns(1); template["nameservers"].forEach(nameserver => { set("nameserver",nameserver); - dns("1"); + dns(2); }); dns: diff --git a/v2/pkg/tmplexec/multiproto/testcases/multiprotodynamic.yaml b/v2/pkg/tmplexec/multiproto/testcases/multiprotodynamic.yaml index 12f429ce52..278dc8303d 100644 --- a/v2/pkg/tmplexec/multiproto/testcases/multiprotodynamic.yaml +++ b/v2/pkg/tmplexec/multiproto/testcases/multiprotodynamic.yaml @@ -16,6 +16,7 @@ dns: - trim_suffix(cname,'.ghost.io.') internal: true + http: - method: GET # http request path: diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d94f93d39d..28e003f3e4 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" + folderutil "github.com/projectdiscovery/utils/folder" ) var ( @@ -473,25 +474,63 @@ func (options *Options) ParseHeadlessOptionalArguments() map[string]string { // LoadHelperFile loads a helper file needed for the template // this respects the sandbox rules and only loads files from // allowed directories -func (options *Options) LoadHelperFile(filePath, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) { +func (options *Options) LoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) { if !options.AllowLocalFileAccess { - filePath = filepath.Clean(filePath) - templateAbsPath, err := filepath.Abs(templatePath) + // if global file access is disabled try loading with restrictions + absPath, err := options.GetValidAbsPath(helperFile, templatePath) if err != nil { - return nil, errorutil.NewWithErr(err).Msgf("could not get absolute path") + return nil, err } - templateDirectory := config.DefaultConfig.TemplatesDirectory - templatePathDir := filepath.Dir(templateAbsPath) - if !(templatePathDir != "/" && strings.HasPrefix(filePath, templatePathDir)) && !strings.HasPrefix(filePath, templateDirectory) { - return nil, errorutil.New("denied payload file path specified") + helperFile = absPath + } + f, err := os.Open(helperFile) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not open file %v", helperFile) + } + return f, nil +} + +// GetValidAbsPath returns absolute path of helper file if it is allowed to be loaded +// this respects the sandbox rules and only loads files from allowed directories +func (o *Options) GetValidAbsPath(helperFilePath, templatePath string) (string, error) { + // Conditions to allow helper file + // 1. If helper file is present in nuclei-templates directory + // 2. If helper file and template file are in same directory given that its not root directory + + // resolve and clean helper file path + // ResolveNClean uses a custom base path instead of CWD + resolvedPath, err := fileutil.ResolveNClean(helperFilePath, config.DefaultConfig.GetTemplateDir()) + if err == nil { + // As per rule 1, if helper file is present in nuclei-templates directory, allow it + if strings.HasPrefix(resolvedPath, config.DefaultConfig.GetTemplateDir()) { + return resolvedPath, nil } } - if catalog != nil { - return catalog.OpenFile(filePath) + + // CleanPath resolves using CWD and cleans the path + helperFilePath, err = fileutil.CleanPath(helperFilePath) + if err != nil { + return "", errorutil.NewWithErr(err).Msgf("could not clean helper file path %v", helperFilePath) } - f, err := os.Open(filePath) + + templatePath, err = fileutil.CleanPath(templatePath) if err != nil { - return nil, errorutil.NewWithErr(err).Msgf("could not open file %v", filePath) + return "", errorutil.NewWithErr(err).Msgf("could not clean template path %v", templatePath) } - return f, nil + + // As per rule 2, if template and helper file exist in same directory or helper file existed in any child dir of template dir + // and both of them are present in user home directory, allow it + // Review: should we keep this rule ? add extra option to disable this ? + if isHomeDir(helperFilePath) && isHomeDir(templatePath) && strings.HasPrefix(filepath.Dir(helperFilePath), filepath.Dir(templatePath)) { + return helperFilePath, nil + } + + // all other cases are denied + return "", errorutil.New("access to helper file %v denied", helperFilePath) +} + +// isRootDir checks if given is root directory +func isHomeDir(path string) bool { + homeDir := folderutil.HomeDirOrDefault("") + return strings.HasPrefix(path, homeDir) } diff --git a/v2/pkg/utils/index.go b/v2/pkg/utils/index.go new file mode 100644 index 0000000000..42ad9ba74e --- /dev/null +++ b/v2/pkg/utils/index.go @@ -0,0 +1,16 @@ +package utils + +// TransformIndex transforms user given index (start from 1) to array index (start from 0) +// in safe way without panic i.e negative index or index out of range +func TransformIndex[T any](arr []T, index int) int { + if index <= 1 { + // negative index + return 0 + } + if index >= len(arr) { + // index out of range + return len(arr) - 1 + } + // valid index + return index - 1 +} diff --git a/v2/pkg/utils/sign.go b/v2/pkg/utils/sign.go deleted file mode 100644 index 7839e629da..0000000000 --- a/v2/pkg/utils/sign.go +++ /dev/null @@ -1,70 +0,0 @@ -package utils - -import ( - "os" - "path/filepath" - - "github.com/pkg/errors" - "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" - "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" - stringsutil "github.com/projectdiscovery/utils/strings" -) - -func SignTemplate(sign *signer.Signer, templatePath string) error { - templateData, err := os.ReadFile(templatePath) - if err != nil { - return err - } - signatureData, err := signer.Sign(sign, templateData) - if err != nil { - return err - } - dataWithoutSignature := signer.RemoveSignatureFromData(templateData) - return AppendToFile(templatePath, dataWithoutSignature, signatureData) -} - -func ProcessFile(sign *signer.Signer, filePath string) error { - ext := filepath.Ext(filePath) - if !stringsutil.EqualFoldAny(ext, extensions.YAML) { - return nil - } - err := SignTemplate(sign, filePath) - if err != nil { - return errors.Wrapf(err, "could not sign template: %s", filePath) - } - - ok, err := VerifyTemplateSignature(sign, filePath) - if err != nil { - return errors.Wrapf(err, "could not verify template: %s", filePath) - } - if !ok { - return errors.Wrapf(err, "template signature doesn't match: %s", filePath) - } - - return nil -} - -func AppendToFile(path string, data []byte, digest string) error { - file, err := os.Create(path) - if err != nil { - return err - } - defer file.Close() - - if _, err := file.Write(data); err != nil { - return err - } - - if _, err := file.WriteString("\n" + digest); err != nil { - return err - } - return nil -} - -func VerifyTemplateSignature(sign *signer.Signer, templatePath string) (bool, error) { - templateData, err := os.ReadFile(templatePath) - if err != nil { - return false, err - } - return signer.Verify(sign, templateData) -}