Skip to content

Commit

Permalink
Add more support for fs.FS in template parsing (#5421)
Browse files Browse the repository at this point in the history
* misc update

* chore(deps): bump github.com/gin-gonic/gin from 1.9.0 to 1.9.1 (#4252)

Bumps [github.com/gin-gonic/gin](https://github.com/gin-gonic/gin) from 1.9.0 to 1.9.1.
- [Release notes](https://github.com/gin-gonic/gin/releases)
- [Changelog](https://github.com/gin-gonic/gin/blob/master/CHANGELOG.md)
- [Commits](gin-gonic/gin@v1.9.0...v1.9.1)

---
updated-dependencies:
- dependency-name: github.com/gin-gonic/gin
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps): bump github.com/docker/docker (#4316)

Bumps [github.com/docker/docker](https://github.com/docker/docker) from 24.0.5+incompatible to 24.0.7+incompatible.
- [Release notes](https://github.com/docker/docker/releases)
- [Commits](moby/moby@v24.0.5...v24.0.7)

---
updated-dependencies:
- dependency-name: github.com/docker/docker
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* fix README_CN.md typos (#4369)

* version update

* Add more support for `fs.FS` in the disk catalog

This adds more support for `fs.FS` in the disk catalog.  This
fixes some places where direct `os` file-related calls were being
made to use the catalog interface instead.

Note that the JavaScript compiler *still* does not work in any
context where the `pkg/js/libs/fs` package is used.  In particular,
the `ReadFilesFromDir` function is hard-coded to use the `os`
package and not respect the catalog.

* Remove some testing artifacts

* Wrap up

* Unwind other changes

* Add a LoadHelperFileFunction to Options

* Use a direct func

* Tweak validation

* Use a function type

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: sandeep <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Xc1Ym <[email protected]>
Co-authored-by: Sandeep Singh <[email protected]>
  • Loading branch information
5 people authored Sep 20, 2024
1 parent 3eee967 commit 694835c
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 13 deletions.
13 changes: 12 additions & 1 deletion pkg/catalog/disk/find.go
Original file line number Diff line number Diff line change
Expand Up @@ -216,12 +216,17 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s
// is a file, it returns true otherwise false with no errors.
func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
if c.templatesFS != nil {
absPath = strings.TrimPrefix(absPath, "/")
absPath = strings.Trim(absPath, "/")
}
var info fs.File
if c.templatesFS == nil {
info, err = os.Open(absPath)
} else {
// If we were given no path, then it's not a file, it's the root, and we can quietly return.
if absPath == "" {
return "", false, nil
}

info, err = c.templatesFS.Open(absPath)
}
if err != nil {
Expand Down Expand Up @@ -263,6 +268,12 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
},
)
} else {
// For the special case of the root directory, we need to pass "." to `fs.WalkDir`.
if absPath == "" {
absPath = "."
}
absPath = strings.TrimSuffix(absPath, "/")

err = fs.WalkDir(
c.templatesFS,
absPath,
Expand Down
34 changes: 24 additions & 10 deletions pkg/catalog/disk/path.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package disk

import (
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
Expand All @@ -21,24 +22,31 @@ func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) {
if filepath.IsAbs(templateName) {
return templateName, nil
}
if c.templatesFS != nil {
if potentialPath, err := c.tryResolve(templateName); err != errNoValidCombination {
return potentialPath, nil
}
}
if second != "" {
secondBasePath := filepath.Join(filepath.Dir(second), templateName)
if potentialPath, err := c.tryResolve(secondBasePath); err != errNoValidCombination {
return potentialPath, nil
}
}

curDirectory, err := os.Getwd()
if err != nil {
return "", err
}
if c.templatesFS == nil {
curDirectory, err := os.Getwd()
if err != nil {
return "", err
}

templatePath := filepath.Join(curDirectory, templateName)
if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination {
return potentialPath, nil
templatePath := filepath.Join(curDirectory, templateName)
if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination {
return potentialPath, nil
}
}

templatePath = filepath.Join(config.DefaultConfig.GetTemplateDir(), templateName)
templatePath := filepath.Join(config.DefaultConfig.GetTemplateDir(), templateName)
if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination {
return potentialPath, nil
}
Expand All @@ -50,8 +58,14 @@ var errNoValidCombination = errors.New("no valid combination found")

// tryResolve attempts to load locate the target by iterating across all the folders tree
func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
if fileutil.FileOrFolderExists(fullPath) {
return fullPath, nil
if c.templatesFS == nil {
if fileutil.FileOrFolderExists(fullPath) {
return fullPath, nil
}
} else {
if _, err := fs.Stat(c.templatesFS, fullPath); err == nil {
return fullPath, nil
}
}
return "", errNoValidCombination
}
Expand Down
7 changes: 7 additions & 0 deletions pkg/protocols/common/generators/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat
return errors.New("invalid number of lines in payload")
}

// For historical reasons, "validate" checks to see if the payload file exist.
// If we're using a custom helper function, then we need to skip any validation beyond just checking the string syntax.
// Actually attempting to load the file will determine whether or not it exists.
if g.options.LoadHelperFileFunction != nil {
return nil
}

// check if it's a file and try to load it
if fileutil.FileExists(payloadType) {
continue
Expand Down
21 changes: 19 additions & 2 deletions pkg/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ var (
ErrNoMoreRequests = io.EOF
)

// LoadHelperFileFunction can be used to load a helper file.
type LoadHelperFileFunction func(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error)

// Options contains the configuration options for nuclei scanner.
type Options struct {
// Tags contains a list of tags to execute templates for. Multiple paths
Expand Down Expand Up @@ -408,6 +411,9 @@ type Options struct {
HttpApiEndpoint string
// ListTemplateProfiles lists all available template profiles
ListTemplateProfiles bool
// LoadHelperFileFunction is a function that will be used to execute LoadHelperFile.
// If none is provided, then the default implementation will be used.
LoadHelperFileFunction LoadHelperFileFunction
// timeouts contains various types of timeouts used in nuclei
// these timeouts are derived from dial-timeout (-timeout) with known multipliers
// This is internally managed and does not need to be set by user by explicitly setting
Expand Down Expand Up @@ -540,10 +546,21 @@ func (options *Options) ParseHeadlessOptionalArguments() map[string]string {
return optionalArguments
}

// LoadHelperFile loads a helper file needed for the template
// LoadHelperFile loads a helper file needed for the template.
//
// If LoadHelperFileFunction is set, then that function will be used.
// Otherwise, the default implementation will be used, which respects the sandbox rules and only loads files from allowed directories.
func (options *Options) LoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) {
if options.LoadHelperFileFunction != nil {
return options.LoadHelperFileFunction(helperFile, templatePath, catalog)
}
return options.defaultLoadHelperFile(helperFile, templatePath, catalog)
}

// defaultLoadHelperFile loads a helper file needed for the template
// this respects the sandbox rules and only loads files from
// allowed directories
func (options *Options) LoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) {
func (options *Options) defaultLoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) {
if !options.AllowLocalFileAccess {
// if global file access is disabled try loading with restrictions
absPath, err := options.GetValidAbsPath(helperFile, templatePath)
Expand Down

0 comments on commit 694835c

Please sign in to comment.