diff --git a/go.mod b/go.mod index 3cd545221..72f2f9199 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,9 @@ require ( github.com/gitleaks/go-gitdiff v0.8.0 // indirect github.com/go-enry/go-oniguruma v1.2.1 // indirect github.com/h2non/filetype v1.1.3 // indirect + github.com/hhatto/gocloc v0.4.3 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/jessevdk/go-flags v1.5.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.16 // indirect diff --git a/go.sum b/go.sum index 50d2a0efe..48a35af5f 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,7 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gitleaks/go-gitdiff v0.8.0 h1:7aExTZm+K/M/EQKOyYcub8rIAdWK6ONxPGuRzxmWW+0= github.com/gitleaks/go-gitdiff v0.8.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= +github.com/go-enry/go-enry/v2 v2.8.0/go.mod h1:GVzIiAytiS5uT/QiuakK7TF1u4xDab87Y8V5EJRpsIQ= github.com/go-enry/go-enry/v2 v2.8.2 h1:uiGmC+3K8sVd/6DOe2AOJEOihJdqda83nPyJNtMR8RI= github.com/go-enry/go-enry/v2 v2.8.2/go.mod h1:GVzIiAytiS5uT/QiuakK7TF1u4xDab87Y8V5EJRpsIQ= github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= @@ -144,10 +145,15 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hhatto/gocloc v0.4.3 h1:bWbEi+cOKDAvWwPsP2lT30638Bg37X+Ru00TG7adpGg= +github.com/hhatto/gocloc v0.4.3/go.mod h1:EPoonh5stxIeraUU70Ogyj9yIpvV6Xirnjhyx+3/cHM= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= @@ -210,6 +216,7 @@ github.com/schollz/progressbar/v3 v3.11.0 h1:3nIBUF1Zw/pGUaRHP7PZWmARP7ZQbWQ6vL6 github.com/schollz/progressbar/v3 v3.11.0/go.mod h1:R2djRgv58sn00AGysc4fN0ip4piOGd3z88K+zVBjczs= github.com/smacker/go-tree-sitter v0.0.0-20220829074436-0a7a807924f2 h1:p+xxTsHssBdE21bzntBWAKjNyZ7BpuxynngfAe4hTHg= github.com/smacker/go-tree-sitter v0.0.0-20220829074436-0a7a807924f2/go.mod h1:q99oHDsbP0xRwmn7Vmob8gbSMNyvJ83OauXPSuHQuKE= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -390,6 +397,7 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -557,6 +565,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/pkg/flag/report_flags.go b/pkg/flag/report_flags.go index 1929676da..4a8926e2d 100644 --- a/pkg/flag/report_flags.go +++ b/pkg/flag/report_flags.go @@ -8,6 +8,7 @@ var ( ReportDetectors = "detectors" ReportDataFlow = "dataflow" + ReportStats = "stats" ) var ( @@ -22,7 +23,7 @@ var ( Name: "report", ConfigName: "report.report", Value: ReportDetectors, - Usage: "specify the kind of report (detectors, dataflow)", + Usage: "specify the kind of report (detectors, dataflow, stats)", } OutputFlag = Flag{ Name: "output", diff --git a/pkg/report/output/dataflow_components_test.go b/pkg/report/output/dataflow/components/components_test.go similarity index 95% rename from pkg/report/output/dataflow_components_test.go rename to pkg/report/output/dataflow/components/components_test.go index 636f0bed0..f77ecac22 100644 --- a/pkg/report/output/dataflow_components_test.go +++ b/pkg/report/output/dataflow/components/components_test.go @@ -1,13 +1,13 @@ -package output_test +package components_test import ( "os" "testing" "github.com/bearer/curio/pkg/commands/process/settings" - "github.com/bearer/curio/pkg/report/output" "github.com/bearer/curio/pkg/report/output/dataflow" "github.com/bearer/curio/pkg/report/output/dataflow/types" + "github.com/bearer/curio/pkg/report/output/detectors" globaltypes "github.com/bearer/curio/pkg/types" "github.com/stretchr/testify/assert" ) @@ -116,7 +116,7 @@ func TestDataflowComponents(t *testing.T) { } file.Close() - detections, err := output.GetDetectorsOutput(globaltypes.Report{ + detections, err := detectors.GetOutput(globaltypes.Report{ Path: file.Name(), }) if err != nil { @@ -124,7 +124,7 @@ func TestDataflowComponents(t *testing.T) { return } - dataflow, err := dataflow.GetOuput(detections, settings.Config{}) + dataflow, err := dataflow.GetOutput(detections, settings.Config{}) if err != nil { t.Fatalf("failed to get detectors output %s", err) return diff --git a/pkg/report/output/dataflow/dataflow.go b/pkg/report/output/dataflow/dataflow.go index 3f85b82c4..37e7b3f94 100644 --- a/pkg/report/output/dataflow/dataflow.go +++ b/pkg/report/output/dataflow/dataflow.go @@ -22,7 +22,7 @@ type DataFlow struct { var allowedDetections []detections.DetectionType = []detections.DetectionType{detections.TypeSchemaClassified, detections.TypeCustomClassified, detections.TypeDependencyClassified, detections.TypeInterfaceClassified} -func GetOuput(input []interface{}, config settings.Config) (*DataFlow, error) { +func GetOutput(input []interface{}, config settings.Config) (*DataFlow, error) { dataTypesHolder := datatypes.New() risksHolder := risks.New(config) componentsHolder := components.New() diff --git a/pkg/report/output/dataflow_datatype_test.go b/pkg/report/output/dataflow/datatypes/datatypes_test.go similarity index 96% rename from pkg/report/output/dataflow_datatype_test.go rename to pkg/report/output/dataflow/datatypes/datatypes_test.go index ff94143a2..b712d8060 100644 --- a/pkg/report/output/dataflow_datatype_test.go +++ b/pkg/report/output/dataflow/datatypes/datatypes_test.go @@ -1,13 +1,13 @@ -package output_test +package datatypes_test import ( "os" "testing" "github.com/bearer/curio/pkg/commands/process/settings" - "github.com/bearer/curio/pkg/report/output" "github.com/bearer/curio/pkg/report/output/dataflow" "github.com/bearer/curio/pkg/report/output/dataflow/types" + "github.com/bearer/curio/pkg/report/output/detectors" globaltypes "github.com/bearer/curio/pkg/types" "github.com/stretchr/testify/assert" ) @@ -161,7 +161,7 @@ func TestDataflowDataType(t *testing.T) { } file.Close() - detections, err := output.GetDetectorsOutput(globaltypes.Report{ + detections, err := detectors.GetOutput(globaltypes.Report{ Path: file.Name(), }) if err != nil { @@ -169,7 +169,7 @@ func TestDataflowDataType(t *testing.T) { return } - dataflow, err := dataflow.GetOuput(detections, test.Config) + dataflow, err := dataflow.GetOutput(detections, test.Config) if err != nil { t.Fatalf("failed to get detectors output %s", err) return diff --git a/pkg/report/output/dataflow_risks_test.go b/pkg/report/output/dataflow/risks/risks_test.go similarity index 96% rename from pkg/report/output/dataflow_risks_test.go rename to pkg/report/output/dataflow/risks/risks_test.go index f3ccada8e..820238be4 100644 --- a/pkg/report/output/dataflow_risks_test.go +++ b/pkg/report/output/dataflow/risks/risks_test.go @@ -1,13 +1,13 @@ -package output_test +package risks_test import ( "os" "testing" "github.com/bearer/curio/pkg/commands/process/settings" - "github.com/bearer/curio/pkg/report/output" "github.com/bearer/curio/pkg/report/output/dataflow" "github.com/bearer/curio/pkg/report/output/dataflow/types" + "github.com/bearer/curio/pkg/report/output/detectors" globaltypes "github.com/bearer/curio/pkg/types" "github.com/stretchr/testify/assert" ) @@ -153,7 +153,7 @@ func TestDataflowRisks(t *testing.T) { } file.Close() - detections, err := output.GetDetectorsOutput(globaltypes.Report{ + detections, err := detectors.GetOutput(globaltypes.Report{ Path: file.Name(), }) if err != nil { @@ -161,7 +161,7 @@ func TestDataflowRisks(t *testing.T) { return } - dataflow, err := dataflow.GetOuput(detections, test.Config) + dataflow, err := dataflow.GetOutput(detections, test.Config) if err != nil { t.Fatalf("failed to get detectors output %s", err) return diff --git a/pkg/report/output/detectors/detectors.go b/pkg/report/output/detectors/detectors.go new file mode 100644 index 000000000..94f655ba8 --- /dev/null +++ b/pkg/report/output/detectors/detectors.go @@ -0,0 +1,26 @@ +package detectors + +import ( + "fmt" + "os" + + "github.com/bearer/curio/pkg/types" + "github.com/rs/zerolog/log" + "github.com/wlredeye/jsonlines" +) + +func GetOutput(report types.Report) ([]interface{}, error) { + var detections []interface{} + f, err := os.Open(report.Path) + if err != nil { + return nil, fmt.Errorf("failed to open report: %w", err) + } + + err = jsonlines.Decode(f, &detections) + if err != nil { + return nil, fmt.Errorf("failed to decode report: %w", err) + } + log.Debug().Msgf("got %d detections", len(detections)) + + return detections, nil +} diff --git a/pkg/report/output/output.go b/pkg/report/output/output.go index 550677987..ddc4e6de1 100644 --- a/pkg/report/output/output.go +++ b/pkg/report/output/output.go @@ -3,26 +3,25 @@ package output import ( "encoding/json" "fmt" - "os" "github.com/bearer/curio/pkg/commands/process/settings" "github.com/bearer/curio/pkg/flag" "github.com/bearer/curio/pkg/report/output/dataflow" + "github.com/bearer/curio/pkg/report/output/detectors" + "github.com/bearer/curio/pkg/report/output/stats" "github.com/bearer/curio/pkg/types" "gopkg.in/yaml.v3" "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/wlredeye/jsonlines" ) func ReportJSON(report types.Report, output *zerolog.Event, config settings.Config) error { - ouputDetections, err := getReportOutput(report, config) + outputDetections, err := getReportOutput(report, config) if err != nil { return err } - jsonBytes, err := json.Marshal(&ouputDetections) + jsonBytes, err := json.Marshal(&outputDetections) if err != nil { return fmt.Errorf("failed to json marshal detections: %w", err) } @@ -49,41 +48,45 @@ func ReportYAML(report types.Report, output *zerolog.Event, config settings.Conf } func getReportOutput(report types.Report, config settings.Config) (any, error) { - var ouputDetections any + var output any var err error if config.Report.Report == flag.ReportDetectors { - ouputDetections, err = GetDetectorsOutput(report) + output, err = detectors.GetOutput(report) if err != nil { return nil, err } } else if config.Report.Report == flag.ReportDataFlow { - detections, err := GetDetectorsOutput(report) + detectorsOutput, err := detectors.GetOutput(report) if err != nil { return nil, err } - ouputDetections, err = dataflow.GetOuput(detections, config) + output, err = dataflow.GetOutput(detectorsOutput, config) + if err != nil { + return nil, err + } + } else if config.Report.Report == flag.ReportStats { + lineOfCodeOutput, err := stats.GoclocDetectorOutput(config.Scan.Target) if err != nil { return nil, err } - } - return ouputDetections, nil -} + detectorsOutput, err := detectors.GetOutput(report) + if err != nil { + return nil, err + } -func GetDetectorsOutput(report types.Report) ([]interface{}, error) { - var detections []interface{} - f, err := os.Open(report.Path) - if err != nil { - return nil, fmt.Errorf("failed to open report: %w", err) - } + dataflowOutput, err := dataflow.GetOutput(detectorsOutput, config) + if err != nil { + return nil, err + } - err = jsonlines.Decode(f, &detections) - if err != nil { - return nil, fmt.Errorf("failed to decode report: %w", err) + output, err = stats.GetOutput(lineOfCodeOutput, dataflowOutput, config) + if err != nil { + return nil, err + } } - log.Debug().Msgf("got %d detections", len(detections)) - return detections, nil + return output, nil } diff --git a/pkg/report/output/stats/gocloc_detector.go b/pkg/report/output/stats/gocloc_detector.go new file mode 100644 index 000000000..36075653a --- /dev/null +++ b/pkg/report/output/stats/gocloc_detector.go @@ -0,0 +1,77 @@ +package stats + +import ( + "regexp" + "strings" + + "github.com/hhatto/gocloc" + "github.com/jessevdk/go-flags" +) + +type CmdOptions struct { + Byfile bool `long:"by-file" description:"report results for every encountered source file"` + SortTag string `long:"sort" default:"code" description:"sort based on a certain column"` + OutputType string `long:"output-type" default:"default" description:"output type [values: default,cloc-xml,sloccount,json]"` + ExcludeExt string `long:"exclude-ext" description:"exclude file name extensions (separated commas)"` + IncludeLang string `long:"include-lang" description:"include language name (separated commas)"` + Match string `long:"match" description:"include file name (regex)"` + NotMatch string `long:"not-match" description:"exclude file name (regex)"` + MatchDir string `long:"match-d" description:"include dir name (regex)"` + NotMatchDir string `long:"not-match-d" description:"exclude dir name (regex)"` + Debug bool `long:"debug" description:"dump debug log for developer"` + SkipDuplicated bool `long:"skip-duplicated" description:"skip duplicated files"` + ShowLang bool `long:"show-lang" description:"print about all languages and extensions"` +} + +func GoclocDetectorOutput(path string) (*gocloc.Result, error) { + var opts CmdOptions + clocOpts := gocloc.NewClocOptions() + args := []string{ + "--output-type=json", + path, + } + + paths, err := flags.ParseArgs(&opts, args) + if err != nil { + return nil, err + } + + languages := gocloc.NewDefinedLanguages() + + // setup option for exclude extensions + for _, ext := range strings.Split(opts.ExcludeExt, ",") { + e, ok := gocloc.Exts[ext] + if ok { + clocOpts.ExcludeExts[e] = struct{}{} + } else { + clocOpts.ExcludeExts[ext] = struct{}{} + } + } + + // directory and file matching options + if opts.Match != "" { + clocOpts.ReMatch = regexp.MustCompile(opts.Match) + } + if opts.NotMatch != "" { + clocOpts.ReNotMatch = regexp.MustCompile(opts.NotMatch) + } + if opts.MatchDir != "" { + clocOpts.ReMatchDir = regexp.MustCompile(opts.MatchDir) + } + if opts.NotMatchDir != "" { + clocOpts.ReNotMatchDir = regexp.MustCompile(opts.NotMatchDir) + } + + // setup option for include languages + for _, lang := range strings.Split(opts.IncludeLang, ",") { + if _, ok := languages.Langs[lang]; ok { + clocOpts.IncludeLangs[lang] = struct{}{} + } + } + + clocOpts.Debug = opts.Debug + clocOpts.SkipDuplicated = opts.SkipDuplicated + + processor := gocloc.NewProcessor(languages, clocOpts) + return processor.Analyze(paths) +} diff --git a/pkg/report/output/stats/stats.go b/pkg/report/output/stats/stats.go new file mode 100644 index 000000000..86f8f0e92 --- /dev/null +++ b/pkg/report/output/stats/stats.go @@ -0,0 +1,41 @@ +package stats + +import ( + "github.com/bearer/curio/pkg/commands/process/settings" + "github.com/bearer/curio/pkg/report/output/dataflow" + "github.com/hhatto/gocloc" +) + +type DataType struct { + Name string `json:"name"` + Occurrences int `json:"occurrences"` +} + +type Stats struct { + NumberOfLines int32 `json:"number_of_lines"` + NumberOfDataTypes int `json:"number_of_data_types"` + DataTypes []DataType `json:"data_types"` +} + +func GetOutput(inputgocloc *gocloc.Result, inputDataflow *dataflow.DataFlow, config settings.Config) (*Stats, error) { + numberOfDataTypesFound := len(inputDataflow.Datatypes) + data_types := []DataType{} + + for _, data_type := range inputDataflow.Datatypes { + occurrences := 0 + for _, detector := range data_type.Detectors { + occurrences += len(detector.Locations) + } + + data_types = append(data_types, DataType{ + Name: data_type.Name, + Occurrences: occurrences, + }) + } + + return &Stats{ + NumberOfLines: inputgocloc.Total.Code, + NumberOfDataTypes: numberOfDataTypesFound, + DataTypes: data_types, + }, nil +}