Semgrep 0.80.0 update

.github/workflows/python_test.yml

@@ -6,6 +6,9 @@ on:
   pull_request:
     branches: [ master ]
 
+env:
+  SEMGREP_R2C_INTERNAL_EXPLICIT_SEMGREPIGNORE: ./tests/assets/files/.semgrepignore
+
 jobs:
   build:
 

libsast/__init__.py

@@ -8,7 +8,7 @@
 __title__ = 'libsast'
 __authors__ = 'Ajin Abraham'
 __copyright__ = 'Copyright 2020 Ajin Abraham, OpenSecurity'
-__version__ = '1.4.5'
+__version__ = '1.4.6'
 __version_info__ = tuple(int(i) for i in __version__.split('.'))
 __all__ = [
     'Scanner',

libsast/common.py

@@ -4,7 +4,7 @@
 from threading import Thread
 
 from libsast.exceptions import (
-    YamlRuleLoadException,
+    YamlRuleLoadError,
     YamlRuleParseError,
 )
 
@@ -67,5 +67,5 @@ def read_yaml(file_obj, text=False):
         raise YamlRuleParseError(
             f'YAML Parse Error: {repr(exp)}')
     except Exception as gen:
-        raise YamlRuleLoadException(
+        raise YamlRuleLoadError(
             f'Failed to load YAML file: {repr(gen)}')

libsast/core_matcher/choice_matcher.py

@@ -46,7 +46,7 @@ def validate_rules(self):
         """Validate Rules before scanning."""
         for rule in self.scan_rules:
             if not isinstance(rule, dict):
-                raise exceptions.InvalidRuleFormatException(
+                raise exceptions.InvalidRuleFormatError(
                     'Choice Matcher Rule format is invalid.')
             if not rule.get('id'):
                 raise exceptions.TypeKeyMissingError(
@@ -91,7 +91,7 @@ def choice_matcher(self, scan_paths, rule):
                         matches.add(match[0])
             self.add_finding(rule, matches, all_matches)
         except Exception:
-            raise exceptions.RuleProcessingException('Rule processing error.')
+            raise exceptions.RuleProcessingError('Rule processing error.')
 
     def add_finding(self, rule, matches, all_matches):
         """Add Choice Findings."""

libsast/core_matcher/helpers.py

@@ -6,7 +6,7 @@
 from libsast.exceptions import (
     InvalidRuleError,
     MissingRuleError,
-    RuleDownloadException,
+    RuleDownloadError,
 )
 from libsast.common import read_yaml
 from libsast.standards import get_mapping
@@ -26,7 +26,7 @@ def download_rule(url):
             r.raise_for_status()
             return r.text
     except requests.exceptions.RequestException:
-        raise RuleDownloadException(f'Failed to download from: {url}')
+        raise RuleDownloadError(f'Failed to download from: {url}')
 
 
 def get_rules(rule_loc):  # noqa: R701

libsast/core_matcher/pattern_matcher.py

@@ -50,7 +50,7 @@ def validate_rules(self):
         """Validate Rules before scanning."""
         for rule in self.scan_rules:
             if not isinstance(rule, dict):
-                raise exceptions.InvalidRuleFormatException(
+                raise exceptions.InvalidRuleFormatError(
                     'Pattern Matcher Rule format is invalid.')
             if not rule.get('type'):
                 raise exceptions.TypeKeyMissingError(
@@ -62,7 +62,7 @@ def validate_rules(self):
             pattern_name = rule['type']
             if pattern_name not in all_mts:
                 supported = ', '.join(all_mts)
-                raise exceptions.MatcherNotFoundException(
+                raise exceptions.MatcherNotFoundError(
                     f'Matcher \'{pattern_name}\' is not supported.'
                     f' Available matchers are {supported}',
                 )
@@ -89,7 +89,7 @@ def pattern_matcher(self, data, file_path, ext):
                 if matches:
                     self.add_finding(file_path, rule, matches)
         except Exception:
-            raise exceptions.RuleProcessingException('Rule processing error.')
+            raise exceptions.RuleProcessingError('Rule processing error.')
 
     def add_finding(self, file_path, rule, matches):
         """Add Code Analysis Findings."""

libsast/core_sgrep/helpers.py

@@ -3,7 +3,6 @@
 import json
 import platform
 import multiprocessing
-from io import StringIO
 
 
 def invoke_semgrep(paths, scan_rules, **kwargs):
@@ -17,22 +16,31 @@ def invoke_semgrep(paths, scan_rules, **kwargs):
         cpu_count = multiprocessing.cpu_count()
     except NotImplementedError:
         cpu_count = 1  # CPU count is not implemented on Windows
-    util.set_flags(False, True, False)  # Verbose, Quiet, Force_color
-    io_capture = StringIO()
-    output_handler = OutputHandler(
-        OutputSettings(
-            output_format=OutputFormat.JSON,
-            output_destination=None,
-            error_on_findings=False,
-            verbose_errors=False,
-            strict=False,
-            timeout_threshold=3,
-            json_stats=False,
-            output_per_finding_max_lines_limit=None,
-        ),
-        stdout=io_capture,
+    # Semgrep output formatting
+    util.set_flags(
+        verbose=False,
+        debug=False,
+        quiet=True,
+        force_color=False)
+    output_settings = OutputSettings(
+        output_format=OutputFormat.JSON,
+        output_destination=None,
+        error_on_findings=False,
+        verbose_errors=False,
+        strict=False,
+        timeout_threshold=3,
+        json_stats=False,
+        output_per_finding_max_lines_limit=None,
     )
-    semgrep_main.main(
+    output_handler = OutputHandler(output_settings)
+    (
+        filtered_matches_by_rule,
+        _all_targets,
+        _filtered_rules,
+        _profiler,
+        _profiling_data,
+        _shown_severities,
+    ) = semgrep_main.main(
         output_handler=output_handler,
         target=[pt.as_posix() for pt in paths],
         jobs=cpu_count,
@@ -43,5 +51,7 @@ def invoke_semgrep(paths, scan_rules, **kwargs):
         timeout_threshold=3,
         **kwargs,
     )
-    output_handler.close()
-    return json.loads(io_capture.getvalue())
+    output_handler.rule_matches = [
+        m for ms in filtered_matches_by_rule.values() for m in ms
+    ]
+    return json.loads(output_handler._build_output())

libsast/exceptions.py

@@ -2,74 +2,74 @@
 """The libsast Exceptions."""
 
 
-class LibsastException(Exception):
+class LibsastError(Exception):
     """Base class for all exceptions thrown by libsast."""
 
     def __init__(self, message=None):
         super().__init__(message)
 
 
-class InvalidPathError(LibsastException):
+class InvalidPathError(LibsastError):
     """Invalid Path Supplied to libsast."""
 
     pass
 
 
-class YamlRuleParseError(LibsastException):
+class YamlRuleParseError(LibsastError):
     """Failed to parse YAML rule."""
 
     pass
 
 
-class YamlRuleLoadException(LibsastException):
+class YamlRuleLoadError(LibsastError):
     """Failed to load YAML rule file."""
 
     pass
 
 
-class MissingRuleError(LibsastException):
+class MissingRuleError(LibsastError):
     """Rule not provided."""
 
     pass
 
 
-class InvalidRuleError(LibsastException):
+class InvalidRuleError(LibsastError):
     """No rule directory, file or url specified."""
 
     pass
 
 
-class TypeKeyMissingError(LibsastException):
+class TypeKeyMissingError(LibsastError):
     """Pattern Matcher rule does not have the key 'type'."""
 
     pass
 
 
-class InvalidRuleFormatException(LibsastException):
+class InvalidRuleFormatError(LibsastError):
     """Pattern Matcher rule file is invalid."""
 
     pass
 
 
-class PatternKeyMissingError(LibsastException):
+class PatternKeyMissingError(LibsastError):
     """Pattern Matcher rule does not have the key 'pattern'."""
 
     pass
 
 
-class RuleDownloadException(LibsastException):
+class RuleDownloadError(LibsastError):
     """Failed to download rule."""
 
     pass
 
 
-class RuleProcessingException(LibsastException):
+class RuleProcessingError(LibsastError):
     """Failed to download rule."""
 
     pass
 
 
-class MatcherNotFoundException(LibsastException):
+class MatcherNotFoundError(LibsastError):
     """Pattern Matcher not found."""
 
     pass

setup.py

@@ -8,7 +8,7 @@ def get_requires():
     requires = [
         'requests>=2.22.0',
         'pyyaml>=5.3',
-        'semgrep==0.53.0;platform_system!="Windows"',
+        'semgrep==0.80.0;platform_system!="Windows"',
     ]
     return requires
 

tests/unit/test_rules.py

@@ -61,7 +61,7 @@ def test_load_invalid_url():
     files_dir = base_dir / 'assets' / 'files'
     options = {'match_rules': rule_url}
     paths = [files_dir.as_posix()]
-    with pytest.raises(libsast.exceptions.RuleDownloadException):
+    with pytest.raises(libsast.exceptions.RuleDownloadError):
         libsast.Scanner(options, paths).scan()
 
 
@@ -91,7 +91,7 @@ def test_load_file_invalid_type():
     rule_file = base_dir / 'assets' / 'invalid' / 'invalid_type.yaml'
     options = {'match_rules': rule_file.as_posix()}
     paths = [files_dir.as_posix()]
-    with pytest.raises(libsast.exceptions.MatcherNotFoundException):
+    with pytest.raises(libsast.exceptions.MatcherNotFoundError):
         libsast.Scanner(options, paths).scan()
 
 

tox.ini

@@ -8,7 +8,8 @@ deps =
     pytest
 commands = 
     py.test -v --cache-clear tests
-
+setenv =
+    SEMGREP_R2C_INTERNAL_EXPLICIT_SEMGREPIGNORE = ./tests/assets/files/.semgrepignore
 [testenv:lint]
 skip_install = true
 deps =