Merge pull request #31 from ajinabraham/60cpus

Fix #30

libsast/__init__.py

@@ -8,7 +8,7 @@
 __title__ = 'libsast'
 __authors__ = 'Ajin Abraham'
 __copyright__ = 'Copyright 2020 Ajin Abraham, OpenSecurity'
-__version__ = '2.0.0'
+__version__ = '2.0.1'
 __version_info__ = tuple(int(i) for i in __version__.split('.'))
 __all__ = [
     'Scanner',

libsast/core_matcher/choice_matcher.py

@@ -1,5 +1,7 @@
 # -*- coding: utf_8 -*-
 """Choice Macher."""
+import os
+import sys
 from pathlib import Path
 from multiprocessing import (
     Pool,
@@ -46,7 +48,11 @@ def scan(self, paths: list) -> dict:
             choice_args.append((scan_paths, rule))
 
         # Multiprocess Pool
-        with Pool() as pool:
+        worker_count = os.cpu_count()
+        if sys.platform == 'win32':
+            # Work around https://bugs.python.org/issue26903
+            worker_count = min(worker_count, 61)
+        with Pool(worker_count) as pool:
             results = pool.starmap(
                 self.choice_matcher,
                 choice_args,

libsast/core_matcher/pattern_matcher.py

@@ -1,5 +1,7 @@
 # -*- coding: utf_8 -*-
 """Pattern Macher."""
+import os
+import sys
 from copy import deepcopy
 from operator import itemgetter
 from multiprocessing import (
@@ -48,7 +50,11 @@ def scan(self, paths: list) -> dict:
                 continue
             files_to_scan.add(sfile)
         # Multiprocess Pool
-        with Pool() as pool:
+        worker_count = os.cpu_count()
+        if sys.platform == 'win32':
+            # Work around https://bugs.python.org/issue26903
+            worker_count = min(worker_count, 61)
+        with Pool(worker_count) as pool:
             results = pool.map(
                 self.pattern_matcher,
                 files_to_scan,

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "libsast"
-version = "2.0.0"
+version = "2.0.1"
 description = "A generic SAST library built on top of semgrep and regex"
 keywords = ["libsast", "SAST", "Python SAST", "SAST API", "Regex SAST", "Pattern Matcher"]
 authors = ["Ajin Abraham <[email protected]>"]