diff --git a/setup.py b/setup.py
index ae19c361..4030ce71 100644
--- a/setup.py
+++ b/setup.py
@@ -53,8 +53,7 @@
     'grpcio',
     'ruamel.yaml>=0.15.89',
     'pyzmq>=17.1.0',
-    'aiohttp',
-    'sklearn']
+    'aiohttp']
 
 # using pip install gnes[xx] is depreciated
 # extras_dep is kept for legacy issue, will be removed soon
@@ -73,7 +72,7 @@
     'transformers': ['pytorch-transformers'],
     'onnx': ['onnxruntime'],
     'audio': ['librosa>=0.7.0'],
-    'scipy': ['scipy'],
+    'scipy': ['scipy', 'sklearn'],
     'flask': ['flask']
 }