feat(base): later import module now override the earlier ones

README.md

@@ -30,6 +30,7 @@
   <a href="#overview">Overview</a> •
   <a href="#install-gnes">Install</a> •
   <a href="#getting-started">Getting Started</a> •
+  <a href="https://github.com/gnes-ai/hub">GNES Hub</a> •
   <a href="#documentation">Documentation</a> •
   <a href="#tutorial">Tutorial</a> •
   <a href="#contributing">Contributing</a> •
@@ -70,7 +71,7 @@ GNES enables large-scale index and semantic search for **text-to-text**, **image
     </tr>
     <tr>
       <td width="33%"><sub>Searching for texts, image or even short-videos? Using Python/C/Java/Go/HTTP as the client? Doesn't matter which content form you have or which language do you use, GNES can handle them all. </sub></td>
-      <td width="33%"><sub>When built-in models do not meet your requirments, simply build your own with <i>one Python file and one YAML file</i>. No need to rebuilt GNES framework, as your models will be loaded as plugins and directly rollout online.</sub></td>
+      <td width="33%"><sub>When built-in models do not meet your requirments, simply build your own with <a href="https://github.com/gnes-ai/hub">GNES Hub</a>. Pack your model as a docker container and use it as a plugin.</sub></td>
       <td width="33%"><sub>We love to learn the best practice from the community, helping our GNES to achieve the next level of availability, resiliency, performance, and durability. If you have any ideas or suggestions, feel free to contribute.</sub></td>
     </tr>
 </table>
@@ -83,6 +84,22 @@ GNES enables large-scale index and semantic search for **text-to-text**, **image
 </a>
 </p>
 
+### GNES Hub
+<center>
+<table>
+  <tr>
+  <td><a href="https://github.com/gnes-ai/hub">
+      <img src=".github/gnes-hub-github.svg" width="33%" alt="component overview">
+      </a></td>
+  <td>
+  <p>GNES Hub ship AI/ML models as Docker containers and use Docker containers as plugins. It offers a clean and sustainable way to port external algorithms (with the dependencies) into the [GNES framework](https://github.com/gnes-ai/gnes). </p>
+  <p>GNES Hub is hosted on the Docker Hub.</p>
+</td>
+  </tr>
+</table>
+</center>
+
+
 <h2 align="center">Install GNES</h2>
 
 There are two ways to get GNES, either as a Docker image or as a PyPi package. **For cloud users, we highly recommend using GNES via Docker**. 

gnes/base/__init__.py

@@ -36,9 +36,9 @@ def register_all_class(cls2file_map: Dict, module_name: str):
     for k, v in cls2file_map.items():
         try:
             getattr(importlib.import_module('gnes.%s.%s' % (module_name, v)), k)
-        except ImportError:
-            # print(e)
-            pass
+        except ImportError as ex:
+            default_logger = set_logger('GNES')
+            default_logger.warning('fail to register %s, due to %s' % (k, ex))
     load_contrib_module()
 
 
@@ -112,7 +112,7 @@ def register_class(cls):
 
             reg_cls_set.add(cls.__name__)
             setattr(cls, '_registered_class', reg_cls_set)
-            yaml.register_class(cls)
+        yaml.register_class(cls)
         return cls
 
     @staticmethod

tests/contrib/fake_faiss.py

@@ -0,0 +1,11 @@
+from gnes.indexer.base import BaseVectorIndexer
+
+
+class FaissIndexer(BaseVectorIndexer):
+
+    def __init__(self, bar: int, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.is_trained = True
+        self.bar = bar
+        self.logger.info('look at me, I override the original GNES faiss indexer')
+

tests/contrib/fake_faiss.yml

@@ -0,0 +1,5 @@
+!FaissIndexer
+parameters:
+  bar: 531
+gnes_config:
+  name: foo_contrib_encoder

tests/contrib/fake_faiss2.py

@@ -0,0 +1,11 @@
+from gnes.indexer.base import BaseVectorIndexer
+
+
+class FaissIndexer(BaseVectorIndexer):
+
+    def __init__(self, bar: int, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.is_trained = True
+        self.bar = bar
+        self.logger.info('look at me, I override the overrided faiss indexer!!!')
+

tests/test_service_mgr.py

@@ -3,10 +3,11 @@
 
 import grpc
 
-from gnes.cli.parser import set_router_parser, set_frontend_parser, set_encoder_parser
+from gnes.cli.parser import set_router_parser, set_frontend_parser, set_encoder_parser, set_indexer_parser
 from gnes.proto import gnes_pb2_grpc, RequestGenerator
 from gnes.service.base import ServiceManager, SocketType, ParallelType
 from gnes.service.frontend import FrontendService
+from gnes.service.indexer import IndexerService
 from gnes.service.router import RouterService
 
 
@@ -85,6 +86,29 @@ def test_external_module(self):
         self.assertTrue(os.path.exists('foo_contrib_encoder.bin'))
         os.remove('foo_contrib_encoder.bin')
 
+    def test_override_module(self):
+        args = set_indexer_parser().parse_args([
+            '--yaml_path', os.path.join(self.dir_path, 'contrib', 'fake_faiss.yml'),
+            '--py_path', os.path.join(self.dir_path, 'contrib', 'fake_faiss.py'),
+        ])
+
+        with ServiceManager(IndexerService, args):
+            pass
+        self.assertTrue(os.path.exists('foo_contrib_encoder.bin'))
+        os.remove('foo_contrib_encoder.bin')
+
+    def test_override_twice_module(self):
+        args = set_indexer_parser().parse_args([
+            '--yaml_path', os.path.join(self.dir_path, 'contrib', 'fake_faiss.yml'),
+            '--py_path', os.path.join(self.dir_path, 'contrib', 'fake_faiss.py'),
+            os.path.join(self.dir_path, 'contrib', 'fake_faiss2.py')
+        ])
+
+        with ServiceManager(IndexerService, args):
+            pass
+        self.assertTrue(os.path.exists('foo_contrib_encoder.bin'))
+        os.remove('foo_contrib_encoder.bin')
+
     def test_grpc_with_pub(self):
         self._test_grpc_multiple_pub('thread', 1)
         self._test_grpc_multiple_pub('process', 1)