diff --git a/.github/gnes-hub-github.svg b/.github/gnes-hub-github.svg new file mode 100644 index 00000000..75a51ad0 --- /dev/null +++ b/.github/gnes-hub-github.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/README.md b/README.md index 96f2077c..98949cbc 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Overview • Install • Getting Started • + GNES Hub • Documentation • Tutorial • Contributing • @@ -70,7 +71,7 @@ GNES enables large-scale index and semantic search for **text-to-text**, **image Searching for texts, image or even short-videos? Using Python/C/Java/Go/HTTP as the client? Doesn't matter which content form you have or which language do you use, GNES can handle them all. - When built-in models do not meet your requirments, simply build your own with one Python file and one YAML file. No need to rebuilt GNES framework, as your models will be loaded as plugins and directly rollout online. + When built-in models do not meet your requirments, simply build your own with GNES Hub. Pack your model as a docker container and use it as a plugin. We love to learn the best practice from the community, helping our GNES to achieve the next level of availability, resiliency, performance, and durability. If you have any ideas or suggestions, feel free to contribute. @@ -83,6 +84,22 @@ GNES enables large-scale index and semantic search for **text-to-text**, **image

+### GNES Hub +
+ + + + + +
+ component overview + +

GNES Hub ship AI/ML models as Docker containers and use Docker containers as plugins. It offers a clean and sustainable way to port external algorithms (with the dependencies) into the [GNES framework](https://github.com/gnes-ai/gnes).

+

GNES Hub is hosted on the Docker Hub.

+
+
+ +

Install GNES

There are two ways to get GNES, either as a Docker image or as a PyPi package. **For cloud users, we highly recommend using GNES via Docker**. diff --git a/gnes/base/__init__.py b/gnes/base/__init__.py index fcfcdca8..51ba279c 100644 --- a/gnes/base/__init__.py +++ b/gnes/base/__init__.py @@ -36,9 +36,9 @@ def register_all_class(cls2file_map: Dict, module_name: str): for k, v in cls2file_map.items(): try: getattr(importlib.import_module('gnes.%s.%s' % (module_name, v)), k) - except ImportError: - # print(e) - pass + except ImportError as ex: + default_logger = set_logger('GNES') + default_logger.warning('fail to register %s, due to %s' % (k, ex)) load_contrib_module() @@ -112,7 +112,7 @@ def register_class(cls): reg_cls_set.add(cls.__name__) setattr(cls, '_registered_class', reg_cls_set) - yaml.register_class(cls) + yaml.register_class(cls) return cls @staticmethod diff --git a/tests/contrib/fake_faiss.py b/tests/contrib/fake_faiss.py new file mode 100644 index 00000000..06108c24 --- /dev/null +++ b/tests/contrib/fake_faiss.py @@ -0,0 +1,11 @@ +from gnes.indexer.base import BaseVectorIndexer + + +class FaissIndexer(BaseVectorIndexer): + + def __init__(self, bar: int, *args, **kwargs): + super().__init__(*args, **kwargs) + self.is_trained = True + self.bar = bar + self.logger.info('look at me, I override the original GNES faiss indexer') + diff --git a/tests/contrib/fake_faiss.yml b/tests/contrib/fake_faiss.yml new file mode 100644 index 00000000..194616fa --- /dev/null +++ b/tests/contrib/fake_faiss.yml @@ -0,0 +1,5 @@ +!FaissIndexer +parameters: + bar: 531 +gnes_config: + name: foo_contrib_encoder \ No newline at end of file diff --git a/tests/contrib/fake_faiss2.py b/tests/contrib/fake_faiss2.py new file mode 100644 index 00000000..68003011 --- /dev/null +++ b/tests/contrib/fake_faiss2.py @@ -0,0 +1,11 @@ +from gnes.indexer.base import BaseVectorIndexer + + +class FaissIndexer(BaseVectorIndexer): + + def __init__(self, bar: int, *args, **kwargs): + super().__init__(*args, **kwargs) + self.is_trained = True + self.bar = bar + self.logger.info('look at me, I override the overrided faiss indexer!!!') + diff --git a/tests/test_service_mgr.py b/tests/test_service_mgr.py index 9843adbd..5c65926c 100644 --- a/tests/test_service_mgr.py +++ b/tests/test_service_mgr.py @@ -3,10 +3,11 @@ import grpc -from gnes.cli.parser import set_router_parser, set_frontend_parser, set_encoder_parser +from gnes.cli.parser import set_router_parser, set_frontend_parser, set_encoder_parser, set_indexer_parser from gnes.proto import gnes_pb2_grpc, RequestGenerator from gnes.service.base import ServiceManager, SocketType, ParallelType from gnes.service.frontend import FrontendService +from gnes.service.indexer import IndexerService from gnes.service.router import RouterService @@ -85,6 +86,29 @@ def test_external_module(self): self.assertTrue(os.path.exists('foo_contrib_encoder.bin')) os.remove('foo_contrib_encoder.bin') + def test_override_module(self): + args = set_indexer_parser().parse_args([ + '--yaml_path', os.path.join(self.dir_path, 'contrib', 'fake_faiss.yml'), + '--py_path', os.path.join(self.dir_path, 'contrib', 'fake_faiss.py'), + ]) + + with ServiceManager(IndexerService, args): + pass + self.assertTrue(os.path.exists('foo_contrib_encoder.bin')) + os.remove('foo_contrib_encoder.bin') + + def test_override_twice_module(self): + args = set_indexer_parser().parse_args([ + '--yaml_path', os.path.join(self.dir_path, 'contrib', 'fake_faiss.yml'), + '--py_path', os.path.join(self.dir_path, 'contrib', 'fake_faiss.py'), + os.path.join(self.dir_path, 'contrib', 'fake_faiss2.py') + ]) + + with ServiceManager(IndexerService, args): + pass + self.assertTrue(os.path.exists('foo_contrib_encoder.bin')) + os.remove('foo_contrib_encoder.bin') + def test_grpc_with_pub(self): self._test_grpc_multiple_pub('thread', 1) self._test_grpc_multiple_pub('process', 1)