Merge pull request #38 from gnes-ai/fix-composer-release

 fix(compose): accept parser argument only

README.md

@@ -60,8 +60,38 @@ GNES enables large-scale index and semantic search for text-to-text, image-to-im
 <img src=".github/gnes-component-overview.svg" alt="component overview">
 </p>
 
+<h2 align="center">Install GNES</h2>
 
-<h2 align="center">Getting Started</h2>
+There are two ways to get GNES, either as a docker image or as a PyPi package. 
+For cloud users, we highly recommend using GNES as a docker image. 
+
+## Run GNES as a Docker Image
+
+#### via [Docker cloud](https://cloud.docker.com/u/gnes/repository/list)
+
+```bash
+docker pull gnes/gnes:latest
+docker run gnes/gnes:latest --help
+```
+
+#### via Tencent Container service
+
+We also provide a public mirror hosted on Tencent Cloud, from which Chinese mainland users can pull the image faster.
+
+```bash
+docker login --username=xxx ccr.ccs.tencentyun.com  # login to Tencent Cloud so that we can pull from it
+docker pull ccr.ccs.tencentyun.com/gnes/gnes:latest
+docker run ccr.ccs.tencentyun.com/gnes/gnes:latest --help
+```
+
+> You may pull and run different versions by changing the `latest` to a version tag, e.g. `v0.0.24`.
+
+## Install via `pip`
+
+Install
+
+
+<h2 align="center">Quick Start</h2>
 
 As a cloud-native application, GNES requires an **orchestration engine** to coordinate all micro-services. Currently, we support Kubernetes, Docker Swarm and a built-in solution.  Click on one of the icons below to get started.
 

gnes/base/__init__.py

@@ -26,20 +26,22 @@
 
 import ruamel.yaml.constructor
 
-from ..helper import set_logger, profiling, yaml, parse_arg
+from ..helper import set_logger, profiling, yaml, parse_arg, load_contrib_module
 
 __all__ = ['TrainableBase']
 
 T = TypeVar('T', bound='TrainableBase')
 
 
+
 def register_all_class(cls2file_map: Dict, module_name: str):
     import importlib
     for k, v in cls2file_map.items():
         try:
             getattr(importlib.import_module('gnes.%s.%s' % (module_name, v)), k)
         except ImportError:
             pass
+    load_contrib_module()
 
 
 def import_class_by_str(name: str):
@@ -339,3 +341,6 @@ def _dump_instance_to_yaml(data):
         if p:
             r['gnes_config'] = p
         return r
+
+
+

gnes/cli/api.py

@@ -35,8 +35,13 @@ def route(args):
 
 
 def compose(args):
-    from ..composer.base import YamlGraph
-    YamlGraph(args).build_all()
+    from ..composer.base import YamlComposer
+    from ..composer.flask import YamlComposerFlask
+
+    if args.flask:
+        YamlComposerFlask(args).run()
+    else:
+        YamlComposer(args).build_all()
 
 
 def frontend(args):

gnes/cli/parser.py

@@ -18,22 +18,27 @@
 
 import argparse
 
-IDX_PORT_DELTA = 2
-
 
 def set_base_parser():
     from .. import __version__
+    from termcolor import colored
     # create the top-level parser
     parser = argparse.ArgumentParser(
-        description='GNES v%s: Generic Neural Elastic Search '
-                    'is an end-to-end solution for semantic text search' % __version__)
+        description='%s, a cloud-native semantic search system '
+                    'based on deep neural network. '
+                    'It enables large-scale index and semantic search for text-to-text, image-to-image, '
+                    'video-to-video and any content form. Visit %s for tutorials and documentations.' % (
+                        colored('GNES v%s: Generic Neural Elastic Search' % __version__, 'green'),
+                        colored('https://gnes.ai', 'cyan', attrs=['underline'])))
     parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)
     parser.add_argument('--verbose', action='store_true', default=False,
                         help='turn on detailed logging for debug')
     return parser
 
 
 def set_composer_parser(parser=None):
+    from pkg_resources import resource_stream
+
     if not parser:
         parser = set_base_parser()
     parser.add_argument('--port',
@@ -45,7 +50,8 @@ def set_composer_parser(parser=None):
                         default='GNES instance',
                         help='name of the instance')
     parser.add_argument('--yaml_path', type=argparse.FileType('r'),
-                        required=True,
+                        default=resource_stream(
+                            'gnes', '/'.join(('resources', 'config', 'compose', 'default.yml'))),
                         help='yaml config of the service')
     parser.add_argument('--html_path', type=argparse.FileType('w', encoding='utf8'),
                         default='./gnes-board.html',
@@ -69,6 +75,19 @@ def set_composer_parser(parser=None):
     return parser
 
 
+def set_composer_flask_parser(parser=None):
+    if not parser:
+        parser = set_base_parser()
+    set_composer_parser(parser)
+    parser.add_argument('--flask', action='store_true', default=False,
+                        help='using Flask to serve GNES composer in interactive mode')
+    parser.add_argument('--cors', type=str, default='*',
+                        help='setting "Access-Control-Allow-Origin" for HTTP requests')
+    parser.add_argument('--http_port', type=int, default=8080,
+                        help='server port for receiving HTTP requests')
+    return parser
+
+
 def set_service_parser(parser=None):
     from ..service.base import SocketType, BaseService
     if not parser:
@@ -166,7 +185,7 @@ def set_indexer_service_parser(parser=None):
 
     # encoder's port_out is indexer's port_in
     parser.set_defaults(port_in=parser.get_default('port_out'),
-                        port_out=parser.get_default('port_out') + IDX_PORT_DELTA,
+                        port_out=parser.get_default('port_out') + 2,
                         socket_in=SocketType.PULL_CONNECT,
                         socket_out=SocketType.PUB_BIND)
     return parser
@@ -244,7 +263,9 @@ def set_http_service_parser(parser=None):
 def get_main_parser():
     # create the top-level parser
     parser = set_base_parser()
-    sp = parser.add_subparsers(dest='cli')
+    sp = parser.add_subparsers(dest='cli', title='GNES sub-commands',
+                               description='use "gnes [sub-command] --help" '
+                                           'to get detailed information about each sub-command')
 
     set_grpc_frontend_parser(sp.add_parser('frontend', help='start a grpc frontend service'))
     set_indexer_service_parser(sp.add_parser('index', help='start an indexer service'))
@@ -253,5 +274,5 @@ def get_main_parser():
     set_preprocessor_service_parser(sp.add_parser('preprocess', help='start a preprocessor service'))
     set_http_service_parser(sp.add_parser('client_http', help='start a http service'))
     set_cli_client_parser(sp.add_parser('client_cli', help='start a grpc client'))
-    set_composer_parser(sp.add_parser('compose', help='start a GNES composer to simplify config generation'))
+    set_composer_flask_parser(sp.add_parser('compose', help='start a GNES Board and visualize YAML config'))
     return parser

gnes/client/cli.py

@@ -1,3 +1,19 @@
+#  Tencent is pleased to support the open source community by making GNES available.
+#
+#  Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
 import zipfile
 
 import grpc

gnes/client/http.py

@@ -21,7 +21,6 @@
 from concurrent.futures import ThreadPoolExecutor
 
 import grpc
-from aiohttp import web
 from google.protobuf.json_format import MessageToJson
 
 from ..helper import set_logger
@@ -34,6 +33,7 @@ def __init__(self, args=None):
         self.logger = set_logger(self.__class__.__name__, self.args.verbose)
 
     def start(self):
+        from aiohttp import web
         loop = asyncio.get_event_loop()
         executor = ThreadPoolExecutor(max_workers=self.args.max_workers)