Merge pull request #15 from gnes-ai/add-buildin-orchestration

feat(composer): add composer and mermaid renderer

gnes/base/__init__.py

@@ -49,7 +49,7 @@ def _import(module_name, class_name):
         if class_name in cls2file:
             return getattr(importlib.import_module('gnes.%s.%s' % (module_name, cls2file[class_name])), class_name)
 
-    search_modules = ['encoder', 'indexer', 'preprocessor', 'router', 'module']
+    search_modules = ['encoder', 'indexer', 'preprocessor', 'router']
 
     for m in search_modules:
         r = _import(m, name)
@@ -245,7 +245,8 @@ def load_yaml(cls: Type[T], filename: Union[str, TextIO]) -> T:
             with open(filename) as fp:
                 return yaml.load(fp)
         else:
-            return yaml.load(filename)
+            with filename:
+                return yaml.load(filename)
 
     @staticmethod
     @profiling

gnes/cli/api.py

@@ -34,6 +34,11 @@ def route(args):
         es.join()
 
 
+def compose(args):
+    from ..composer.base import YamlGraph
+    YamlGraph(args).build_all()
+
+
 def frontend(args):
     from ..service.grpc import GRPCFrontend
     import threading

gnes/cli/parser.py

@@ -33,6 +33,30 @@ def set_base_parser():
     return parser
 
 
+def set_composer_parser(parser=None):
+    if not parser:
+        parser = set_base_parser()
+    parser.add_argument('--port',
+                        type=int,
+                        default=8800,
+                        help='host port of the grpc service')
+    parser.add_argument('--name',
+                        type=str,
+                        default='GNES instance',
+                        help='name of the instance')
+    parser.add_argument('--yaml_path', type=argparse.FileType('r'),
+                        help='yaml config of the service')
+    parser.add_argument('--html_path', type=argparse.FileType('w', encoding='utf8'),
+                        help='output path of the HTML file, will contain all possible generations')
+    parser.add_argument('--shell_path', type=argparse.FileType('w', encoding='utf8'),
+                        help='output path of the shell-based starting script')
+    parser.add_argument('--swarm_path', type=argparse.FileType('w', encoding='utf8'),
+                        help='output path of the docker-compose file for Docker Swarm')
+    parser.add_argument('--k8s_path', type=argparse.FileType('w', encoding='utf8'),
+                        help='output path of the docker-compose file for Docker Swarm')
+    return parser
+
+
 def set_service_parser(parser=None):
     from ..service.base import SocketType, BaseService
     if not parser:
@@ -127,8 +151,6 @@ def set_indexer_service_parser(parser=None):
     if not parser:
         parser = set_base_parser()
     set_loadable_service_parser(parser)
-    parser.add_argument('--top_k', type=int, default=10,
-                        help='number of top results to retrieve')
     parser.set_defaults(yaml_path=pkg_resources.resource_stream(
         'gnes', '/'.join(('resources', 'config', 'indexer', 'default.yml'))))
 
@@ -148,8 +170,8 @@ def _set_grpc_parser(parser=None):
                         default='0.0.0.0',
                         help='host address of the grpc service')
     parser.add_argument('--grpc_port',
-                        type=str,
-                        default='8800',
+                        type=int,
+                        default=8800,
                         help='host port of the grpc service')
     return parser
 
@@ -168,7 +190,7 @@ def set_grpc_frontend_parser(parser=None):
     return parser
 
 
-def set_grpc_client_parser(parser=None):
+def set_cli_client_parser(parser=None):
     import sys
     if not parser:
         parser = set_base_parser()
@@ -220,5 +242,6 @@ def get_main_parser():
     set_router_service_parser(sp.add_parser('route', help='start a router service'))
     set_preprocessor_service_parser(sp.add_parser('preprocess', help='start a preprocessor service'))
     set_http_service_parser(sp.add_parser('client_http', help='start a http service'))
-    set_grpc_client_parser(sp.add_parser('client_cli', help='start a grpc client'))
+    set_cli_client_parser(sp.add_parser('client_cli', help='start a grpc client'))
+    set_composer_parser(sp.add_parser('compose', help='start a GNES composer to simplify config generation'))
     return parser