From 033a4b9c65bab6f9c392eedd9c4e3d233d00c3be Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Tue, 16 Jul 2019 13:15:26 +0800 Subject: [PATCH 1/9] feat(composer): add composer and mermaid renderer --- gnes/base/__init__.py | 5 +- gnes/cli/parser.py | 29 ++- gnes/composer/__init__.py | 0 gnes/composer/base.py | 351 ++++++++++++++++++++++++++ gnes/encoder/text/bert.py | 2 - gnes/resources/static/gnes-graph.html | 24 ++ tests/test_compose.py | 35 +++ tests/yaml/topology1.yml | 4 + tests/yaml/topology2.yml | 6 + tests/yaml/topology3.yml | 7 + tests/yaml/topology4.yml | 9 + tests/yaml/topology5.yml | 10 + tests/yaml/topology6.yml | 13 + 13 files changed, 485 insertions(+), 10 deletions(-) create mode 100644 gnes/composer/__init__.py create mode 100644 gnes/composer/base.py create mode 100644 gnes/resources/static/gnes-graph.html create mode 100644 tests/test_compose.py create mode 100644 tests/yaml/topology1.yml create mode 100644 tests/yaml/topology2.yml create mode 100644 tests/yaml/topology3.yml create mode 100644 tests/yaml/topology4.yml create mode 100644 tests/yaml/topology5.yml create mode 100644 tests/yaml/topology6.yml diff --git a/gnes/base/__init__.py b/gnes/base/__init__.py index 91040ca2..9fc44f65 100644 --- a/gnes/base/__init__.py +++ b/gnes/base/__init__.py @@ -49,7 +49,7 @@ def _import(module_name, class_name): if class_name in cls2file: return getattr(importlib.import_module('gnes.%s.%s' % (module_name, cls2file[class_name])), class_name) - search_modules = ['encoder', 'indexer', 'preprocessor', 'router', 'module'] + search_modules = ['encoder', 'indexer', 'preprocessor', 'router'] for m in search_modules: r = _import(m, name) @@ -245,7 +245,8 @@ def load_yaml(cls: Type[T], filename: Union[str, TextIO]) -> T: with open(filename) as fp: return yaml.load(fp) else: - return yaml.load(filename) + with filename: + return yaml.load(filename) @staticmethod @profiling diff --git a/gnes/cli/parser.py b/gnes/cli/parser.py index 0a37c942..083410a9 100644 --- a/gnes/cli/parser.py +++ b/gnes/cli/parser.py @@ -33,6 +33,24 @@ def set_base_parser(): return parser +def set_composer_parser(parser=None): + if not parser: + parser = set_base_parser() + parser.add_argument('--port', + type=int, + default=8800, + help='host port of the grpc service') + parser.add_argument('--name', + type=str, + default='GNES instance', + help='name of the instance') + parser.add_argument('--yaml_path', type=argparse.FileType('r'), + help='yaml config of the service') + parser.add_argument('--html_path', type=argparse.FileType('w', encoding='utf8'), + help='render the network graph in HTML with mermaid.js') + return parser + + def set_service_parser(parser=None): from ..service.base import SocketType, BaseService if not parser: @@ -127,8 +145,6 @@ def set_indexer_service_parser(parser=None): if not parser: parser = set_base_parser() set_loadable_service_parser(parser) - parser.add_argument('--top_k', type=int, default=10, - help='number of top results to retrieve') parser.set_defaults(yaml_path=pkg_resources.resource_stream( 'gnes', '/'.join(('resources', 'config', 'indexer', 'default.yml')))) @@ -148,8 +164,8 @@ def _set_grpc_parser(parser=None): default='0.0.0.0', help='host address of the grpc service') parser.add_argument('--grpc_port', - type=str, - default='8800', + type=int, + default=8800, help='host port of the grpc service') return parser @@ -164,7 +180,7 @@ def set_grpc_frontend_parser(parser=None): return parser -def set_grpc_client_parser(parser=None): +def set_cli_client_parser(parser=None): import sys if not parser: parser = set_base_parser() @@ -216,5 +232,6 @@ def get_main_parser(): set_router_service_parser(sp.add_parser('route', help='start a router service')) set_preprocessor_service_parser(sp.add_parser('preprocess', help='start a preprocessor service')) set_http_service_parser(sp.add_parser('client_http', help='start a http service')) - set_grpc_client_parser(sp.add_parser('client_cli', help='start a grpc client')) + set_cli_client_parser(sp.add_parser('client_cli', help='start a grpc client')) + set_composer_parser(sp.add_parser('compose', help='start a GNES composer')) return parser diff --git a/gnes/composer/__init__.py b/gnes/composer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gnes/composer/base.py b/gnes/composer/base.py new file mode 100644 index 00000000..6c928ff6 --- /dev/null +++ b/gnes/composer/base.py @@ -0,0 +1,351 @@ +import copy +import random +from typing import Dict, List + +import pkg_resources +from ruamel.yaml import YAML +from ruamel.yaml.comments import CommentedMap + +from ..helper import set_logger +from ..service.base import SocketType + +_yaml = YAML() + + +class YamlGraph: + comp2file = { + 'Encoder': 'service.encoder.EncoderService', + 'Router': 'service.router.RouterService', + 'Indexer': 'service.indexer.IndexerService', + 'Frontend': 'service.grpc.GRPCFrontend', + 'Preprocessor': 'service.preprocessor.PreprocessorService' + } + + class Layer: + default_values = { + 'name': None, + 'yaml_path': None, + 'dump_path': None, + 'replicas': 1, + 'income': 'pull' + } + + def __init__(self, id: int = 0): + self.id = id + self.components = [] + + @staticmethod + def get_value(comp: Dict, key: str): + return comp.get(key, YamlGraph.Layer.default_values[key]) + + @property + def is_homogenous(self): + return len(self.components) == 1 + + @property + def is_single_component(self): + return self.is_homogenous and self.get_value(self.components[0], 'replicas') == 1 + + @property + def is_homo_multi_component(self): + return self.is_homogenous and not self.is_single_component + + @property + def is_heto_single_component(self): + return not self.is_homogenous and all(self.get_value(c, 'replicas') == 1 for c in self.components) + + @property + def get_component_name(self, unique: bool = False): + r = [c['name'] for c in self.components] + if unique: + r = list(set(r)) + return r + + def append(self, comp): + self.components.append(comp) + + def __repr__(self): + return str(self.components) + + def __init__(self, args): + + self._layers = [] # type: List['YamlGraph.Layer'] + self.logger = set_logger(self.__class__.__name__) + tmp = _yaml.load(args.yaml_path) + self.name = tmp.get('name', args.name) + self.port = tmp.get('port', args.port) + self.args = args + self._num_layer = 0 + + if 'component' in tmp: + self.add_layer() + self.add_comp(CommentedMap({'name': 'Frontend'})) + for comp in tmp['component']: + self.add_layer() + if isinstance(comp, list): + [self.add_comp(c) for c in comp if self.add_comp(c)] + elif self.check_fields(comp): + self.add_comp(comp) + else: + raise ValueError(comp) + else: + self.logger.error('yaml file defines an empty graph! no "component" field exists!') + + def check_fields(self, comp: Dict) -> bool: + if 'name' not in comp: + raise AttributeError('a component must have a name (choices: %s)' % ', '.join(self.comp2file.keys())) + if comp['name'] not in self.comp2file: + raise AttributeError( + 'a component must be one of: %s, but given %s' % (', '.join(self.comp2file.keys()), comp['name'])) + if 'yaml_path' not in comp and 'dump_path' not in comp: + self.logger.warning( + 'you did not specify "yaml_path" and "dump_path", ' + 'will use a default config and would probably result in an empty model') + for k in comp: + if k not in self.Layer.default_values: + self.logger.warning('your yaml contains an unrecognized key named "%s"' % k) + return True + + def add_layer(self, layer: 'Layer' = None) -> None: + self._layers.append(copy.deepcopy(layer) or self.Layer(id=self._num_layer)) + self._num_layer += 1 + + def add_comp(self, comp: Dict) -> None: + self._layers[-1].append(comp) + + def build_layers(self) -> List['YamlGraph.Layer']: + all_layers = [] # type: List['YamlGraph.Layer'] + for idx, layer in enumerate(self._layers[1:] + [self._layers[0]], 1): + last_layer = self._layers[idx - 1] + for l in self._add_router(last_layer, layer): + all_layers.append(copy.deepcopy(l)) + # # add frontend + # for l in self._add_router(all_layers[-1], all_layers[0]): + # all_layers.append(copy.deepcopy(l)) + all_layers[0] = copy.deepcopy(self._layers[0]) + return all_layers + + @staticmethod + def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True): + mermaid_graph = [] + for l_idx, layer in enumerate(all_layers[1:] + [all_layers[0]], 1): + last_layer = all_layers[l_idx - 1] + + for c_idx, c in enumerate(last_layer.components): + # if len(last_layer.components) > 1: + # self.mermaid_graph.append('\tsubgraph %s%d' % (c['name'], c_idx)) + for j in range(YamlGraph.Layer.get_value(c, 'replicas')): + for c1_idx, c1 in enumerate(layer.components): + if c1['port_in'] == c['port_out']: + p = '((%s%s))' if c['name'] == 'Router' else '[%s%s]' + p1 = '((%s%s))' if c1['name'] == 'Router' else '[%s%s]' + for j1 in range(YamlGraph.Layer.get_value(c1, 'replicas')): + id, id1 = '%s%s%s' % (last_layer.id, c_idx, j), '%s%s%s' % (layer.id, c1_idx, j1) + conn_type = ( + c['socket_out'].split('_')[0] + '/' + c1['socket_in'].split('_')[0]).lower() + s_id = '%s%s' % (c_idx if len(last_layer.components) > 1 else '', + j if YamlGraph.Layer.get_value(c, 'replicas') > 1 else '') + s1_id = '%s%s' % (c1_idx if len(layer.components) > 1 else '', + j1 if YamlGraph.Layer.get_value(c1, 'replicas') > 1 else '') + mermaid_graph.append( + '\t%s%s%s-- %s -->%s%s%s' % ( + c['name'], id, p % (c['name'], s_id), conn_type, c1['name'], id1, + p1 % (c1['name'], s1_id))) + # if len(last_layer.components) > 1: + # self.mermaid_graph.append('\tend') + + style = 'style Frontend000 fill:#ffd889ff,stroke:#f66,stroke-width:2px,stroke-dasharray:5' + mermaid_str = '\n'.join(['graph TD' if show_topdown else 'graph LR'] + [style] + mermaid_graph) + return mermaid_str + + def build_html(self, mermaid_str: str): + if not self.args.html_path: + print(mermaid_str) + else: + r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'static', 'gnes-graph.html'))) + with r, self.args.html_path as fp: + html = r.read().decode().replace('@mermaid-graph', mermaid_str) + fp.write(html) + + @staticmethod + def _get_random_port(min_port: int = 49152, max_port: int = 65536) -> str: + return str(random.randrange(min_port, max_port)) + + def _add_router(self, last_layer: 'YamlGraph.Layer', layer: 'YamlGraph.Layer') -> List['YamlGraph.Layer']: + def rule1(): + # a shortcut fn: push connect the last and current + last_layer.components[0]['socket_out'] = str(SocketType.PUSH_BIND) + layer.components[0]['socket_in'] = str(SocketType.PULL_CONNECT) + + def rule2(): + # a shortcut fn: pub connect the last and the current + last_layer.components[0]['socket_out'] = str(SocketType.PUB_BIND) + layer.components[0]['socket_in'] = str(SocketType.SUB_CONNECT) + + def rule3(): + # a shortcut fn: (N)-2-(N) with push pull connection + router_layer = YamlGraph.Layer(id=self._num_layer) + self._num_layer += 1 + last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT) + r = CommentedMap({'name': 'Router', + 'yaml_path': None, + 'socket_in': str(SocketType.PULL_BIND), + 'socket_out': str(SocketType.PUSH_BIND), + 'port_in': last_layer.components[0]['port_out'], + 'port_out': self._get_random_port()}) + layer.components[0]['socket_in'] = str(SocketType.PULL_CONNECT) + layer.components[0]['port_in'] = r['port_out'] + router_layer.append(r) + router_layers.append(router_layer) + + def rule4(): + # a shortcut fn: (N)-to-(1)&(1)&(1) + last_layer.components[0]['socket_out'] = str(SocketType.PUB_BIND) + for c in layer.components: + c['socket_in'] = str(SocketType.SUB_CONNECT) + + def rule5(): + # a shortcut fn: based on c3(): (N)-2-(N) with pub sub connection + rule3() + router_layers[0].components[0]['socket_out'] = str(SocketType.PUB_BIND) + for c in layer.components: + c['socket_in'] = str(SocketType.SUB_CONNECT) + + def rule6(): + last_layer.components[0]['socket_out'] = str(SocketType.PUB_BIND) + router_layer = YamlGraph.Layer(id=self._num_layer) + self._num_layer += 1 + for c in layer.components: + r = CommentedMap({'name': 'Router', + 'yaml_path': None, + 'socket_in': str(SocketType.SUB_CONNECT), + 'socket_out': str(SocketType.PUSH_BIND), + 'port_in': last_layer.components[0]['port_out'], + 'port_out': self._get_random_port()}) + c['socket_in'] = str(SocketType.PULL_CONNECT) + c['port_in'] = r['port_out'] + router_layer.append(r) + router_layers.append(router_layer) + + def rule7(): + last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT) + + router_layer = YamlGraph.Layer(id=self._num_layer) + self._num_layer += 1 + r0 = CommentedMap({'name': 'Router', + 'yaml_path': None, + 'socket_in': str(SocketType.PULL_BIND), + 'socket_out': str(SocketType.PUB_BIND), + 'port_in': self._get_random_port(), + 'port_out': self._get_random_port()}) + router_layer.append(r0) + router_layers.append(router_layer) + last_layer.components[0]['port_out'] = r0['port_in'] + + router_layer = YamlGraph.Layer(id=self._num_layer) + self._num_layer += 1 + for c in layer.components: + r = CommentedMap({'name': 'Router', + 'yaml_path': None, + 'socket_in': str(SocketType.SUB_CONNECT), + 'socket_out': str(SocketType.PUSH_BIND), + 'port_in': r0['port_out'], + 'port_out': self._get_random_port()}) + c['socket_in'] = str(SocketType.PULL_CONNECT) + c['port_in'] = r['port_out'] + router_layer.append(r) + router_layers.append(router_layer) + + def rule8(): + last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT) + router_layer = YamlGraph.Layer(id=self._num_layer) + self._num_layer += 1 + r = CommentedMap({'name': 'Router', + 'yaml_path': None, + 'socket_in': str(SocketType.PULL_BIND), + 'socket_out': str(SocketType.PUSH_BIND), + 'port_in': self._get_random_port(), + 'port_out': self._get_random_port()}) + router_layer.append(r) + + for c in last_layer.components: + c['socket_out'] = str(SocketType.PUSH_CONNECT) + c['port_out'] = self._get_random_port() + r_c = CommentedMap({'name': 'Router', + 'yaml_path': None, + 'socket_in': str(SocketType.PULL_BIND), + 'socket_out': str(SocketType.PUSH_BIND), + 'port_in': c['port_out'], + 'port_out': r['port_in']}) + router_layer.append(r_c) + + for c in layer.components: + c['socket_in'] = str(SocketType.PULL_CONNECT) + c['port_in'] = r['port_out'] + router_layers.append(router_layer) + + router_layer = YamlGraph.Layer(id=self._num_layer) + self._num_layer += 1 + router_layer.append(r) + router_layers.append(router_layer) + + def rule9(): + # a shortcut fn: push connect the last and current + last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT) + layer.components[0]['socket_in'] = str(SocketType.PULL_BIND) + + router_layers = [] # type: List['self.Layer'] + # bind the last out to current in + last_layer.components[0]['port_out'] = self._get_random_port() + layer.components[0]['port_in'] = last_layer.components[0]['port_out'] + if last_layer.is_single_component: + # 1-to-? + if layer.is_single_component: + # 1-to-(1) + # no router is needed + rule1() + elif layer.is_homo_multi_component: + # 1-to-(N) + last_income = self.Layer.get_value(last_layer.components[0], 'income') + if last_income == 'pull': + rule1() + elif last_income == 'sub': + rule2() + else: + raise NotImplementedError('replica type: %s is not recognized!' % last_income) + elif layer.is_heto_single_component: + # 1-to-(1)&(1)&(1) + rule4() + else: + # 1-to-(N)&(N)&(N) + rule6() + elif last_layer.is_homo_multi_component: + # (N)-to-? + last_income = self.Layer.get_value(last_layer.components[0], 'income') + + if layer.is_single_component: + if last_income == 'pull': + # (N)-to-1 + rule9() + elif last_income == 'sub': + # (N)-to-1 with a sync barrier + rule3() + else: + raise NotImplementedError('replica type: %s is not recognized!' % last_income) + elif layer.is_homo_multi_component: + # (N)-to-(N) + # need a router anyway + if self.Layer.get_value(layer.components[0], 'income') == 'sub': + rule5() + else: + rule3() + elif layer.is_heto_single_component: + # (N)-to-(1)&(1)&(1) + rule5() + else: + rule7() + elif last_layer.is_heto_single_component: + rule3() + else: + rule8() + return [last_layer, *router_layers] diff --git a/gnes/encoder/text/bert.py b/gnes/encoder/text/bert.py index 4a10abb8..0bb1997f 100644 --- a/gnes/encoder/text/bert.py +++ b/gnes/encoder/text/bert.py @@ -41,7 +41,6 @@ def post_init(self): def encode(self, text: List[str], *args, **kwargs) -> np.ndarray: return self.bc_encoder.encode(text, *args, **kwargs) # type: np.ndarray - def close(self): self.bc_encoder.close() @@ -70,6 +69,5 @@ def post_init(self): self.bert_server.start() self.bert_server.is_ready.wait() - def close(self): self.bert_server.close() diff --git a/gnes/resources/static/gnes-graph.html b/gnes/resources/static/gnes-graph.html new file mode 100644 index 00000000..6ff01dad --- /dev/null +++ b/gnes/resources/static/gnes-graph.html @@ -0,0 +1,24 @@ + + + + + + + GNES graph visualization + + +
+
+ @mermaid-graph +
+
+ + + + \ No newline at end of file diff --git a/tests/test_compose.py b/tests/test_compose.py new file mode 100644 index 00000000..011dc923 --- /dev/null +++ b/tests/test_compose.py @@ -0,0 +1,35 @@ +import os +import unittest + +from gnes.cli.parser import set_composer_parser +from gnes.composer.base import YamlGraph + + +class TestCompose(unittest.TestCase): + def setUp(self): + self.dirname = os.path.dirname(__file__) + self.html_path = os.path.join(self.dirname, 'test.html') + + def test_all(self): + paths = [os.path.join(self.dirname, 'yaml', 'topology%d.yml' % j) for j in range(1, 7)] + b_a = [(3, 3), (4, 4), (4, 5), (4, 7), (4, 7), (4, 9)] + for p, j in zip(paths, b_a): + self._test_topology(p, *j) + + def _test_topology(self, yaml_path: str, num_layer_before: int, num_layer_after: int): + args = set_composer_parser().parse_args([ + '--yaml_path', yaml_path, + '--html_path', self.html_path + ]) + a = YamlGraph(args) + self.assertEqual(len(a._layers), num_layer_before) + r = a.build_layers() + self.assertEqual(len(r), num_layer_after) + for c in r: + print(c) + a.build_html(a.build_mermaid(r)) + os.path.exists(self.html_path) + + # def tearDown(self): + # if os.path.exists(self.html_path): + # os.remove(self.html_path) diff --git a/tests/yaml/topology1.yml b/tests/yaml/topology1.yml new file mode 100644 index 00000000..7544de58 --- /dev/null +++ b/tests/yaml/topology1.yml @@ -0,0 +1,4 @@ +port: 5566 +component: +- name: Preprocessor +- name: Encoder \ No newline at end of file diff --git a/tests/yaml/topology2.yml b/tests/yaml/topology2.yml new file mode 100644 index 00000000..2d142290 --- /dev/null +++ b/tests/yaml/topology2.yml @@ -0,0 +1,6 @@ +port: 5566 +component: +- name: Preprocessor +- name: Encoder + replicas: 2 +- name: Indexer \ No newline at end of file diff --git a/tests/yaml/topology3.yml b/tests/yaml/topology3.yml new file mode 100644 index 00000000..914c0e80 --- /dev/null +++ b/tests/yaml/topology3.yml @@ -0,0 +1,7 @@ +port: 5566 +component: +- name: Preprocessor + replicas: 2 +- name: Encoder + replicas: 3 +- name: Indexer \ No newline at end of file diff --git a/tests/yaml/topology4.yml b/tests/yaml/topology4.yml new file mode 100644 index 00000000..5031d71f --- /dev/null +++ b/tests/yaml/topology4.yml @@ -0,0 +1,9 @@ +port: 5566 +component: +- name: Preprocessor + replicas: 2 +- name: Encoder + replicas: 3 +- name: Indexer + replicas: 4 + income: sub \ No newline at end of file diff --git a/tests/yaml/topology5.yml b/tests/yaml/topology5.yml new file mode 100644 index 00000000..1a842d5e --- /dev/null +++ b/tests/yaml/topology5.yml @@ -0,0 +1,10 @@ +port: 5566 +component: +- name: Preprocessor + replicas: 2 +- name: Encoder +- + - name: Indexer + replicas: 4 + - name: Indexer + replicas: 3 \ No newline at end of file diff --git a/tests/yaml/topology6.yml b/tests/yaml/topology6.yml new file mode 100644 index 00000000..2159bb33 --- /dev/null +++ b/tests/yaml/topology6.yml @@ -0,0 +1,13 @@ +port: 5566 +component: +- name: Preprocessor + replicas: 2 +- name: Encoder + replicas: 3 +- + - name: Indexer + yaml_path: indexer-binary.yml + replicas: 4 + - name: Indexer + yaml_path: indexer-fulltext.yml + replicas: 3 \ No newline at end of file From 08aa30f4c4a4467e25d32f0508fd01a80c7ee7c0 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Tue, 16 Jul 2019 14:36:49 +0800 Subject: [PATCH 2/9] feat(composer): add shell script generator --- gnes/composer/base.py | 36 +++++++++++++++++++++------ gnes/resources/compose/gnes-shell.sh | 7 ++++++ gnes/resources/static/gnes-graph.html | 2 +- tests/test_compose.py | 1 + 4 files changed, 37 insertions(+), 9 deletions(-) create mode 100755 gnes/resources/compose/gnes-shell.sh diff --git a/gnes/composer/base.py b/gnes/composer/base.py index 6c928ff6..c599d30f 100644 --- a/gnes/composer/base.py +++ b/gnes/composer/base.py @@ -5,6 +5,7 @@ import pkg_resources from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap +from termcolor import colored from ..helper import set_logger from ..service.base import SocketType @@ -14,11 +15,11 @@ class YamlGraph: comp2file = { - 'Encoder': 'service.encoder.EncoderService', - 'Router': 'service.router.RouterService', - 'Indexer': 'service.indexer.IndexerService', - 'Frontend': 'service.grpc.GRPCFrontend', - 'Preprocessor': 'service.preprocessor.PreprocessorService' + 'Encoder': 'encode', + 'Router': 'route', + 'Indexer': 'index', + 'Frontend': 'frontend', + 'Preprocessor': 'preprocess' } class Layer: @@ -99,8 +100,8 @@ def check_fields(self, comp: Dict) -> bool: 'a component must be one of: %s, but given %s' % (', '.join(self.comp2file.keys()), comp['name'])) if 'yaml_path' not in comp and 'dump_path' not in comp: self.logger.warning( - 'you did not specify "yaml_path" and "dump_path", ' - 'will use a default config and would probably result in an empty model') + 'found empty "yaml_path" and "dump_path", ' + 'i will use a default config and would probably result in an empty model') for k in comp: if k not in self.Layer.default_values: self.logger.warning('your yaml contains an unrecognized key named "%s"' % k) @@ -125,6 +126,25 @@ def build_layers(self) -> List['YamlGraph.Layer']: all_layers[0] = copy.deepcopy(self._layers[0]) return all_layers + @staticmethod + def build_shell(all_layers: List['YamlGraph.Layer']): + shell_lines = [] + taboo = {'name', 'replicas'} + for layer in all_layers: + for c in layer.components: + rep_c = YamlGraph.Layer.get_value(c, 'replicas') + shell_lines.append('printf "starting service %s with %s replicas...\\n"' % ( + colored(c['name'], 'green'), colored(rep_c, 'yellow'))) + for j in range(rep_c): + cmd = YamlGraph.comp2file[c['name']] + print(c) + args = ' '.join(['--%s %s' % (a, v if ' ' not in v else ('"%s"' % v)) for a, v in c.items() if a not in taboo and v]) + shell_lines.append('gnes %s %s &' % (cmd, args)) + + r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-shell.sh'))) + with r: + return r.read().decode().replace('{{gnes-template}}', '\n'.join(shell_lines)) + @staticmethod def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True): mermaid_graph = [] @@ -164,7 +184,7 @@ def build_html(self, mermaid_str: str): else: r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'static', 'gnes-graph.html'))) with r, self.args.html_path as fp: - html = r.read().decode().replace('@mermaid-graph', mermaid_str) + html = r.read().decode().replace('{{gnes-template}}', mermaid_str) fp.write(html) @staticmethod diff --git a/gnes/resources/compose/gnes-shell.sh b/gnes/resources/compose/gnes-shell.sh new file mode 100755 index 00000000..0704e2db --- /dev/null +++ b/gnes/resources/compose/gnes-shell.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -e + +trap 'kill $(jobs -p)' EXIT + +{{gnes-template}} \ No newline at end of file diff --git a/gnes/resources/static/gnes-graph.html b/gnes/resources/static/gnes-graph.html index 6ff01dad..4ef9788d 100644 --- a/gnes/resources/static/gnes-graph.html +++ b/gnes/resources/static/gnes-graph.html @@ -13,7 +13,7 @@
- @mermaid-graph + {{gnes-template}}
diff --git a/tests/test_compose.py b/tests/test_compose.py index 011dc923..76a0d3a9 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -28,6 +28,7 @@ def _test_topology(self, yaml_path: str, num_layer_before: int, num_layer_after: for c in r: print(c) a.build_html(a.build_mermaid(r)) + print(a.build_shell(r)) os.path.exists(self.html_path) # def tearDown(self): From 64aef41361814b4c0b58fe0d3e99d846f4b38c68 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Tue, 16 Jul 2019 14:59:50 +0800 Subject: [PATCH 3/9] fix(composer): fix styling according to codacy --- gnes/composer/base.py | 42 +++++++++++++++++----------- gnes/resources/compose/gnes-shell.sh | 4 ++- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/gnes/composer/base.py b/gnes/composer/base.py index c599d30f..62122f07 100644 --- a/gnes/composer/base.py +++ b/gnes/composer/base.py @@ -31,8 +31,8 @@ class Layer: 'income': 'pull' } - def __init__(self, id: int = 0): - self.id = id + def __init__(self, layer_id: int = 0): + self.layer_id = layer_id self.components = [] @staticmethod @@ -84,7 +84,8 @@ def __init__(self, args): for comp in tmp['component']: self.add_layer() if isinstance(comp, list): - [self.add_comp(c) for c in comp if self.add_comp(c)] + for c in comp: + self.add_comp(c) elif self.check_fields(comp): self.add_comp(comp) else: @@ -108,7 +109,7 @@ def check_fields(self, comp: Dict) -> bool: return True def add_layer(self, layer: 'Layer' = None) -> None: - self._layers.append(copy.deepcopy(layer) or self.Layer(id=self._num_layer)) + self._layers.append(copy.deepcopy(layer) or self.Layer(layer_id=self._num_layer)) self._num_layer += 1 def add_comp(self, comp: Dict) -> None: @@ -127,7 +128,7 @@ def build_layers(self) -> List['YamlGraph.Layer']: return all_layers @staticmethod - def build_shell(all_layers: List['YamlGraph.Layer']): + def build_shell(all_layers: List['YamlGraph.Layer'], log_fn: str = None, job_background: bool = False): shell_lines = [] taboo = {'name', 'replicas'} for layer in all_layers: @@ -135,11 +136,13 @@ def build_shell(all_layers: List['YamlGraph.Layer']): rep_c = YamlGraph.Layer.get_value(c, 'replicas') shell_lines.append('printf "starting service %s with %s replicas...\\n"' % ( colored(c['name'], 'green'), colored(rep_c, 'yellow'))) - for j in range(rep_c): + for _ in range(rep_c): cmd = YamlGraph.comp2file[c['name']] print(c) - args = ' '.join(['--%s %s' % (a, v if ' ' not in v else ('"%s"' % v)) for a, v in c.items() if a not in taboo and v]) - shell_lines.append('gnes %s %s &' % (cmd, args)) + args = ' '.join(['--%s %s' % (a, v if ' ' not in v else ('"%s"' % v)) for a, v in c.items() if + a not in taboo and v]) + shell_lines.append('gnes %s %s %s %s' % ( + cmd, args, '>> %s 2>&1' % log_fn if log_fn else '', '&' if job_background else '')) r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-shell.sh'))) with r: @@ -160,7 +163,7 @@ def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True p = '((%s%s))' if c['name'] == 'Router' else '[%s%s]' p1 = '((%s%s))' if c1['name'] == 'Router' else '[%s%s]' for j1 in range(YamlGraph.Layer.get_value(c1, 'replicas')): - id, id1 = '%s%s%s' % (last_layer.id, c_idx, j), '%s%s%s' % (layer.id, c1_idx, j1) + id, id1 = '%s%s%s' % (last_layer.layer_id, c_idx, j), '%s%s%s' % (layer.layer_id, c1_idx, j1) conn_type = ( c['socket_out'].split('_')[0] + '/' + c1['socket_in'].split('_')[0]).lower() s_id = '%s%s' % (c_idx if len(last_layer.components) > 1 else '', @@ -174,8 +177,13 @@ def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True # if len(last_layer.components) > 1: # self.mermaid_graph.append('\tend') - style = 'style Frontend000 fill:#ffd889ff,stroke:#f66,stroke-width:2px,stroke-dasharray:5' - mermaid_str = '\n'.join(['graph TD' if show_topdown else 'graph LR'] + [style] + mermaid_graph) + style = ['classDef FrontendCLS fill:#ffd889ff,stroke:#f66,stroke-width:2px,stroke-dasharray:5;', + 'classDef EncoderCLS fill:#ffd889ff,stroke:#f66,stroke-width:2px;', + 'classDef IndexerCLS fill:#ffd889ff,stroke:#f66,stroke-width:2px;', + 'classDef RouterCLS fill:#ffd889ff,stroke:#f66,stroke-width:2px;', + 'classDef PreprocessorCLS fill:#ffd889ff,stroke:#f66,stroke-width:2px;'] + + mermaid_str = '\n'.join(['graph TD' if show_topdown else 'graph LR'] + style + mermaid_graph) return mermaid_str def build_html(self, mermaid_str: str): @@ -204,7 +212,7 @@ def rule2(): def rule3(): # a shortcut fn: (N)-2-(N) with push pull connection - router_layer = YamlGraph.Layer(id=self._num_layer) + router_layer = YamlGraph.Layer(layer_id=self._num_layer) self._num_layer += 1 last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT) r = CommentedMap({'name': 'Router', @@ -233,7 +241,7 @@ def rule5(): def rule6(): last_layer.components[0]['socket_out'] = str(SocketType.PUB_BIND) - router_layer = YamlGraph.Layer(id=self._num_layer) + router_layer = YamlGraph.Layer(layer_id=self._num_layer) self._num_layer += 1 for c in layer.components: r = CommentedMap({'name': 'Router', @@ -250,7 +258,7 @@ def rule6(): def rule7(): last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT) - router_layer = YamlGraph.Layer(id=self._num_layer) + router_layer = YamlGraph.Layer(layer_id=self._num_layer) self._num_layer += 1 r0 = CommentedMap({'name': 'Router', 'yaml_path': None, @@ -262,7 +270,7 @@ def rule7(): router_layers.append(router_layer) last_layer.components[0]['port_out'] = r0['port_in'] - router_layer = YamlGraph.Layer(id=self._num_layer) + router_layer = YamlGraph.Layer(layer_id=self._num_layer) self._num_layer += 1 for c in layer.components: r = CommentedMap({'name': 'Router', @@ -278,7 +286,7 @@ def rule7(): def rule8(): last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT) - router_layer = YamlGraph.Layer(id=self._num_layer) + router_layer = YamlGraph.Layer(layer_id=self._num_layer) self._num_layer += 1 r = CommentedMap({'name': 'Router', 'yaml_path': None, @@ -304,7 +312,7 @@ def rule8(): c['port_in'] = r['port_out'] router_layers.append(router_layer) - router_layer = YamlGraph.Layer(id=self._num_layer) + router_layer = YamlGraph.Layer(layer_id=self._num_layer) self._num_layer += 1 router_layer.append(r) router_layers.append(router_layer) diff --git a/gnes/resources/compose/gnes-shell.sh b/gnes/resources/compose/gnes-shell.sh index 0704e2db..b92cf312 100755 --- a/gnes/resources/compose/gnes-shell.sh +++ b/gnes/resources/compose/gnes-shell.sh @@ -4,4 +4,6 @@ set -e trap 'kill $(jobs -p)' EXIT -{{gnes-template}} \ No newline at end of file +{{gnes-template}} + +wait \ No newline at end of file From 6ec4233d37c191705c406f035f5554d9333901b1 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Tue, 16 Jul 2019 19:59:26 +0800 Subject: [PATCH 4/9] feat(composer): add swarm and bash generator --- gnes/cli/parser.py | 6 + gnes/composer/base.py | 177 ++++++++++++++++++---- gnes/resources/static/gnes-graph.html | 207 +++++++++++++++++++++++++- tests/test_compose.py | 3 +- tests/yaml/topology1.yml | 2 +- tests/yaml/topology2.yml | 2 +- tests/yaml/topology3.yml | 2 +- tests/yaml/topology4.yml | 2 +- tests/yaml/topology5.yml | 2 +- tests/yaml/topology6.yml | 2 +- 10 files changed, 361 insertions(+), 44 deletions(-) diff --git a/gnes/cli/parser.py b/gnes/cli/parser.py index 083410a9..e26b6ca8 100644 --- a/gnes/cli/parser.py +++ b/gnes/cli/parser.py @@ -48,6 +48,12 @@ def set_composer_parser(parser=None): help='yaml config of the service') parser.add_argument('--html_path', type=argparse.FileType('w', encoding='utf8'), help='render the network graph in HTML with mermaid.js') + parser.add_argument('--shell_path', type=argparse.FileType('w', encoding='utf8'), + help='output path of the shell-based starting script') + parser.add_argument('--swarm_path', type=argparse.FileType('w', encoding='utf8'), + help='output path of the docker-compose file for Docker Swarm') + parser.add_argument('--k8s_path', type=argparse.FileType('w', encoding='utf8'), + help='output path of the docker-compose file for Docker Swarm') return parser diff --git a/gnes/composer/base.py b/gnes/composer/base.py index 62122f07..cc6c46e5 100644 --- a/gnes/composer/base.py +++ b/gnes/composer/base.py @@ -1,9 +1,10 @@ import copy import random -from typing import Dict, List +from collections import defaultdict +from typing import Dict, List, Set import pkg_resources -from ruamel.yaml import YAML +from ruamel.yaml import YAML, StringIO from ruamel.yaml.comments import CommentedMap from termcolor import colored @@ -73,15 +74,18 @@ def __init__(self, args): self._layers = [] # type: List['YamlGraph.Layer'] self.logger = set_logger(self.__class__.__name__) tmp = _yaml.load(args.yaml_path) + stream = StringIO() + _yaml.dump(tmp, stream) + self.original_yaml = stream.getvalue() self.name = tmp.get('name', args.name) self.port = tmp.get('port', args.port) self.args = args self._num_layer = 0 - if 'component' in tmp: + if 'services' in tmp: self.add_layer() - self.add_comp(CommentedMap({'name': 'Frontend'})) - for comp in tmp['component']: + self.add_comp(CommentedMap({'name': 'Frontend', 'grpc_port': self.port})) + for comp in tmp['services']: self.add_layer() if isinstance(comp, list): for c in comp: @@ -128,7 +132,86 @@ def build_layers(self) -> List['YamlGraph.Layer']: return all_layers @staticmethod - def build_shell(all_layers: List['YamlGraph.Layer'], log_fn: str = None, job_background: bool = False): + def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gnes/gnes:latest', + volumes: Dict = None) -> str: + swarm_lines = {'version': '3.4', 'services': {}} + taboo = {'name', 'replicas', 'yaml_path', 'dump_path'} + config_dict = {} + network_dict = {'gnes-network': {'driver': 'overlay', 'attachable': True}} + for l_idx, layer in enumerate(all_layers): + for c_idx, c in enumerate(layer.components): + c_name = '%s%d%d' % (c['name'], l_idx, c_idx) + args = ['--%s %s' % (a, str(v) if ' ' not in str(v) else ('"%s"' % str(v))) for a, v in c.items() if + a not in taboo and v] + if 'yaml_path' in c and c['yaml_path'] is not None: + args.append('--yaml_path /%s_yaml' % c_name) + config_dict['%s_yaml' % c_name] = {'file': c['yaml_path']} + + if l_idx + 1 < len(all_layers): + next_layer = all_layers[l_idx + 1] + _l_idx = l_idx + 1 + else: + next_layer = all_layers[0] + _l_idx = 0 + + host_out_name = '' + for _c_idx, _c in enumerate(next_layer.components): + if _c['port_in'] == c['port_out']: + host_out_name = '%s%d%d' % (_c['name'], _l_idx, _c_idx) + break + + if l_idx - 1 >= 0: + last_layer = all_layers[l_idx - 1] + _l_idx = l_idx - 1 + else: + last_layer = all_layers[-1] + _l_idx = len(all_layers) - 1 + + host_in_name = '' + for _c_idx, _c in enumerate(last_layer.components): + if _c['port_out'] == c['port_in']: + host_in_name = '%s%d%d' % (_c['name'], _l_idx, _c_idx) + break + + args += ['--host_in %s' % host_in_name, + '--host_out %s' % host_out_name] + + cmd = '%s %s' % (YamlGraph.comp2file[c['name']], ' '.join(args)) + swarm_lines['services'][c_name] = { + 'image': docker_img, + 'command': cmd, + } + + rep_c = YamlGraph.Layer.get_value(c, 'replicas') + if rep_c > 1: + swarm_lines['services'][c_name]['deploy'] = { + 'replicas': YamlGraph.Layer.get_value(c, 'replicas'), + 'restart_policy': { + 'condition': 'on-failure', + 'max_attempts': 3, + } + } + + if 'yaml_path' in c and c['yaml_path'] is not None: + swarm_lines['services'][c_name]['configs'] = ['%s_yaml' % c_name] + + if c['name'] == 'Frontend': + swarm_lines['services'][c_name]['ports'] = ['%d:%d' % (c['grpc_port'], c['grpc_port'])] + + if volumes: + swarm_lines['volumes'] = volumes + swarm_lines['configs'] = config_dict + swarm_lines['networks'] = network_dict + stream = StringIO() + _yaml.dump(swarm_lines, stream) + return stream.getvalue() + + @staticmethod + def build_kubernetes(all_layers: List['YamlGraph.Layer'], *args, **kwargs): + pass + + @staticmethod + def build_shell(all_layers: List['YamlGraph.Layer'], log_fn: str = None, job_background: bool = False) -> str: shell_lines = [] taboo = {'name', 'replicas'} for layer in all_layers: @@ -138,9 +221,9 @@ def build_shell(all_layers: List['YamlGraph.Layer'], log_fn: str = None, job_bac colored(c['name'], 'green'), colored(rep_c, 'yellow'))) for _ in range(rep_c): cmd = YamlGraph.comp2file[c['name']] - print(c) - args = ' '.join(['--%s %s' % (a, v if ' ' not in v else ('"%s"' % v)) for a, v in c.items() if - a not in taboo and v]) + args = ' '.join( + ['--%s %s' % (a, str(v) if ' ' not in str(v) else ('"%s"' % str(v))) for a, v in c.items() if + a not in taboo and v]) shell_lines.append('gnes %s %s %s %s' % ( cmd, args, '>> %s 2>&1' % log_fn if log_fn else '', '&' if job_background else '')) @@ -149,8 +232,9 @@ def build_shell(all_layers: List['YamlGraph.Layer'], log_fn: str = None, job_bac return r.read().decode().replace('{{gnes-template}}', '\n'.join(shell_lines)) @staticmethod - def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True): + def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True) -> str: mermaid_graph = [] + cls_dict = defaultdict(set) # type: Dict[str, Set] for l_idx, layer in enumerate(all_layers[1:] + [all_layers[0]], 1): last_layer = all_layers[l_idx - 1] @@ -163,7 +247,8 @@ def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True p = '((%s%s))' if c['name'] == 'Router' else '[%s%s]' p1 = '((%s%s))' if c1['name'] == 'Router' else '[%s%s]' for j1 in range(YamlGraph.Layer.get_value(c1, 'replicas')): - id, id1 = '%s%s%s' % (last_layer.layer_id, c_idx, j), '%s%s%s' % (layer.layer_id, c1_idx, j1) + _id, _id1 = '%s%s%s' % (last_layer.layer_id, c_idx, j), '%s%s%s' % ( + layer.layer_id, c1_idx, j1) conn_type = ( c['socket_out'].split('_')[0] + '/' + c1['socket_in'].split('_')[0]).lower() s_id = '%s%s' % (c_idx if len(last_layer.components) > 1 else '', @@ -172,33 +257,64 @@ def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True j1 if YamlGraph.Layer.get_value(c1, 'replicas') > 1 else '') mermaid_graph.append( '\t%s%s%s-- %s -->%s%s%s' % ( - c['name'], id, p % (c['name'], s_id), conn_type, c1['name'], id1, + c['name'], _id, p % (c['name'], s_id), conn_type, c1['name'], _id1, p1 % (c1['name'], s1_id))) + cls_dict[c['name'] + 'CLS'].add('%s%s' % (c['name'], _id)) + cls_dict[c1['name'] + 'CLS'].add('%s%s' % (c1['name'], _id1)) # if len(last_layer.components) > 1: # self.mermaid_graph.append('\tend') - style = ['classDef FrontendCLS fill:#ffd889ff,stroke:#f66,stroke-width:2px,stroke-dasharray:5;', - 'classDef EncoderCLS fill:#ffd889ff,stroke:#f66,stroke-width:2px;', - 'classDef IndexerCLS fill:#ffd889ff,stroke:#f66,stroke-width:2px;', - 'classDef RouterCLS fill:#ffd889ff,stroke:#f66,stroke-width:2px;', - 'classDef PreprocessorCLS fill:#ffd889ff,stroke:#f66,stroke-width:2px;'] - - mermaid_str = '\n'.join(['graph TD' if show_topdown else 'graph LR'] + style + mermaid_graph) + style = ['classDef FrontendCLS fill:#FFFFCB,stroke:#277CE8,stroke-width:1px,stroke-dasharray:5;', + 'classDef EncoderCLS fill:#27E1E8,stroke:#277CE8,stroke-width:1px;', + 'classDef IndexerCLS fill:#27E1E8,stroke:#277CE8,stroke-width:1px;', + 'classDef RouterCLS fill:#2BFFCB,stroke:#277CE8,stroke-width:1px;', + 'classDef PreprocessorCLS fill:#27E1E8,stroke:#277CE8,stroke-width:1px;'] + class_def = ['class %s %s;' % (','.join(v), k) for k, v in cls_dict.items()] + mermaid_str = '\n'.join(['graph %s' % 'TD' if show_topdown else 'LR'] + mermaid_graph + style + class_def) return mermaid_str - def build_html(self, mermaid_str: str): - if not self.args.html_path: - print(mermaid_str) - else: - r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'static', 'gnes-graph.html'))) - with r, self.args.html_path as fp: - html = r.read().decode().replace('{{gnes-template}}', mermaid_str) - fp.write(html) + @staticmethod + def build_html(generate_dict: Dict[str, str]) -> str: + r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'static', 'gnes-graph.html'))) + with r: + html = r.read().decode() + for k, v in generate_dict.items(): + if v: + html = html.replace('{{gnes-%s}}' % k, v) + return html + + def build_all(self): + def std_or_print(f, content): + if content: + if f: + with f as fp: + fp.write(content) + self.logger.info('generated content is written to %s' % f) + else: + self.logger.warning('no file path is defined, i will just print it to stdout') + print(content) + + all_layers = self.build_layers() + rep_dict = { + 'mermaid': self.build_mermaid(all_layers), + 'shell': self.build_shell(all_layers), + 'yaml': self.original_yaml, + 'docker': self.build_dockerswarm(all_layers), + 'k8s': self.build_kubernetes(all_layers), + } + std_or_print(self.args.shell_path, rep_dict['shell']) + std_or_print(self.args.swarm_path, rep_dict['docker']) + std_or_print(self.args.k8s_path, rep_dict['k8s']) + std_or_print(self.args.html_path, self.build_html(rep_dict)) @staticmethod def _get_random_port(min_port: int = 49152, max_port: int = 65536) -> str: return str(random.randrange(min_port, max_port)) + @staticmethod + def _get_random_host(comp_name: str) -> str: + return str(comp_name + str(random.randrange(0, 100))) + def _add_router(self, last_layer: 'YamlGraph.Layer', layer: 'YamlGraph.Layer') -> List['YamlGraph.Layer']: def rule1(): # a shortcut fn: push connect the last and current @@ -294,17 +410,16 @@ def rule8(): 'socket_out': str(SocketType.PUSH_BIND), 'port_in': self._get_random_port(), 'port_out': self._get_random_port()}) - router_layer.append(r) for c in last_layer.components: c['socket_out'] = str(SocketType.PUSH_CONNECT) - c['port_out'] = self._get_random_port() r_c = CommentedMap({'name': 'Router', 'yaml_path': None, 'socket_in': str(SocketType.PULL_BIND), - 'socket_out': str(SocketType.PUSH_BIND), - 'port_in': c['port_out'], + 'socket_out': str(SocketType.PUSH_CONNECT), + 'port_in': self._get_random_port(), 'port_out': r['port_in']}) + c['port_out'] = r_c['port_in'] router_layer.append(r_c) for c in layer.components: diff --git a/gnes/resources/static/gnes-graph.html b/gnes/resources/static/gnes-graph.html index 4ef9788d..b2697b30 100644 --- a/gnes/resources/static/gnes-graph.html +++ b/gnes/resources/static/gnes-graph.html @@ -3,22 +3,217 @@ + + - GNES graph visualization + GNES Board -
-
- {{gnes-template}} + +
+
+
+
+
+ {{gnes-mermaid}} +
+
+
+
+
+
+
+ +
+
+ 
+                    
+{{gnes-yaml}}
+                    
+
+
+
+
+
+
+
+ +
+
+ 
+                    
+{{gnes-shell}}
+                    
+
+
+
-
+
+
+
+ +
+
+ 
+                    
+{{gnes-docker}}
+                    
+
+
+
+
+
+
+
+ +
+
+ 
+                    
+{{gnes-k8s}}
+                    
+
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/tests/test_compose.py b/tests/test_compose.py index 76a0d3a9..1ab21c3c 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -27,9 +27,10 @@ def _test_topology(self, yaml_path: str, num_layer_before: int, num_layer_after: self.assertEqual(len(r), num_layer_after) for c in r: print(c) - a.build_html(a.build_mermaid(r)) + a.build_all() print(a.build_shell(r)) os.path.exists(self.html_path) + print(a.build_dockerswarm(r)) # def tearDown(self): # if os.path.exists(self.html_path): diff --git a/tests/yaml/topology1.yml b/tests/yaml/topology1.yml index 7544de58..b7437d59 100644 --- a/tests/yaml/topology1.yml +++ b/tests/yaml/topology1.yml @@ -1,4 +1,4 @@ port: 5566 -component: +services: - name: Preprocessor - name: Encoder \ No newline at end of file diff --git a/tests/yaml/topology2.yml b/tests/yaml/topology2.yml index 2d142290..2245aa21 100644 --- a/tests/yaml/topology2.yml +++ b/tests/yaml/topology2.yml @@ -1,5 +1,5 @@ port: 5566 -component: +services: - name: Preprocessor - name: Encoder replicas: 2 diff --git a/tests/yaml/topology3.yml b/tests/yaml/topology3.yml index 914c0e80..09f156b3 100644 --- a/tests/yaml/topology3.yml +++ b/tests/yaml/topology3.yml @@ -1,5 +1,5 @@ port: 5566 -component: +services: - name: Preprocessor replicas: 2 - name: Encoder diff --git a/tests/yaml/topology4.yml b/tests/yaml/topology4.yml index 5031d71f..db779638 100644 --- a/tests/yaml/topology4.yml +++ b/tests/yaml/topology4.yml @@ -1,5 +1,5 @@ port: 5566 -component: +services: - name: Preprocessor replicas: 2 - name: Encoder diff --git a/tests/yaml/topology5.yml b/tests/yaml/topology5.yml index 1a842d5e..06892eef 100644 --- a/tests/yaml/topology5.yml +++ b/tests/yaml/topology5.yml @@ -1,5 +1,5 @@ port: 5566 -component: +services: - name: Preprocessor replicas: 2 - name: Encoder diff --git a/tests/yaml/topology6.yml b/tests/yaml/topology6.yml index 2159bb33..25dcfae7 100644 --- a/tests/yaml/topology6.yml +++ b/tests/yaml/topology6.yml @@ -1,5 +1,5 @@ port: 5566 -component: +services: - name: Preprocessor replicas: 2 - name: Encoder From 743ec3b06ec17f36c80e290fa2a2708e2fffba43 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Tue, 16 Jul 2019 20:01:40 +0800 Subject: [PATCH 5/9] fix(composer): fix unit test and add tear down --- gnes/cli/parser.py | 2 +- tests/test_compose.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gnes/cli/parser.py b/gnes/cli/parser.py index e26b6ca8..d87fb026 100644 --- a/gnes/cli/parser.py +++ b/gnes/cli/parser.py @@ -47,7 +47,7 @@ def set_composer_parser(parser=None): parser.add_argument('--yaml_path', type=argparse.FileType('r'), help='yaml config of the service') parser.add_argument('--html_path', type=argparse.FileType('w', encoding='utf8'), - help='render the network graph in HTML with mermaid.js') + help='output path of the HTML file, will contain all possible generations') parser.add_argument('--shell_path', type=argparse.FileType('w', encoding='utf8'), help='output path of the shell-based starting script') parser.add_argument('--swarm_path', type=argparse.FileType('w', encoding='utf8'), diff --git a/tests/test_compose.py b/tests/test_compose.py index 1ab21c3c..013f2fee 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -32,6 +32,6 @@ def _test_topology(self, yaml_path: str, num_layer_before: int, num_layer_after: os.path.exists(self.html_path) print(a.build_dockerswarm(r)) - # def tearDown(self): - # if os.path.exists(self.html_path): - # os.remove(self.html_path) + def tearDown(self): + if os.path.exists(self.html_path): + os.remove(self.html_path) From 054981ce11138fba59fec1fe69ee8e974585ece0 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Tue, 16 Jul 2019 20:11:03 +0800 Subject: [PATCH 6/9] fix(composer): fix gnes board naming --- gnes/composer/base.py | 2 +- gnes/resources/static/{gnes-graph.html => gnes-board.html} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename gnes/resources/static/{gnes-graph.html => gnes-board.html} (99%) diff --git a/gnes/composer/base.py b/gnes/composer/base.py index cc6c46e5..ce8d4c41 100644 --- a/gnes/composer/base.py +++ b/gnes/composer/base.py @@ -275,7 +275,7 @@ def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True @staticmethod def build_html(generate_dict: Dict[str, str]) -> str: - r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'static', 'gnes-graph.html'))) + r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'static', 'gnes-board.html'))) with r: html = r.read().decode() for k, v in generate_dict.items(): diff --git a/gnes/resources/static/gnes-graph.html b/gnes/resources/static/gnes-board.html similarity index 99% rename from gnes/resources/static/gnes-graph.html rename to gnes/resources/static/gnes-board.html index b2697b30..6b70e635 100644 --- a/gnes/resources/static/gnes-graph.html +++ b/gnes/resources/static/gnes-board.html @@ -146,7 +146,7 @@
From d5bffc19fedd3b6584313d29ce648ce986c810a8 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Tue, 16 Jul 2019 20:18:41 +0800 Subject: [PATCH 7/9] fix(composer): in bash mode always run job in background --- gnes/composer/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gnes/composer/base.py b/gnes/composer/base.py index ce8d4c41..eff06db2 100644 --- a/gnes/composer/base.py +++ b/gnes/composer/base.py @@ -211,7 +211,7 @@ def build_kubernetes(all_layers: List['YamlGraph.Layer'], *args, **kwargs): pass @staticmethod - def build_shell(all_layers: List['YamlGraph.Layer'], log_fn: str = None, job_background: bool = False) -> str: + def build_shell(all_layers: List['YamlGraph.Layer'], log_path: str = None) -> str: shell_lines = [] taboo = {'name', 'replicas'} for layer in all_layers: @@ -224,8 +224,8 @@ def build_shell(all_layers: List['YamlGraph.Layer'], log_fn: str = None, job_bac args = ' '.join( ['--%s %s' % (a, str(v) if ' ' not in str(v) else ('"%s"' % str(v))) for a, v in c.items() if a not in taboo and v]) - shell_lines.append('gnes %s %s %s %s' % ( - cmd, args, '>> %s 2>&1' % log_fn if log_fn else '', '&' if job_background else '')) + shell_lines.append('gnes %s %s %s &' % ( + cmd, args, '>> %s 2>&1' % log_path if log_path else '')) r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-shell.sh'))) with r: From 70ba3fca920ee15a26d1d2f6e8feacd263d3890b Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Tue, 16 Jul 2019 20:26:18 +0800 Subject: [PATCH 8/9] fix(composer): in bash mode always run job in background --- gnes/composer/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnes/composer/base.py b/gnes/composer/base.py index eff06db2..29e5f0f4 100644 --- a/gnes/composer/base.py +++ b/gnes/composer/base.py @@ -1,7 +1,7 @@ import copy import random from collections import defaultdict -from typing import Dict, List, Set +from typing import Dict, List import pkg_resources from ruamel.yaml import YAML, StringIO @@ -234,7 +234,7 @@ def build_shell(all_layers: List['YamlGraph.Layer'], log_path: str = None) -> st @staticmethod def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True) -> str: mermaid_graph = [] - cls_dict = defaultdict(set) # type: Dict[str, Set] + cls_dict = defaultdict(set) for l_idx, layer in enumerate(all_layers[1:] + [all_layers[0]], 1): last_layer = all_layers[l_idx - 1] From 3423ec832c04aad22a6d897be36486e0cb3372a1 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Tue, 16 Jul 2019 20:29:27 +0800 Subject: [PATCH 9/9] fix(composer): add compose api to api.py --- gnes/cli/api.py | 5 +++++ gnes/cli/parser.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/gnes/cli/api.py b/gnes/cli/api.py index 2915b4bd..d609a8a7 100644 --- a/gnes/cli/api.py +++ b/gnes/cli/api.py @@ -34,6 +34,11 @@ def route(args): es.join() +def compose(args): + from ..composer.base import YamlGraph + YamlGraph(args).build_all() + + def frontend(args): from ..service.grpc import GRPCFrontend import threading diff --git a/gnes/cli/parser.py b/gnes/cli/parser.py index d87fb026..3f54a7b1 100644 --- a/gnes/cli/parser.py +++ b/gnes/cli/parser.py @@ -239,5 +239,5 @@ def get_main_parser(): set_preprocessor_service_parser(sp.add_parser('preprocess', help='start a preprocessor service')) set_http_service_parser(sp.add_parser('client_http', help='start a http service')) set_cli_client_parser(sp.add_parser('client_cli', help='start a grpc client')) - set_composer_parser(sp.add_parser('compose', help='start a GNES composer')) + set_composer_parser(sp.add_parser('compose', help='start a GNES composer to simplify config generation')) return parser