fix(compose): fix compose bug of pub-sub rule, duplicate yaml_path

README.md

@@ -157,6 +157,7 @@ Either way, if you see the following message after `$ gnes` or `$ docker run gne
   * [Runtime](#runtime)
 - [Build your first GNES app](#build-your-first-gnes-app)
 - [Scale your GNES app](#scale-your-gnes-app)
+- [Customize GNES on your need](#customize-gnes-on-your-need)
 - [Take-home messages](#take-home-messages)
   * [👨‍💻️What's next?](#-whats-next)
 
@@ -422,7 +423,7 @@ services:
    </pre>
 </td>
 <td width="70%">
-  <img src=".github/mermaid-diagram-20190724110437.svg" alt="GNES workflow of example 3" width="60%">
+  <img src=".github/mermaid-diagram-20190724110437.svg" alt="GNES workflow of example 3" width="50%">
 </td>
 </tr>
 <tr>
@@ -442,7 +443,7 @@ services:
 </tr>
 <tr>
 <td width="30%">
-Vector-only indexing with 3 shards 
+Index-time with 3 vector-index shards 
    <pre lang="yaml">
 port: 5566
 services:
@@ -458,7 +459,7 @@ services:
 </tr>
 <tr>
 <td width="30%">
-Query-time with 2 vector-index shards and 3 full-text-index shards
+Query-time with 2 vector-index shards followed by 3 full-text-index shards
    <pre lang="yaml">
 port: 5566
 services:

gnes/composer/base.py

@@ -22,7 +22,6 @@
 from pkg_resources import resource_stream
 from ruamel.yaml import YAML, StringIO
 from ruamel.yaml.comments import CommentedMap
-from termcolor import colored
 
 from .. import __version__
 from ..cli.parser import set_grpc_frontend_parser, \
@@ -104,7 +103,7 @@ def __init__(self, args):
             tmp = _yaml.load(args.yaml_path)
             stream = StringIO()
             _yaml.dump(tmp, stream)
-            self.original_yaml = stream.getvalue()
+            self.original_yaml = stream.getvalue().strip()
 
         self._name = tmp.get('name', args.name)
         self._port = tmp.get('port', args.port)
@@ -173,7 +172,7 @@ def build_dockerswarm(all_layers: List['YamlComposer.Layer'], docker_img: str =
             for c_idx, c in enumerate(layer.components):
                 c_name = '%s%d%d' % (c['name'], l_idx, c_idx)
                 args = ['--%s %s' % (a, str(v) if ' ' not in str(v) else ('"%s"' % str(v))) for a, v in c.items() if
-                        a in YamlComposer.comp2args[c['name']] and v]
+                        a in YamlComposer.comp2args[c['name']] and a != 'yaml_path' and v]
                 if 'yaml_path' in c and c['yaml_path'] is not None:
                     args.append('--yaml_path /%s_yaml' % c_name)
                     config_dict['%s_yaml' % c_name] = {'file': c['yaml_path']}
@@ -235,7 +234,7 @@ def build_dockerswarm(all_layers: List['YamlComposer.Layer'], docker_img: str =
         swarm_lines['configs'] = config_dict
         stream = StringIO()
         _yaml.dump(swarm_lines, stream)
-        return stream.getvalue()
+        return stream.getvalue().strip()
 
     @staticmethod
     def build_kubernetes(all_layers: List['YamlComposer.Layer'], *args, **kwargs):
@@ -247,8 +246,8 @@ def build_shell(all_layers: List['YamlComposer.Layer'], log_redirect: str = None
         for layer in all_layers:
             for c in layer.components:
                 rep_c = YamlComposer.Layer.get_value(c, 'replicas')
-                shell_lines.append('printf "starting service %s with %s replicas...\\n"' % (
-                    colored(c['name'], 'green'), colored(rep_c, 'yellow')))
+                shell_lines.append('printf "starting service \\e[1;33m%s\\e[0m with \e[1;33m%s\e[0m replicas...\\n"' % (
+                    c['name'], rep_c))
                 for _ in range(rep_c):
                     cmd = YamlComposer.comp2file[c['name']]
                     args = ' '.join(
@@ -258,7 +257,7 @@ def build_shell(all_layers: List['YamlComposer.Layer'], log_redirect: str = None
                         cmd, args, '>> %s 2>&1' % log_redirect if log_redirect else ''))
 
         with resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-shell.sh'))) as r:
-            return r.read().decode().replace('{{gnes-template}}', '\n'.join(shell_lines))
+            return r.read().decode().replace('{{gnes-template}}', '\n'.join(shell_lines)).strip()
 
     @staticmethod
     def build_mermaid(all_layers: List['YamlComposer.Layer'], mermaid_leftright: bool = False) -> str:
@@ -301,7 +300,7 @@ def build_mermaid(all_layers: List['YamlComposer.Layer'], mermaid_leftright: boo
         class_def = ['class %s %s;' % (','.join(v), k) for k, v in cls_dict.items()]
         mermaid_str = '\n'.join(
             ['graph %s' % ('LR' if mermaid_leftright else 'TD')] + mermaid_graph + style + class_def)
-        return mermaid_str
+        return mermaid_str.strip()
 
     @staticmethod
     def build_html(generate_dict: Dict[str, str]) -> str:
@@ -310,7 +309,7 @@ def build_html(generate_dict: Dict[str, str]) -> str:
             for k, v in generate_dict.items():
                 if v:
                     html = html.replace('{{gnes-%s}}' % k, v)
-        return html
+        return html.strip()
 
     def build_all(self):
         def std_or_print(f, content):
@@ -516,13 +515,13 @@ def rule9():
                 rule1()
             elif layer.is_homo_multi_component:
                 # 1-to-(N)
-                last_income = self.Layer.get_value(last_layer.components[0], 'income')
-                if last_income == 'pull':
+                income = self.Layer.get_value(layer.components[0], 'income')
+                if income == 'pull':
                     rule1()
-                elif last_income == 'sub':
+                elif income == 'sub':
                     rule2()
                 else:
-                    raise NotImplementedError('replica type: %s is not recognized!' % last_income)
+                    raise NotImplementedError('replica type: %s is not recognized!' % income)
             elif layer.is_heto_single_component:
                 # 1-to-(1)&(1)&(1)
                 rule4()

gnes/resources/compose/gnes-board.html

@@ -294,9 +294,9 @@
                             You can use it to start a Docker Swarm distributed on multiple machines.</p>
                         <hr class="my-4">
                         <p> 1. Install Docker and Docker Swarm<br>
-                            2. Create a new file say <code>my-compose.yml</code><br>
+                            2. Create a new file say <code>my-gnes.yml</code><br>
                             3. Copy the following content to it<br>
-                            4. Run <code>docker stack deploy --compose-file my-compose.yml</code>.</p>
+                            4. Run <code>docker stack deploy --compose-file my-gnes.yml</code>.</p>
                     </div>
                     <div class="card">
                         <div class="card-header">

tests/test_compose.py

@@ -33,6 +33,16 @@ def _test_topology(self, yaml_path: str, num_layer_before: int, num_layer_after:
         os.path.exists(self.html_path)
         print(a.build_dockerswarm(r))
 
+    @unittest.SkipTest
+    def test_flask_local(self):
+        yaml_path = os.path.join(self.dirname, 'yaml', 'topology1.yml')
+        args = set_composer_flask_parser().parse_args([
+            '--flask',
+            '--yaml_path', yaml_path,
+            '--html_path', self.html_path
+        ])
+        YamlComposerFlask(args).run()
+
     def test_flask(self):
         yaml_path = os.path.join(self.dirname, 'yaml', 'topology1.yml')
         args = set_composer_flask_parser().parse_args([