refactor(compose): move compose template to resources

docker-compose/component/encoder.bas-pca.yml

@@ -13,7 +13,7 @@ component:
             model_dir: /ext_data/chinese_L-12_H-768_A-12
             max_seq_len: 40
             pooling_layer: -12
-        property:
+        gnes_config:
           is_trained: true
       bert_client:
         !BertEncoder
@@ -22,11 +22,11 @@ component:
             port: 5555
             port_out: 5556
             ignore_all_checks: true
-        property:
+        gnes_config:
           is_trained: true
   - !PCALocalEncoder
     parameter:
       output_dim: 300
       num_locals: 30
-    property:
+    gnes_config:
       batch_size: 2048

docker-compose/component/encoder.bas.yml

@@ -13,7 +13,7 @@ component:
             model_dir: /ext_data/chinese_L-12_H-768_A-12
             max_seq_len: 40
             pooling_layer: -12
-        property:
+        gnes_config:
           is_trained: true
       bert_client:
         !BertEncoder
@@ -22,13 +22,13 @@ component:
             port: 5555
             port_out: 5556
             ignore_all_checks: true
-        property:
+        gnes_config:
           is_trained: true
   - !PCALocalEncoder
     parameter:
       output_dim: 300
       num_locals: 30
-    property:
+    gnes_config:
       batch_size: 2048
   - !PQEncoder
     parameter:

docker-compose/component/encoder.inception.yml

@@ -26,13 +26,13 @@ component:
         - torch.nn.functional.adaptive_avg_pool2d(x, (1, 1))
         - torch.nn.functional.dropout(x, training=False)
         - x.view(x.size(0), -1)
-    property:
+    gnes_config:
       is_trained: true
   - !PCALocalEncoder
     parameter:
       output_dim: 200
       num_locals: 10
-    property:
+    gnes_config:
       batch_size: 2048
   - !PQEncoder
     parameter:

docker-compose/component/encoder.resnet.yml

@@ -15,13 +15,13 @@ component:
         - layer4
         - avgpool
         - x.reshape(x.size(0), -1)
-    property:
+    gnes_config:
       is_trained: true
   - !PCALocalEncoder
     parameter:
       output_dim: 200
       num_locals: 10
-    property:
+    gnes_config:
       batch_size: 2048
   - !PQEncoder
     parameter:

docker-compose/component/encoder.vgg.yml

@@ -9,13 +9,13 @@ component:
         - avg_pool
         - x.view(x.size(0), -1)
         - classifier[0]
-    property:
+    gnes_config:
       is_trained: true
   - !PCALocalEncoder
     parameter:
       output_dim: 200
       num_locals: 10
-    property:
+    gnes_config:
       batch_size: 2048
   - !PQEncoder
     parameter:

docker-compose/component/encoder.w2v.yml

@@ -3,13 +3,13 @@ component:
   - !Word2VecEncoder
     parameter:
       model_dir: /ext_data/sgns.wiki.bigram-char.refine
-    property:
+    gnes_config:
       is_trained: true
   - !PCALocalEncoder
     parameter:
       output_dim: 200
       num_locals: 10
-    property:
+    gnes_config:
       batch_size: 2048
   - !PQEncoder
     parameter:

docker-compose/component/img_preprocessor_fasterRCNN.yml

@@ -4,5 +4,5 @@ parameter:
   model_name: fasterrcnn_resnet50_fpn
   target_img_size: 224
   _use_cuda: false
-property:
+gnes_config:
   is_trained: true

docker-compose/component/preprocessor.yml

@@ -3,5 +3,5 @@ parameter:
   start_doc_id: 0
   random_doc_id: True
   deliminator: "[.。！？!?]+"
-property:
+gnes_config:
   is_trained: true

docs/chapter/microservice.rst

@@ -4,7 +4,7 @@ Microservices
 GNES is all-in-microservice: encoder, indexer, preprocessor and router are all running statelessly and independently in their own containers. They communicate via versioned APIs and collaborate under the orchestration of Docker Swarm/Kubernetes etc. Scaling, load-balancing, automated recovering, they come off-the-shelf in GNES.
 
 
-.. image:: ../.github/gnes-all-in-ms.svg
+.. image:: ../../.github/gnes-all-in-ms.svg
    :align: center
    :width: 500px
 

gnes/composer/base.py

@@ -21,7 +21,7 @@ class YamlGraph:
         'Encoder': 'encode',
         'Router': 'route',
         'Indexer': 'index',
-        'Frontend': 'frontend',
+        'gRPCFrontend': 'frontend',
         'Preprocessor': 'preprocess'
     }
 
@@ -78,15 +78,19 @@ def __init__(self, args):
             tmp = _yaml.load(args.yaml_path)
             stream = StringIO()
             _yaml.dump(tmp, stream)
-        self.original_yaml = stream.getvalue()
-        self.name = tmp.get('name', args.name)
-        self.port = tmp.get('port', args.port)
+            self.original_yaml = stream.getvalue()
+
+        self._name = tmp.get('name', args.name)
+        self._port = tmp.get('port', args.port)
+        self._networks = tmp.get('networks', {})
+        self._volumes = tmp.get('volumes', {})
+
         self.args = args
         self._num_layer = 0
 
         if 'services' in tmp:
             self.add_layer()
-            self.add_comp(CommentedMap({'name': 'Frontend', 'grpc_port': self.port}))
+            self.add_comp(CommentedMap({'name': 'gRPCFrontend', 'grpc_port': self._port}))
             for comp in tmp['services']:
                 self.add_layer()
                 if isinstance(comp, list):
@@ -135,11 +139,11 @@ def build_layers(self) -> List['YamlGraph.Layer']:
 
     @staticmethod
     def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gnes/gnes:latest',
-                          volumes: Dict = None) -> str:
-        swarm_lines = {'version': '3.4', 'services': {}}
+                          volumes: Dict = None, networks: Dict = None) -> str:
+        with pkg_resources.resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-swarm.yml'))) as r:
+            swarm_lines = _yaml.load(r)
         taboo = {'name', 'replicas', 'yaml_path'}
         config_dict = {}
-        network_dict = {'gnes-network': {'driver': 'overlay', 'attachable': True}}
         for l_idx, layer in enumerate(all_layers):
             for c_idx, c in enumerate(layer.components):
                 c_name = '%s%d%d' % (c['name'], l_idx, c_idx)
@@ -149,18 +153,17 @@ def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gn
                     args.append('--yaml_path /%s_yaml' % c_name)
                     config_dict['%s_yaml' % c_name] = {'file': c['yaml_path']}
 
-                if l_idx + 1 < len(all_layers):
-                    next_layer = all_layers[l_idx + 1]
-                    _l_idx = l_idx + 1
-                else:
-                    next_layer = all_layers[0]
-                    _l_idx = 0
-
-                host_out_name = ''
-                for _c_idx, _c in enumerate(next_layer.components):
-                    if _c['port_in'] == c['port_out']:
-                        host_out_name = '%s%d%d' % (_c['name'], _l_idx, _c_idx)
-                        break
+                # if l_idx + 1 < len(all_layers):
+                #     next_layer = all_layers[l_idx + 1]
+                #     _l_idx = l_idx + 1
+                # else:
+                #     next_layer = all_layers[0]
+                #     _l_idx = 0
+                # host_out_name = ''
+                # for _c_idx, _c in enumerate(next_layer.components):
+                #     if _c['port_in'] == c['port_out']:
+                #         host_out_name = '%s%d%d' % (_c['name'], _l_idx, _c_idx)
+                #         break
 
                 if l_idx - 1 >= 0:
                     last_layer = all_layers[l_idx - 1]
@@ -179,31 +182,32 @@ def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gn
                 # '--host_out %s' % host_out_name]
 
                 cmd = '%s %s' % (YamlGraph.comp2file[c['name']], ' '.join(args))
-                swarm_lines['services'][c_name] = {
+                swarm_lines['services'][c_name] = CommentedMap({
                     'image': docker_img,
                     'command': cmd,
-                }
+                })
 
                 rep_c = YamlGraph.Layer.get_value(c, 'replicas')
                 if rep_c > 1:
-                    swarm_lines['services'][c_name]['deploy'] = {
+                    swarm_lines['services'][c_name]['deploy'] = CommentedMap({
                         'replicas': YamlGraph.Layer.get_value(c, 'replicas'),
                         'restart_policy': {
                             'condition': 'on-failure',
                             'max_attempts': 3,
                         }
-                    }
+                    })
 
                 if 'yaml_path' in c and c['yaml_path'] is not None:
                     swarm_lines['services'][c_name]['configs'] = ['%s_yaml' % c_name]
 
-                if c['name'] == 'Frontend':
+                if c['name'] == 'gRPCFrontend':
                     swarm_lines['services'][c_name]['ports'] = ['%d:%d' % (c['grpc_port'], c['grpc_port'])]
 
         if volumes:
             swarm_lines['volumes'] = volumes
+        if networks:
+            swarm_lines['networks'] = volumes
         swarm_lines['configs'] = config_dict
-        swarm_lines['networks'] = network_dict
         stream = StringIO()
         _yaml.dump(swarm_lines, stream)
         return stream.getvalue()
@@ -278,8 +282,7 @@ def build_mermaid(all_layers: List['YamlGraph.Layer'], mermaid_leftright: bool =
 
     @staticmethod
     def build_html(generate_dict: Dict[str, str]) -> str:
-        r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'static', 'gnes-board.html')))
-        with r:
+        with pkg_resources.resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-board.html'))) as r:
             html = r.read().decode()
             for k, v in generate_dict.items():
                 if v:
@@ -302,7 +305,8 @@ def std_or_print(f, content):
             'mermaid': self.build_mermaid(all_layers, self.args.mermaid_leftright),
             'shell': self.build_shell(all_layers, self.args.shell_log_redirect),
             'yaml': self.original_yaml,
-            'docker': self.build_dockerswarm(all_layers, self.args.docker_img),
+            'docker': self.build_dockerswarm(all_layers, self.args.docker_img,
+                                             volumes=self._volumes, networks=self._networks),
             'k8s': self.build_kubernetes(all_layers),
             'timestamp': time.strftime("%a, %d %b %Y %H:%M:%S"),
             'version': __version__

gnes/resources/compose/gnes-shell.sh

@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+## Prerequirment of this script
+## You need to install GNES locally on this local machine
+## pip install gnes
+
 set -e
 
 trap 'kill $(jobs -p)' EXIT

gnes/resources/compose/gnes-swarm.yml

@@ -0,0 +1,8 @@
+version: '3.4'
+services: {}
+volumes: {}
+networks:
+  gnes-net:
+    driver: overlay
+    attachable: true
+configs: {}

gnes/resources/config/encoder/default.yml

@@ -13,7 +13,7 @@ component:
             model_dir: /ext_data/chinese_L-12_H-768_A-12
             max_seq_len: NONE
             pooling_layer: -12
-        property:
+        gnes_config:
           is_trained: true
       bert_client:
         !BertEncoder
@@ -22,13 +22,13 @@ component:
             port: 5555
             port_out: 5556
             ignore_all_checks: true
-        property:
+        gnes_config:
           is_trained: true
   - !PCALocalEncoder
     parameter:
       output_dim: 32
       num_locals: 8
-    property:
+    gnes_config:
       batch_size: 2048
   - !PQEncoder
     parameter:

gnes/resources/config/gnes/default.yml

@@ -16,7 +16,7 @@ component:
                 model_dir: /ext_data/chinese_L-12_H-768_A-12
                 max_seq_len: NONE
                 pooling_layer: -12
-            property:
+            gnes_config:
               is_trained: true
           bert_client:
             !BertEncoder
@@ -25,13 +25,13 @@ component:
                 port: 5555
                 port_out: 5556
                 ignore_all_checks: true
-            property:
+            gnes_config:
               is_trained: true
       - !PCALocalEncoder
         parameter:
           output_dim: 32
           num_locals: 8
-        property:
+        gnes_config:
           batch_size: 2048
       - !PQEncoder
         parameter:
@@ -44,5 +44,5 @@ component:
   text_indexer: !LVDBIndexer
     parameter:
       data_path: ./test_db
-property:
+gnes_config:
   work_dir: /ext_data/demo-model

gnes/resources/config/preprocessor/default.yml

@@ -3,5 +3,5 @@ parameter:
   start_doc_id: 0
   random_doc_id: True
   deliminator: "[.。！？!?]+"
-property:
+gnes_config:
   is_trained: true

tests/yaml/base-elmo-nes.yml

@@ -8,13 +8,13 @@ component:
           model_dir: $ELMO_CI_MODEL
           pooling_layer: -1
           pooling_stragy: REDUCE_MEAN
-        property:
+        gnes_config:
           is_trained: true
       - !PCALocalEncoder
         parameter:
           output_dim: 32
           num_locals: 8
-        property:
+        gnes_config:
           batch_size: 2048
       - !PQEncoder
         parameter: