From abb0841cdcd07c3bedeee03682b3022743449bda Mon Sep 17 00:00:00 2001 From: Larry Yan Date: Wed, 17 Jul 2019 13:22:11 +0800 Subject: [PATCH 1/4] feat(encoder): add convolution variational autoencoder --- gnes/encoder/image/cvae.py | 69 +++++++++++++++ gnes/encoder/image/cvae_cores/model.py | 114 +++++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 gnes/encoder/image/cvae.py create mode 100644 gnes/encoder/image/cvae_cores/model.py diff --git a/gnes/encoder/image/cvae.py b/gnes/encoder/image/cvae.py new file mode 100644 index 00000000..e3b0e711 --- /dev/null +++ b/gnes/encoder/image/cvae.py @@ -0,0 +1,69 @@ +# Tencent is pleased to support the open source community by making GNES available. +# +# Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List +import numpy as np +from gnes.helper import batch_iterator +from ..base import BaseImageEncoder +from PIL import Image + + +class CVAEEncoder(BaseImageEncoder): + + def __init__(self, model_dir: str, + latent_dim: int = 300, + batch_size: int = 64, + select_method: str = 'MEAN', + use_gpu: bool = True, + *args, **kwargs): + super().__init__(*args, **kwargs) + + self.model_dir = model_dir + self.latent_dim = latent_dim + self.batch_size = batch_size + self.select_method = select_method + self.use_gpu = use_gpu + + def post_init(self): + import tensorflow as tf + from .cave_cores.model import CVAE + + self._model = CVAE(self.latent_dim) + self.inputs = tf.placeholder(tf.float32, + (None, 120, 120, 3)) + + self.mean, self.var = self._model.encode(self.inputs) + + config = tf.ConfigProto(log_device_placement=False) + if self.use_gpu: + config.gpu_options.allow_growth = True + self.sess = tf.Session(config=config) + self.saver = tf.train.Saver() + self.saver.restore(self.sess, self.model_dir) + + def encode(self, img: List['np.ndarray'], *args, **kwargs) -> np.ndarray: + ret = [] + img = [(np.array(Image.fromarray(im).resize((120, 120)), + dtype=np.float32)/255) for im in img] + for _im in batch_iterator(img, self.batch_size): + _mean, _var = self.sess.run((self.mean, self.var), + feed_dict={self.inputs: _im}) + if self.select_method == 'MEAN': + ret.append(_mean) + elif self.select_method == 'VAR': + ret.append(_var) + elif self.select_method == 'MEAN_VAR': + ret.append(np.concatenate([_mean, _var]), axis=1) + return np.concatenate(ret, axis=0).astype(np.float32) diff --git a/gnes/encoder/image/cvae_cores/model.py b/gnes/encoder/image/cvae_cores/model.py new file mode 100644 index 00000000..2df695d4 --- /dev/null +++ b/gnes/encoder/image/cvae_cores/model.py @@ -0,0 +1,114 @@ +# Tencent is pleased to support the open source community by making GNES available. +# +# Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tensorflow as tf +import numpy as np + + +class CVAE(tf.keras.Model): + def __init__(self, latent_dim): + super(CVAE, self).__init__() + self.latent_dim = latent_dim + self.inference_net = tf.keras.Sequential( + [ + tf.keras.layers.InputLayer(input_shape=(120, 120, 3)), + tf.keras.layers.Conv2D( + filters=32, kernel_size=3, strides=(2, 2), + padding='SAME', + activation='relu'), + tf.keras.layers.Conv2D( + filters=32, kernel_size=3, strides=(2, 2), + padding='SAME', + activation='relu'), + tf.keras.layers.Conv2D( + filters=32, kernel_size=3, strides=(2, 2), + padding='SAME', + activation='relu'), + tf.keras.layers.Flatten(), + # No activation + tf.keras.layers.Dense(latent_dim + latent_dim), + ] + ) + + self.generative_net = tf.keras.Sequential( + [ + tf.keras.layers.InputLayer(input_shape=(latent_dim,)), + tf.keras.layers.Dense(units=15*15*32, + activation=tf.nn.relu), + tf.keras.layers.Reshape(target_shape=(15, 15, 32)), + tf.keras.layers.Conv2DTranspose( + filters=32, + kernel_size=3, + strides=(2, 2), + padding="SAME", + activation='relu'), + tf.keras.layers.Conv2DTranspose( + filters=32, + kernel_size=3, + strides=(2, 2), + padding="SAME", + activation='relu'), + tf.keras.layers.Conv2DTranspose( + filters=32, + kernel_size=3, + strides=(2, 2), + padding="SAME", + activation='relu'), + # No activation + tf.keras.layers.Conv2DTranspose( + filters=3, kernel_size=3, strides=(1, 1), padding="SAME"), + ] + ) + + def sample(self, eps=None): + if eps is None: + eps = tf.random_normal(shape=(100, self.latent_dim)) + return self.decode(eps, apply_sigmoid=True) + + def encode(self, x): + mean, logvar = tf.split(self.inference_net(x), num_or_size_splits=2, axis=1) + return mean, logvar + + def reparameterize(self, mean, logvar): + eps = tf.random_normal(shape=tf.shape(mean)) + return eps * tf.exp(logvar * .5) + mean + + def decode(self, z, apply_sigmoid=False): + logits = self.generative_net(z) + if apply_sigmoid: + probs = tf.sigmoid(logits) + return probs + + return logits + + def compute_loss(self, x): + mean, logvar = self.encode(x) + z = self.reparameterize(mean, logvar) + x_logit = self.decode(z) + + cross_ent = tf.nn.sigmoid_cross_entropy_with_logits(logits=x_logit, + labels=x) + logpx_z = -tf.reduce_sum(cross_ent, axis=[1, 2, 3]) + logpz = CVAE.log_normal_pdf(z, 0., 0.) + logqz_x = CVAE.log_normal_pdf(z, mean, logvar) + + return -tf.reduce_mean(logpx_z + logpz - logqz_x) + + @staticmethod + def log_normal_pdf(sample, mean, logvar, raxis=1): + log2pi = tf.math.log(2. * np.pi) + return tf.reduce_sum( + -.5 * ((sample - mean) ** 2. * tf.exp(-logvar) + logvar + log2pi), + axis=raxis) From 2213905b1a587e069a4e4cd66b23344ac64c6465 Mon Sep 17 00:00:00 2001 From: Larry Yan Date: Fri, 19 Jul 2019 10:32:05 +0800 Subject: [PATCH 2/4] fix(encoder): modify CVAE --- gnes/encoder/image/cvae_cores/model.py | 5 +- show.html | 512 +++++++++++++++++++++++++ 2 files changed, 515 insertions(+), 2 deletions(-) create mode 100644 show.html diff --git a/gnes/encoder/image/cvae_cores/model.py b/gnes/encoder/image/cvae_cores/model.py index 2df695d4..d928473d 100644 --- a/gnes/encoder/image/cvae_cores/model.py +++ b/gnes/encoder/image/cvae_cores/model.py @@ -81,7 +81,8 @@ def encode(self, x): mean, logvar = tf.split(self.inference_net(x), num_or_size_splits=2, axis=1) return mean, logvar - def reparameterize(self, mean, logvar): + @staticmethod + def reparameterize(mean, logvar): eps = tf.random_normal(shape=tf.shape(mean)) return eps * tf.exp(logvar * .5) + mean @@ -95,7 +96,7 @@ def decode(self, z, apply_sigmoid=False): def compute_loss(self, x): mean, logvar = self.encode(x) - z = self.reparameterize(mean, logvar) + z = CVAE.reparameterize(mean, logvar) x_logit = self.decode(z) cross_ent = tf.nn.sigmoid_cross_entropy_with_logits(logits=x_logit, diff --git a/show.html b/show.html new file mode 100644 index 00000000..582e1411 --- /dev/null +++ b/show.html @@ -0,0 +1,512 @@ + + + + + + + + + GNES Board + + + +
+
+
+
+
+ YAML config +
+
+ + 
+                    
+port: 5566
+services:
+- name: Preprocessor
+  replicas: 2
+- name: Encoder
+  replicas: 3
+- - name: Indexer
+    yaml_path: indexer-binary.yml
+    replicas: 4
+    income: sub
+  - name: Indexer
+    yaml_path: indexer-fulltext.yml
+    replicas: 3
+    income: sub
+
+                    
+
+
+
+
+
+
+ +
+
+ +

This is the workflow generated from your input YAML config, which helps you + to understand how microservices work together in GNES.

+
+
+
+ Workflow +
+
+
+ graph TD + Frontend000(Frontend)-- push/pull -->Preprocessor100(Preprocessor0) + Frontend000(Frontend)-- push/pull -->Preprocessor101(Preprocessor1) + Preprocessor100(Preprocessor0)-- push/pull -->Router400((Router)) + Preprocessor101(Preprocessor1)-- push/pull -->Router400((Router)) + Router400((Router))-- push/pull -->Encoder200(Encoder0) + Router400((Router))-- push/pull -->Encoder201(Encoder1) + Router400((Router))-- push/pull -->Encoder202(Encoder2) + Encoder200(Encoder0)-- push/pull -->Router500((Router)) + Encoder201(Encoder1)-- push/pull -->Router500((Router)) + Encoder202(Encoder2)-- push/pull -->Router500((Router)) + Router500((Router))-- pub/sub -->Indexer300(Indexer00) + Router500((Router))-- pub/sub -->Indexer301(Indexer01) + Router500((Router))-- pub/sub -->Indexer302(Indexer02) + Router500((Router))-- pub/sub -->Indexer303(Indexer03) + Router500((Router))-- pub/sub -->Indexer310(Indexer10) + Router500((Router))-- pub/sub -->Indexer311(Indexer11) + Router500((Router))-- pub/sub -->Indexer312(Indexer12) + Indexer300(Indexer00)-- push/pull -->Router600((Router0)) + Indexer301(Indexer01)-- push/pull -->Router600((Router0)) + Indexer302(Indexer02)-- push/pull -->Router600((Router0)) + Indexer303(Indexer03)-- push/pull -->Router600((Router0)) + Indexer310(Indexer10)-- push/pull -->Router610((Router1)) + Indexer311(Indexer11)-- push/pull -->Router610((Router1)) + Indexer312(Indexer12)-- push/pull -->Router610((Router1)) + Router600((Router0))-- push/pull -->Router700((Router)) + Router610((Router1))-- push/pull -->Router700((Router)) + Router700((Router))-- push/pull -->Frontend000(Frontend) +classDef FrontendCLS fill:#ffb347,stroke:#277CE8,stroke-width:1px,stroke-dasharray:5; +classDef EncoderCLS fill:#27E1E8,stroke:#277CE8,stroke-width:1px; +classDef IndexerCLS fill:#27E1E8,stroke:#277CE8,stroke-width:1px; +classDef RouterCLS fill:#2BFFCB,stroke:#277CE8,stroke-width:1px; +classDef PreprocessorCLS fill:#27E1E8,stroke:#277CE8,stroke-width:1px; +class Frontend000 FrontendCLS; +class Preprocessor101,Preprocessor100 PreprocessorCLS; +class Router600,Router700,Router400,Router500,Router610 RouterCLS; +class Encoder201,Encoder200,Encoder202 EncoderCLS; +class Indexer311,Indexer300,Indexer301,Indexer312,Indexer303,Indexer302,Indexer310 IndexerCLS; +
+
+
+
+
+
+ +

This is a Bash script generated from your YAML config. + You can use it to start a GNES server on a local machine.

+
+

1. Install GNES via pip install gnes
+ 2. Create a new file say run.sh
+ 3. Copy the following content to it and run it via bash ./run.sh.

+
+
+
+ Shell script +
+
+ + 
+                    
+#!/usr/bin/env bash
+
+set -e
+
+trap 'kill $(jobs -p)' EXIT
+
+printf "starting service Frontend with 1 replicas...\n"
+gnes frontend --grpc_port 5566 --port_out 56795 --socket_out PUSH_BIND --port_in 49250 --socket_in PULL_CONNECT  &
+printf "starting service Preprocessor with 2 replicas...\n"
+gnes preprocess --port_in 56795 --socket_in PULL_CONNECT --port_out 59865 --socket_out PUSH_CONNECT  &
+gnes preprocess --port_in 56795 --socket_in PULL_CONNECT --port_out 59865 --socket_out PUSH_CONNECT  &
+printf "starting service Router with 1 replicas...\n"
+gnes route --socket_in PULL_BIND --socket_out PUSH_BIND --port_in 59865 --port_out 50001  &
+printf "starting service Encoder with 3 replicas...\n"
+gnes encode --port_in 50001 --socket_in PULL_CONNECT --port_out 50865 --socket_out PUSH_CONNECT  &
+gnes encode --port_in 50001 --socket_in PULL_CONNECT --port_out 50865 --socket_out PUSH_CONNECT  &
+gnes encode --port_in 50001 --socket_in PULL_CONNECT --port_out 50865 --socket_out PUSH_CONNECT  &
+printf "starting service Router with 1 replicas...\n"
+gnes route --socket_in PULL_BIND --socket_out PUB_BIND --port_in 50865 --port_out 64586  &
+printf "starting service Indexer with 4 replicas...\n"
+gnes index --yaml_path indexer-binary.yml --income sub --port_in 64586 --socket_in SUB_CONNECT --port_out 63366 --socket_out PUSH_CONNECT  &
+gnes index --yaml_path indexer-binary.yml --income sub --port_in 64586 --socket_in SUB_CONNECT --port_out 63366 --socket_out PUSH_CONNECT  &
+gnes index --yaml_path indexer-binary.yml --income sub --port_in 64586 --socket_in SUB_CONNECT --port_out 63366 --socket_out PUSH_CONNECT  &
+gnes index --yaml_path indexer-binary.yml --income sub --port_in 64586 --socket_in SUB_CONNECT --port_out 63366 --socket_out PUSH_CONNECT  &
+printf "starting service Indexer with 3 replicas...\n"
+gnes index --yaml_path indexer-fulltext.yml --income sub --socket_in SUB_CONNECT --port_in 64586 --socket_out PUSH_CONNECT --port_out 59025  &
+gnes index --yaml_path indexer-fulltext.yml --income sub --socket_in SUB_CONNECT --port_in 64586 --socket_out PUSH_CONNECT --port_out 59025  &
+gnes index --yaml_path indexer-fulltext.yml --income sub --socket_in SUB_CONNECT --port_in 64586 --socket_out PUSH_CONNECT --port_out 59025  &
+printf "starting service Router with 1 replicas...\n"
+gnes route --socket_in PULL_BIND --socket_out PUSH_CONNECT --port_in 63366 --port_out 54726  &
+printf "starting service Router with 1 replicas...\n"
+gnes route --socket_in PULL_BIND --socket_out PUSH_CONNECT --port_in 59025 --port_out 54726  &
+printf "starting service Router with 1 replicas...\n"
+gnes route --socket_in PULL_BIND --socket_out PUSH_BIND --port_in 54726 --port_out 49250  &
+
+wait
+                    
+
+
+
+
+ +
+
+ +

This is a docker-compose YAML file generated from your YAML config. + You can use it to start a Docker Swarm distributed on multiple machines.

+
+

1. Install Docker and Docker Swarm
+ 2. Create a new file say my-compose.yml
+ 3. Copy the following content to it + 4. Run docker stack deploy --compose-file my-compose.yml.

+
+
+
+ Docker-Swarm/Compose config +
+
+ + 
+                    
+version: '3.4'
+services:
+  Frontend00:
+    image: gnes/gnes:latest
+    command: frontend --grpc_port 5566 --port_out 56795 --socket_out PUSH_BIND --port_in
+      49250 --socket_in PULL_CONNECT --host_in Router70
+    ports:
+    - 5566:5566
+  Preprocessor10:
+    image: gnes/gnes:latest
+    command: preprocess --port_in 56795 --socket_in PULL_CONNECT --port_out 59865
+      --socket_out PUSH_CONNECT --host_in Frontend00
+    deploy:
+      replicas: 2
+      restart_policy:
+        condition: on-failure
+        max_attempts: 3
+  Router20:
+    image: gnes/gnes:latest
+    command: route --socket_in PULL_BIND --socket_out PUSH_BIND --port_in 59865 --port_out
+      50001 --host_in Preprocessor10
+  Encoder30:
+    image: gnes/gnes:latest
+    command: encode --port_in 50001 --socket_in PULL_CONNECT --port_out 50865 --socket_out
+      PUSH_CONNECT --host_in Router20
+    deploy:
+      replicas: 3
+      restart_policy:
+        condition: on-failure
+        max_attempts: 3
+  Router40:
+    image: gnes/gnes:latest
+    command: route --socket_in PULL_BIND --socket_out PUB_BIND --port_in 50865 --port_out
+      64586 --host_in Encoder30
+  Indexer50:
+    image: gnes/gnes:latest
+    command: index --income sub --port_in 64586 --socket_in SUB_CONNECT --port_out
+      63366 --socket_out PUSH_CONNECT --yaml_path /Indexer50_yaml --host_in Router40
+    deploy:
+      replicas: 4
+      restart_policy:
+        condition: on-failure
+        max_attempts: 3
+    configs:
+    - Indexer50_yaml
+  Indexer51:
+    image: gnes/gnes:latest
+    command: index --income sub --socket_in SUB_CONNECT --port_in 64586 --socket_out
+      PUSH_CONNECT --port_out 59025 --yaml_path /Indexer51_yaml --host_in Router40
+    deploy:
+      replicas: 3
+      restart_policy:
+        condition: on-failure
+        max_attempts: 3
+    configs:
+    - Indexer51_yaml
+  Router60:
+    image: gnes/gnes:latest
+    command: route --socket_in PULL_BIND --socket_out PUSH_CONNECT --port_in 63366
+      --port_out 54726 --host_in Indexer50
+  Router61:
+    image: gnes/gnes:latest
+    command: route --socket_in PULL_BIND --socket_out PUSH_CONNECT --port_in 59025
+      --port_out 54726 --host_in Indexer51
+  Router70:
+    image: gnes/gnes:latest
+    command: route --socket_in PULL_BIND --socket_out PUSH_BIND --port_in 54726 --port_out
+      49250 --host_in Router60
+configs:
+  Indexer50_yaml:
+    file: indexer-binary.yml
+  Indexer51_yaml:
+    file: indexer-fulltext.yml
+networks:
+  gnes-network:
+    driver: overlay
+    attachable: true
+
+                    
+
+
+
+
+ +
+
+
+ Docker-Swarm/Docker-compose config +
+
+ + 
+                    
+{{gnes-k8s}}
+                    
+
+
+
+
+
+ + +
+ +
+
+ + + + + + + + + + + + + \ No newline at end of file From 2448411db6c21b5526fcc389c4e9112c09b11146 Mon Sep 17 00:00:00 2001 From: Larry Yan Date: Fri, 19 Jul 2019 10:32:39 +0800 Subject: [PATCH 3/4] fix(encoder): modify CVAE --- show.html | 512 ------------------------------------------------------ 1 file changed, 512 deletions(-) delete mode 100644 show.html diff --git a/show.html b/show.html deleted file mode 100644 index 582e1411..00000000 --- a/show.html +++ /dev/null @@ -1,512 +0,0 @@ - - - - - - - - - GNES Board - - - -
-
-
-
-
- YAML config -
-
- - 
-                    
-port: 5566
-services:
-- name: Preprocessor
-  replicas: 2
-- name: Encoder
-  replicas: 3
-- - name: Indexer
-    yaml_path: indexer-binary.yml
-    replicas: 4
-    income: sub
-  - name: Indexer
-    yaml_path: indexer-fulltext.yml
-    replicas: 3
-    income: sub
-
-                    
-
-
-
-
-
-
- -
-
- -

This is the workflow generated from your input YAML config, which helps you - to understand how microservices work together in GNES.

-
-
-
- Workflow -
-
-
- graph TD - Frontend000(Frontend)-- push/pull -->Preprocessor100(Preprocessor0) - Frontend000(Frontend)-- push/pull -->Preprocessor101(Preprocessor1) - Preprocessor100(Preprocessor0)-- push/pull -->Router400((Router)) - Preprocessor101(Preprocessor1)-- push/pull -->Router400((Router)) - Router400((Router))-- push/pull -->Encoder200(Encoder0) - Router400((Router))-- push/pull -->Encoder201(Encoder1) - Router400((Router))-- push/pull -->Encoder202(Encoder2) - Encoder200(Encoder0)-- push/pull -->Router500((Router)) - Encoder201(Encoder1)-- push/pull -->Router500((Router)) - Encoder202(Encoder2)-- push/pull -->Router500((Router)) - Router500((Router))-- pub/sub -->Indexer300(Indexer00) - Router500((Router))-- pub/sub -->Indexer301(Indexer01) - Router500((Router))-- pub/sub -->Indexer302(Indexer02) - Router500((Router))-- pub/sub -->Indexer303(Indexer03) - Router500((Router))-- pub/sub -->Indexer310(Indexer10) - Router500((Router))-- pub/sub -->Indexer311(Indexer11) - Router500((Router))-- pub/sub -->Indexer312(Indexer12) - Indexer300(Indexer00)-- push/pull -->Router600((Router0)) - Indexer301(Indexer01)-- push/pull -->Router600((Router0)) - Indexer302(Indexer02)-- push/pull -->Router600((Router0)) - Indexer303(Indexer03)-- push/pull -->Router600((Router0)) - Indexer310(Indexer10)-- push/pull -->Router610((Router1)) - Indexer311(Indexer11)-- push/pull -->Router610((Router1)) - Indexer312(Indexer12)-- push/pull -->Router610((Router1)) - Router600((Router0))-- push/pull -->Router700((Router)) - Router610((Router1))-- push/pull -->Router700((Router)) - Router700((Router))-- push/pull -->Frontend000(Frontend) -classDef FrontendCLS fill:#ffb347,stroke:#277CE8,stroke-width:1px,stroke-dasharray:5; -classDef EncoderCLS fill:#27E1E8,stroke:#277CE8,stroke-width:1px; -classDef IndexerCLS fill:#27E1E8,stroke:#277CE8,stroke-width:1px; -classDef RouterCLS fill:#2BFFCB,stroke:#277CE8,stroke-width:1px; -classDef PreprocessorCLS fill:#27E1E8,stroke:#277CE8,stroke-width:1px; -class Frontend000 FrontendCLS; -class Preprocessor101,Preprocessor100 PreprocessorCLS; -class Router600,Router700,Router400,Router500,Router610 RouterCLS; -class Encoder201,Encoder200,Encoder202 EncoderCLS; -class Indexer311,Indexer300,Indexer301,Indexer312,Indexer303,Indexer302,Indexer310 IndexerCLS; -
-
-
-
-
-
- -

This is a Bash script generated from your YAML config. - You can use it to start a GNES server on a local machine.

-
-

1. Install GNES via pip install gnes
- 2. Create a new file say run.sh
- 3. Copy the following content to it and run it via bash ./run.sh.

-
-
-
- Shell script -
-
- - 
-                    
-#!/usr/bin/env bash
-
-set -e
-
-trap 'kill $(jobs -p)' EXIT
-
-printf "starting service Frontend with 1 replicas...\n"
-gnes frontend --grpc_port 5566 --port_out 56795 --socket_out PUSH_BIND --port_in 49250 --socket_in PULL_CONNECT  &
-printf "starting service Preprocessor with 2 replicas...\n"
-gnes preprocess --port_in 56795 --socket_in PULL_CONNECT --port_out 59865 --socket_out PUSH_CONNECT  &
-gnes preprocess --port_in 56795 --socket_in PULL_CONNECT --port_out 59865 --socket_out PUSH_CONNECT  &
-printf "starting service Router with 1 replicas...\n"
-gnes route --socket_in PULL_BIND --socket_out PUSH_BIND --port_in 59865 --port_out 50001  &
-printf "starting service Encoder with 3 replicas...\n"
-gnes encode --port_in 50001 --socket_in PULL_CONNECT --port_out 50865 --socket_out PUSH_CONNECT  &
-gnes encode --port_in 50001 --socket_in PULL_CONNECT --port_out 50865 --socket_out PUSH_CONNECT  &
-gnes encode --port_in 50001 --socket_in PULL_CONNECT --port_out 50865 --socket_out PUSH_CONNECT  &
-printf "starting service Router with 1 replicas...\n"
-gnes route --socket_in PULL_BIND --socket_out PUB_BIND --port_in 50865 --port_out 64586  &
-printf "starting service Indexer with 4 replicas...\n"
-gnes index --yaml_path indexer-binary.yml --income sub --port_in 64586 --socket_in SUB_CONNECT --port_out 63366 --socket_out PUSH_CONNECT  &
-gnes index --yaml_path indexer-binary.yml --income sub --port_in 64586 --socket_in SUB_CONNECT --port_out 63366 --socket_out PUSH_CONNECT  &
-gnes index --yaml_path indexer-binary.yml --income sub --port_in 64586 --socket_in SUB_CONNECT --port_out 63366 --socket_out PUSH_CONNECT  &
-gnes index --yaml_path indexer-binary.yml --income sub --port_in 64586 --socket_in SUB_CONNECT --port_out 63366 --socket_out PUSH_CONNECT  &
-printf "starting service Indexer with 3 replicas...\n"
-gnes index --yaml_path indexer-fulltext.yml --income sub --socket_in SUB_CONNECT --port_in 64586 --socket_out PUSH_CONNECT --port_out 59025  &
-gnes index --yaml_path indexer-fulltext.yml --income sub --socket_in SUB_CONNECT --port_in 64586 --socket_out PUSH_CONNECT --port_out 59025  &
-gnes index --yaml_path indexer-fulltext.yml --income sub --socket_in SUB_CONNECT --port_in 64586 --socket_out PUSH_CONNECT --port_out 59025  &
-printf "starting service Router with 1 replicas...\n"
-gnes route --socket_in PULL_BIND --socket_out PUSH_CONNECT --port_in 63366 --port_out 54726  &
-printf "starting service Router with 1 replicas...\n"
-gnes route --socket_in PULL_BIND --socket_out PUSH_CONNECT --port_in 59025 --port_out 54726  &
-printf "starting service Router with 1 replicas...\n"
-gnes route --socket_in PULL_BIND --socket_out PUSH_BIND --port_in 54726 --port_out 49250  &
-
-wait
-                    
-
-
-
-
- -
-
- -

This is a docker-compose YAML file generated from your YAML config. - You can use it to start a Docker Swarm distributed on multiple machines.

-
-

1. Install Docker and Docker Swarm
- 2. Create a new file say my-compose.yml
- 3. Copy the following content to it - 4. Run docker stack deploy --compose-file my-compose.yml.

-
-
-
- Docker-Swarm/Compose config -
-
- - 
-                    
-version: '3.4'
-services:
-  Frontend00:
-    image: gnes/gnes:latest
-    command: frontend --grpc_port 5566 --port_out 56795 --socket_out PUSH_BIND --port_in
-      49250 --socket_in PULL_CONNECT --host_in Router70
-    ports:
-    - 5566:5566
-  Preprocessor10:
-    image: gnes/gnes:latest
-    command: preprocess --port_in 56795 --socket_in PULL_CONNECT --port_out 59865
-      --socket_out PUSH_CONNECT --host_in Frontend00
-    deploy:
-      replicas: 2
-      restart_policy:
-        condition: on-failure
-        max_attempts: 3
-  Router20:
-    image: gnes/gnes:latest
-    command: route --socket_in PULL_BIND --socket_out PUSH_BIND --port_in 59865 --port_out
-      50001 --host_in Preprocessor10
-  Encoder30:
-    image: gnes/gnes:latest
-    command: encode --port_in 50001 --socket_in PULL_CONNECT --port_out 50865 --socket_out
-      PUSH_CONNECT --host_in Router20
-    deploy:
-      replicas: 3
-      restart_policy:
-        condition: on-failure
-        max_attempts: 3
-  Router40:
-    image: gnes/gnes:latest
-    command: route --socket_in PULL_BIND --socket_out PUB_BIND --port_in 50865 --port_out
-      64586 --host_in Encoder30
-  Indexer50:
-    image: gnes/gnes:latest
-    command: index --income sub --port_in 64586 --socket_in SUB_CONNECT --port_out
-      63366 --socket_out PUSH_CONNECT --yaml_path /Indexer50_yaml --host_in Router40
-    deploy:
-      replicas: 4
-      restart_policy:
-        condition: on-failure
-        max_attempts: 3
-    configs:
-    - Indexer50_yaml
-  Indexer51:
-    image: gnes/gnes:latest
-    command: index --income sub --socket_in SUB_CONNECT --port_in 64586 --socket_out
-      PUSH_CONNECT --port_out 59025 --yaml_path /Indexer51_yaml --host_in Router40
-    deploy:
-      replicas: 3
-      restart_policy:
-        condition: on-failure
-        max_attempts: 3
-    configs:
-    - Indexer51_yaml
-  Router60:
-    image: gnes/gnes:latest
-    command: route --socket_in PULL_BIND --socket_out PUSH_CONNECT --port_in 63366
-      --port_out 54726 --host_in Indexer50
-  Router61:
-    image: gnes/gnes:latest
-    command: route --socket_in PULL_BIND --socket_out PUSH_CONNECT --port_in 59025
-      --port_out 54726 --host_in Indexer51
-  Router70:
-    image: gnes/gnes:latest
-    command: route --socket_in PULL_BIND --socket_out PUSH_BIND --port_in 54726 --port_out
-      49250 --host_in Router60
-configs:
-  Indexer50_yaml:
-    file: indexer-binary.yml
-  Indexer51_yaml:
-    file: indexer-fulltext.yml
-networks:
-  gnes-network:
-    driver: overlay
-    attachable: true
-
-                    
-
-
-
-
- -
-
-
- Docker-Swarm/Docker-compose config -
-
- - 
-                    
-{{gnes-k8s}}
-                    
-
-
-
-
-
- - -
- -
-
- - - - - - - - - - - - - \ No newline at end of file From 4977aa3cf5e247c967c7b5d0729916fa69cc0381 Mon Sep 17 00:00:00 2001 From: Jem Date: Fri, 19 Jul 2019 13:38:02 +0800 Subject: [PATCH 4/4] fix(vector indexer): reorder relevance and chunk weight --- gnes/service/indexer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnes/service/indexer.py b/gnes/service/indexer.py index 7322ae44..928cf960 100644 --- a/gnes/service/indexer.py +++ b/gnes/service/indexer.py @@ -61,7 +61,7 @@ def _handler_chunk_search(self, msg: 'gnes_pb2.Message'): results = self._model.query(vecs, top_k=msg.request.search.top_k) q_weights = [qc.weight for qc in msg.request.search.query.chunks] for all_topks, qc_weight in zip(results, q_weights): - for _doc_id, _offset, _relevance, _weight in all_topks: + for _doc_id, _offset, _weight, _relevance in all_topks: r = msg.response.search.topk_results.add() r.chunk.doc_id = _doc_id r.chunk.offset_1d = _offset