diff --git a/docker-compose/.env b/docker-compose/.env deleted file mode 100644 index 759af67c..00000000 --- a/docker-compose/.env +++ /dev/null @@ -1,12 +0,0 @@ -GNES_STACK_NAME=gnes-grpc -DOCKER_IMG_URL="ccr.ccs.tencentyun.com/gnes/aipd-gnes:master" -MODEL_DIR=${PWD}/pretrained_gnes -OUTPUT_DIR=${MODEL_DIR}/output_data -ENCODER_YAML_PATH=${MODEL_DIR}/encoder.yml -INDEXER_YAML_PATH=${MODEL_DIR}/indexer.yml -PREPROCESSOR_YAML_PATH=${MODEL_DIR}/preprocessor.yml -BATCH_SIZE=100 -NUM_INDEXER=31 -NUM_ENCODER=3 -GRPC_PORT=8800 -HTTP_PORT=80 diff --git a/docker-compose/basic/index-compose.yml b/docker-compose/basic/index-compose.yml deleted file mode 100644 index 830d5e8a..00000000 --- a/docker-compose/basic/index-compose.yml +++ /dev/null @@ -1,118 +0,0 @@ -version: '3.4' -services: - preprocess: - image: ${DOCKER_IMG_URL} - command: > - preprocess --port_in ${PREPROCESSOR_IN} --port_out ${PREPROCESSOR_OUT} - --socket_in PULL_BIND --socket_out PUSH_CONNECT - --host_out encoder_serviced - --yaml_path /preprocessor_config --dump_path /dump/preprocessor.bin - networks: - gnes-net: - aliases: - - preprocessor_serviced - configs: - - preprocessor_config - - encoder: - image: ${DOCKER_IMG_URL} - command: > - encode --port_in ${PREPROCESSOR_OUT} --port_out ${ENCODER_OUT} - --socket_in PULL_CONNECT --socket_out PUB_BIND - --host_in preprocessor_serviced --host_out middleman-route - --yaml_path /encoder_config --dump_path /dump/model.bin - volumes: - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - networks: - gnes-net: - aliases: - - encoder_serviced - deploy: - replicas: ${NUM_ENCODER} - restart_policy: - condition: on-failure - max_attempts: 3 - - depends_on: - - income-route - - middleman-route - - configs: - - encoder_config - - vector_indexer: - image: ${DOCKER_IMG_URL} - command: > - index --port_in ${MIDDLEMAN_ROUTE_OUT} --port_out ${OUTGOING_ROUTE_IN} - --socket_in SUB_CONNECT --socket_out PUSH_CONNECT - --host_in middleman-route --host_out outgoing-route - --yaml_path /indexer_config --dump_path /out_data/index.bin - volumes: - - index_data:/out_data - - networks: - gnes-net: - aliases: - - indexer_vector_serviced - - configs: - - indexer_vector_config - - fulltext_indexer: - image: ${DOCKER_IMG_URL} - command: > - index --port_in ${MIDDLEMAN_ROUTE_OUT} --port_out ${OUTGOING_ROUTE_IN} - --socket_in SUB_CONNECT --socket_out PUSH_CONNECT - --host_in middleman-route --host_out outgoing-route - --yaml_path /indexer_config --dump_path /out_data/index.bin - volumes: - - index_data:/out_data - - networks: - gnes-net: - aliases: - - indexer_fulltext_serviced - - configs: - - indexer_fulltext_config - - frontend: - image: ${DOCKER_IMG_URL} - command: > - frontend --port_in ${OUTGOING_ROUTE_OUT} --port_out ${PREPROCESSOR_IN} - --host_in outgoing-route --host_out preprocess - --socket_in SUB_CONNECT --socket_out PUSH_CONNECT - --grpc_port ${GRPC_PORT} - - networks: - gnes-net: - aliases: - - grpc_serviced - - depends_on: - - preprocess - - outgoing-route - - ports: - - ${GRPC_PORT}:${GRPC_PORT} - - -volumes: - index_data: - name: "${GNES_STACK_NAME}_shard_{{ .Task.Slot }}_data" - -networks: - gnes-net: - driver: overlay - attachable: true - -configs: - encoder_config: - file: "${ENCODER_YAML_PATH}" - indexer_vector_config: - file: "${INDEXER_VECTOR_YAML_PATH}" - indexer_fulltext_config: - file: "${INDEXER_FULLTEXT_YAML_PATH}" - preprocessor_config: - file: "${PREPROCESSOR_YAML_PATH}" \ No newline at end of file diff --git a/docker-compose/basic/query-compose.yml b/docker-compose/basic/query-compose.yml deleted file mode 100644 index 7cdf9e5f..00000000 --- a/docker-compose/basic/query-compose.yml +++ /dev/null @@ -1,150 +0,0 @@ -version: '3.4' -services: - preprocess: - image: ${DOCKER_IMG_URL} - command: > - preprocess --port_in ${PREPROCESSOR_IN} --port_out ${INCOME_ROUTE_IN} - --socket_in PULL_BIND --socket_out PUSH_CONNECT - --host_out income-route - --yaml_path /preprocessor_config --dump_path /dump/preprocessor.bin - networks: - gnes-net: - aliases: - - preprocessor_serviced - configs: - - preprocessor_config - - income-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type RouterService --port_in ${INCOME_ROUTE_IN} --port_out ${INCOME_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUSH_BIND - networks: - gnes-net: - aliases: - - income_route - - middleman-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type=RouterService --port_in ${MIDDLEMAN_ROUTE_IN} --port_out ${MIDDLEMAN_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUB_BIND - networks: - gnes-net: - aliases: - - middleman_route - - outgoing-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type=ReduceRouterService --port_in ${OUTGOING_ROUTE_IN} --port_out ${OUTGOING_ROUTE_OUT} - --socket_in SUB_BIND --socket_out PUB_BIND --num_part=$NUM_INDEXER - networks: - gnes-net: - aliases: - - outgoing_route - - encoder: - image: ${DOCKER_IMG_URL} - command: > - encode --port_in ${INCOME_ROUTE_OUT} --port_out ${MIDDLEMAN_ROUTE_IN} - --socket_in PULL_CONNECT --socket_out PUSH_CONNECT - --host_in income-route --host_out middleman-route - --yaml_path /encoder_config --dump_path /dump/model.bin - volumes: - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - networks: - gnes-net: - aliases: - - encoder_serviced - deploy: - replicas: 1 - restart_policy: - condition: on-failure - max_attempts: 3 - - depends_on: - - income-route - - middleman-route - - configs: - - encoder_config - - indexer: - image: ${DOCKER_IMG_URL} - command: > - index --port_in ${MIDDLEMAN_ROUTE_OUT} --port_out ${OUTGOING_ROUTE_IN} - --socket_in SUB_CONNECT --socket_out PUB_CONNECT - --host_in middleman-route --host_out outgoing-route - --yaml_path /indexer_config --dump_path /out_data/index.bin - volumes: - - index_data:/out_data - #- type: volume - # source: "index_data" - # target: "/out_data" - - #- "${OUTPUT_DIR}/indexer_dump.{{.Task.Slot}}:/out_data" - - deploy: - replicas: ${NUM_INDEXER} - restart_policy: - condition: on-failure - max_attempts: 3 - - environment: - - NODE_ID={{.Node.ID}} - - SERVICE_ID={{.Service.ID}} - - SERVICE_NAME={{.Service.Name}} - - SERVICE_LABELS={{.Service.Labels}} - - TASK_ID={{.Task.ID}} - - TASK_NAME={{.Task.Name}} - - TASK_SLOT={{.Task.Slot}} - - networks: - gnes-net: - aliases: - - index_serviced - depends_on: - - middleman-route - - outgoing-route - - configs: - - indexer_config - - frontend: - image: ${DOCKER_IMG_URL} - command: > - frontend --port_in ${OUTGOING_ROUTE_OUT} --port_out ${PREPROCESSOR_IN} - --host_in outgoing-route --host_out preprocess --socket_in SUB_CONNECT - --grpc_port ${GRPC_PORT} - - networks: - gnes-net: - aliases: - - grpc_serviced - - depends_on: - - income-route - - outgoing-route - - ports: - - ${GRPC_PORT}:${GRPC_PORT} - - -volumes: - index_data: - name: "${GNES_STACK_NAME}_shard_{{ .Task.Slot }}_data" - -networks: - gnes-net: - driver: overlay - attachable: true - -configs: - encoder_config: - file: "${ENCODER_YAML_PATH}" - indexer_config: - file: "${INDEXER_YAML_PATH}" - preprocessor_config: - file: "${PREPROCESSOR_YAML_PATH}" \ No newline at end of file diff --git a/docker-compose/basic/train-compose.yml b/docker-compose/basic/train-compose.yml deleted file mode 100644 index 32b50033..00000000 --- a/docker-compose/basic/train-compose.yml +++ /dev/null @@ -1,59 +0,0 @@ -version: '3.4' -services: - preprocess: - image: ${DOCKER_IMG_URL} - command: > - preprocess --port_in ${PREPROCESSOR_IN} --port_out ${INCOME_ROUTE_IN} - --socket_in PULL_BIND --socket_out PUSH_CONNECT - --host_out encoder_serviced - --yaml_path /preprocessor_config --dump_path /dump/preprocessor.bin - networks: - gnes-net: - aliases: - - preprocessor_serviced - configs: - - preprocessor_config - - encoder: - image: ${DOCKER_IMG_URL} - networks: - gnes-net: - aliases: - - encoder_serviced - command: > - encode --port_in ${INCOME_ROUTE_IN} --port_out ${OUTGOING_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUB_BIND - --yaml_path /encoder_config --dump_path /dump/model.bin - volumes: - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - configs: - - encoder_config - - frontend: - image: ${DOCKER_IMG_URL} - command: > - frontend --port_in ${OUTGOING_ROUTE_OUT} --port_out ${PREPROCESSOR_IN} - --host_in encoder_serviced --host_out preprocessor_serviced - --socket_in SUB_CONNECT --socket_out PUSH_CONNECT - --grpc_port ${GRPC_PORT} - - networks: - gnes-net: - aliases: - - grpc_serviced - - ports: - - ${GRPC_PORT}:${GRPC_PORT} - - -networks: - gnes-net: - driver: overlay - attachable: true - -configs: - encoder_config: - file: "${ENCODER_YAML_PATH}" - preprocessor_config: - file: "${PREPROCESSOR_YAML_PATH}" diff --git a/docker-compose/elastic/index-compose.yml b/docker-compose/elastic/index-compose.yml deleted file mode 100644 index 96fa489e..00000000 --- a/docker-compose/elastic/index-compose.yml +++ /dev/null @@ -1,155 +0,0 @@ -version: '3.4' -services: - preprocess: - image: ${DOCKER_IMG_URL} - command: > - preprocess --port_in ${PREPROCESSOR_IN} --port_out ${INCOME_ROUTE_IN} - --socket_in PULL_BIND --socket_out PUSH_CONNECT - --host_out income-route - --yaml_path /preprocessor_config --dump_path /dump/preprocessor.bin - networks: - gnes-net: - aliases: - - preprocessor_serviced - configs: - - preprocessor_config - depends_on: - - income-route - - income-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type MapRouterService --port_in ${INCOME_ROUTE_IN} --port_out ${INCOME_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUSH_BIND --batch_size=128 - networks: - gnes-net: - aliases: - - income_route - - middleman-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type=RouterService --port_in ${MIDDLEMAN_ROUTE_IN} --port_out ${MIDDLEMAN_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUSH_BIND - networks: - gnes-net: - aliases: - - middleman_route - - outgoing-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type=RouterService --port_in ${OUTGOING_ROUTE_IN} --port_out ${OUTGOING_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUB_BIND - networks: - gnes-net: - aliases: - - outgoing_route - - encoder: - image: ${DOCKER_IMG_URL} - command: > - encode --port_in ${INCOME_ROUTE_OUT} --port_out ${MIDDLEMAN_ROUTE_IN} - --socket_in PULL_CONNECT --socket_out PUSH_CONNECT - --host_in income-route --host_out middleman-route - --yaml_path /encoder_config --dump_path /dump/model.bin - volumes: - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - networks: - gnes-net: - aliases: - - encoder_serviced - deploy: - replicas: ${NUM_ENCODER} - restart_policy: - condition: on-failure - max_attempts: 3 - - depends_on: - - income-route - - middleman-route - - configs: - - encoder_config - - indexer: - image: ${DOCKER_IMG_URL} - command: > - index --port_in ${MIDDLEMAN_ROUTE_OUT} --port_out ${OUTGOING_ROUTE_IN} - --socket_in PULL_CONNECT --socket_out PUSH_CONNECT - --host_in middleman-route --host_out outgoing-route - --yaml_path /indexer_config --dump_path /out_data/index.bin - volumes: - - index_data:/out_data - #- type: volume - # source: "index_data" - # target: "/out_data" - - #- "${OUTPUT_DIR}/indexer_dump.{{.Task.Slot}}:/out_data" - - deploy: - replicas: ${NUM_INDEXER} - restart_policy: - condition: on-failure - max_attempts: 3 - - environment: - - NODE_ID={{.Node.ID}} - - SERVICE_ID={{.Service.ID}} - - SERVICE_NAME={{.Service.Name}} - - SERVICE_LABELS={{.Service.Labels}} - - TASK_ID={{.Task.ID}} - - TASK_NAME={{.Task.Name}} - - TASK_SLOT={{.Task.Slot}} - - networks: - gnes-net: - aliases: - - index_serviced - depends_on: - - middleman-route - - outgoing-route - - configs: - - indexer_config - - frontend: - image: ${DOCKER_IMG_URL} - command: > - frontend --port_in ${OUTGOING_ROUTE_OUT} --port_out ${PREPROCESSOR_IN} - --host_in outgoing-route --host_out preprocess - --socket_in SUB_CONNECT --socket_out PUSH_CONNECT - --grpc_port ${GRPC_PORT} - - networks: - gnes-net: - aliases: - - grpc_serviced - - depends_on: - - preprocess - - outgoing-route - - ports: - - ${GRPC_PORT}:${GRPC_PORT} - - -volumes: - index_data: - name: "${GNES_STACK_NAME}_shard_{{ .Task.Slot }}_data" - -networks: - gnes-net: - driver: overlay - attachable: true - -configs: - encoder_config: - file: "${ENCODER_YAML_PATH}" - indexer_vector_config: - file: "${INDEXER_VECTOR_YAML_PATH}" - indexer_fulltext_config: - file: "${INDEXER_FULLTEXT_YAML_PATH}" - preprocessor_config: - file: "${PREPROCESSOR_YAML_PATH}" \ No newline at end of file diff --git a/docker-compose/elastic/query-compose.yml b/docker-compose/elastic/query-compose.yml deleted file mode 100644 index 7cdf9e5f..00000000 --- a/docker-compose/elastic/query-compose.yml +++ /dev/null @@ -1,150 +0,0 @@ -version: '3.4' -services: - preprocess: - image: ${DOCKER_IMG_URL} - command: > - preprocess --port_in ${PREPROCESSOR_IN} --port_out ${INCOME_ROUTE_IN} - --socket_in PULL_BIND --socket_out PUSH_CONNECT - --host_out income-route - --yaml_path /preprocessor_config --dump_path /dump/preprocessor.bin - networks: - gnes-net: - aliases: - - preprocessor_serviced - configs: - - preprocessor_config - - income-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type RouterService --port_in ${INCOME_ROUTE_IN} --port_out ${INCOME_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUSH_BIND - networks: - gnes-net: - aliases: - - income_route - - middleman-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type=RouterService --port_in ${MIDDLEMAN_ROUTE_IN} --port_out ${MIDDLEMAN_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUB_BIND - networks: - gnes-net: - aliases: - - middleman_route - - outgoing-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type=ReduceRouterService --port_in ${OUTGOING_ROUTE_IN} --port_out ${OUTGOING_ROUTE_OUT} - --socket_in SUB_BIND --socket_out PUB_BIND --num_part=$NUM_INDEXER - networks: - gnes-net: - aliases: - - outgoing_route - - encoder: - image: ${DOCKER_IMG_URL} - command: > - encode --port_in ${INCOME_ROUTE_OUT} --port_out ${MIDDLEMAN_ROUTE_IN} - --socket_in PULL_CONNECT --socket_out PUSH_CONNECT - --host_in income-route --host_out middleman-route - --yaml_path /encoder_config --dump_path /dump/model.bin - volumes: - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - networks: - gnes-net: - aliases: - - encoder_serviced - deploy: - replicas: 1 - restart_policy: - condition: on-failure - max_attempts: 3 - - depends_on: - - income-route - - middleman-route - - configs: - - encoder_config - - indexer: - image: ${DOCKER_IMG_URL} - command: > - index --port_in ${MIDDLEMAN_ROUTE_OUT} --port_out ${OUTGOING_ROUTE_IN} - --socket_in SUB_CONNECT --socket_out PUB_CONNECT - --host_in middleman-route --host_out outgoing-route - --yaml_path /indexer_config --dump_path /out_data/index.bin - volumes: - - index_data:/out_data - #- type: volume - # source: "index_data" - # target: "/out_data" - - #- "${OUTPUT_DIR}/indexer_dump.{{.Task.Slot}}:/out_data" - - deploy: - replicas: ${NUM_INDEXER} - restart_policy: - condition: on-failure - max_attempts: 3 - - environment: - - NODE_ID={{.Node.ID}} - - SERVICE_ID={{.Service.ID}} - - SERVICE_NAME={{.Service.Name}} - - SERVICE_LABELS={{.Service.Labels}} - - TASK_ID={{.Task.ID}} - - TASK_NAME={{.Task.Name}} - - TASK_SLOT={{.Task.Slot}} - - networks: - gnes-net: - aliases: - - index_serviced - depends_on: - - middleman-route - - outgoing-route - - configs: - - indexer_config - - frontend: - image: ${DOCKER_IMG_URL} - command: > - frontend --port_in ${OUTGOING_ROUTE_OUT} --port_out ${PREPROCESSOR_IN} - --host_in outgoing-route --host_out preprocess --socket_in SUB_CONNECT - --grpc_port ${GRPC_PORT} - - networks: - gnes-net: - aliases: - - grpc_serviced - - depends_on: - - income-route - - outgoing-route - - ports: - - ${GRPC_PORT}:${GRPC_PORT} - - -volumes: - index_data: - name: "${GNES_STACK_NAME}_shard_{{ .Task.Slot }}_data" - -networks: - gnes-net: - driver: overlay - attachable: true - -configs: - encoder_config: - file: "${ENCODER_YAML_PATH}" - indexer_config: - file: "${INDEXER_YAML_PATH}" - preprocessor_config: - file: "${PREPROCESSOR_YAML_PATH}" \ No newline at end of file diff --git a/docker-compose/elastic/train-compose.yml b/docker-compose/elastic/train-compose.yml deleted file mode 100644 index 32b50033..00000000 --- a/docker-compose/elastic/train-compose.yml +++ /dev/null @@ -1,59 +0,0 @@ -version: '3.4' -services: - preprocess: - image: ${DOCKER_IMG_URL} - command: > - preprocess --port_in ${PREPROCESSOR_IN} --port_out ${INCOME_ROUTE_IN} - --socket_in PULL_BIND --socket_out PUSH_CONNECT - --host_out encoder_serviced - --yaml_path /preprocessor_config --dump_path /dump/preprocessor.bin - networks: - gnes-net: - aliases: - - preprocessor_serviced - configs: - - preprocessor_config - - encoder: - image: ${DOCKER_IMG_URL} - networks: - gnes-net: - aliases: - - encoder_serviced - command: > - encode --port_in ${INCOME_ROUTE_IN} --port_out ${OUTGOING_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUB_BIND - --yaml_path /encoder_config --dump_path /dump/model.bin - volumes: - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - configs: - - encoder_config - - frontend: - image: ${DOCKER_IMG_URL} - command: > - frontend --port_in ${OUTGOING_ROUTE_OUT} --port_out ${PREPROCESSOR_IN} - --host_in encoder_serviced --host_out preprocessor_serviced - --socket_in SUB_CONNECT --socket_out PUSH_CONNECT - --grpc_port ${GRPC_PORT} - - networks: - gnes-net: - aliases: - - grpc_serviced - - ports: - - ${GRPC_PORT}:${GRPC_PORT} - - -networks: - gnes-net: - driver: overlay - attachable: true - -configs: - encoder_config: - file: "${ENCODER_YAML_PATH}" - preprocessor_config: - file: "${PREPROCESSOR_YAML_PATH}" diff --git a/docker-compose/http/index-compose.yml b/docker-compose/http/index-compose.yml deleted file mode 100644 index eef4d4fe..00000000 --- a/docker-compose/http/index-compose.yml +++ /dev/null @@ -1,169 +0,0 @@ -version: '3.4' -services: - preprocess: - image: ${DOCKER_IMG_URL} - command: > - preprocess --port_in ${PREPROCESSOR_IN} --port_out ${INCOME_ROUTE_IN} - --socket_in PULL_BIND --socket_out PUSH_CONNECT - --host_out income-route - --yaml_path /preprocessor_config --dump_path /dump/preprocessor.bin - networks: - gnes-net: - aliases: - - preprocessor_serviced - configs: - - preprocessor_config - depends_on: - - income-route - - income-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type MapRouterService --port_in ${INCOME_ROUTE_IN} --port_out ${INCOME_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUSH_BIND --batch_size=128 - networks: - gnes-net: - aliases: - - income_route - - middleman-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type=RouterService --port_in ${MIDDLEMAN_ROUTE_IN} --port_out ${MIDDLEMAN_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUSH_BIND - networks: - gnes-net: - aliases: - - middleman_route - - outgoing-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type=RouterService --port_in ${OUTGOING_ROUTE_IN} --port_out ${OUTGOING_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUB_BIND - networks: - gnes-net: - aliases: - - outgoing_route - - encoder: - image: ${DOCKER_IMG_URL} - command: > - encode --port_in ${INCOME_ROUTE_OUT} --port_out ${MIDDLEMAN_ROUTE_IN} - --socket_in PULL_CONNECT --socket_out PUSH_CONNECT - --host_in income-route --host_out middleman-route - --yaml_path /encoder_config --dump_path /dump/model.bin - volumes: - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - networks: - gnes-net: - aliases: - - encoder_serviced - deploy: - replicas: ${NUM_ENCODER} - restart_policy: - condition: on-failure - max_attempts: 3 - - depends_on: - - income-route - - middleman-route - - configs: - - encoder_config - - indexer: - image: ${DOCKER_IMG_URL} - command: > - index --port_in ${MIDDLEMAN_ROUTE_OUT} --port_out ${OUTGOING_ROUTE_IN} - --socket_in PULL_CONNECT --socket_out PUSH_CONNECT - --host_in middleman-route --host_out outgoing-route - --yaml_path /indexer_config --dump_path /out_data/index.bin - volumes: - - index_data:/out_data - #- type: volume - # source: "index_data" - # target: "/out_data" - - #- "${OUTPUT_DIR}/indexer_dump.{{.Task.Slot}}:/out_data" - - deploy: - replicas: ${NUM_INDEXER} - restart_policy: - condition: on-failure - max_attempts: 3 - - environment: - - NODE_ID={{.Node.ID}} - - SERVICE_ID={{.Service.ID}} - - SERVICE_NAME={{.Service.Name}} - - SERVICE_LABELS={{.Service.Labels}} - - TASK_ID={{.Task.ID}} - - TASK_NAME={{.Task.Name}} - - TASK_SLOT={{.Task.Slot}} - - networks: - gnes-net: - aliases: - - index_serviced - depends_on: - - middleman-route - - outgoing-route - - configs: - - indexer_config - - frontend: - image: ${DOCKER_IMG_URL} - command: > - frontend --port_in ${OUTGOING_ROUTE_OUT} --port_out ${PREPROCESSOR_IN} - --host_in outgoing-route --host_out preprocess - --socket_in SUB_CONNECT --socket_out PUSH_CONNECT - --grpc_port ${GRPC_PORT} - - networks: - gnes-net: - aliases: - - grpc_serviced - - depends_on: - - preprocess - - outgoing-route - - ports: - - ${GRPC_PORT}:${GRPC_PORT} - - http_service: - image: ${DOCKER_IMG_URL} - command: > - client_http --http_port=${HTTP_PORT} --grpc_port ${GRPC_PORT} - --grpc_host frontend --max_workers=10 - networks: - gnes-net: - aliases: - - http_serviced - ports: - - ${HTTP_PORT}:${HTTP_PORT} - depends_on: - - encoder - - indexer - - outgoing-proxy - - income-proxy - -volumes: - index_data: - name: "${GNES_STACK_NAME}_shard_{{ .Task.Slot }}_data" - -networks: - gnes-net: - driver: overlay - attachable: true - -configs: - encoder_config: - file: "${ENCODER_YAML_PATH}" - indexer_config: - file: "${INDEXER_YAML_PATH}" - preprocessor_config: - file: "${PREPROCESSOR_YAML_PATH}" \ No newline at end of file diff --git a/docker-compose/http/query-compose.yml b/docker-compose/http/query-compose.yml deleted file mode 100644 index 674daad4..00000000 --- a/docker-compose/http/query-compose.yml +++ /dev/null @@ -1,167 +0,0 @@ -version: '3.4' -services: - preprocess: - image: ${DOCKER_IMG_URL} - command: > - preprocess --port_in ${PREPROCESSOR_IN} --port_out ${INCOME_ROUTE_IN} - --socket_in PULL_BIND --socket_out PUSH_CONNECT - --host_out income-route - --yaml_path /preprocessor_config --dump_path /dump/preprocessor.bin - networks: - gnes-net: - aliases: - - preprocessor_serviced - configs: - - preprocessor_config - - income-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type RouterService --port_in ${INCOME_ROUTE_IN} --port_out ${INCOME_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUSH_BIND - networks: - gnes-net: - aliases: - - income_route - - middleman-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type=RouterService --port_in ${MIDDLEMAN_ROUTE_IN} --port_out ${MIDDLEMAN_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUB_BIND - networks: - gnes-net: - aliases: - - middleman_route - - outgoing-route: - image: ${DOCKER_IMG_URL} - command: > - route --router_type=ReduceRouterService --port_in ${OUTGOING_ROUTE_IN} --port_out ${OUTGOING_ROUTE_OUT} - --socket_in SUB_BIND --socket_out PUB_BIND --num_part=$NUM_INDEXER - networks: - gnes-net: - aliases: - - outgoing_route - - encoder: - image: ${DOCKER_IMG_URL} - command: > - encode --port_in ${INCOME_ROUTE_OUT} --port_out ${MIDDLEMAN_ROUTE_IN} - --socket_in PULL_CONNECT --socket_out PUSH_CONNECT - --host_in income-route --host_out middleman-route - --yaml_path /encoder_config --dump_path /dump/model.bin - volumes: - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - networks: - gnes-net: - aliases: - - encoder_serviced - deploy: - replicas: 1 - restart_policy: - condition: on-failure - max_attempts: 3 - - depends_on: - - income-route - - middleman-route - - configs: - - encoder_config - - indexer: - image: ${DOCKER_IMG_URL} - command: > - index --port_in ${MIDDLEMAN_ROUTE_OUT} --port_out ${OUTGOING_ROUTE_IN} - --socket_in SUB_CONNECT --socket_out PUB_CONNECT - --host_in middleman-route --host_out outgoing-route - --yaml_path /indexer_config --dump_path /out_data/index.bin - volumes: - - index_data:/out_data - #- type: volume - # source: "index_data" - # target: "/out_data" - - #- "${OUTPUT_DIR}/indexer_dump.{{.Task.Slot}}:/out_data" - - deploy: - replicas: ${NUM_INDEXER} - restart_policy: - condition: on-failure - max_attempts: 3 - - environment: - - NODE_ID={{.Node.ID}} - - SERVICE_ID={{.Service.ID}} - - SERVICE_NAME={{.Service.Name}} - - SERVICE_LABELS={{.Service.Labels}} - - TASK_ID={{.Task.ID}} - - TASK_NAME={{.Task.Name}} - - TASK_SLOT={{.Task.Slot}} - - networks: - gnes-net: - aliases: - - index_serviced - depends_on: - - middleman-route - - outgoing-route - - configs: - - indexer_config - - frontend: - image: ${DOCKER_IMG_URL} - command: > - frontend --port_in ${OUTGOING_ROUTE_OUT} --port_out ${PREPROCESSOR_IN} - --host_in outgoing-route --host_out preprocess --socket_in SUB_CONNECT - --grpc_port ${GRPC_PORT} - - networks: - gnes-net: - aliases: - - grpc_serviced - - depends_on: - - income-route - - outgoing-route - - ports: - - ${GRPC_PORT}:${GRPC_PORT} - - - http_service: - image: ${DOCKER_IMG_URL} - command: > - client_http --http_port=${HTTP_PORT} --grpc_port ${GRPC_PORT} - --grpc_host frontend --max_workers=10 - networks: - gnes-net: - aliases: - - http_serviced - ports: - - ${HTTP_PORT}:${HTTP_PORT} - depends_on: - - encoder - - indexer - - outgoing-proxy - - income-proxy - -volumes: - index_data: - name: "${GNES_STACK_NAME}_shard_{{ .Task.Slot }}_data" - -networks: - gnes-net: - driver: overlay - attachable: true - -configs: - encoder_config: - file: "${ENCODER_YAML_PATH}" - indexer_config: - file: "${INDEXER_YAML_PATH}" - preprocessor_config: - file: "${PREPROCESSOR_YAML_PATH}" \ No newline at end of file diff --git a/docker-compose/http/train-compose.yml b/docker-compose/http/train-compose.yml deleted file mode 100644 index b240915d..00000000 --- a/docker-compose/http/train-compose.yml +++ /dev/null @@ -1,76 +0,0 @@ -version: '3.4' -services: - preprocess: - image: ${DOCKER_IMG_URL} - command: > - preprocess --port_in ${PREPROCESSOR_IN} --port_out ${INCOME_ROUTE_IN} - --socket_in PULL_BIND --socket_out PUSH_CONNECT - --host_out encoder_serviced - --yaml_path /preprocessor_config --dump_path /dump/preprocessor.bin - networks: - gnes-net: - aliases: - - preprocessor_serviced - configs: - - preprocessor_config - - encoder: - image: ${DOCKER_IMG_URL} - networks: - gnes-net: - aliases: - - encoder_serviced - command: > - encode --port_in ${INCOME_ROUTE_IN} --port_out ${OUTGOING_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUB_BIND - --yaml_path /encoder_config --dump_path /dump/model.bin - volumes: - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - configs: - - encoder_config - - frontend: - image: ${DOCKER_IMG_URL} - command: > - frontend --port_in ${OUTGOING_ROUTE_OUT} --port_out ${PREPROCESSOR_IN} - --host_in encoder_serviced --host_out preprocessor_serviced - --socket_in SUB_CONNECT --socket_out PUSH_CONNECT - --grpc_port ${GRPC_PORT} - - networks: - gnes-net: - aliases: - - grpc_serviced - - ports: - - ${GRPC_PORT}:${GRPC_PORT} - - - http_service: - image: ${DOCKER_IMG_URL} - command: > - client_http --http_port=${HTTP_PORT} --grpc_port ${GRPC_PORT} - --grpc_host frontend --max_workers=10 - networks: - gnes-net: - aliases: - - http_serviced - ports: - - ${HTTP_PORT}:${HTTP_PORT} - depends_on: - - encoder - - indexer - - outgoing-proxy - - income-proxy - -networks: - gnes-net: - driver: overlay - attachable: true - -configs: - encoder_config: - file: "${ENCODER_YAML_PATH}" - preprocessor_config: - file: "${PREPROCESSOR_YAML_PATH}" diff --git a/docker-compose/swarm/index-compose.yml b/docker-compose/swarm/index-compose.yml deleted file mode 100644 index 5e1a549d..00000000 --- a/docker-compose/swarm/index-compose.yml +++ /dev/null @@ -1,134 +0,0 @@ -version: '3.4' -services: - income-route: - image: ${DOCKER_IMG_URL} - command: > - route --route_type MapRouteService --port_in ${INCOME_ROUTE_IN} --port_out ${INCOME_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUSH_BIND --batch_size=${BATCH_SIZE} - networks: - gnes-net: - aliases: - - income_route - ports: - - ${HOST_PORT_IN}:${INCOME_ROUTE_IN} - deploy: - placement: - constraints: [node.role == worker] - replicas: 1 - restart_policy: - condition: on-failure - max_attempts: 3 - - middleman-route: - image: ${DOCKER_IMG_URL} - command: > - route --route_type=RouteService --port_in ${MIDDLEMAN_ROUTE_IN} --port_out ${MIDDLEMAN_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUSH_BIND - --unk_msg_route DEFAULT - networks: - gnes-net: - aliases: - - middleman_route - deploy: - placement: - constraints: [node.role == worker] - - outgoing-route: - image: ${DOCKER_IMG_URL} - command: > - route --route_type=RouteService --port_in ${OUTGOING_ROUTE_IN} --port_out ${OUTGOING_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUB_BIND - --unk_msg_route DEFAULT - networks: - gnes-net: - aliases: - - outgoing_route - ports: - - ${HOST_PORT_OUT}:${OUTGOING_ROUTE_OUT} - deploy: - placement: - constraints: [node.role == worker] - - encoder: - image: ${DOCKER_IMG_URL} - command: > - encode --port_in ${INCOME_ROUTE_OUT} --port_out ${MIDDLEMAN_ROUTE_IN} - --socket_in PULL_CONNECT --socket_out PUSH_CONNECT - --host_in income-route --host_out middleman-route - --mode INDEX --yaml_path /encoder.yml --dump_path /dump/model.bin - volumes: - - "${ENCODER_YAML_PATH}:/encoder.yml" - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - networks: - gnes-net: - aliases: - - encoder_serviced - deploy: - placement: - constraints: [node.role == worker] - replicas: 1 - restart_policy: - condition: on-failure - max_attempts: 3 - - depends_on: - - income-route - - middleman-route - - indexer: - image: ${DOCKER_IMG_URL} - command: > - index --port_in ${MIDDLEMAN_ROUTE_OUT} --port_out ${OUTGOING_ROUTE_IN} - --socket_in PULL_CONNECT --socket_out PUSH_CONNECT - --host_in middleman-route --host_out outgoing-route - --mode INDEX --yaml_path /indexer.yml --dump_path /out_data/index.bin - volumes: - - "${INDEXER_YAML_PATH}:/indexer.yml" - - index_data:/out_data - #- type: volume - # source: "index_data" - # target: "/out_data" - - #- "${OUTPUT_DIR}/indexer_dump.{{.Task.Slot}}:/out_data" - - deploy: - placement: - constraints: [node.role == worker] - replicas: 3 - restart_policy: - condition: on-failure - max_attempts: 3 - - environment: - - NODE_ID={{.Node.ID}} - - SERVICE_ID={{.Service.ID}} - - SERVICE_NAME={{.Service.Name}} - - SERVICE_LABELS={{.Service.Labels}} - - TASK_ID={{.Task.ID}} - - TASK_NAME={{.Task.Name}} - - TASK_SLOT={{.Task.Slot}} - - networks: - gnes-net: - aliases: - - index_serviced - depends_on: - - middleman-route - - outgoing-route - -volumes: - index_data: - driver: vieux/sshfs:latest - driver_opts: - sshcmd: "ubuntu@172.17.0.8:/data/larry/save_data/shard_{{ .Task.Slot }}_data" - password: "!QAZ2wsx#EDC" - allow_other: "" - name: 'shard_{{ .Task.Slot }}_data' - #external: - # name: 'index_shard_{{ .Task.Slot }}_data' - -networks: - gnes-net: - driver: overlay - attachable: true diff --git a/docker-compose/swarm/query-compose.yml b/docker-compose/swarm/query-compose.yml deleted file mode 100644 index 3c0dc6e9..00000000 --- a/docker-compose/swarm/query-compose.yml +++ /dev/null @@ -1,132 +0,0 @@ -version: '3.4' -services: - income-route: - image: ${DOCKER_IMG_URL} - command: > - route --route_type MapRouteService --port_in ${INCOME_ROUTE_IN} --port_out ${INCOME_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUSH_BIND - networks: - gnes-net: - aliases: - - income_route - ports: - - ${HOST_PORT_IN}:${INCOME_ROUTE_IN} - deploy: - placement: - constraints: [node.role == worker] - replicas: 1 - restart_policy: - condition: on-failure - max_attempts: 3 - - middleman-route: - image: ${DOCKER_IMG_URL} - command: > - route --route_type=RouteService --port_in ${MIDDLEMAN_ROUTE_IN} --port_out ${MIDDLEMAN_ROUTE_OUT} - --socket_in PULL_BIND --socket_out PUB_BIND - --unk_msg_route DEFAULT - networks: - gnes-net: - aliases: - - middleman_route - deploy: - placement: - constraints: [node.role == worker] - - outgoing-route: - image: ${DOCKER_IMG_URL} - command: > - route --route_type=ReduceRouteService --port_in ${OUTGOING_ROUTE_IN} --port_out ${OUTGOING_ROUTE_OUT} - --socket_in SUB_BIND --socket_out PUB_BIND --num_part=$NUM_REPLICAS - --unk_msg_route DEFAULT - networks: - gnes-net: - aliases: - - outgoing_route - ports: - - ${HOST_PORT_OUT}:${OUTGOING_ROUTE_OUT} - deploy: - placement: - constraints: [node.role == worker] - - encoder: - image: ${DOCKER_IMG_URL} - command: > - encode --port_in ${INCOME_ROUTE_OUT} --port_out ${MIDDLEMAN_ROUTE_IN} - --socket_in PULL_CONNECT --socket_out PUSH_CONNECT - --host_in income-route --host_out middleman-route - --mode QUERY --yaml_path /encoder.yml --dump_path /dump/model.bin - volumes: - - "${ENCODER_YAML_PATH}:/encoder.yml" - - "${MODEL_DIR}:/ext_data" - - "${OUTPUT_DIR}:/dump" - networks: - gnes-net: - aliases: - - encoder_serviced - deploy: - placement: - constraints: [node.role == worker] - replicas: 1 - restart_policy: - condition: on-failure - max_attempts: 3 - - depends_on: - - income-route - - middleman-route - - indexer: - image: ${DOCKER_IMG_URL} - command: > - index --port_in ${MIDDLEMAN_ROUTE_OUT} --port_out ${OUTGOING_ROUTE_IN} - --socket_in SUB_CONNECT --socket_out PUB_CONNECT - --host_in middleman-route --host_out outgoing-route - --mode QUERY --yaml_path /indexer.yml --dump_path /out_data/index.bin - volumes: - - "${INDEXER_YAML_PATH}:/indexer.yml" - - index_data:/out_data - #- type: volume - # source: "index_data" - # target: "/out_data" - - #- "${OUTPUT_DIR}/indexer_dump.{{.Task.Slot}}:/out_data" - - deploy: - replicas: ${NUM_REPLICAS} - restart_policy: - condition: on-failure - max_attempts: 3 - - environment: - - NODE_ID={{.Node.ID}} - - SERVICE_ID={{.Service.ID}} - - SERVICE_NAME={{.Service.Name}} - - SERVICE_LABELS={{.Service.Labels}} - - TASK_ID={{.Task.ID}} - - TASK_NAME={{.Task.Name}} - - TASK_SLOT={{.Task.Slot}} - - networks: - gnes-net: - aliases: - - index_serviced - depends_on: - - middleman-route - - outgoing-route - -volumes: - index_data: - driver: vieux/sshfs:latest - driver_opts: - sshcmd: "ubuntu@172.17.0.8:/data/larry/save_data/shard_{{ .Task.Slot }}_data" - password: "!QAZ2wsx#EDC" - allow_other: "" - name: 'shard_{{ .Task.Slot }}_data' - #external: - # name: 'index_shard_{{ .Task.Slot }}_data' - -networks: - gnes-net: - driver: overlay - attachable: true diff --git a/gnes/cli/parser.py b/gnes/cli/parser.py index d6382ead..7dab3078 100644 --- a/gnes/cli/parser.py +++ b/gnes/cli/parser.py @@ -90,12 +90,14 @@ def set_composer_flask_parser(parser=None): def set_service_parser(parser=None): from ..service.base import SocketType, BaseService + import random if not parser: parser = set_base_parser() - parser.add_argument('--port_in', type=int, default=5310, - help='port for input data') - parser.add_argument('--port_out', type=int, default=5311, - help='port for output data') + min_port, max_port = 49152, 65536 + parser.add_argument('--port_in', type=int, default=random.randrange(min_port, max_port), + help='port for input data, default a random port between [49152, 65536]') + parser.add_argument('--port_out', type=int, default=random.randrange(min_port, max_port), + help='port for output data, default a random port between [49152, 65536]') parser.add_argument('--host_in', type=str, default=BaseService.default_host, help='host address for input') parser.add_argument('--host_out', type=str, default=BaseService.default_host, diff --git a/gnes/proto/__init__.py b/gnes/proto/__init__.py index 6586b287..060ea4c3 100644 --- a/gnes/proto/__init__.py +++ b/gnes/proto/__init__.py @@ -97,7 +97,7 @@ def new_envelope(client_id: str, evlp.client_id = client_id evlp.request_id = request_id evlp.part_id = part_id - evlp.num_part = num_part + evlp.num_part.append(num_part) evlp.timeout = timeout add_route(evlp, client_id) return evlp diff --git a/gnes/proto/gnes.proto b/gnes/proto/gnes.proto index 94475df0..96911ac5 100644 --- a/gnes/proto/gnes.proto +++ b/gnes/proto/gnes.proto @@ -83,7 +83,7 @@ message Envelope { // for multi-part message uint32 part_id = 3; - uint32 num_part = 4; + repeated uint32 num_part = 4; uint32 timeout = 5; diff --git a/gnes/proto/gnes_pb2.py b/gnes/proto/gnes_pb2.py index f8af7170..8657e180 100644 --- a/gnes/proto/gnes_pb2.py +++ b/gnes/proto/gnes_pb2.py @@ -21,7 +21,7 @@ package='gnes', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n\ngnes.proto\x12\x04gnes\x1a\x1fgoogle/protobuf/timestamp.proto\"9\n\x07NdArray\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x11\n\x05shape\x18\x02 \x03(\rB\x02\x10\x01\x12\r\n\x05\x64type\x18\x03 \x01(\t\"\xbc\x01\n\x05\x43hunk\x12\x0e\n\x06\x64oc_id\x18\x01 \x01(\x04\x12\x0e\n\x04text\x18\x02 \x01(\tH\x00\x12\x1d\n\x04\x62lob\x18\x03 \x01(\x0b\x32\r.gnes.NdArrayH\x00\x12\x11\n\toffset_1d\x18\x04 \x01(\r\x12)\n\toffset_nd\x18\x05 \x01(\x0b\x32\x16.gnes.Chunk.Coordinate\x12\x0e\n\x06weight\x18\x06 \x01(\x02\x1a\x1b\n\nCoordinate\x12\r\n\x01x\x18\x01 \x03(\rB\x02\x10\x01\x42\t\n\x07\x63ontent\"\xe2\x02\n\x08\x44ocument\x12\x0e\n\x06\x64oc_id\x18\x01 \x01(\x04\x12\x1b\n\x06\x63hunks\x18\x02 \x03(\x0b\x32\x0b.gnes.Chunk\x12\'\n\x10\x63hunk_embeddings\x18\x03 \x01(\x0b\x32\r.gnes.NdArray\x12(\n\x08\x64oc_type\x18\x04 \x01(\x0e\x32\x16.gnes.Document.DocType\x12\x11\n\tmeta_info\x18\x05 \x01(\x0c\x12\x12\n\x08raw_text\x18\x06 \x01(\tH\x00\x12\"\n\traw_image\x18\x07 \x01(\x0b\x32\r.gnes.NdArrayH\x00\x12\"\n\traw_video\x18\x08 \x01(\x0b\x32\r.gnes.NdArrayH\x00\x12\x13\n\traw_bytes\x18\t \x01(\x0cH\x00\x12\x0e\n\x06weight\x18\n \x01(\x02\"6\n\x07\x44ocType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04TEXT\x10\x01\x12\t\n\x05IMAGE\x10\x02\x12\t\n\x05VIDEO\x10\x03\x42\n\n\x08raw_data\"\xd4\x01\n\x08\x45nvelope\x12\x11\n\tclient_id\x18\x01 \x01(\t\x12\x12\n\nrequest_id\x18\x02 \x01(\t\x12\x0f\n\x07part_id\x18\x03 \x01(\r\x12\x10\n\x08num_part\x18\x04 \x01(\r\x12\x0f\n\x07timeout\x18\x05 \x01(\r\x12$\n\x06routes\x18\x06 \x03(\x0b\x32\x14.gnes.Envelope.route\x1aG\n\x05route\x12\x0f\n\x07service\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"y\n\x07Message\x12 \n\x08\x65nvelope\x18\x01 \x01(\x0b\x32\x0e.gnes.Envelope\x12 \n\x07request\x18\x02 \x01(\x0b\x32\r.gnes.RequestH\x00\x12\"\n\x08response\x18\x03 \x01(\x0b\x32\x0e.gnes.ResponseH\x00\x42\x06\n\x04\x62ody\"\xf6\x03\n\x07Request\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12+\n\x05train\x18\x02 \x01(\x0b\x32\x1a.gnes.Request.TrainRequestH\x00\x12+\n\x05index\x18\x03 \x01(\x0b\x32\x1a.gnes.Request.IndexRequestH\x00\x12,\n\x06search\x18\x04 \x01(\x0b\x32\x1a.gnes.Request.QueryRequestH\x00\x12/\n\x07\x63ontrol\x18\x05 \x01(\x0b\x32\x1c.gnes.Request.ControlRequestH\x00\x1a;\n\x0cTrainRequest\x12\x1c\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x0e.gnes.Document\x12\r\n\x05\x66lush\x18\x02 \x01(\x08\x1a,\n\x0cIndexRequest\x12\x1c\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x0e.gnes.Document\x1a<\n\x0cQueryRequest\x12\x1d\n\x05query\x18\x01 \x01(\x0b\x32\x0e.gnes.Document\x12\r\n\x05top_k\x18\x02 \x01(\r\x1am\n\x0e\x43ontrolRequest\x12\x35\n\x07\x63ommand\x18\x01 \x01(\x0e\x32$.gnes.Request.ControlRequest.Command\"$\n\x07\x43ommand\x12\r\n\tTERMINATE\x10\x00\x12\n\n\x06STATUS\x10\x01\x42\x06\n\x04\x62ody\"\xc4\x06\n\x08Response\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12-\n\x05train\x18\x02 \x01(\x0b\x32\x1c.gnes.Response.TrainResponseH\x00\x12-\n\x05index\x18\x03 \x01(\x0b\x32\x1c.gnes.Response.IndexResponseH\x00\x12.\n\x06search\x18\x04 \x01(\x0b\x32\x1c.gnes.Response.QueryResponseH\x00\x12\x31\n\x07\x63ontrol\x18\x05 \x01(\x0b\x32\x1e.gnes.Response.ControlResponseH\x00\x1a\x36\n\rTrainResponse\x12%\n\x06status\x18\x01 \x01(\x0e\x32\x15.gnes.Response.Status\x1a\x36\n\rIndexResponse\x12%\n\x06status\x18\x01 \x01(\x0e\x32\x15.gnes.Response.Status\x1a\x38\n\x0f\x43ontrolResponse\x12%\n\x06status\x18\x01 \x01(\x0e\x32\x15.gnes.Response.Status\x1a\x81\x03\n\rQueryResponse\x12%\n\x06status\x18\x01 \x01(\x0e\x32\x15.gnes.Response.Status\x12\r\n\x05top_k\x18\x02 \x01(\r\x12?\n\x0ctopk_results\x18\x03 \x03(\x0b\x32).gnes.Response.QueryResponse.ScoredResult\x12\x39\n\x05level\x18\x04 \x01(\x0e\x32*.gnes.Response.QueryResponse.ResponseLevel\x1a{\n\x0cScoredResult\x12\x1c\n\x05\x63hunk\x18\x01 \x01(\x0b\x32\x0b.gnes.ChunkH\x00\x12\x1d\n\x03\x64oc\x18\x02 \x01(\x0b\x32\x0e.gnes.DocumentH\x00\x12\r\n\x05score\x18\x03 \x01(\x02\x12\x17\n\x0fscore_explained\x18\x04 \x01(\tB\x06\n\x04\x62ody\"A\n\rResponseLevel\x12\t\n\x05\x43HUNK\x10\x00\x12\x17\n\x13\x44OCUMENT_NOT_FILLED\x10\x01\x12\x0c\n\x08\x44OCUMENT\x10\x02\"-\n\x06Status\x12\x0b\n\x07SUCCESS\x10\x00\x12\t\n\x05\x45RROR\x10\x01\x12\x0b\n\x07PENDING\x10\x02\x42\x06\n\x04\x62ody2\xe3\x01\n\x07GnesRPC\x12(\n\x05Train\x12\r.gnes.Request\x1a\x0e.gnes.Response\"\x00\x12(\n\x05Index\x12\r.gnes.Request\x1a\x0e.gnes.Response\"\x00\x12(\n\x05Query\x12\r.gnes.Request\x1a\x0e.gnes.Response\"\x00\x12\'\n\x04\x43\x61ll\x12\r.gnes.Request\x1a\x0e.gnes.Response\"\x00\x12\x31\n\nStreamCall\x12\r.gnes.Request\x1a\x0e.gnes.Response\"\x00(\x01\x30\x01\x62\x06proto3') + serialized_pb=_b('\n\ngnes.proto\x12\x04gnes\x1a\x1fgoogle/protobuf/timestamp.proto\"9\n\x07NdArray\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x11\n\x05shape\x18\x02 \x03(\rB\x02\x10\x01\x12\r\n\x05\x64type\x18\x03 \x01(\t\"\xbc\x01\n\x05\x43hunk\x12\x0e\n\x06\x64oc_id\x18\x01 \x01(\x04\x12\x0e\n\x04text\x18\x02 \x01(\tH\x00\x12\x1d\n\x04\x62lob\x18\x03 \x01(\x0b\x32\r.gnes.NdArrayH\x00\x12\x11\n\toffset_1d\x18\x04 \x01(\r\x12)\n\toffset_nd\x18\x05 \x01(\x0b\x32\x16.gnes.Chunk.Coordinate\x12\x0e\n\x06weight\x18\x06 \x01(\x02\x1a\x1b\n\nCoordinate\x12\r\n\x01x\x18\x01 \x03(\rB\x02\x10\x01\x42\t\n\x07\x63ontent\"\xe2\x02\n\x08\x44ocument\x12\x0e\n\x06\x64oc_id\x18\x01 \x01(\x04\x12\x1b\n\x06\x63hunks\x18\x02 \x03(\x0b\x32\x0b.gnes.Chunk\x12\'\n\x10\x63hunk_embeddings\x18\x03 \x01(\x0b\x32\r.gnes.NdArray\x12(\n\x08\x64oc_type\x18\x04 \x01(\x0e\x32\x16.gnes.Document.DocType\x12\x11\n\tmeta_info\x18\x05 \x01(\x0c\x12\x12\n\x08raw_text\x18\x06 \x01(\tH\x00\x12\"\n\traw_image\x18\x07 \x01(\x0b\x32\r.gnes.NdArrayH\x00\x12\"\n\traw_video\x18\x08 \x01(\x0b\x32\r.gnes.NdArrayH\x00\x12\x13\n\traw_bytes\x18\t \x01(\x0cH\x00\x12\x0e\n\x06weight\x18\n \x01(\x02\"6\n\x07\x44ocType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04TEXT\x10\x01\x12\t\n\x05IMAGE\x10\x02\x12\t\n\x05VIDEO\x10\x03\x42\n\n\x08raw_data\"\xd4\x01\n\x08\x45nvelope\x12\x11\n\tclient_id\x18\x01 \x01(\t\x12\x12\n\nrequest_id\x18\x02 \x01(\t\x12\x0f\n\x07part_id\x18\x03 \x01(\r\x12\x10\n\x08num_part\x18\x04 \x03(\r\x12\x0f\n\x07timeout\x18\x05 \x01(\r\x12$\n\x06routes\x18\x06 \x03(\x0b\x32\x14.gnes.Envelope.route\x1aG\n\x05route\x12\x0f\n\x07service\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"y\n\x07Message\x12 \n\x08\x65nvelope\x18\x01 \x01(\x0b\x32\x0e.gnes.Envelope\x12 \n\x07request\x18\x02 \x01(\x0b\x32\r.gnes.RequestH\x00\x12\"\n\x08response\x18\x03 \x01(\x0b\x32\x0e.gnes.ResponseH\x00\x42\x06\n\x04\x62ody\"\xf6\x03\n\x07Request\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12+\n\x05train\x18\x02 \x01(\x0b\x32\x1a.gnes.Request.TrainRequestH\x00\x12+\n\x05index\x18\x03 \x01(\x0b\x32\x1a.gnes.Request.IndexRequestH\x00\x12,\n\x06search\x18\x04 \x01(\x0b\x32\x1a.gnes.Request.QueryRequestH\x00\x12/\n\x07\x63ontrol\x18\x05 \x01(\x0b\x32\x1c.gnes.Request.ControlRequestH\x00\x1a;\n\x0cTrainRequest\x12\x1c\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x0e.gnes.Document\x12\r\n\x05\x66lush\x18\x02 \x01(\x08\x1a,\n\x0cIndexRequest\x12\x1c\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x0e.gnes.Document\x1a<\n\x0cQueryRequest\x12\x1d\n\x05query\x18\x01 \x01(\x0b\x32\x0e.gnes.Document\x12\r\n\x05top_k\x18\x02 \x01(\r\x1am\n\x0e\x43ontrolRequest\x12\x35\n\x07\x63ommand\x18\x01 \x01(\x0e\x32$.gnes.Request.ControlRequest.Command\"$\n\x07\x43ommand\x12\r\n\tTERMINATE\x10\x00\x12\n\n\x06STATUS\x10\x01\x42\x06\n\x04\x62ody\"\xc4\x06\n\x08Response\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12-\n\x05train\x18\x02 \x01(\x0b\x32\x1c.gnes.Response.TrainResponseH\x00\x12-\n\x05index\x18\x03 \x01(\x0b\x32\x1c.gnes.Response.IndexResponseH\x00\x12.\n\x06search\x18\x04 \x01(\x0b\x32\x1c.gnes.Response.QueryResponseH\x00\x12\x31\n\x07\x63ontrol\x18\x05 \x01(\x0b\x32\x1e.gnes.Response.ControlResponseH\x00\x1a\x36\n\rTrainResponse\x12%\n\x06status\x18\x01 \x01(\x0e\x32\x15.gnes.Response.Status\x1a\x36\n\rIndexResponse\x12%\n\x06status\x18\x01 \x01(\x0e\x32\x15.gnes.Response.Status\x1a\x38\n\x0f\x43ontrolResponse\x12%\n\x06status\x18\x01 \x01(\x0e\x32\x15.gnes.Response.Status\x1a\x81\x03\n\rQueryResponse\x12%\n\x06status\x18\x01 \x01(\x0e\x32\x15.gnes.Response.Status\x12\r\n\x05top_k\x18\x02 \x01(\r\x12?\n\x0ctopk_results\x18\x03 \x03(\x0b\x32).gnes.Response.QueryResponse.ScoredResult\x12\x39\n\x05level\x18\x04 \x01(\x0e\x32*.gnes.Response.QueryResponse.ResponseLevel\x1a{\n\x0cScoredResult\x12\x1c\n\x05\x63hunk\x18\x01 \x01(\x0b\x32\x0b.gnes.ChunkH\x00\x12\x1d\n\x03\x64oc\x18\x02 \x01(\x0b\x32\x0e.gnes.DocumentH\x00\x12\r\n\x05score\x18\x03 \x01(\x02\x12\x17\n\x0fscore_explained\x18\x04 \x01(\tB\x06\n\x04\x62ody\"A\n\rResponseLevel\x12\t\n\x05\x43HUNK\x10\x00\x12\x17\n\x13\x44OCUMENT_NOT_FILLED\x10\x01\x12\x0c\n\x08\x44OCUMENT\x10\x02\"-\n\x06Status\x12\x0b\n\x07SUCCESS\x10\x00\x12\t\n\x05\x45RROR\x10\x01\x12\x0b\n\x07PENDING\x10\x02\x42\x06\n\x04\x62ody2\xe3\x01\n\x07GnesRPC\x12(\n\x05Train\x12\r.gnes.Request\x1a\x0e.gnes.Response\"\x00\x12(\n\x05Index\x12\r.gnes.Request\x1a\x0e.gnes.Response\"\x00\x12(\n\x05Query\x12\r.gnes.Request\x1a\x0e.gnes.Response\"\x00\x12\'\n\x04\x43\x61ll\x12\r.gnes.Request\x1a\x0e.gnes.Response\"\x00\x12\x31\n\nStreamCall\x12\r.gnes.Request\x1a\x0e.gnes.Response\"\x00(\x01\x30\x01\x62\x06proto3') , dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,]) @@ -441,8 +441,8 @@ serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='num_part', full_name='gnes.Envelope.num_part', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, + number=4, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), diff --git a/gnes/router/base.py b/gnes/router/base.py index 547a4cb2..174ab360 100644 --- a/gnes/router/base.py +++ b/gnes/router/base.py @@ -33,4 +33,7 @@ def apply(self, msg: 'gnes_pb2.Message', *args, **kwargs) -> Generator: class BaseReduceRouter(BaseRouter): def apply(self, msg: 'gnes_pb2.Message', accum_msgs: List['gnes_pb2.Message'], *args, **kwargs) -> None: merge_routes(msg, accum_msgs) - msg.envelope.num_part = 1 + if len(msg.envelope.num_part) > 1: + msg.envelope.num_part.pop() + else: + self.logger.error('can not reduce the message further, as num_part="%s"' % msg.envelope.num_part) diff --git a/gnes/router/map/simple.py b/gnes/router/map/simple.py index 81db685b..1c2ee97f 100644 --- a/gnes/router/map/simple.py +++ b/gnes/router/map/simple.py @@ -27,7 +27,7 @@ def __init__(self, num_part: int, *args, **kwargs): self.num_part = num_part def apply(self, msg: 'gnes_pb2.Message', *args, **kwargs) -> Generator: - msg.envelope.num_part = self.num_part + msg.envelope.num_part.append(self.num_part) yield msg @@ -46,5 +46,5 @@ def apply(self, msg: 'gnes_pb2.Message', *args, **kwargs) -> Generator: _msg.request.index.ClearField('docs') _msg.request.index.docs.extend(b) _msg.envelope.part_id = p_idx - _msg.envelope.num_part = num_part + _msg.envelope.num_part.append(num_part) yield _msg diff --git a/gnes/service/grpc.py b/gnes/service/grpc.py index 8f234501..4745e0b8 100644 --- a/gnes/service/grpc.py +++ b/gnes/service/grpc.py @@ -103,7 +103,7 @@ def add_envelope(self, body: 'gnes_pb2.Request', zmq_client: 'ZmqClient'): msg.envelope.request_id = str(uuid.uuid4()) self.logger.warning('request_id is missing, filled it with a random uuid!') msg.envelope.part_id = 1 - msg.envelope.num_part = 1 + msg.envelope.num_part.append(1) msg.envelope.timeout = 5000 r = msg.envelope.routes.add() r.service = zmq_client.__class__.__name__ diff --git a/gnes/service/router.py b/gnes/service/router.py index be713eeb..272f7682 100644 --- a/gnes/service/router.py +++ b/gnes/service/router.py @@ -32,7 +32,7 @@ def post_init(self): self._pending = defaultdict(list) # type: Dict[str, List] def _is_msg_complete(self, msg: 'gnes_pb2.Message', num_req: int) -> bool: - return (hasattr(self.args, 'num_part') and num_req == self.args.num_part) or (num_req == msg.envelope.num_part) + return (hasattr(self.args, 'num_part') and num_req == self.args.num_part[-1]) or (num_req == msg.envelope.num_part[-1]) @handler.register(NotImplementedError) def _handler_default(self, msg: 'gnes_pb2.Message'): diff --git a/tests/test_router.py b/tests/test_router.py index 03b0270b..b9430569 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -58,11 +58,12 @@ def test_publish_router(self): with RouterService(args), ZmqClient(c_args) as c1, ZmqClient(c_args) as c2: msg = gnes_pb2.Message() msg.request.index.docs.extend([gnes_pb2.Document() for _ in range(5)]) + msg.envelope.num_part.append(1) c1.send_message(msg) r = c1.recv_message() - self.assertEqual(r.envelope.num_part, 2) + self.assertSequenceEqual(r.envelope.num_part, [1, 2]) r = c2.recv_message() - self.assertEqual(r.envelope.num_part, 2) + self.assertSequenceEqual(r.envelope.num_part, [1, 2]) def test_reduce_router(self): args = set_router_service_parser().parse_args([ @@ -77,12 +78,12 @@ def test_reduce_router(self): with RouterService(args), ZmqClient(c_args) as c1, ZmqClient(c_args) as c2: msg = gnes_pb2.Message() msg.request.index.docs.extend([gnes_pb2.Document() for _ in range(5)]) - msg.envelope.num_part = 3 + msg.envelope.num_part.extend([1, 3]) c1.send_message(msg) c1.send_message(msg) c1.send_message(msg) r = c1.recv_message() - self.assertEqual(r.envelope.num_part, 1) + self.assertSequenceEqual(r.envelope.num_part, [1]) print(r.envelope.routes) def test_chunk_reduce_router(self): @@ -112,7 +113,7 @@ def test_chunk_reduce_router(self): s.score_explained = '1-c3' s.chunk.doc_id = 1 - msg.envelope.num_part = 2 + msg.envelope.num_part.extend([1, 2]) c1.send_message(msg) msg.response.search.ClearField('topk_results') @@ -131,10 +132,9 @@ def test_chunk_reduce_router(self): s.score = 0.3 s.score_explained = '2-c3' s.chunk.doc_id = 3 - msg.envelope.num_part = 2 c1.send_message(msg) r = c1.recv_message() - self.assertEqual(r.envelope.num_part, 1) + self.assertSequenceEqual(r.envelope.num_part, [1]) self.assertEqual(len(r.response.search.topk_results), 3) self.assertGreaterEqual(r.response.search.topk_results[0].score, r.response.search.topk_results[-1].score) print(r.response.search.topk_results) @@ -173,7 +173,7 @@ def test_doc_reduce_router(self): s.score = 0.3 s.chunk.doc_id = 3 - msg.envelope.num_part = 2 + msg.envelope.num_part.extend([1, 2]) c1.send_message(msg) msg.response.search.ClearField('topk_results') @@ -194,12 +194,11 @@ def test_doc_reduce_router(self): s.doc.raw_text = 'd3' msg.response.search.top_k = 5 - msg.envelope.num_part = 2 c1.send_message(msg) r = c1.recv_message() print(r.response.search.topk_results) - self.assertEqual(r.envelope.num_part, 1) + self.assertSequenceEqual(r.envelope.num_part, [1]) self.assertEqual(len(r.response.search.topk_results), 3) self.assertGreaterEqual(r.response.search.topk_results[0].score, r.response.search.topk_results[-1].score) @@ -216,12 +215,12 @@ def test_concat_router(self): with RouterService(args), ZmqClient(c_args) as c1: msg = gnes_pb2.Message() msg.request.search.query.chunk_embeddings.CopyFrom(array2blob(np.random.random([5, 2]))) - msg.envelope.num_part = 3 + msg.envelope.num_part.extend([1, 3]) c1.send_message(msg) c1.send_message(msg) c1.send_message(msg) r = c1.recv_message() - self.assertEqual(r.envelope.num_part, 1) + self.assertSequenceEqual(r.envelope.num_part, [1]) print(r.envelope.routes) self.assertEqual(r.request.search.query.chunk_embeddings.shape, [5, 6]) @@ -229,11 +228,100 @@ def test_concat_router(self): d = msg.request.index.docs.add() d.chunk_embeddings.CopyFrom(array2blob(np.random.random([5, 2 * j]))) - msg.envelope.num_part = 3 c1.send_message(msg) c1.send_message(msg) c1.send_message(msg) r = c1.recv_message() - self.assertEqual(r.envelope.num_part, 1) + self.assertSequenceEqual(r.envelope.num_part, [1]) for j in range(1, 4): self.assertEqual(r.request.index.docs[j - 1].chunk_embeddings.shape, [5, 6 * j]) + + def test_multimap_multireduce(self): + # p1 -> + # p21 -> + # r311 + # r312 + # -> r41 + # -> r5 + # p22 -> + # r321 + # r322 + # -> r42 + # -> r5 + # -> client + p1 = set_router_service_parser().parse_args([ + '--yaml_path', self.publish_router_yaml, + '--socket_in', str(SocketType.PULL_CONNECT), + '--socket_out', str(SocketType.PUB_BIND), + ]) + r5 = set_router_service_parser().parse_args([ + '--yaml_path', self.reduce_router_yaml, + '--socket_in', str(SocketType.PULL_BIND), + '--socket_out', str(SocketType.PUSH_CONNECT), + ]) + r41 = set_router_service_parser().parse_args([ + '--yaml_path', self.reduce_router_yaml, + '--socket_in', str(SocketType.PULL_BIND), + '--socket_out', str(SocketType.PUSH_CONNECT), + '--port_out', str(r5.port_in) + ]) + r42 = set_router_service_parser().parse_args([ + '--yaml_path', self.reduce_router_yaml, + '--socket_in', str(SocketType.PULL_BIND), + '--socket_out', str(SocketType.PUSH_CONNECT), + '--port_out', str(r5.port_in) + ]) + p21 = set_router_service_parser().parse_args([ + '--yaml_path', self.publish_router_yaml, + '--socket_in', str(SocketType.SUB_CONNECT), + '--socket_out', str(SocketType.PUB_BIND), + '--port_in', str(p1.port_out) + ]) + p22 = set_router_service_parser().parse_args([ + '--yaml_path', self.publish_router_yaml, + '--socket_in', str(SocketType.SUB_CONNECT), + '--socket_out', str(SocketType.PUB_BIND), + '--port_in', str(p1.port_out) + ]) + r311 = set_router_service_parser().parse_args([ + '--socket_in', str(SocketType.SUB_CONNECT), + '--socket_out', str(SocketType.PUSH_CONNECT), + '--port_in', str(p21.port_out), + '--port_out', str(r41.port_in) + ]) + r312 = set_router_service_parser().parse_args([ + '--socket_in', str(SocketType.SUB_CONNECT), + '--socket_out', str(SocketType.PUSH_CONNECT), + '--port_in', str(p21.port_out), + '--port_out', str(r41.port_in) + ]) + r321 = set_router_service_parser().parse_args([ + '--socket_in', str(SocketType.SUB_CONNECT), + '--socket_out', str(SocketType.PUSH_CONNECT), + '--port_in', str(p22.port_out), + '--port_out', str(r42.port_in) + ]) + r322 = set_router_service_parser().parse_args([ + '--socket_in', str(SocketType.SUB_CONNECT), + '--socket_out', str(SocketType.PUSH_CONNECT), + '--port_in', str(p22.port_out), + '--port_out', str(r42.port_in) + ]) + + c_args = _set_client_parser().parse_args([ + '--port_in', str(r5.port_out), + '--port_out', str(p1.port_in), + '--socket_in', str(SocketType.PULL_BIND), + '--socket_out', str(SocketType.PUSH_BIND), + ]) + with RouterService(p1), RouterService(r5), \ + RouterService(p21), RouterService(p22), \ + RouterService(r311), RouterService(r312), RouterService(r321), RouterService(r322), \ + RouterService(r41), RouterService(r42), \ + ZmqClient(c_args) as c1: + msg = gnes_pb2.Message() + msg.envelope.num_part.append(1) + c1.send_message(msg) + r = c1.recv_message() + self.assertSequenceEqual(r.envelope.num_part, [1]) + print(r.envelope.routes) diff --git a/tutorials/gnes-compose-yaml-spec.md b/tutorials/gnes-compose-yaml-spec.md index 2fc5cfed..cba26aaf 100644 --- a/tutorials/gnes-compose-yaml-spec.md +++ b/tutorials/gnes-compose-yaml-spec.md @@ -74,9 +74,8 @@ When specifying `services`, please kindly note the difference between a list and SequentialParallel - + 
-```yaml
 services:
 - name: Preprocessor
   yaml_path: text-prep.yml
@@ -86,10 +85,9 @@ services:
   yaml_path: faiss-indexer.yml
 - name: Indexer
   yaml_path: fulltext-indexer.yml
-```
- + 
 services:
 - name: Preprocessor
@@ -105,14 +103,14 @@ services:
 
 
 
-
+
 
-  GNES workflow of example 1
+  GNES workflow of example 1
   
 
-
+
 
-  GNES workflow of example 2
+  GNES workflow of example 2
   
 
 
diff --git a/tutorials/img/mermaid-diagram-20190729105252.svg b/tutorials/img/mermaid-diagram-20190729105252.svg
new file mode 100644
index 00000000..6d415a5e
--- /dev/null
+++ b/tutorials/img/mermaid-diagram-20190729105252.svg
@@ -0,0 +1,399 @@
+
push/pull
push/pull
push/pull
push/pull
push/pull
gRPCFrontend
Preprocessor
Encoder
Indexer
Indexer

\ No newline at end of file
diff --git a/tutorials/img/mermaid-diagram-20190729105330.svg b/tutorials/img/mermaid-diagram-20190729105330.svg
new file mode 100644
index 00000000..2c56802a
--- /dev/null
+++ b/tutorials/img/mermaid-diagram-20190729105330.svg
@@ -0,0 +1,399 @@
+
push/pull
push/pull
pub/sub
pub/sub
push/pull
push/pull
push/pull
gRPCFrontend
Preprocessor
Encoder
Indexer0
Indexer1
Router

\ No newline at end of file
diff --git a/docker-compose/component/encoder.bas-pca.yml b/yaml-example/component/encoder.bas-pca.yml
similarity index 100%
rename from docker-compose/component/encoder.bas-pca.yml
rename to yaml-example/component/encoder.bas-pca.yml
diff --git a/docker-compose/component/encoder.bas.yml b/yaml-example/component/encoder.bas.yml
similarity index 100%
rename from docker-compose/component/encoder.bas.yml
rename to yaml-example/component/encoder.bas.yml
diff --git a/docker-compose/component/encoder.inception.yml b/yaml-example/component/encoder.inception.yml
similarity index 100%
rename from docker-compose/component/encoder.inception.yml
rename to yaml-example/component/encoder.inception.yml
diff --git a/docker-compose/component/encoder.resnet.yml b/yaml-example/component/encoder.resnet.yml
similarity index 100%
rename from docker-compose/component/encoder.resnet.yml
rename to yaml-example/component/encoder.resnet.yml
diff --git a/docker-compose/component/encoder.vgg.yml b/yaml-example/component/encoder.vgg.yml
similarity index 100%
rename from docker-compose/component/encoder.vgg.yml
rename to yaml-example/component/encoder.vgg.yml
diff --git a/docker-compose/component/encoder.w2v.yml b/yaml-example/component/encoder.w2v.yml
similarity index 100%
rename from docker-compose/component/encoder.w2v.yml
rename to yaml-example/component/encoder.w2v.yml
diff --git a/docker-compose/component/img_preprocessor_fasterRCNN.yml b/yaml-example/component/img_preprocessor_fasterRCNN.yml
similarity index 100%
rename from docker-compose/component/img_preprocessor_fasterRCNN.yml
rename to yaml-example/component/img_preprocessor_fasterRCNN.yml
diff --git a/docker-compose/component/img_preprocessor_singleton.yml b/yaml-example/component/img_preprocessor_singleton.yml
similarity index 100%
rename from docker-compose/component/img_preprocessor_singleton.yml
rename to yaml-example/component/img_preprocessor_singleton.yml
diff --git a/docker-compose/component/img_preprocessor_vanilla_sldwin.yml b/yaml-example/component/img_preprocessor_vanilla_sldwin.yml
similarity index 100%
rename from docker-compose/component/img_preprocessor_vanilla_sldwin.yml
rename to yaml-example/component/img_preprocessor_vanilla_sldwin.yml
diff --git a/docker-compose/component/img_preprocessor_weight_sldwin.yml b/yaml-example/component/img_preprocessor_weight_sldwin.yml
similarity index 100%
rename from docker-compose/component/img_preprocessor_weight_sldwin.yml
rename to yaml-example/component/img_preprocessor_weight_sldwin.yml
diff --git a/docker-compose/component/indexer.fulltext.yml b/yaml-example/component/indexer.fulltext.yml
similarity index 100%
rename from docker-compose/component/indexer.fulltext.yml
rename to yaml-example/component/indexer.fulltext.yml
diff --git a/docker-compose/component/indexer.vector.yml b/yaml-example/component/indexer.vector.yml
similarity index 100%
rename from docker-compose/component/indexer.vector.yml
rename to yaml-example/component/indexer.vector.yml
diff --git a/docker-compose/component/preprocessor.yml b/yaml-example/component/preprocessor.yml
similarity index 100%
rename from docker-compose/component/preprocessor.yml
rename to yaml-example/component/preprocessor.yml