From afd5ddabe2f328423a3b572a0bce55831aaff277 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Fri, 26 Jul 2019 15:57:06 +0800 Subject: [PATCH] docs(tutorial): add yaml explain --- README.md | 2 +- docs/conf.py | 4 +- docs/index.rst | 10 +- docs/requirements.txt | 3 +- .../img/mermaid-diagram-20190726144822.svg | 399 ------------------ setup.py | 2 +- .../gnes-yaml-specifications.md | 39 +- .../img/mermaid-diagram-20190726150531.svg | 399 ++++++++++++++++++ .../img/mermaid-diagram-20190726150644.svg | 399 ++++++++++++++++++ .../img/mermaid-diagram-20190726154922.svg | 399 ++++++++++++++++++ 10 files changed, 1243 insertions(+), 413 deletions(-) delete mode 100644 docs/tutorial/img/mermaid-diagram-20190726144822.svg rename {docs/tutorial => tutorials}/gnes-yaml-specifications.md (68%) create mode 100644 tutorials/img/mermaid-diagram-20190726150531.svg create mode 100644 tutorials/img/mermaid-diagram-20190726150644.svg create mode 100644 tutorials/img/mermaid-diagram-20190726154922.svg diff --git a/README.md b/README.md index 8fc62d30..4346ac26 100644 --- a/README.md +++ b/README.md @@ -545,7 +545,7 @@ The official documentation of GNES is hosted on [doc.gnes.ai](https://doc.gnes.a > 🚧 Tutorial is still under construction. Stay tuned! Meanwhile, we sincerely welcome you to contribute your own learning experience / case study with GNES! -- How to write your GNES YAML config +- [How to write your GNES YAML config](tutorials/gnes-yaml-specifications.md) - How to write a component-wise YAML config - Understanding preprocessor, encoder, indexer and router - Index and query text data with GNES diff --git a/docs/conf.py b/docs/conf.py index a22f4b37..242842b6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,9 +54,11 @@ 'sphinx.ext.viewcode', 'sphinxcontrib.apidoc', 'sphinxarg.ext', - 'recommonmark' + 'recommonmark', + 'sphinx_markdown_tables', ] + apidoc_module_dir = '../gnes' apidoc_output_dir = 'api' apidoc_excluded_paths = ['tests'] diff --git a/docs/index.rst b/docs/index.rst index 02aa6ca6..7a0408cc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -51,14 +51,14 @@ Highlights .. toctree:: :maxdepth: 3 - :caption: Microservices API + :caption: GNES Microservices API chapter/microservice.rst .. toctree:: :maxdepth: 3 - :caption: Module API + :caption: GNES Modules api/gnes @@ -83,11 +83,13 @@ Tutorials 🚧 Tutorial is still under construction. Stay tuned! Meanwhile, we sincerely welcome you to contribute your own learning experience / case study with GNES! +Miscs +----- + .. toctree:: :maxdepth: 1 + :caption: Miscs - chapter/overview.md - chapter/yaml-config.md chapter/troubleshooting.md chapter/protobuf-dev.md diff --git a/docs/requirements.txt b/docs/requirements.txt index 1f36fde0..b9d3f9f0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ sphinx-argparse -sphinxcontrib-apidoc \ No newline at end of file +sphinxcontrib-apidoc +sphinx-markdown-tables \ No newline at end of file diff --git a/docs/tutorial/img/mermaid-diagram-20190726144822.svg b/docs/tutorial/img/mermaid-diagram-20190726144822.svg deleted file mode 100644 index 5c0babe0..00000000 --- a/docs/tutorial/img/mermaid-diagram-20190726144822.svg +++ /dev/null @@ -1,399 +0,0 @@ -
push/pull
push/pull
push/pull
push/pull
push/pull
gRPCFrontend
Preprocessor
Encoder
Indexer
Indexer
\ No newline at end of file diff --git a/setup.py b/setup.py index 3a134bb3..b84abd37 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ 'flair': ['flair>=0.4.1'], 'annoy': ['annoy==1.15.2'], 'chinese': ['jieba'], - 'vision': ['opencv-python>=4.0.0', 'torchvision==0.3.0', 'imagehash>=4.0'], + 'vision': ['opencv-python>=4.0.0', 'imagehash>=4.0'], 'leveldb': ['plyvel>=1.0.5'], 'test': ['pylint', 'memory_profiler>=0.55.0', 'psutil>=5.6.1', 'gputil>=1.4.0'], 'http': ['flask', 'flask-compress', 'flask-cors', 'flask-json', 'aiohttp==3.5.4'] diff --git a/docs/tutorial/gnes-yaml-specifications.md b/tutorials/gnes-yaml-specifications.md similarity index 68% rename from docs/tutorial/gnes-yaml-specifications.md rename to tutorials/gnes-yaml-specifications.md index 5f798bf3..2a0e1e19 100644 --- a/docs/tutorial/gnes-yaml-specifications.md +++ b/tutorials/gnes-yaml-specifications.md @@ -6,9 +6,9 @@ Essentially, GNES only requires two types of YAML config: - GNES-compose YAML - Component-wise YAML -All other YAML files, including the docker-compose YAML config and Kubernetes config generated from the [GNES Board](https://board.gnes.ai) or `$ gnes compose` command are not a part of this tutorial. Interested readers are welcome to read their [YAML specification](https://docs.docker.com/compose/compose-file/) respectively. +All other YAML files, including the docker-compose YAML config and Kubernetes config generated from the [GNES Board](https://board.gnes.ai) or `gnes compose` command are not a part of this tutorial. Interested readers are welcome to read their [YAML specification](https://docs.docker.com/compose/compose-file/) respectively. -## GNES-compose YAML +## GNES-compose YAML specification The GNES-compose YAML defines a high-level service topology behind the GNES app. It is designed for simplicity and clarity, allowing the user to quickly get started with GNES. @@ -37,7 +37,7 @@ In this example, our GNES app is composed of three microservices: `Preprocessor` | `networks`| map/dict | network mapping used in Docker-Compose/Swarm mode, default `{gnes-net: {driver: overlay, attachable: true}}` | -### The service specification +### `services` specification For each service defined in `services`, it accepts the following arguments. @@ -48,7 +48,7 @@ For each service defined in `services`, it accepts the following arguments. | `replicas` | int | the number of replica services you want to start, default `1` | | `income` | str | choose from `[pull, sub]`. Only applicable when `replicas>1`, where `pull` means competing for the income data and only one replica read the data; `sub` is like a broadcast, all replicas receive the same data, default `pull` | -#### Sequential and parallel services +### Sequential and parallel services When specifying `services`, please kindly note the difference between a list and a nested list: @@ -59,6 +59,7 @@ When specifying `services`, please kindly note the difference between a list and
+```yaml
 services:
 - name: Preprocessor
   yaml_path: text-prep.yml
@@ -68,6 +69,7 @@ services:
   yaml_path: faiss-indexer.yml
 - name: Indexer
   yaml_path: fulltext-indexer.yml
+```
    
@@ -88,16 +90,41 @@ services: - GNES workflow of example 2 + GNES workflow of example 1 - GNES workflow of example 2 + GNES workflow of example 2 +In the parallel mode, two heterogeneous indexers receive the incoming data from the last encoder in a simultaneous manner. This is particular useful when you have multiple heterogeneous models (e.g. VGG and inception, or Faiss indexer and key-value database), you don't want to wait for one after another as their executions are essentially independent. + +You may also scale up the parallel mode by setting `replicas`, e.g. + +```yaml +services: +- name: Preprocessor + yaml_path: text-prep.yml +- name: Encoder + yaml_path: gpt2.yml +- + - name: Indexer + yaml_path: faiss-indexer.yml + replicas: 2 + - name: Indexer + yaml_path: fulltext-indexer.yml + replicas: 3 +``` + +which results a topology like the following: + +![](img/mermaid-diagram-20190726154922.svg) + +## What's Next? +The GNES-compose YAML describes a high-level picture of the GNES topology, the detailed specification of each component is defined in `yaml_path` respectively, namely the *component-wise YAML config*. In the next tutorial, you will learn how to write a component-wise YAML config. diff --git a/tutorials/img/mermaid-diagram-20190726150531.svg b/tutorials/img/mermaid-diagram-20190726150531.svg new file mode 100644 index 00000000..1a44052f --- /dev/null +++ b/tutorials/img/mermaid-diagram-20190726150531.svg @@ -0,0 +1,399 @@ +
push/pull
push/pull
pub/sub
pub/sub
push/pull
push/pull
push/pull
gRPCFrontend
Preprocessor
Encoder
Indexer0
Indexer1
Router
\ No newline at end of file diff --git a/tutorials/img/mermaid-diagram-20190726150644.svg b/tutorials/img/mermaid-diagram-20190726150644.svg new file mode 100644 index 00000000..d10d8fe6 --- /dev/null +++ b/tutorials/img/mermaid-diagram-20190726150644.svg @@ -0,0 +1,399 @@ +
push/pull
push/pull
push/pull
push/pull
push/pull
gRPCFrontend
Preprocessor
Encoder
Indexer
Indexer
\ No newline at end of file diff --git a/tutorials/img/mermaid-diagram-20190726154922.svg b/tutorials/img/mermaid-diagram-20190726154922.svg new file mode 100644 index 00000000..dac6724e --- /dev/null +++ b/tutorials/img/mermaid-diagram-20190726154922.svg @@ -0,0 +1,399 @@ +
push/pull
push/pull
pub/sub
pub/sub
push/pull
push/pull
push/pull
push/pull
push/pull
push/pull
push/pull
push/pull
push/pull
push/pull
push/pull
gRPCFrontend
Preprocessor
Encoder
Router0
Router1
Indexer00
Indexer01
Indexer10
Indexer11
Indexer12
Router
\ No newline at end of file