Merge branch 'tab/doc-data-source' of https://github.com/singularity-…

…data/risingwave into tab/doc-data-source

.github/workflow-template/jobs/e2e-risedev.yml

@@ -98,7 +98,7 @@ jobs:
       - name: e2e test, batch, Rust frontend, 1 node
         timeout-minutes: 2
         run: |
-          RW_IMPLICIT_FLUSH=1 ~/cargo-make/makers ci-start ci-1cn-1fe
+          ~/cargo-make/makers ci-start ci-1cn-1fe
           sqllogictest -p 4566 './e2e_test/v2/ddl/**/*.slt'
           sqllogictest -p 4566 './e2e_test/v2/batch/**/*.slt'
 
@@ -108,7 +108,7 @@ jobs:
       - name: e2e test w/ Rust frontend ci-3node, batch, distributed
         timeout-minutes: 2
         run: |
-          RW_DIST_QUERY=1 RW_IMPLICIT_FLUSH=1 ~/cargo-make/makers ci-start ci-3cn-1fe
+          RW_DIST_QUERY=1 ~/cargo-make/makers ci-start ci-3cn-1fe
           sqllogictest -p 4566 './e2e_test/v2/batch_distributed/**/*.slt'
 
       - name: Kill cluster

.github/workflow-template/template.yml

@@ -5,10 +5,9 @@ name:
 on:
 
 env:
-  RUST_TOOLCHAIN: nightly-2022-03-09
-  CACHE_KEY_SUFFIX: v20220331
+  RUST_TOOLCHAIN: nightly-2022-04-09
+  CACHE_KEY_SUFFIX: v20220412
   CARGO_TERM_COLOR: always
-  RW_CI: true
   RW_SQLLOGICTEST_URL: https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.3.3/sqllogictest-linux-amd64.tar.gz
   RW_CARGO_MAKE_URL: https://github.com/sagiegurari/cargo-make/releases/download/0.35.10/cargo-make-v0.35.10-x86_64-unknown-linux-musl.zip
   RW_CARGO_MAKE_DIRECTORY: cargo-make-v0.35.10-x86_64-unknown-linux-musl

.github/workflows/dashboard_main.yml

@@ -1,12 +1,12 @@
-name: CI
+name: Dashboard (main)
 on:
   push:
     branches: [main]
     paths: [dashboard/**]
 jobs:
   dashboard-ui-deploy:
     runs-on: ubuntu-latest
-    steps: 
+    steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
@@ -24,4 +24,4 @@ jobs:
           BRANCH: dashboard-artifact
           FOLDER: dashboard/out
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SQUASH_HISTORY: true
+          SQUASH_HISTORY: true

.github/workflows/dashboard_pr.yml

@@ -1,12 +1,12 @@
-name: CI
+name: Dashboard
 on:
   pull_request:
     branches: [main]
     paths: [dashboard/**]
 jobs:
   dashboard-ui-build:
     runs-on: ubuntu-latest
-    steps: 
+    steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
@@ -16,4 +16,4 @@ jobs:
         run: |
           npm install
           npm run build
-          npm run build-static
+          npm run build-static

.github/workflows/docker_build.yml

@@ -17,7 +17,7 @@ jobs:
   build_docker_images:
     runs-on: ubuntu-latest
     steps:
-      - 
+      -
         name: Checkout
         uses: actions/checkout@v2
       -

.github/workflows/labeler.yml

@@ -5,8 +5,9 @@ on:
     types: [opened, edited]
 
 jobs:
-  build:
+  pr-labeler:
     runs-on: ubuntu-latest
+    name: pr-labeler
     steps:
     - uses: srvaroa/labeler@master
       env:

.github/workflows/license_check.yml

@@ -0,0 +1,15 @@
+name: License checker
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  license-header-check:
+    runs-on: ubuntu-latest
+    name: license-header-check
+    steps:
+    - name: Check License Header
+      uses: apache/skywalking-eyes@main

.github/workflows/main.yml

@@ -13,10 +13,9 @@ on:
       - "v*"
   workflow_dispatch:
 env:
-  RUST_TOOLCHAIN: nightly-2022-03-09
-  CACHE_KEY_SUFFIX: v20220331
+  RUST_TOOLCHAIN: nightly-2022-04-09
+  CACHE_KEY_SUFFIX: v20220412
   CARGO_TERM_COLOR: always
-  RW_CI: true
   RW_SQLLOGICTEST_URL: https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.3.3/sqllogictest-linux-amd64.tar.gz
   RW_CARGO_MAKE_URL: https://github.com/sagiegurari/cargo-make/releases/download/0.35.10/cargo-make-v0.35.10-x86_64-unknown-linux-musl.zip
   RW_CARGO_MAKE_DIRECTORY: cargo-make-v0.35.10-x86_64-unknown-linux-musl
@@ -268,15 +267,15 @@ jobs:
       - name: e2e test, batch, Rust frontend, 1 node
         timeout-minutes: 2
         run: |
-          RW_IMPLICIT_FLUSH=1 ~/cargo-make/makers ci-start ci-1cn-1fe
+          ~/cargo-make/makers ci-start ci-1cn-1fe
           sqllogictest -p 4566 './e2e_test/v2/ddl/**/*.slt'
           sqllogictest -p 4566 './e2e_test/v2/batch/**/*.slt'
       - name: Kill cluster
         run: ~/cargo-make/makers ci-kill
       - name: e2e test w/ Rust frontend ci-3node, batch, distributed
         timeout-minutes: 2
         run: |
-          RW_DIST_QUERY=1 RW_IMPLICIT_FLUSH=1 ~/cargo-make/makers ci-start ci-3cn-1fe
+          RW_DIST_QUERY=1 ~/cargo-make/makers ci-start ci-3cn-1fe
           sqllogictest -p 4566 './e2e_test/v2/batch_distributed/**/*.slt'
       - name: Kill cluster
         run: ~/cargo-make/makers ci-kill
@@ -388,15 +387,15 @@ jobs:
       - name: e2e test, batch, Rust frontend, 1 node
         timeout-minutes: 2
         run: |
-          RW_IMPLICIT_FLUSH=1 ~/cargo-make/makers ci-start ci-1cn-1fe
+          ~/cargo-make/makers ci-start ci-1cn-1fe
           sqllogictest -p 4566 './e2e_test/v2/ddl/**/*.slt'
           sqllogictest -p 4566 './e2e_test/v2/batch/**/*.slt'
       - name: Kill cluster
         run: ~/cargo-make/makers ci-kill
       - name: e2e test w/ Rust frontend ci-3node, batch, distributed
         timeout-minutes: 2
         run: |
-          RW_DIST_QUERY=1 RW_IMPLICIT_FLUSH=1 ~/cargo-make/makers ci-start ci-3cn-1fe
+          RW_DIST_QUERY=1 ~/cargo-make/makers ci-start ci-3cn-1fe
           sqllogictest -p 4566 './e2e_test/v2/batch_distributed/**/*.slt'
       - name: Kill cluster
         run: ~/cargo-make/makers ci-kill

.github/workflows/pull-request.yml

@@ -13,10 +13,9 @@ on:
     branches:
       - "forks/*"
 env:
-  RUST_TOOLCHAIN: nightly-2022-03-09
-  CACHE_KEY_SUFFIX: v20220331
+  RUST_TOOLCHAIN: nightly-2022-04-09
+  CACHE_KEY_SUFFIX: v20220412
   CARGO_TERM_COLOR: always
-  RW_CI: true
   RW_SQLLOGICTEST_URL: https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.3.3/sqllogictest-linux-amd64.tar.gz
   RW_CARGO_MAKE_URL: https://github.com/sagiegurari/cargo-make/releases/download/0.35.10/cargo-make-v0.35.10-x86_64-unknown-linux-musl.zip
   RW_CARGO_MAKE_DIRECTORY: cargo-make-v0.35.10-x86_64-unknown-linux-musl
@@ -223,15 +222,15 @@ jobs:
       - name: e2e test, batch, Rust frontend, 1 node
         timeout-minutes: 2
         run: |
-          RW_IMPLICIT_FLUSH=1 ~/cargo-make/makers ci-start ci-1cn-1fe
+          ~/cargo-make/makers ci-start ci-1cn-1fe
           sqllogictest -p 4566 './e2e_test/v2/ddl/**/*.slt'
           sqllogictest -p 4566 './e2e_test/v2/batch/**/*.slt'
       - name: Kill cluster
         run: ~/cargo-make/makers ci-kill
       - name: e2e test w/ Rust frontend ci-3node, batch, distributed
         timeout-minutes: 2
         run: |
-          RW_DIST_QUERY=1 RW_IMPLICIT_FLUSH=1 ~/cargo-make/makers ci-start ci-3cn-1fe
+          RW_DIST_QUERY=1 ~/cargo-make/makers ci-start ci-3cn-1fe
           sqllogictest -p 4566 './e2e_test/v2/batch_distributed/**/*.slt'
       - name: Kill cluster
         run: ~/cargo-make/makers ci-kill

CONTRIBUTING.md

@@ -194,7 +194,7 @@ cargo install --git https://github.com/risinglightdb/sqllogictest-rs --features
 To run end-to-end test, you will need to start a full cluster first:
 
 ```shell
-RW_IMPLICIT_FLUSH=1 ./risedev d
+./risedev d
 ```
 
 Then run some e2e tests:

docs/README.md

@@ -4,7 +4,7 @@ This directory contains RisingWave design documents that are intended to be used
 
 ## Table of Contents
 
-* Architecture Design
+* [Architecture Design](./architecture-design.md)
 * [An Overview of RisingWave Streaming Engine](./streaming-overview.md)
 * [An Overview of RisingWave State Store](./state-store-overview.md)
 * [Meta Service](./meta-service.md)

docs/architecture-design.md

@@ -0,0 +1,83 @@
+# Architecture Design
+
+## Motivation 
+
+This doc serves as one of the materials for newcomers to learn the high-level architecture and the functionalities of each component.
+
+## Architecture 
+
+There are currently 3 types of nodes in the cluster: 
+
+* **Frontend**: Frontend is a stateless proxy that accepts user queries through Postgres protocol. It is responsible for parsing, validation, optimization, and answering the results of each individual query. 
+* **ComputeNode**: ComputeNode is responsible to execute the optimized query plan. 
+* **MetaServer**: The central metadata management service. It also acts as a failure detector that periodically sends heartbeats to frontends and compute-nodes in the cluster.
+
+![Architecture](./images/architecture-design/architecture.svg)
+
+The topmost component is the Postgres client. It issues queries through [TCP-based Postgres wire protocol](https://www.postgresql.org/docs/current/protocol.html).
+
+The leftmost component is the streaming data source. [Kafka](https://kafka.apache.org) is the most representative system for streaming sources. Alternatively, [Redpanda](https://redpanda.com/), [Apache Pulsar](https://pulsar.apache.org/), [AWS Kinesis](aws.amazon.com/kinesis), [Google Pub/Sub](https://cloud.google.com/pubsub/docs/overview) are also widely-used. Streams from Kafka will be consumed and processed through the pipeline in the database. 
+
+The bottom-most component is AWS S3, or MinIO (an open-sourced s3-compatible system). We employed disaggregated architecture in order to elastically scale the compute-nodes without migrating the storage.
+
+## Execution Mode 
+
+There are 2 execution modes in our system serving different analytics purposes. 
+
+### Batch-Query Mode 
+
+The first is the *batch-query mode*. Users issue such a query via a *SELECT statement* and the system answers immediately. This is the most typical RDBMS use case. 
+
+Let's begin with a simple SELECT and see how it is executed. 
+
+```sql
+SELECT SUM(t.quantity) FROM t group by t.company;
+```
+
+![Batch-Query](./images/architecture-design/batch-query.svg)
+
+The query will be sliced into multiple *plan fragments*, each is an independent scheduling unit and probably with different parallelism. For simplicity, parallelism is usually set to the number of compute-nodes in the cluster.
+
+Each parallel unit is called a *task*. Specifically, PlanFragment 2 will be distributed as 3 tasks to 3 compute-nodes.
+
+![Plan-Fragments](./images/architecture-design/plan-fragments.svg)
+
+Behind the TableScan operator, there's a storage engine called Hummock that stores the internal states, materialized views, and the tables. Note that only the materialized views and tables are queryable. The internal states are invisible to users.
+
+To know more about Hummock, you can check out "[An Overview of RisingWave State Store](./state-store-overview.md)".
+
+### Streaming Mode 
+
+The other is the *streaming mode*. Users build streaming pipelines via [CREATE MATERIALIZED VIEW statement](https://www.postgresql.org/docs/current/sql-creatematerializedview.html). 
+For example: 
+
+```sql
+CREATE MATERIALIZED VIEW mv1 AS SELECT SUM(t.quantity) as q FROM t group by t.company;
+```
+
+![Stream-Pipeline](./images/architecture-design/stream-pipeline.png)
+
+When the data source (Kafka, e.g.) propagates a bunch of records into the system, the materialized view will refresh automatically.
+
+Assuming a sequence [(2, "AMERICA"), (3, "ASIA"), (4, "AMERICA"), (5, "ASIA")]. After the sequence flowed through the DAG, the MV will be updated to: 
+
+| A | B
+| - | -
+| 6 | AMERICA
+| 8 | ASIA
+
+While another sequence [(6, "EUROPE"), (7, "EUROPE")] comes, the MV will soon be: 
+
+| A | B
+| - | -
+| 6 | AMERICA
+| 8 | ASIA
+| 13 | EUROPE
+
+`mv1` can also act as other MV's source. For example, mv2, mv3 can reuse the processing results of mv1 thus deduplicating the computation.
+
+The durability of materialized views in RisingWave is built upon a snapshot-based mechanism. Every time a snapshot is triggered, the internal states of each operator will be flushed to S3. Upon failover, the operator recovers from the latest S3 checkpoint. 
+
+Since the streaming states can be extremely large that are probably unable (or ineffective) to be entirely held in memory, we have designed Hummock to be highly scalable. Compared to [Flink's rocksdb-based state store](https://nightlies.apache.org/flink/flink-docs-release-1.14/docs/ops/state/state_backends/#the-embeddedrocksdbstatebackend), Hummock is cloud-native  and provides super elasticity.
+
+For more details of streaming engine, please refer to "[An Overview of RisingWave Streaming Engine](./streaming-overview.md)".

docs/data-model-and-encoding.md

@@ -29,7 +29,7 @@ Primitive data types:
 Composite data types (WIP):
 
 - `Struct`: A structure with a list of named, strong-typed fields.
-- `List`: A variable-length list of values with same data type
+- `List`: A variable-length list of values with same data type.
 
 ## In-Memory Encoding
 

docs/meta-service.md

@@ -12,7 +12,7 @@
 
 ## Background
 
-RisingWave provides both real-time analytical query as well as high-concurrent access to Materialized Views. Therefore, both the front-end and compute nodes are designed to be scalable, and they may share the same set of host machines or not, depending on cluster size and whether the user cares about resource isolation.
+RisingWave provides both real-time analytical query as well as high-concurrent access to Materialized Views. Therefore, both the frontend and compute nodes are designed to be scalable, and they may share the same set of host machines or not, depending on cluster size and whether the user cares about resource isolation.
 
 Meanwhile, components such as metadata provider, scheduler, monitoring are more suitable for a centralized design. For example, a typical on-premise deployment may look like below, where the dotted boxes represent minimal unit of deployment (VM or container).
 
@@ -35,7 +35,7 @@ Catalog is the metadata of relational tables in databases.
 - **Table & Materialized Views**: Definition of tables & materialized views along with the columns on them.
 - **Source**: User-defined external data sources.
 
-To execute a DDL statement like `CREATE` or `DROP TABLE`, the front-end sends an RPC to meta node and waits the updated catalog to take effect.
+To execute a DDL statement like `CREATE` or `DROP TABLE`, the frontend sends an RPC to meta node and waits the updated catalog to take effect.
 
 ### Storage
 
@@ -48,7 +48,7 @@ There are 2 choices for how to distribute information across multiple nodes.
 * *Push*: When metadata changes, the meta node tells all nodes to update, and master node must wait for others to acknowledge before continuing. 
 * *Pull*: When data changes, the master node does nothing. Other nodes may not have the latest information, so they need to ask the master node every time.
 
-Currently, for simplicity, we choose the push-style approach for all kinds of metadata. This is implemented as `NotificationService` on meta service and `ObserverManager` on front-end and compute nodes. 
+Currently, for simplicity, we choose the push-style approach for all kinds of metadata. This is implemented as `NotificationService` on meta service and `ObserverManager` on frontend and compute nodes.
 
 ![Notification](./images/meta-service/notification.svg)
 

docs/mv-on-mv.md

@@ -17,7 +17,7 @@ create materialized view mv3 as select count(v1) as count_v1 from mv1;
 
 ### Broadcast operator
 
-In physical representation, we introduced a new dispatcher operator, *Broadcast*. Broadcast will dispatch every message to multiple downstream. To simplify our design, we can assume that every MViewOperator has a `Broadcast` output, with 0 or more downstreams. The implementation of Broadcast is trivial.
+In physical representation, we introduced a new dispatcher operator, *Broadcast*. Broadcast will dispatch every message to multiple downstreams. To simplify our design, we can assume that every MViewOperator has a `Broadcast` output, with 0 or more downstreams. The implementation of Broadcast is trivial.
 
 ![fig1](../docs/images/mv-on-mv/mv-on-mv-01.svg)
 
@@ -35,8 +35,8 @@ The full process of creation is:
 2. StreamManager creates the new actors.
 3. StreamManager chooses a change epoch e1, pins a snapshot of mv1 at e1, and sends a barrier with e1 and change info.
 4. The broadcast operator receives the barrier, then creates a SnapshotStream of mv1 with e1, and creates a Chain operator, then connects them all. (only changes in the memory).
-5. The broadcast operator sends a normal barrier e1 to all downstreams, and continue.
-6. The chain operator consumes all messages from snapshot and receives EOF, then consumes buffered messages from upstream.
+5. The broadcast operator sends a normal barrier e1 to all downstreams, and continues.
+6. The Chain operator consumes all messages from snapshot and receives EOF, then consumes buffered messages from upstream.
 7. StreamManager discovered that mv2 has almost caught up with the progress of mv1, and the creation success.
 
 ## Drop mview online
@@ -48,5 +48,5 @@ The full process of drop mv3 is:
 1. The frontend parses the query and sends the plan to StreamManager.
 2. StreamManager chooses a change epoch e1, and sends a barrier with e1 and change info.
 3. The broadcast operator sends a normal barrier e1 to all downstreams.
-4. The broadcast operator removes the dropped output from its outputs, and continue.
+4. The broadcast operator removes the dropped output from its outputs, and continues.
 5. StreamManager discovered that mv3 has the epoch e1, then drops extra fragments physically.