Merge branch 'main' into ir_fix_multi_frontend

alibaba · Jul 19, 2023 · c2d2c42 · c2d2c42
2 parents 956322f + c1df6f2
commit c2d2c42
Show file tree

Hide file tree

Showing 273 changed files with 38,260 additions and 610 deletions.
diff --git a/.github/workflows/flex.yml b/.github/workflows/flex.yml
@@ -22,11 +22,11 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test-hqps-engine:
+  build-flex:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.3
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.4
     steps:
     - uses: actions/checkout@v3
 
@@ -46,4 +46,13 @@ jobs:
       run: |
         cd ${GITHUB_WORKSPACE}/flex
         mkdir build && cd build
-        cmake .. && sudo make -j$(nproc)
+        cmake .. && sudo make -j$(nproc)
+
+    - name: GRIN on mutable csr test
+      run: |
+        git submodule update --init
+        cd flex/engines/graph_db/grin
+        mkdir build && cd build
+        cmake .. && sudo make -j$(nproc)
+        export FLEX_DATA_DIR=../../../../storages/rt_mutable_graph/modern_graph/
+        ./run_grin_test
diff --git a/.github/workflows/gaia.yml b/.github/workflows/gaia.yml
@@ -94,7 +94,7 @@ jobs:
       run: |
         cd ${GITHUB_WORKSPACE}
         git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git /tmp/gstest
-        cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_exprimental_pattern_ci.sh
+        cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_exprimental_advanced_ci.sh
 
     - name: Ir Integration Test on Csr Store
       run: |

diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml
@@ -0,0 +1,114 @@
+name: GraphScope GIE HighQPS DB CI
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'flex/**'
+      - 'interactive_engine/**'
+      - '.github/workflows/hqps-db-ci.yml'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'flex/**'
+      - 'interactive_engine/**'
+      - '.github/workflows/hqps-db-ci.yml'
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  test-hqps-engine:
+    runs-on: ubuntu-20.04
+    if: ${{ github.repository == 'alibaba/GraphScope' }}
+    container:
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.4
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/cache@v3
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+            ${{ runner.os }}-maven-
+
+    - uses: actions/cache@v3
+      with:
+        path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            ~/.cache/sccache
+        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Install latest libgrape-lite
+      if: false
+      run: |
+        git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
+        cd /tmp/libgrape-lite
+        mkdir -p build && cd build
+        cmake ..
+        make -j$(nproc)
+        make install
+
+    - name: Setup tmate session
+      if: false
+      uses: mxschmitt/action-tmate@v3
+
+    - name: Build
+      env:
+        GIE_HOME: ${{ github.workspace }}/interactive_engine/
+        HOME: /home/graphscope/
+      run: |
+        cd ${GITHUB_WORKSPACE}/flex
+        mkdir build && cd build
+        cmake .. && sudo make -j$(nproc)
+        sudo make install
+
+        # cargo
+        . /home/graphscope/.cargo/env
+        which cargo
+
+        # build compiler
+        cd ${GIE_HOME}/compiler
+        make build
+
+    - name: Run codegen test.
+      env:
+        GS_TEST_DIR: ${{ github.workspace }}/gstest
+        GIE_HOME: ${{ github.workspace }}/interactive_engine/
+        HOME : /home/graphscope/
+      run: |
+        # download dataset
+        git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR}
+
+        #flex_test_dir=${GS_TEST_DIR}/flex
+        ## preprocess bulk_load.yaml to use gstest_dir
+        #sed -i "s|workspaces/gstest|$GS_TEST_DIR|" ${flex_test_dir}/ldbc-sf01-long-date/audit_bulk_load.yaml
+
+        # create tmp ir.compiler.properties
+        touch /tmp/ir.compiler.properties
+        echo "engine.type: hiactor" >> /tmp/ir.compiler.properties
+        echo "graph.schema: ${GIE_HOME}/executor/ir/core/resource/modern_schema.json" >> /tmp/ir.compiler.properties
+        echo "graph.store: exp" >> /tmp/ir.compiler.properties
+        echo "graph.planner: {\"isOn\":true,\"opt\":\"RBO\",\"rules\":[\"FilterMatchRule\"]}" >> /tmp/ir.compiler.properties
+
+        cd ${GITHUB_WORKSPACE}/flex/bin
+
+        for i in 2 3 5 6 8 9 11 12;
+        do
+          cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codgen/"
+          cmd=${cmd}" -o=/tmp/plugin --ir_conf=/tmp/ir.compiler.properties "
+          cmd=${cmd}" --graph_schema_path=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json"
+          echo $cmd
+          eval ${cmd}
+        done
+
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,7 @@
 [submodule "learning_engine/graph-learn"]
 	path = learning_engine/graph-learn
 	url = https://github.com/alibaba/graph-learn.git
+
+[submodule "flex/grin"]
+	path = flex/grin
+	url = https://github.com/GraphScope/GRIN.git
diff --git a/analytical_engine/CMakeLists.txt b/analytical_engine/CMakeLists.txt
@@ -278,10 +278,10 @@ if(Arrow_FOUND AND Arrow_VERSION VERSION_GREATER 9.0.1)
 endif()
 
 # Generate proto
-execute_process(COMMAND python3 python/graphscope/proto/proto_generator.py "${PROJECT_SOURCE_DIR}" --cpp
-        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/..)
+execute_process(COMMAND python3 ../proto/proto_generator.py "${PROJECT_SOURCE_DIR}/proto" --cpp
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
 
-file(GLOB PROTO_CPP_FILES "graphscope/proto/*.cc")
+file(GLOB PROTO_CPP_FILES "proto/*.cc")
 file(GLOB CORE_RPC_SRC_FILES "core/server/*.cc")
 
 # Add targets
@@ -569,7 +569,7 @@ endif()
 install_gsa_headers("${PROJECT_SOURCE_DIR}/apps")
 install_gsa_headers("${PROJECT_SOURCE_DIR}/benchmarks")
 install_gsa_headers("${PROJECT_SOURCE_DIR}/core")
-install_gsa_headers("${PROJECT_SOURCE_DIR}/graphscope")
+install_gsa_headers("${PROJECT_SOURCE_DIR}/proto")
 install_gsa_app_frames("${PROJECT_SOURCE_DIR}/frame")
 
 install_gsa_dependency_modules("${PROJECT_SOURCE_DIR}/cmake")

diff --git a/analytical_engine/core/app/app_invoker.h b/analytical_engine/core/app/app_invoker.h
@@ -29,8 +29,8 @@
 #endif
 #include "core/config.h"
 #include "core/error.h"
-#include "graphscope/proto/data_types.pb.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/data_types.pb.h"
+#include "proto/types.pb.h"
 
 namespace bl = boost::leaf;
 

diff --git a/analytical_engine/core/context/vertex_data_context.h b/analytical_engine/core/context/vertex_data_context.h
@@ -50,7 +50,7 @@
 #include "core/server/rpc_utils.h"
 #include "core/utils/mpi_utils.h"
 #include "core/utils/transform_utils.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/types.pb.h"
 
 #define CONTEXT_TYPE_VERTEX_DATA "vertex_data"
 #define CONTEXT_TYPE_LABELED_VERTEX_DATA "labeled_vertex_data"

diff --git a/analytical_engine/core/error.h b/analytical_engine/core/error.h
@@ -24,7 +24,7 @@
 
 #include "vineyard/graph/utils/error.h"  // IWYU pragma: export
 
-#include "graphscope/proto/error_codes.pb.h"  // IWYU pragma: export
+#include "proto/error_codes.pb.h"  // IWYU pragma: export
 
 namespace gs {
 

diff --git a/analytical_engine/core/fragment/arrow_projected_fragment.h b/analytical_engine/core/fragment/arrow_projected_fragment.h
@@ -41,7 +41,7 @@
 #include "core/config.h"
 #include "core/fragment/arrow_projected_fragment_base.h"  // IWYU pragma: export
 #include "core/vertex_map/arrow_projected_vertex_map.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/types.pb.h"
 
 namespace arrow {
 class Array;

diff --git a/analytical_engine/core/fragment/dynamic_fragment.h b/analytical_engine/core/fragment/dynamic_fragment.h
@@ -42,7 +42,7 @@
 #include "core/object/dynamic.h"
 #include "core/utils/convert_utils.h"
 #include "core/utils/partitioner.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/types.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/fragment/fragment_reporter.h b/analytical_engine/core/fragment/fragment_reporter.h
@@ -39,7 +39,7 @@
 #include "core/server/rpc_utils.h"
 #include "core/utils/convert_utils.h"
 #include "core/utils/msgpack_utils.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/types.pb.h"
 
 namespace bl = boost::leaf;
 

diff --git a/analytical_engine/core/grape_instance.cc b/analytical_engine/core/grape_instance.cc
@@ -65,9 +65,9 @@
 #include "core/server/command_detail.h"
 #include "core/server/rpc_utils.h"
 #include "core/utils/mpi_utils.h"
-#include "graphscope/proto/attr_value.pb.h"
-#include "graphscope/proto/graph_def.pb.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/attr_value.pb.h"
+#include "proto/graph_def.pb.h"
+#include "proto/types.pb.h"
 
 namespace bl = boost::leaf;
 

diff --git a/analytical_engine/core/grape_instance.h b/analytical_engine/core/grape_instance.h
@@ -38,7 +38,7 @@
 #include "core/object/object_manager.h"
 #include "core/server/dispatcher.h"
 #include "core/server/rpc_utils.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/types.pb.h"
 
 namespace bl = boost::leaf;
 

diff --git a/analytical_engine/core/io/property_parser.h b/analytical_engine/core/io/property_parser.h
@@ -34,8 +34,8 @@
 #include "vineyard/common/util/status.h"
 
 #include "core/server/rpc_utils.h"
-#include "graphscope/proto/attr_value.pb.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/attr_value.pb.h"
+#include "proto/types.pb.h"
 
 namespace bl = boost::leaf;
 

diff --git a/analytical_engine/core/object/dynamic.h b/analytical_engine/core/object/dynamic.h
@@ -32,7 +32,7 @@
 // IWYU pragma: end_exports
 
 #include "grape/serialization/in_archive.h"
-#include "graphscope/proto/graph_def.pb.h"
+#include "proto/graph_def.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/object/fragment_wrapper.h b/analytical_engine/core/object/fragment_wrapper.h
@@ -61,8 +61,8 @@
 #include "core/object/i_fragment_wrapper.h"
 #include "core/server/rpc_utils.h"
 #include "core/utils/transform_utils.h"
-#include "graphscope/proto/graph_def.pb.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/graph_def.pb.h"
+#include "proto/types.pb.h"
 
 namespace bl = boost::leaf;
 

diff --git a/analytical_engine/core/server/command_detail.cc b/analytical_engine/core/server/command_detail.cc
@@ -21,8 +21,8 @@
 #include "grape/serialization/in_archive.h"
 #include "grape/serialization/out_archive.h"
 
-#include "graphscope/proto/attr_value.pb.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/attr_value.pb.h"
+#include "proto/types.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/server/command_detail.h b/analytical_engine/core/server/command_detail.h
@@ -19,8 +19,8 @@
 #include <map>
 #include <utility>
 
-#include "graphscope/proto/attr_value.pb.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/attr_value.pb.h"
+#include "proto/types.pb.h"
 
 namespace grape {
 class InArchive;

diff --git a/analytical_engine/core/server/dispatcher.cc b/analytical_engine/core/server/dispatcher.cc
@@ -37,8 +37,8 @@
 #include "core/error.h"
 #include "core/io/property_parser.h"
 #include "core/server/command_detail.h"
-#include "graphscope/proto/attr_value.pb.h"
-#include "graphscope/proto/graph_def.pb.h"
+#include "proto/attr_value.pb.h"
+#include "proto/graph_def.pb.h"
 
 namespace bl = boost::leaf;
 

diff --git a/analytical_engine/core/server/dispatcher.h b/analytical_engine/core/server/dispatcher.h
@@ -25,9 +25,9 @@
 #include "grape/worker/comm_spec.h"
 #include "vineyard/common/util/blocking_queue.h"
 
-#include "graphscope/proto/error_codes.pb.h"
-#include "graphscope/proto/graph_def.pb.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/error_codes.pb.h"
+#include "proto/graph_def.pb.h"
+#include "proto/types.pb.h"
 
 namespace bl = boost::leaf;
 

diff --git a/analytical_engine/core/server/graphscope_service.cc b/analytical_engine/core/server/graphscope_service.cc
@@ -26,11 +26,11 @@
 #include "grpcpp/support/sync_stream.h"
 
 #include "core/server/rpc_utils.h"
-#include "graphscope/proto/attr_value.pb.h"
-#include "graphscope/proto/error_codes.pb.h"
-#include "graphscope/proto/graph_def.pb.h"
-#include "graphscope/proto/message.pb.h"
-#include "graphscope/proto/op_def.pb.h"
+#include "proto/attr_value.pb.h"
+#include "proto/error_codes.pb.h"
+#include "proto/graph_def.pb.h"
+#include "proto/message.pb.h"
+#include "proto/op_def.pb.h"
 
 namespace gs {
 struct CommandDetail;

diff --git a/analytical_engine/core/server/graphscope_service.h b/analytical_engine/core/server/graphscope_service.h
@@ -29,9 +29,9 @@
 #include "grpcpp/support/sync_stream.h"
 
 #include "core/server/dispatcher.h"
-#include "graphscope/proto/engine_service.grpc.pb.h"
-#include "graphscope/proto/message.pb.h"
-#include "graphscope/proto/op_def.pb.h"
+#include "proto/engine_service.grpc.pb.h"
+#include "proto/message.pb.h"
+#include "proto/op_def.pb.h"
 
 namespace gs {
 namespace rpc {

diff --git a/analytical_engine/core/server/rpc_utils.h b/analytical_engine/core/server/rpc_utils.h
@@ -28,10 +28,10 @@
 
 #include "core/config.h"
 #include "core/server/command_detail.h"
-#include "graphscope/proto/attr_value.pb.h"
-#include "graphscope/proto/graph_def.pb.h"
-#include "graphscope/proto/op_def.pb.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/attr_value.pb.h"
+#include "proto/graph_def.pb.h"
+#include "proto/op_def.pb.h"
+#include "proto/types.pb.h"
 
 namespace bl = boost::leaf;
 

diff --git a/analytical_engine/frame/cython_app_frame.cc b/analytical_engine/frame/cython_app_frame.cc
@@ -47,8 +47,8 @@
 #include "core/app/pregel/pregel_property_app_base.h"
 #include "core/error.h"
 #include "frame/ctx_wrapper_builder.h"
-#include "graphscope/proto/data_types.pb.h"
-#include "graphscope/proto/types.pb.h"
+#include "proto/data_types.pb.h"
+#include "proto/types.pb.h"
 
 #include QUOTE(_APP_HEADER)