Merge branch 'fix_gie_docs' of https://github.com/shirly121/GraphScope …

…into fix_gie_docs Committed-by: longbinlai from Dev container
alibaba · Aug 22, 2023 · 5d7048d · 5d7048d
2 parents 48b31bb + 92f10ea
commit 5d7048d
Show file tree

Hide file tree

Showing 47 changed files with 1,842 additions and 318 deletions.
diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml
@@ -81,23 +81,34 @@ jobs:
         cd ${GIE_HOME}/compiler
         make build
 
-    - name: Run codegen test.
+    - name: Prepare dataset
       env:
         GS_TEST_DIR: ${{ github.workspace }}/gstest
-        GIE_HOME: ${{ github.workspace }}/interactive_engine/
-        HOME : /home/graphscope/
       run: |
         # download dataset
-        git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR}
+        git clone -b master --single-branch --depth=1 https://github.com/Graphscope/gstest.git ${GS_TEST_DIR}
+
+    - name: Sample Query test
+      env:
+        GS_TEST_DIR: ${{ github.workspace }}/gstest
+      run: |
+        cd ${GITHUB_WORKSPACE}/flex/build
+        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
+        ./tests/hqps/query_test ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \
+        ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml \
+        /tmp/csr-data-dir/
 
-        #flex_test_dir=${GS_TEST_DIR}/flex
-        ## preprocess bulk_load.yaml to use gstest_dir
-        #sed -i "s|workspaces/gstest|$GS_TEST_DIR|" ${flex_test_dir}/ldbc-sf01-long-date/audit_bulk_load.yaml
+    - name: Run codegen test.
+      env:
+        GS_TEST_DIR: ${{ github.workspace }}/gstest
+        HOME : /home/graphscope/
+      run: |
+        GIE_HOME=${GITHUB_WORKSPACE}/interactive_engine
 
         # create tmp ir.compiler.properties
         touch /tmp/ir.compiler.properties
         echo "engine.type: hiactor" >> /tmp/ir.compiler.properties
-        echo "graph.schema: ${GIE_HOME}/executor/ir/core/resource/modern_schema.json" >> /tmp/ir.compiler.properties
+        echo "graph.schema: ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json" >> /tmp/ir.compiler.properties
         echo "graph.store: exp" >> /tmp/ir.compiler.properties
         echo "graph.planner: {\"isOn\":true,\"opt\":\"RBO\",\"rules\":[\"FilterMatchRule\"]}" >> /tmp/ir.compiler.properties
 
@@ -108,7 +119,16 @@ jobs:
           cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codgen/"
           cmd=${cmd}" -o=/tmp/plugin --ir_conf=/tmp/ir.compiler.properties "
           cmd=${cmd}" --graph_schema_path=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json"
+          cmd=${cmd}" --gie_home=${GIE_HOME}"
           echo $cmd
           eval ${cmd}
         done
+    - name: Run End-to-End cypher adhoc query test
+      env:
+        GS_TEST_DIR: ${{ github.workspace }}/gstest
+        HOME : /home/graphscope/
+      run: |
+        cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
+        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
+        bash hqps_cypher_test.sh ${GS_TEST_DIR}
 
diff --git a/.gitignore b/.gitignore
@@ -90,3 +90,8 @@ core.*
 
 # Flex related
 flex/docs/
+flex/interactive/data/*/indices/
+flex/interactive/data/*/plugins/
+flex/interactive/logs/*
+flex/interactive/examples/sf0.1-raw/
+flex/interactive/.running
diff --git a/flex/.devcontainer.json b/flex/.devcontainer.json
@@ -3,7 +3,7 @@
 {
     "name": "GraphScope",
     // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-    "image": "registry.cn-hongkong.aliyuncs.com/graphscope/hiactor-server:v0.0.1",
+    "image": "registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.4",
     // Features to add to the dev container. More info: https://containers.dev/features.
     "features": {
         "ghcr.io/devcontainers/features/common-utils:2": {

diff --git a/flex/bin/load_plan_and_gen.sh b/flex/bin/load_plan_and_gen.sh
@@ -68,18 +68,15 @@ fi
 #fi
 
 cypher_to_plan() {
-  if [ $# -ne 4 ]; then
-    echo "Usage: $0 <input_file> <output_file> <ir_compiler_properties> <graph_schema_path>, but receive: "$#
-    exit 1
-  fi
-  # check GIE_HOME set
-  if [ -z ${GIE_HOME} ]; then
-    echo "GIE_HOME not set."
+  if [ $# -ne 5 ]; then
+    echo "Usage: $0 <input_file> <output_file> <ir_compiler_properties> <graph_schema_path> <gie_home>, but receive: "$#
     exit 1
   fi
   input_path=$1
   output_path=$2
   ir_compiler_properties=$3
+  graph_schema_path=$4
+  GIE_HOME=$5
   # find java executable
   echo "IR compiler properties = ${ir_compiler_properties}"
   #check file exists
@@ -137,23 +134,25 @@ cypher_to_plan() {
 
 compile_hqps_so() {
   #check input params size eq 2 or 3
-  if [ $# -ne 4 ] && [ $# -ne 5 ]; then
-    echo "Usage: $0 <input_file> <work_dir> <ir_compiler_properties_file> <graph_schema_file> [output_dir]"
+  if [ $# -ne 5 ] && [ $# -ne 6 ]; then
+    echo "Usage: $0 <input_file> <work_dir> <ir_compiler_properties_file> <graph_schema_file> <GIE_HOME>[output_dir]"
     exit 1
   fi
   input_path=$1
   work_dir=$2
   ir_compiler_properties=$3
   graph_schema_path=$4
-  if [ $# -eq 5 ]; then
-    output_dir=$5
+  gie_home=$5
+  if [ $# -eq 6 ]; then
+    output_dir=$6
   else
     output_dir=${work_dir}
   fi
   echo "Input path = ${input_path}"
   echo "Work dir = ${work_dir}"
   echo "ir compiler properties = ${ir_compiler_properties}"
   echo "graph schema path = ${graph_schema_path}"
+  echo "GIE_HOME = ${gie_home}"
   echo "Output dir = ${output_dir}"
 
   last_file_name=$(basename ${input_path})
@@ -189,15 +188,15 @@ compile_hqps_so() {
 
   #only do codegen when receives a .pb file.
   if [[ $last_file_name == *.pb ]]; then
-    cmd="${CODEGEN_RUNNER} ${input_path} ${output_cc_path}"
+    cmd="${CODEGEN_RUNNER} -e hqps -i ${input_path} -o ${output_cc_path}"
     echo "Codegen command = ${cmd}"
     eval ${cmd}
     echo "----------------------------"
   elif [[ $last_file_name == *.cypher ]]; then
     echo "Generating code from cypher query"
     # first do .cypher to .pb
     output_pb_path="${cur_dir}/${query_name}.pb"
-    cypher_to_plan ${input_path} ${output_pb_path} ${ir_compiler_properties} ${graph_schema_path}
+    cypher_to_plan ${input_path} ${output_pb_path} ${ir_compiler_properties} ${graph_schema_path} ${gie_home}
     echo "----------------------------"
     echo "Codegen from cypher query done."
     echo "----------------------------"
@@ -398,6 +397,20 @@ compile_pegasus_so() {
   echo "Finish copying, output to ${dst_so_path}"
 }
 
+function usage(){
+  cat << EOF
+  Usage: $0 [options]
+  Options:
+    -e, --engine_type=ENGINE_TYPE
+    -i, --input=INPUT
+    -w, --work_dir=WORK_DIR
+    --ir_conf=IR_CONF
+    --graph_schema_path=GRAPH_SCHEMA_PATH
+    --gie_home=GIE_HOME
+    [-o, --output_dir=OUTPUT_DIR]
+EOF
+}
+
 # input path
 # output dir
 run() {
@@ -415,8 +428,8 @@ run() {
       WORK_DIR="${i#*=}"
       shift # past argument=value
       ;;
-    -o=* | --output_dir=*)
-      OUTPUT_DIR="${i#*=}"
+    --gie_home=*)
+      GIE_HOME="${i#*=}"
       shift # past argument=value
       ;;
     --ir_conf=*)
@@ -427,6 +440,10 @@ run() {
       GRAPH_SCHEMA_PATH="${i#*=}"
       shift # past argument=value
       ;;
+    -o=* | --output_dir=*)
+      OUTPUT_DIR="${i#*=}"
+      shift # past argument=value
+      ;;
     -* | --*)
       echo "Unknown option $i"
       exit 1
@@ -436,12 +453,14 @@ run() {
     esac
   done
 
+  echo "Engine type            ="${ENGINE_TYPE}
   echo "Input                  ="${INPUT}
   echo "Work dir               ="${WORK_DIR}
-  echo "Output path            ="${OUTPUT_DIR}
   echo "ir conf                ="${IR_CONF}
   echo "graph_schema_path      ="${GRAPH_SCHEMA_PATH}
-  echo "Engine type            ="${ENGINE_TYPE}
+  echo "GIE_HOME               ="${GIE_HOME}
+  echo "Output path            ="${OUTPUT_DIR}
+
 
   # check input exist
   if [ ! -f ${INPUT} ]; then
@@ -452,7 +471,7 @@ run() {
   # if engine_type equals hqps
   if [ ${ENGINE_TYPE} == "hqps" ]; then
     echo "Engine type is hqps, generating dynamic library for hqps engine."
-    compile_hqps_so ${INPUT} ${WORK_DIR} ${IR_CONF} ${GRAPH_SCHEMA_PATH} ${OUTPUT_DIR}
+    compile_hqps_so ${INPUT} ${WORK_DIR} ${IR_CONF} ${GRAPH_SCHEMA_PATH} ${GIE_HOME} ${OUTPUT_DIR} 
 
   # else if engine_type equals pegasus
   elif [ ${ENGINE_TYPE} == "pegasus" ]; then
@@ -465,10 +484,9 @@ run() {
   exit 0
 }
 
-if [ $# -lt 5 ]; then
-  echo "Usage: $0 input_file work_dir output_dir"
-  echo "Example: $0 -e=hqps/pegasus -i=../query/1.pb -o=/plugin/ --ir_conf=../conf/ir.conf --graph_schema_path=../conf/graph_schema.json -w=/tmp/codegen"
-  echo "your num args: "$#
+if [ $# -lt 6 ]; then
+  echo "only receives: $# args"
+  usage
   exit 1
 fi