diff --git a/analytical_engine/CMakeLists.txt b/analytical_engine/CMakeLists.txt index 55425acd8d5b..2913b5473076 100644 --- a/analytical_engine/CMakeLists.txt +++ b/analytical_engine/CMakeLists.txt @@ -278,10 +278,10 @@ if(Arrow_FOUND AND Arrow_VERSION VERSION_GREATER 9.0.1) endif() # Generate proto -execute_process(COMMAND python3 python/graphscope/proto/proto_generator.py "${PROJECT_SOURCE_DIR}" --cpp - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/..) +execute_process(COMMAND python3 ../proto/proto_generator.py "${PROJECT_SOURCE_DIR}/proto" --cpp + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) -file(GLOB PROTO_CPP_FILES "graphscope/proto/*.cc") +file(GLOB PROTO_CPP_FILES "proto/*.cc") file(GLOB CORE_RPC_SRC_FILES "core/server/*.cc") # Add targets @@ -569,7 +569,7 @@ endif() install_gsa_headers("${PROJECT_SOURCE_DIR}/apps") install_gsa_headers("${PROJECT_SOURCE_DIR}/benchmarks") install_gsa_headers("${PROJECT_SOURCE_DIR}/core") -install_gsa_headers("${PROJECT_SOURCE_DIR}/graphscope") +install_gsa_headers("${PROJECT_SOURCE_DIR}/proto") install_gsa_app_frames("${PROJECT_SOURCE_DIR}/frame") install_gsa_dependency_modules("${PROJECT_SOURCE_DIR}/cmake") diff --git a/analytical_engine/core/app/app_invoker.h b/analytical_engine/core/app/app_invoker.h index be4c38ff5808..eb3cf8c21942 100644 --- a/analytical_engine/core/app/app_invoker.h +++ b/analytical_engine/core/app/app_invoker.h @@ -29,8 +29,8 @@ #endif #include "core/config.h" #include "core/error.h" -#include "graphscope/proto/data_types.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/data_types.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/context/vertex_data_context.h b/analytical_engine/core/context/vertex_data_context.h index 1ae381d38d3d..c1671853899c 100644 --- a/analytical_engine/core/context/vertex_data_context.h +++ b/analytical_engine/core/context/vertex_data_context.h @@ -50,7 +50,7 @@ #include "core/server/rpc_utils.h" #include "core/utils/mpi_utils.h" #include "core/utils/transform_utils.h" -#include "graphscope/proto/types.pb.h" +#include "proto/types.pb.h" #define CONTEXT_TYPE_VERTEX_DATA "vertex_data" #define CONTEXT_TYPE_LABELED_VERTEX_DATA "labeled_vertex_data" diff --git a/analytical_engine/core/error.h b/analytical_engine/core/error.h index e2bfb51dc48b..a5a1e0c6f60f 100644 --- a/analytical_engine/core/error.h +++ b/analytical_engine/core/error.h @@ -24,7 +24,7 @@ #include "vineyard/graph/utils/error.h" // IWYU pragma: export -#include "graphscope/proto/error_codes.pb.h" // IWYU pragma: export +#include "proto/error_codes.pb.h" // IWYU pragma: export namespace gs { diff --git a/analytical_engine/core/fragment/arrow_projected_fragment.h b/analytical_engine/core/fragment/arrow_projected_fragment.h index 8afeee239a7a..8a1e2f368110 100644 --- a/analytical_engine/core/fragment/arrow_projected_fragment.h +++ b/analytical_engine/core/fragment/arrow_projected_fragment.h @@ -41,7 +41,7 @@ #include "core/config.h" #include "core/fragment/arrow_projected_fragment_base.h" // IWYU pragma: export #include "core/vertex_map/arrow_projected_vertex_map.h" -#include "graphscope/proto/types.pb.h" +#include "proto/types.pb.h" namespace arrow { class Array; diff --git a/analytical_engine/core/fragment/dynamic_fragment.h b/analytical_engine/core/fragment/dynamic_fragment.h index fa9e1db8fae5..734e745e2227 100644 --- a/analytical_engine/core/fragment/dynamic_fragment.h +++ b/analytical_engine/core/fragment/dynamic_fragment.h @@ -42,7 +42,7 @@ #include "core/object/dynamic.h" #include "core/utils/convert_utils.h" #include "core/utils/partitioner.h" -#include "graphscope/proto/types.pb.h" +#include "proto/types.pb.h" namespace gs { diff --git a/analytical_engine/core/fragment/fragment_reporter.h b/analytical_engine/core/fragment/fragment_reporter.h index 1ac8d424020f..0cb332b4a721 100644 --- a/analytical_engine/core/fragment/fragment_reporter.h +++ b/analytical_engine/core/fragment/fragment_reporter.h @@ -39,7 +39,7 @@ #include "core/server/rpc_utils.h" #include "core/utils/convert_utils.h" #include "core/utils/msgpack_utils.h" -#include "graphscope/proto/types.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/grape_instance.cc b/analytical_engine/core/grape_instance.cc index 97f701931ac5..c49357f73875 100644 --- a/analytical_engine/core/grape_instance.cc +++ b/analytical_engine/core/grape_instance.cc @@ -65,9 +65,9 @@ #include "core/server/command_detail.h" #include "core/server/rpc_utils.h" #include "core/utils/mpi_utils.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/graph_def.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/graph_def.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/grape_instance.h b/analytical_engine/core/grape_instance.h index 460eaa8ff784..03e32fbb6fa8 100644 --- a/analytical_engine/core/grape_instance.h +++ b/analytical_engine/core/grape_instance.h @@ -38,7 +38,7 @@ #include "core/object/object_manager.h" #include "core/server/dispatcher.h" #include "core/server/rpc_utils.h" -#include "graphscope/proto/types.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/io/property_parser.h b/analytical_engine/core/io/property_parser.h index a55e3201e29d..acf3e1da25ca 100644 --- a/analytical_engine/core/io/property_parser.h +++ b/analytical_engine/core/io/property_parser.h @@ -34,8 +34,8 @@ #include "vineyard/common/util/status.h" #include "core/server/rpc_utils.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/object/dynamic.h b/analytical_engine/core/object/dynamic.h index 35d7c6c48ef3..d1b6a30fb8f8 100644 --- a/analytical_engine/core/object/dynamic.h +++ b/analytical_engine/core/object/dynamic.h @@ -32,7 +32,7 @@ // IWYU pragma: end_exports #include "grape/serialization/in_archive.h" -#include "graphscope/proto/graph_def.pb.h" +#include "proto/graph_def.pb.h" namespace gs { diff --git a/analytical_engine/core/object/fragment_wrapper.h b/analytical_engine/core/object/fragment_wrapper.h index a178dc435e87..aded8aaaea68 100644 --- a/analytical_engine/core/object/fragment_wrapper.h +++ b/analytical_engine/core/object/fragment_wrapper.h @@ -61,8 +61,8 @@ #include "core/object/i_fragment_wrapper.h" #include "core/server/rpc_utils.h" #include "core/utils/transform_utils.h" -#include "graphscope/proto/graph_def.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/graph_def.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/server/command_detail.cc b/analytical_engine/core/server/command_detail.cc index 47e50b51a06a..49a09539f3c1 100644 --- a/analytical_engine/core/server/command_detail.cc +++ b/analytical_engine/core/server/command_detail.cc @@ -21,8 +21,8 @@ #include "grape/serialization/in_archive.h" #include "grape/serialization/out_archive.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/types.pb.h" namespace gs { diff --git a/analytical_engine/core/server/command_detail.h b/analytical_engine/core/server/command_detail.h index c33db5e77ae2..efc75ca65f25 100644 --- a/analytical_engine/core/server/command_detail.h +++ b/analytical_engine/core/server/command_detail.h @@ -19,8 +19,8 @@ #include #include -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/types.pb.h" namespace grape { class InArchive; diff --git a/analytical_engine/core/server/dispatcher.cc b/analytical_engine/core/server/dispatcher.cc index e10f4fd74d0b..efa3e535c7d3 100644 --- a/analytical_engine/core/server/dispatcher.cc +++ b/analytical_engine/core/server/dispatcher.cc @@ -37,8 +37,8 @@ #include "core/error.h" #include "core/io/property_parser.h" #include "core/server/command_detail.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/graph_def.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/graph_def.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/server/dispatcher.h b/analytical_engine/core/server/dispatcher.h index 1b2a7d30ce47..ecbffda33805 100644 --- a/analytical_engine/core/server/dispatcher.h +++ b/analytical_engine/core/server/dispatcher.h @@ -25,9 +25,9 @@ #include "grape/worker/comm_spec.h" #include "vineyard/common/util/blocking_queue.h" -#include "graphscope/proto/error_codes.pb.h" -#include "graphscope/proto/graph_def.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/error_codes.pb.h" +#include "proto/graph_def.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/server/graphscope_service.cc b/analytical_engine/core/server/graphscope_service.cc index 84dd07aa35c8..3e646a8ffae6 100644 --- a/analytical_engine/core/server/graphscope_service.cc +++ b/analytical_engine/core/server/graphscope_service.cc @@ -26,11 +26,11 @@ #include "grpcpp/support/sync_stream.h" #include "core/server/rpc_utils.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/error_codes.pb.h" -#include "graphscope/proto/graph_def.pb.h" -#include "graphscope/proto/message.pb.h" -#include "graphscope/proto/op_def.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/error_codes.pb.h" +#include "proto/graph_def.pb.h" +#include "proto/message.pb.h" +#include "proto/op_def.pb.h" namespace gs { struct CommandDetail; diff --git a/analytical_engine/core/server/graphscope_service.h b/analytical_engine/core/server/graphscope_service.h index 435f3f87c74a..8e5850cab290 100644 --- a/analytical_engine/core/server/graphscope_service.h +++ b/analytical_engine/core/server/graphscope_service.h @@ -29,9 +29,9 @@ #include "grpcpp/support/sync_stream.h" #include "core/server/dispatcher.h" -#include "graphscope/proto/engine_service.grpc.pb.h" -#include "graphscope/proto/message.pb.h" -#include "graphscope/proto/op_def.pb.h" +#include "proto/engine_service.grpc.pb.h" +#include "proto/message.pb.h" +#include "proto/op_def.pb.h" namespace gs { namespace rpc { diff --git a/analytical_engine/core/server/rpc_utils.h b/analytical_engine/core/server/rpc_utils.h index bd7aa6b3b7fd..83314d84f46e 100644 --- a/analytical_engine/core/server/rpc_utils.h +++ b/analytical_engine/core/server/rpc_utils.h @@ -28,10 +28,10 @@ #include "core/config.h" #include "core/server/command_detail.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/graph_def.pb.h" -#include "graphscope/proto/op_def.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/graph_def.pb.h" +#include "proto/op_def.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/frame/cython_app_frame.cc b/analytical_engine/frame/cython_app_frame.cc index 61f595bbc398..71914faff1ea 100644 --- a/analytical_engine/frame/cython_app_frame.cc +++ b/analytical_engine/frame/cython_app_frame.cc @@ -47,8 +47,8 @@ #include "core/app/pregel/pregel_property_app_base.h" #include "core/error.h" #include "frame/ctx_wrapper_builder.h" -#include "graphscope/proto/data_types.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/data_types.pb.h" +#include "proto/types.pb.h" #include QUOTE(_APP_HEADER) diff --git a/analytical_engine/frame/cython_pie_app_frame.cc b/analytical_engine/frame/cython_pie_app_frame.cc index c8084a5b31fe..2771d98eeff5 100644 --- a/analytical_engine/frame/cython_pie_app_frame.cc +++ b/analytical_engine/frame/cython_pie_app_frame.cc @@ -45,8 +45,8 @@ #include "core/app/app_invoker.h" #include "core/error.h" #include "frame/ctx_wrapper_builder.h" -#include "graphscope/proto/data_types.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/data_types.pb.h" +#include "proto/types.pb.h" #ifdef _APP_HEADER #include QUOTE(_APP_HEADER) diff --git a/analytical_engine/frame/project_frame.cc b/analytical_engine/frame/project_frame.cc index 112ce3743012..bf156240fff2 100644 --- a/analytical_engine/frame/project_frame.cc +++ b/analytical_engine/frame/project_frame.cc @@ -30,7 +30,7 @@ #include "core/object/fragment_wrapper.h" #include "core/server/rpc_utils.h" #include "core/utils/fragment_traits.h" -#include "graphscope/proto/attr_value.pb.h" +#include "proto/attr_value.pb.h" #if !defined(_PROJECTED_GRAPH_TYPE) #error "_PROJECTED_GRAPH_TYPE is undefined" diff --git a/analytical_engine/frame/property_graph_frame.cc b/analytical_engine/frame/property_graph_frame.cc index 9e0645e676c5..50ac6e9d89e7 100644 --- a/analytical_engine/frame/property_graph_frame.cc +++ b/analytical_engine/frame/property_graph_frame.cc @@ -33,8 +33,8 @@ #include "core/server/rpc_utils.h" #include "core/utils/fragment_traits.h" #include "core/vertex_map/arrow_projected_vertex_map.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/graph_def.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/graph_def.pb.h" #if !defined(_GRAPH_TYPE) #error Missing _GRAPH_TYPE diff --git a/analytical_engine/test/run_java_app.cc b/analytical_engine/test/run_java_app.cc index 118dd5a8feb9..d5b75037a92a 100644 --- a/analytical_engine/test/run_java_app.cc +++ b/analytical_engine/test/run_java_app.cc @@ -49,9 +49,9 @@ #include "core/loader/arrow_fragment_loader.h" #include "core/object/fragment_wrapper.h" #include "core/utils/transform_utils.h" -#include "graphscope/proto/graph_def.pb.h" #include "java_pie/java_pie_projected_parallel_app.h" #include "java_pie/java_pie_property_parallel_app.h" +#include "proto/graph_def.pb.h" namespace bl = boost::leaf; diff --git a/coordinator/gscoordinator/kubernetes_launcher.py b/coordinator/gscoordinator/kubernetes_launcher.py index a83a18914dcf..ebf0a0d225b6 100644 --- a/coordinator/gscoordinator/kubernetes_launcher.py +++ b/coordinator/gscoordinator/kubernetes_launcher.py @@ -1148,7 +1148,7 @@ def _allocate_analytical_engine(self): def _distribute_analytical_process(self, pod_name_list, pod_ip_list): # generate and distribute hostfile - hosts = os.path.join(get_tempdir(), "kube_hosts") + hosts = os.path.join(get_tempdir(), "hosts_of_nodes") with open(hosts, "w") as f: for i, pod_ip in enumerate(pod_ip_list): f.write(f"{pod_ip} {pod_name_list[i]}\n") diff --git a/coordinator/gscoordinator/op_executor.py b/coordinator/gscoordinator/op_executor.py index 59bb3e3e3d88..4c5499ec78b3 100644 --- a/coordinator/gscoordinator/op_executor.py +++ b/coordinator/gscoordinator/op_executor.py @@ -355,7 +355,7 @@ def _create_analytical_grpc_stub(self): time.sleep(delay) delay *= 2 # back off raise RuntimeError( - "Failed to connect to engine in 60s, deployment may failed. Please check coordinator log for details" + "Failed to connect to engine in a reasonable time, deployment may failed. Please check coordinator log for details" ) @property diff --git a/coordinator/gscoordinator/template/CMakeLists.template b/coordinator/gscoordinator/template/CMakeLists.template index 2be48666c9ce..610d79472a16 100644 --- a/coordinator/gscoordinator/template/CMakeLists.template +++ b/coordinator/gscoordinator/template/CMakeLists.template @@ -263,7 +263,7 @@ endif() if(GRAPHSCOPE_ANALYTICAL_HOME) include_directories("${GRAPHSCOPE_ANALYTICAL_INCLUDE_DIRS}") include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/graphscope/apps") - include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/graphscope/proto") + include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/graphscope") # include vineyard---------------------------------------------------------- include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/vineyard") else() diff --git a/interactive_engine/common/pom.xml b/interactive_engine/common/pom.xml index bbd719967862..58ed3c8b109a 100644 --- a/interactive_engine/common/pom.xml +++ b/interactive_engine/common/pom.xml @@ -84,7 +84,7 @@ interactive - ../proto + ../../proto/groot/ sdk/* diff --git a/interactive_engine/executor/store/groot/build.rs b/interactive_engine/executor/store/groot/build.rs index c7217b6d4bc6..37b6fee8aff0 100644 --- a/interactive_engine/executor/store/groot/build.rs +++ b/interactive_engine/executor/store/groot/build.rs @@ -1,7 +1,7 @@ extern crate protoc_grpcio; fn main() { - let proto_root = "../../../proto"; + let proto_root = "../../../../proto/groot"; protoc_grpcio::compile_grpc_protos( &[ proto_root.to_owned() + "/sdk/model.proto", diff --git a/interactive_engine/groot-client/pom.xml b/interactive_engine/groot-client/pom.xml index 7b1baf1d36fb..7bb85f11de01 100644 --- a/interactive_engine/groot-client/pom.xml +++ b/interactive_engine/groot-client/pom.xml @@ -90,7 +90,7 @@ sdk - ../proto + ../../proto/groot sdk/* @@ -103,11 +103,11 @@ graphscope - ../../python + ../../proto - graphscope/proto/graph_def.proto - graphscope/proto/ddl_service.proto - graphscope/proto/write_service.proto + graph_def.proto + ddl_service.proto + write_service.proto diff --git a/interactive_engine/lgraph/CMakeLists.txt b/interactive_engine/lgraph/CMakeLists.txt index 241dd3e6f2a9..3560f9bb72e7 100644 --- a/interactive_engine/lgraph/CMakeLists.txt +++ b/interactive_engine/lgraph/CMakeLists.txt @@ -39,7 +39,7 @@ find_package(gRPC 1.26.0 REQUIRED) find_package(CppKafka 0.3.1 REQUIRED) # Proto file generating -set(PROTO_FILE_DIR ${Lgraph_SOURCE_DIR}/../proto) +set(PROTO_FILE_DIR ${Lgraph_SOURCE_DIR}/../../proto/groot) set(GRPC_FILES ${PROTO_FILE_DIR}/sdk/client.proto ${PROTO_FILE_DIR}/sdk/client_backup_service.proto) @@ -47,7 +47,7 @@ set(PROTO_FILES ${GRPC_FILES} ${PROTO_FILE_DIR}/sdk/model.proto ${PROTO_FILE_DIR}/sdk/common.proto) -set(PROTOBUF_IMPORT_DIRS ${Lgraph_SOURCE_DIR}/../proto) +set(PROTOBUF_IMPORT_DIRS ${Lgraph_SOURCE_DIR}/../../proto/groot) set(PROTO_BUILD_DIR ${Lgraph_BINARY_DIR}/generated) set(PROTO_SRCS_DEST_DIR ${Lgraph_SOURCE_DIR}/src/proto) set(PROTO_HDRS_DEST_DIR ${Lgraph_SOURCE_DIR}/include/lgraph/proto) diff --git a/interactive_engine/proto/ingest_progress_service.proto b/interactive_engine/proto/ingest_progress_service.proto deleted file mode 100644 index baefde0644c1..000000000000 --- a/interactive_engine/proto/ingest_progress_service.proto +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; - -option java_package = "com.alibaba.graphscope.proto.groot"; -option java_multiple_files = true; - -service IngestProgress { - rpc getTailOffsets(GetTailOffsetsRequest) returns(GetTailOffsetsResponse); -} - -message GetTailOffsetsRequest { - repeated int32 queueId = 1; -} - -message GetTailOffsetsResponse { - repeated int64 offsets = 1; -} diff --git a/interactive_engine/proto/ingestor_snapshot_service.proto b/interactive_engine/proto/ingestor_snapshot_service.proto deleted file mode 100644 index c17196f6db66..000000000000 --- a/interactive_engine/proto/ingestor_snapshot_service.proto +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; - -option java_package = "com.alibaba.graphscope.proto.groot"; -option java_multiple_files = true; - -service IngestorSnapshot { - rpc advanceIngestSnapshotId(AdvanceIngestSnapshotIdRequest) returns (AdvanceIngestSnapshotIdResponse); -} - -message AdvanceIngestSnapshotIdRequest { - int64 snapshotId = 1; -} - -message AdvanceIngestSnapshotIdResponse { - int64 previousSnapshotId = 1; -} diff --git a/interactive_engine/sdk-common/pom.xml b/interactive_engine/sdk-common/pom.xml index 52a04ac79e07..82d38cbc6edb 100644 --- a/interactive_engine/sdk-common/pom.xml +++ b/interactive_engine/sdk-common/pom.xml @@ -107,7 +107,7 @@ sdk - ../proto + ../../proto/groot sdk/* @@ -120,11 +120,11 @@ graphscope - ../../python + ../../proto - graphscope/proto/graph_def.proto - graphscope/proto/ddl_service.proto - graphscope/proto/write_service.proto + graph_def.proto + ddl_service.proto + write_service.proto diff --git a/python/graphscope/proto/attr_value.proto b/proto/attr_value.proto similarity index 100% rename from python/graphscope/proto/attr_value.proto rename to proto/attr_value.proto diff --git a/python/graphscope/proto/coordinator_service.proto b/proto/coordinator_service.proto similarity index 97% rename from python/graphscope/proto/coordinator_service.proto rename to proto/coordinator_service.proto index cfe917bed62c..038709bd66ff 100644 --- a/python/graphscope/proto/coordinator_service.proto +++ b/proto/coordinator_service.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package gs.rpc; -import "graphscope/proto/message.proto"; +import "message.proto"; service CoordinatorService { // Connect a session. diff --git a/python/graphscope/proto/data_types.proto b/proto/data_types.proto similarity index 100% rename from python/graphscope/proto/data_types.proto rename to proto/data_types.proto diff --git a/python/graphscope/proto/ddl_service.proto b/proto/ddl_service.proto similarity index 98% rename from python/graphscope/proto/ddl_service.proto rename to proto/ddl_service.proto index af0d49f96a94..e553924bd858 100644 --- a/python/graphscope/proto/ddl_service.proto +++ b/proto/ddl_service.proto @@ -15,7 +15,7 @@ syntax = "proto3"; package gs.rpc.ddl_service.v1; -import "graphscope/proto/graph_def.proto"; +import "graph_def.proto"; option java_package = "com.alibaba.graphscope.proto.ddl"; option java_multiple_files = true; diff --git a/python/graphscope/proto/engine_service.proto b/proto/engine_service.proto similarity index 95% rename from python/graphscope/proto/engine_service.proto rename to proto/engine_service.proto index 76596a826595..da15dbcf86d2 100644 --- a/python/graphscope/proto/engine_service.proto +++ b/proto/engine_service.proto @@ -17,7 +17,7 @@ syntax = "proto3"; package gs.rpc; -import "graphscope/proto/message.proto"; +import "message.proto"; service EngineService { // Drives the graph computation. diff --git a/python/graphscope/proto/error_codes.proto b/proto/error_codes.proto similarity index 100% rename from python/graphscope/proto/error_codes.proto rename to proto/error_codes.proto diff --git a/python/graphscope/proto/graph_def.proto b/proto/graph_def.proto similarity index 100% rename from python/graphscope/proto/graph_def.proto rename to proto/graph_def.proto diff --git a/interactive_engine/proto/cluster.proto b/proto/groot/cluster.proto similarity index 100% rename from interactive_engine/proto/cluster.proto rename to proto/groot/cluster.proto diff --git a/interactive_engine/proto/frontend_snapshot_service.proto b/proto/groot/frontend_snapshot_service.proto similarity index 100% rename from interactive_engine/proto/frontend_snapshot_service.proto rename to proto/groot/frontend_snapshot_service.proto diff --git a/interactive_engine/proto/id_allocate_service.proto b/proto/groot/id_allocate_service.proto similarity index 100% rename from interactive_engine/proto/id_allocate_service.proto rename to proto/groot/id_allocate_service.proto diff --git a/interactive_engine/proto/ingestor_write_service.proto b/proto/groot/ingestor_snapshot_service.proto similarity index 66% rename from interactive_engine/proto/ingestor_write_service.proto rename to proto/groot/ingestor_snapshot_service.proto index f10bfff8e103..6d9258dd7fb7 100644 --- a/interactive_engine/proto/ingestor_write_service.proto +++ b/proto/groot/ingestor_snapshot_service.proto @@ -20,6 +20,31 @@ import "sdk/model.proto"; option java_package = "com.alibaba.graphscope.proto.groot"; option java_multiple_files = true; +service IngestorSnapshot { + rpc advanceIngestSnapshotId(AdvanceIngestSnapshotIdRequest) returns (AdvanceIngestSnapshotIdResponse); +} + +message AdvanceIngestSnapshotIdRequest { + int64 snapshotId = 1; +} + +message AdvanceIngestSnapshotIdResponse { + int64 previousSnapshotId = 1; +} + + +service IngestProgress { + rpc getTailOffsets(GetTailOffsetsRequest) returns(GetTailOffsetsResponse); +} + +message GetTailOffsetsRequest { + repeated int32 queueId = 1; +} + +message GetTailOffsetsResponse { + repeated int64 offsets = 1; +} + service IngestorWrite { rpc writeIngestor(WriteIngestorRequest) returns (WriteIngestorResponse); } @@ -32,4 +57,4 @@ message WriteIngestorRequest { message WriteIngestorResponse { int64 snapshotId = 1; -} +} \ No newline at end of file diff --git a/interactive_engine/proto/meta_service.proto b/proto/groot/meta_service.proto similarity index 100% rename from interactive_engine/proto/meta_service.proto rename to proto/groot/meta_service.proto diff --git a/interactive_engine/proto/metrics_collect_service.proto b/proto/groot/metrics_collect_service.proto similarity index 100% rename from interactive_engine/proto/metrics_collect_service.proto rename to proto/groot/metrics_collect_service.proto diff --git a/interactive_engine/proto/schema_service.proto b/proto/groot/schema_service.proto similarity index 100% rename from interactive_engine/proto/schema_service.proto rename to proto/groot/schema_service.proto diff --git a/interactive_engine/proto/sdk/backup_service.proto b/proto/groot/sdk/backup_service.proto similarity index 100% rename from interactive_engine/proto/sdk/backup_service.proto rename to proto/groot/sdk/backup_service.proto diff --git a/interactive_engine/proto/sdk/client.proto b/proto/groot/sdk/client.proto similarity index 100% rename from interactive_engine/proto/sdk/client.proto rename to proto/groot/sdk/client.proto diff --git a/interactive_engine/proto/sdk/client_backup_service.proto b/proto/groot/sdk/client_backup_service.proto similarity index 100% rename from interactive_engine/proto/sdk/client_backup_service.proto rename to proto/groot/sdk/client_backup_service.proto diff --git a/interactive_engine/proto/sdk/client_ddl_service.proto b/proto/groot/sdk/client_ddl_service.proto similarity index 100% rename from interactive_engine/proto/sdk/client_ddl_service.proto rename to proto/groot/sdk/client_ddl_service.proto diff --git a/interactive_engine/proto/sdk/common.proto b/proto/groot/sdk/common.proto similarity index 100% rename from interactive_engine/proto/sdk/common.proto rename to proto/groot/sdk/common.proto diff --git a/interactive_engine/proto/sdk/model.proto b/proto/groot/sdk/model.proto similarity index 100% rename from interactive_engine/proto/sdk/model.proto rename to proto/groot/sdk/model.proto diff --git a/interactive_engine/proto/sdk/schema.proto b/proto/groot/sdk/schema.proto similarity index 100% rename from interactive_engine/proto/sdk/schema.proto rename to proto/groot/sdk/schema.proto diff --git a/interactive_engine/proto/snapshot_commit_service.proto b/proto/groot/snapshot_commit_service.proto similarity index 100% rename from interactive_engine/proto/snapshot_commit_service.proto rename to proto/groot/snapshot_commit_service.proto diff --git a/interactive_engine/proto/store_backup_service.proto b/proto/groot/store_backup_service.proto similarity index 100% rename from interactive_engine/proto/store_backup_service.proto rename to proto/groot/store_backup_service.proto diff --git a/interactive_engine/proto/store_ingest_service.proto b/proto/groot/store_ingest_service.proto similarity index 100% rename from interactive_engine/proto/store_ingest_service.proto rename to proto/groot/store_ingest_service.proto diff --git a/interactive_engine/proto/store_schema_service.proto b/proto/groot/store_schema_service.proto similarity index 100% rename from interactive_engine/proto/store_schema_service.proto rename to proto/groot/store_schema_service.proto diff --git a/interactive_engine/proto/store_write_service.proto b/proto/groot/store_write_service.proto similarity index 94% rename from interactive_engine/proto/store_write_service.proto rename to proto/groot/store_write_service.proto index f2b4fd7e5b95..35403de8125c 100644 --- a/interactive_engine/proto/store_write_service.proto +++ b/proto/groot/store_write_service.proto @@ -21,7 +21,7 @@ option java_multiple_files = true; import "sdk/model.proto"; service StoreWrite { - rpc writeStore(WriteStoreRequest) returns(WriteStoreResponse); + rpc writeStore(WriteStoreRequest) returns (WriteStoreResponse); } message PartitionToBatchPb { diff --git a/python/graphscope/proto/message.proto b/proto/message.proto similarity index 97% rename from python/graphscope/proto/message.proto rename to proto/message.proto index 3509d7aead60..a6cbff9abf02 100644 --- a/python/graphscope/proto/message.proto +++ b/proto/message.proto @@ -17,9 +17,9 @@ syntax = "proto3"; package gs.rpc; -import "graphscope/proto/error_codes.proto"; -import "graphscope/proto/op_def.proto"; -import "graphscope/proto/types.proto"; +import "error_codes.proto"; +import "op_def.proto"; +import "types.proto"; //////////////////////////////////////////////////////////////////////////////// diff --git a/python/graphscope/proto/op_def.proto b/proto/op_def.proto similarity index 92% rename from python/graphscope/proto/op_def.proto rename to proto/op_def.proto index 036329a07b79..1cd9c6b98d8b 100644 --- a/python/graphscope/proto/op_def.proto +++ b/proto/op_def.proto @@ -17,10 +17,10 @@ syntax = "proto3"; package gs.rpc; -import "graphscope/proto/attr_value.proto"; -import "graphscope/proto/error_codes.proto"; -import "graphscope/proto/graph_def.proto"; -import "graphscope/proto/types.proto"; +import "attr_value.proto"; +import "error_codes.proto"; +import "graph_def.proto"; +import "types.proto"; message OpDef { // Unique key for every OpDef. Usually generated by analytical engine. diff --git a/python/graphscope/proto/proto_generator.py b/proto/proto_generator.py old mode 100755 new mode 100644 similarity index 63% rename from python/graphscope/proto/proto_generator.py rename to proto/proto_generator.py index 503dba2fab5a..e6d3e06d5274 --- a/python/graphscope/proto/proto_generator.py +++ b/proto/proto_generator.py @@ -24,28 +24,27 @@ def gather_all_proto(proto_dir, suffix="*.proto"): - directory = os.path.join(proto_dir, suffix) - files = glob.glob(directory) - return files + pattern = os.path.join(proto_dir, suffix) + return glob.glob(pattern) def create_path(path): """Utility function to create a path.""" - if os.path.isdir(path): - return - os.makedirs(path, exist_ok=True) + if not os.path.isdir(path): + os.makedirs(path, exist_ok=True) def cpp_out(relative_dir, output_dir): files = gather_all_proto(relative_dir) for proto_file in files: + cmd = [ + shutil.which("protoc"), + "-I.", + f"--cpp_out={output_dir}", + proto_file, + ] subprocess.check_call( - [ - shutil.which("protoc"), - "-I%s" % ".", - "--cpp_out=%s" % output_dir, - proto_file, - ], + cmd, stderr=subprocess.STDOUT, ) @@ -53,16 +52,17 @@ def cpp_out(relative_dir, output_dir): def python_out(relative_dir, output_dir): files = gather_all_proto(relative_dir) for proto_file in files: + cmd = [ + sys.executable, + "-m", + "grpc_tools.protoc", + "-I.", + f"--python_out={output_dir}", + f"--mypy_out={output_dir}", + proto_file, + ] subprocess.check_call( - [ - sys.executable, - "-m", - "grpc_tools.protoc", - "-I%s" % ".", - "--python_out=%s" % os.path.join(output_dir), - "--mypy_out=%s" % os.path.join(output_dir), - proto_file, - ], + cmd, stderr=subprocess.STDOUT, ) @@ -74,14 +74,15 @@ def cpp_service_out(relative_dir, output_dir): suffix = "*_service.proto" files = gather_all_proto(relative_dir, suffix) for proto_file in files: + cmd = [ + shutil.which("protoc"), + "-I.", + f"--grpc_out={output_dir}", + f"--plugin=protoc-gen-grpc={plugin_path}", + proto_file, + ] subprocess.check_call( - [ - shutil.which("protoc"), - "-I%s" % ".", - "--grpc_out=%s" % output_dir, - "--plugin=protoc-gen-grpc=%s" % plugin_path, - proto_file, - ], + cmd, stderr=subprocess.STDOUT, ) @@ -90,18 +91,19 @@ def python_service_out(relative_dir, output_dir): suffix = "*_service.proto" files = gather_all_proto(relative_dir, suffix) for proto_file in files: + cmd = [ + sys.executable, + "-m", + "grpc_tools.protoc", + "-I.", + f"--python_out={output_dir}", + f"--mypy_out={output_dir}", + f"--grpc_python_out={output_dir}", + f"--mypy_grpc_out={output_dir}", + proto_file, + ] subprocess.check_call( - [ - sys.executable, - "-m", - "grpc_tools.protoc", - "-I%s" % ".", - "--python_out=%s" % output_dir, - "--mypy_out=%s" % os.path.join(output_dir), - "--grpc_python_out=%s" % output_dir, - "--mypy_grpc_out=%s" % os.path.join(output_dir), - proto_file, - ], + cmd, stderr=subprocess.STDOUT, ) @@ -111,19 +113,16 @@ def python_service_out(relative_dir, output_dir): print("Usage: python proto_generator.py [--cpp] [--python]") sys.exit(1) - # path to 'GraphScope/python/graphscope/proto' - current_dir = os.path.dirname(os.path.abspath(__file__)) - - # path to 'GraphScope/python' - base_dir = os.path.join(current_dir, "../", "../") - os.chdir(base_dir) - output_dir = sys.argv[1] output_dir = os.path.realpath(os.path.realpath(output_dir)) create_path(output_dir) + # path to 'GraphScope/proto' + current_dir = os.path.dirname(os.path.abspath(__file__)) + os.chdir(current_dir) + # must use relative path - relative_dir = os.path.join(".", "graphscope", "proto") + relative_dir = "." if len(sys.argv) <= 2 or len(sys.argv) > 2 and sys.argv[2] == "--cpp": print("Generating cpp proto to: " + output_dir) cpp_out(relative_dir, output_dir) diff --git a/python/graphscope/proto/types.proto b/proto/types.proto similarity index 100% rename from python/graphscope/proto/types.proto rename to proto/types.proto diff --git a/python/graphscope/proto/write_service.proto b/proto/write_service.proto similarity index 100% rename from python/graphscope/proto/write_service.proto rename to proto/write_service.proto diff --git a/python/graphscope/proto/__init__.py b/python/graphscope/proto/__init__.py index 618dda69a7be..e61cf97316fe 100644 --- a/python/graphscope/proto/__init__.py +++ b/python/graphscope/proto/__init__.py @@ -15,3 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/python/setup.py b/python/setup.py index 50d3c9cdc82b..8eab42409929 100644 --- a/python/setup.py +++ b/python/setup.py @@ -36,7 +36,7 @@ # https://github.com/pypa/pip/issues/7953 site.ENABLE_USER_SITE = "--user" in sys.argv[1:] -repo_root = os.path.dirname(os.path.abspath(__file__)) +pkg_root = os.path.dirname(os.path.abspath(__file__)) if platform.system() == "Darwin": # see also: https://github.com/python/cpython/issues/100420 @@ -57,20 +57,20 @@ def finalize_options(self): pass def run(self): + cmd = [ + sys.executable, + os.path.join( + pkg_root, + "..", + "proto", + "proto_generator.py", + ), + os.path.join(pkg_root, "graphscope", "proto"), + "--python", + ] + print(" ".join(cmd)) subprocess.check_call( - [ - sys.executable, - os.path.join( - repo_root, - "..", - "python", - "graphscope", - "proto", - "proto_generator.py", - ), - repo_root, - "--python", - ], + cmd, env=os.environ.copy(), ) @@ -92,17 +92,17 @@ def finalize_options(self): def run(self): if self.inplace: - subprocess.check_call([sys.executable, "-m", "isort", "."], cwd=repo_root) - subprocess.check_call([sys.executable, "-m", "black", "."], cwd=repo_root) - subprocess.check_call([sys.executable, "-m", "flake8", "."], cwd=repo_root) + subprocess.check_call([sys.executable, "-m", "isort", "."], cwd=pkg_root) + subprocess.check_call([sys.executable, "-m", "black", "."], cwd=pkg_root) + subprocess.check_call([sys.executable, "-m", "flake8", "."], cwd=pkg_root) else: subprocess.check_call( - [sys.executable, "-m", "isort", "--check", "--diff", "."], cwd=repo_root + [sys.executable, "-m", "isort", "--check", "--diff", "."], cwd=pkg_root ) subprocess.check_call( - [sys.executable, "-m", "black", "--check", "--diff", "."], cwd=repo_root + [sys.executable, "-m", "black", "--check", "--diff", "."], cwd=pkg_root ) - subprocess.check_call([sys.executable, "-m", "flake8", "."], cwd=repo_root) + subprocess.check_call([sys.executable, "-m", "flake8", "."], cwd=pkg_root) class CustomBuildPy(build_py): @@ -141,7 +141,7 @@ def run(self): graphlearn_shared_lib = "libgraphlearn_shared.so" if not os.path.isfile( os.path.join( - repo_root, + pkg_root, "..", "learning_engine", "graph-learn", @@ -156,18 +156,18 @@ def run(self): bdist_wheel.run(self) -with open(os.path.join(repo_root, "..", "README.md"), "r", encoding="utf-8") as fp: +with open(os.path.join(pkg_root, "..", "README.md"), "r", encoding="utf-8") as fp: long_description = fp.read() def parsed_reqs(): - with open(os.path.join(repo_root, "requirements.txt"), "r", encoding="utf-8") as fp: + with open(os.path.join(pkg_root, "requirements.txt"), "r", encoding="utf-8") as fp: return fp.read().splitlines() def parsed_dev_reqs(): with open( - os.path.join(repo_root, "requirements-dev.txt"), "r", encoding="utf-8" + os.path.join(pkg_root, "requirements-dev.txt"), "r", encoding="utf-8" ) as fp: return fp.read().splitlines() @@ -202,7 +202,7 @@ def build_learning_engine(): import numpy ROOT_PATH = os.path.abspath( - os.path.join(repo_root, "..", "learning_engine", "graph-learn") + os.path.join(pkg_root, "..", "learning_engine", "graph-learn") ) include_dirs = [] @@ -270,7 +270,7 @@ def parse_version(root, **kwargs): from setuptools_scm.git import parse from setuptools_scm.version import meta - version_file = os.path.join(repo_root, "..", "VERSION") + version_file = os.path.join(pkg_root, "..", "VERSION") if os.path.isfile(version_file): with open(version_file, "r", encoding="utf-8") as fp: return meta(fp.read().strip()) @@ -305,7 +305,7 @@ def parse_version(root, **kwargs): ], keywords="Graph, Large-Scale, Distributed Computing", use_scm_version={ - "root": repo_root, + "root": pkg_root, "parse": parse_version, }, setup_requires=[