[Bug Fix] Fix Incremental Query Id Issues in Multiple Frontend (#3026)

Fixes #2974
alibaba · Jul 21, 2023 · 5e3a32b · 5e3a32b
1 parent 95b7130
commit 5e3a32b
Show file tree

Hide file tree

Showing 7 changed files with 22 additions and 10 deletions.
diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties
@@ -3,7 +3,6 @@
 
 # pegasus service config
 pegasus.worker.num: 2
-pegasus.timeout: 240000
 pegasus.batch.size: 1024
 pegasus.output.capacity: 16
 pegasus.hosts: localhost:1234

diff --git a/...ve_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java b/...ve_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java
@@ -33,4 +33,10 @@ public class FrontendConfig {
             Config.intConfig("query.execution.timeout.ms", 3000000);
 
     public static final Config<String> ENGINE_TYPE = Config.stringConfig("engine.type", "pegasus");
+
+    public static final Config<Integer> FRONTEND_SERVER_ID =
+            Config.intConfig("frontend.server.id", 0);
+
+    public static final Config<Integer> FRONTEND_SERVER_NUM =
+            Config.intConfig("frontend.server.num", 1);
 }
diff --git a/...ive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/PegasusConfig.java b/...ive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/PegasusConfig.java
@@ -20,9 +20,6 @@ public class PegasusConfig {
     public static final Config<Integer> PEGASUS_WORKER_NUM =
             Config.intConfig("pegasus.worker.num", 1);
 
-    public static final Config<Integer> PEGASUS_TIMEOUT =
-            Config.intConfig("pegasus.timeout", 240000);
-
     public static final Config<Integer> PEGASUS_BATCH_SIZE =
             Config.intConfig("pegasus.batch.size", 1024);
 

diff --git a/...ve_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/...ve_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java
@@ -19,6 +19,7 @@
 import com.alibaba.graphscope.common.antlr4.Antlr4Parser;
 import com.alibaba.graphscope.common.config.Configs;
 import com.alibaba.graphscope.common.config.FileLoadType;
+import com.alibaba.graphscope.common.config.FrontendConfig;
 import com.alibaba.graphscope.common.config.PlannerConfig;
 import com.alibaba.graphscope.common.ir.planner.rules.FilterMatchRule;
 import com.alibaba.graphscope.common.ir.runtime.PhysicalBuilder;
@@ -63,16 +64,21 @@ public GraphPlanner(Configs graphConfig) {
         this.plannerConfig = PlannerConfig.create(this.graphConfig);
         this.optPlanner = createRelOptPlanner(this.plannerConfig);
         this.rexBuilder = new GraphRexBuilder(new JavaTypeFactoryImpl());
-        this.idGenerator = new AtomicLong(0L);
+        this.idGenerator = new AtomicLong(FrontendConfig.FRONTEND_SERVER_ID.get(graphConfig));
     }
 
     public PlannerInstance instance(ParseTree parsedQuery, IrMeta irMeta) {
-        long id = idGenerator.getAndIncrement();
+        long id = generateInstanceId();
         String name = "ir_plan_" + id;
         GraphOptCluster optCluster = GraphOptCluster.create(this.optPlanner, this.rexBuilder);
         return new PlannerInstance(id, name, parsedQuery, optCluster, irMeta);
     }
 
+    public long generateInstanceId() {
+        long delta = FrontendConfig.FRONTEND_SERVER_NUM.get(graphConfig);
+        return idGenerator.getAndAdd(delta);
+    }
+
     public class PlannerInstance {
         private final long id;
         private final String name;

diff --git a/...src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java b/...src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java
@@ -94,8 +94,7 @@ public ThrowingConsumer<Context> select(Context ctx) {
                             Traversal traversal =
                                     (Traversal) scriptEngine.eval(script, this.context);
                             applyStrategies(traversal);
-
-                            long jobId = graphPlanner.getIdGenerator().getAndIncrement();
+                            long jobId = graphPlanner.generateInstanceId();
                             IrMeta irMeta = metaQueryCallback.beforeExec();
                             QueryStatusCallback statusCallback =
                                     createQueryStatusCallback(script, jobId);

diff --git a/.../src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java b/.../src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java
@@ -119,7 +119,7 @@ protected void evalOpInternal(
 
         String language = AntlrGremlinScriptEngineFactory.LANGUAGE_NAME;
 
-        long jobId = graphPlanner.getIdGenerator().getAndIncrement();
+        long jobId = graphPlanner.generateInstanceId();
         IrMeta irMeta = metaQueryCallback.beforeExec();
         QueryStatusCallback statusCallback = createQueryStatusCallback(script, jobId);
         GremlinExecutor.LifeCycle lifeCycle =

diff --git a/...groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java b/...groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java
@@ -103,7 +103,6 @@ private com.alibaba.graphscope.common.config.Configs getConfigs() {
         // add pegasus config
         addToConfigMapIfExist(PegasusConfig.PEGASUS_HOSTS.getKey(), configMap);
         addToConfigMapIfExist(PegasusConfig.PEGASUS_WORKER_NUM.getKey(), configMap);
-        addToConfigMapIfExist(PegasusConfig.PEGASUS_TIMEOUT.getKey(), configMap);
         addToConfigMapIfExist(PegasusConfig.PEGASUS_BATCH_SIZE.getKey(), configMap);
         addToConfigMapIfExist(PegasusConfig.PEGASUS_OUTPUT_CAPACITY.getKey(), configMap);
         addToConfigMapIfExist(PegasusConfig.PEGASUS_MEMORY_LIMIT.getKey(), configMap);
@@ -116,6 +115,12 @@ private com.alibaba.graphscope.common.config.Configs getConfigs() {
         // add neo4j config
         addToConfigMapIfExist(FrontendConfig.NEO4J_BOLT_SERVER_DISABLED.getKey(), configMap);
         addToConfigMapIfExist(FrontendConfig.NEO4J_BOLT_SERVER_PORT.getKey(), configMap);
+        // add timeout config
+        addToConfigMapIfExist(FrontendConfig.QUERY_EXECUTION_TIMEOUT_MS.getKey(), configMap);
+        // add frontend server id
+        addToConfigMapIfExist(FrontendConfig.FRONTEND_SERVER_ID.getKey(), configMap);
+        // add frontend server num
+        addToConfigMapIfExist(FrontendConfig.FRONTEND_SERVER_NUM.getKey(), configMap);
         return new com.alibaba.graphscope.common.config.Configs(configMap);
     }