From 8493ee755561dc1921967d8314cbff3b74a52547 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Tue, 14 Jan 2025 14:55:09 +0800 Subject: [PATCH 01/12] optimize query 'Match (n) Return n Limit 10' by ScanEarlyStopRule --- .../common/ir/planner/PlannerGroup.java | 8 +- .../ir/planner/rules/ScanEarlyStopRule.java | 120 +++++++++ .../planner/rules/ScanExpandFusionRule.java | 229 ++++++++++++++++++ .../ir/planner/rules/TopKPushDownRule.java | 97 ++++++++ .../graphscope/common/ir/rel/QueryParams.java | 41 ++++ .../graphscope/common/ir/rel/RangeParam.java | 50 ++++ .../ir/rel/graph/GraphLogicalExpand.java | 23 ++ .../common/ir/rel/graph/GraphLogicalGetV.java | 19 ++ .../ir/rel/graph/GraphLogicalSource.java | 37 ++- .../ir/rel/type/order/GraphRelCollations.java | 3 +- .../proto/GraphRelToProtoConverter.java | 20 +- .../ir/planner/rbo/ScanEarlyStopTest.java | 102 ++++++++ 12 files changed, 739 insertions(+), 10 deletions(-) create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/QueryParams.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/RangeParam.java create mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroup.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroup.java index 110c615ab82a..ab46074ed4a3 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroup.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroup.java @@ -152,13 +152,19 @@ private RelOptPlanner createPhysicalPlanner() { config.getRules() .forEach( k -> { - if (k.equals(ExpandGetVFusionRule.class.getSimpleName())) { + if (k.equals(ScanExpandFusionRule.class.getSimpleName())) { + ruleConfigs.add(ScanExpandFusionRule.Config.DEFAULT); + } else if (k.equals(ExpandGetVFusionRule.class.getSimpleName())) { ruleConfigs.add( ExpandGetVFusionRule.BasicExpandGetVFusionRule.Config .DEFAULT); ruleConfigs.add( ExpandGetVFusionRule.PathBaseExpandGetVFusionRule.Config .DEFAULT); + } else if (k.equals(TopKPushDownRule.class.getSimpleName())) { + ruleConfigs.add(TopKPushDownRule.Config.DEFAULT); + } else if (k.equals(ScanEarlyStopRule.class.getSimpleName())) { + ruleConfigs.add(ScanEarlyStopRule.Config.DEFAULT); } }); ruleConfigs.forEach( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java new file mode 100644 index 000000000000..50970bd65e48 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java @@ -0,0 +1,120 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.planner.rules; + +import com.alibaba.graphscope.common.ir.rel.GraphLogicalSort; +import com.alibaba.graphscope.common.ir.rel.QueryParams; +import com.alibaba.graphscope.common.ir.rel.RangeParam; +import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalSource; + +import org.apache.calcite.plan.GraphOptCluster; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.tools.RelBuilderFactory; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * fuse limit to source to stop scan at an early stage + */ +public class ScanEarlyStopRule extends RelRule { + protected ScanEarlyStopRule(RelRule.Config config) { + super(config); + } + + @Override + public void onMatch(RelOptRuleCall relOptRuleCall) { + // fuse limit to source + GraphLogicalSort sort = relOptRuleCall.rel(0); + GraphLogicalSource source = relOptRuleCall.rel(1); + RangeParam range = new RangeParam(sort.offset, sort.fetch); + GraphLogicalSource fused = + GraphLogicalSource.create( + (GraphOptCluster) source.getCluster(), + source.getHints(), + source.getOpt(), + source.getTableConfig(), + source.getAliasName(), + new QueryParams().addParam("range", range), + source.getUniqueKeyFilters(), + source.getFilters()); + relOptRuleCall.transformTo(fused); + } + + public static class Config implements RelRule.Config { + public static RelRule.Config DEFAULT = + new Config() + .withOperandSupplier( + b0 -> + b0.operand(GraphLogicalSort.class) + .predicate( + (GraphLogicalSort sort) -> { + // check sort is the limit + return sort.getCollation() + .getFieldCollations() + .isEmpty(); + }) + .oneInput( + b1 -> + b1.operand(GraphLogicalSource.class) + .anyInputs())); + + private RelRule.OperandTransform operandSupplier; + private @Nullable String description; + private RelBuilderFactory builderFactory; + + @Override + public RelRule toRule() { + return new ScanEarlyStopRule(this); + } + + @Override + public Config withRelBuilderFactory(RelBuilderFactory relBuilderFactory) { + this.builderFactory = relBuilderFactory; + return this; + } + + @Override + public Config withDescription( + @org.checkerframework.checker.nullness.qual.Nullable String s) { + this.description = s; + return this; + } + + @Override + public Config withOperandSupplier(OperandTransform operandTransform) { + this.operandSupplier = operandTransform; + return this; + } + + @Override + public OperandTransform operandSupplier() { + return this.operandSupplier; + } + + @Override + public @org.checkerframework.checker.nullness.qual.Nullable String description() { + return this.description; + } + + @Override + public RelBuilderFactory relBuilderFactory() { + return this.builderFactory; + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java new file mode 100644 index 000000000000..59164b13fdcc --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java @@ -0,0 +1,229 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.planner.rules; + +import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject; +import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalExpand; +import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalGetV; +import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalSource; +import com.alibaba.graphscope.common.ir.rex.RexVariableAliasCollector; +import com.alibaba.graphscope.common.ir.tools.AliasInference; +import com.alibaba.graphscope.common.ir.tools.Utils; +import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; +import com.alibaba.graphscope.common.ir.type.GraphLabelType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import org.apache.calcite.plan.GraphOptCluster; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.commons.lang3.ObjectUtils; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * transform scan(V) + expand(E) + getV + project(E) to scan(E) + project(E) + */ +public class ScanExpandFusionRule extends RelRule { + + protected ScanExpandFusionRule(RelRule.Config config) { + super(config); + } + + @Override + public void onMatch(RelOptRuleCall relOptRuleCall) { + GraphLogicalProject project = relOptRuleCall.rel(0); + GraphLogicalGetV getV = relOptRuleCall.rel(1); + GraphLogicalExpand expand = relOptRuleCall.rel(2); + GraphLogicalSource source = relOptRuleCall.rel(3); + if (!checkLabel(getV, expand, source) + || !checkOpt(getV, expand, source) + || !checkAlias(project, getV, expand, source) + || !checkFilters(getV, expand, source)) { + return; + } + GraphLogicalSource fused = + GraphLogicalSource.create( + (GraphOptCluster) expand.getCluster(), + expand.getHints(), + GraphOpt.Source.EDGE, + expand.getTableConfig(), + expand.getAliasName(), + source.getParams(), + source.getUniqueKeyFilters(), + expand.getFilters()); + RelNode newProject = project.copy(project.getTraitSet(), ImmutableList.of(fused)); + relOptRuleCall.transformTo(newProject); + } + + private boolean checkOpt( + GraphLogicalGetV getV, GraphLogicalExpand expand, GraphLogicalSource source) { + return source.getOpt() == GraphOpt.Source.VERTEX + && (expand.getOpt() == GraphOpt.Expand.OUT && getV.getOpt() == GraphOpt.GetV.END + || expand.getOpt() == GraphOpt.Expand.IN + && getV.getOpt() == GraphOpt.GetV.START + || expand.getOpt() == GraphOpt.Expand.BOTH + && getV.getOpt() == GraphOpt.GetV.BOTH); + } + + private boolean checkAlias( + GraphLogicalProject project, + GraphLogicalGetV getV, + GraphLogicalExpand expand, + GraphLogicalSource source) { + List usedAliasIds = Lists.newArrayList(); + RexVariableAliasCollector collector = + new RexVariableAliasCollector<>(true, var -> var.getAliasId()); + project.getProjects() + .forEach( + expr -> { + usedAliasIds.addAll(expr.accept(collector)); + }); + if (source.getAliasId() != AliasInference.DEFAULT_ID + && usedAliasIds.contains(source.getAliasId())) { + return false; + } + if (getV.getAliasId() != AliasInference.DEFAULT_ID + && usedAliasIds.contains(getV.getAliasId())) { + return false; + } + return true; + } + + private boolean checkLabel( + GraphLogicalGetV getV, GraphLogicalExpand expand, GraphLogicalSource source) { + GraphLabelType sourceType = Utils.getGraphLabels(source.getRowType()); + GraphLabelType getVType = Utils.getGraphLabels(getV.getRowType()); + List sourceCandidates = + sourceType.getLabelsEntry().stream() + .map(k -> k.getLabel()) + .collect(Collectors.toList()); + List getVCandidates = + getVType.getLabelsEntry().stream() + .map(k -> k.getLabel()) + .collect(Collectors.toList()); + List optTables = expand.getTableConfig().getTables(); + for (RelOptTable optTable : optTables) { + GraphLabelType expandType = Utils.getGraphLabels(optTable.getRowType()); + for (GraphLabelType.Entry entry : expandType.getLabelsEntry()) { + switch (expand.getOpt()) { + case OUT: + if (!sourceCandidates.contains(entry.getSrcLabel()) + || !getVCandidates.contains(entry.getDstLabel())) { + return false; + } + break; + case IN: + if (!sourceCandidates.contains(entry.getDstLabel()) + || !getVCandidates.contains(entry.getSrcLabel())) { + return false; + } + break; + case BOTH: + default: + if ((!sourceCandidates.contains(entry.getSrcLabel()) + || !getVCandidates.contains(entry.getDstLabel())) + && (!sourceCandidates.contains(entry.getDstLabel()) + || !getVCandidates.contains(entry.getSrcLabel()))) { + return false; + } + } + } + } + return true; + } + + private boolean checkFilters( + GraphLogicalGetV getV, GraphLogicalExpand expand, GraphLogicalSource source) { + return source.getUniqueKeyFilters() == null + && source.getParams().getParams().isEmpty() + && ObjectUtils.isEmpty(source.getFilters()) + && ObjectUtils.isEmpty(getV.getFilters()); + } + + public static class Config implements RelRule.Config { + public static RelRule.Config DEFAULT = + new Config() + .withOperandSupplier( + b0 -> + b0.operand(GraphLogicalProject.class) + .oneInput( + b1 -> + b1.operand(GraphLogicalGetV.class) + .oneInput( + b2 -> + b2.operand( + GraphLogicalExpand + .class) + .oneInput( + b3 -> + b3.operand( + GraphLogicalSource + .class) + .anyInputs())))); + + private RelRule.OperandTransform operandSupplier; + private @Nullable String description; + private RelBuilderFactory builderFactory; + + @Override + public ScanExpandFusionRule toRule() { + return new ScanExpandFusionRule(this); + } + + @Override + public Config withRelBuilderFactory(RelBuilderFactory relBuilderFactory) { + this.builderFactory = relBuilderFactory; + return this; + } + + @Override + public Config withDescription( + @org.checkerframework.checker.nullness.qual.Nullable String s) { + this.description = s; + return this; + } + + @Override + public Config withOperandSupplier(RelRule.OperandTransform operandTransform) { + this.operandSupplier = operandTransform; + return this; + } + + @Override + public RelRule.OperandTransform operandSupplier() { + return this.operandSupplier; + } + + @Override + public @org.checkerframework.checker.nullness.qual.Nullable String description() { + return this.description; + } + + @Override + public RelBuilderFactory relBuilderFactory() { + return this.builderFactory; + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java new file mode 100644 index 000000000000..04460b06c1f4 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java @@ -0,0 +1,97 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.planner.rules; + +import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject; +import com.alibaba.graphscope.common.ir.rel.GraphLogicalSort; + +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.rules.SortProjectTransposeRule; +import org.apache.calcite.tools.RelBuilderFactory; +import org.checkerframework.checker.nullness.qual.Nullable; + +public class TopKPushDownRule extends SortProjectTransposeRule { + protected TopKPushDownRule(Config config) { + super(config); + } + + public static class Config implements SortProjectTransposeRule.Config { + public static RelRule.Config DEFAULT = + // the sort is the limit operator + new TopKPushDownRule.Config() + .withOperandSupplier( + b0 -> + b0.operand(GraphLogicalSort.class) + .predicate( + (GraphLogicalSort sort) -> + sort.getCollation() + .getFieldCollations() + .isEmpty()) + .oneInput( + b1 -> + b1.operand( + GraphLogicalProject + .class) + .anyInputs())) + .as(Config.class); + + private RelRule.OperandTransform operandSupplier; + private @Nullable String description; + private RelBuilderFactory builderFactory; + + @Override + public TopKPushDownRule toRule() { + return new TopKPushDownRule(this); + } + + @Override + public Config withRelBuilderFactory(RelBuilderFactory relBuilderFactory) { + this.builderFactory = relBuilderFactory; + return this; + } + + @Override + public Config withDescription( + @org.checkerframework.checker.nullness.qual.Nullable String s) { + this.description = s; + return this; + } + + @Override + public Config withOperandSupplier(OperandTransform operandTransform) { + this.operandSupplier = operandTransform; + return this; + } + + @Override + public OperandTransform operandSupplier() { + return this.operandSupplier; + } + + @Override + public @org.checkerframework.checker.nullness.qual.Nullable String description() { + return this.description; + } + + @Override + public RelBuilderFactory relBuilderFactory() { + return this.builderFactory; + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/QueryParams.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/QueryParams.java new file mode 100644 index 000000000000..d5576d79fe5b --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/QueryParams.java @@ -0,0 +1,41 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.rel; + +import com.google.common.collect.Maps; + +import java.util.Collections; +import java.util.Map; + +public class QueryParams { + private Map params; + + public QueryParams() { + this.params = Maps.newHashMap(); + } + + public QueryParams addParam(String key, Object value) { + this.params.put(key, value); + return this; + } + + public Map getParams() { + return Collections.unmodifiableMap(this.params); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/RangeParam.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/RangeParam.java new file mode 100644 index 000000000000..7a9a1a94c468 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/RangeParam.java @@ -0,0 +1,50 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.rel; + +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.util.Pair; + +public class RangeParam extends Pair { + public RangeParam(RexNode offset, RexNode fetch) { + super(offset, fetch); + } + + @Override + public String toString() { + RexNode offset = left; + RexNode fetch = right; + // default value of offset is 0 + String offsetStr = "0"; + if (offset instanceof RexLiteral) { + offsetStr = String.valueOf(((Number) ((RexLiteral) offset).getValue()).intValue()); + } else if (offset != null) { + offsetStr = offset.toString(); + } + // default value of fetch is -1, which mean no upper bound + String fetchStr = "-1"; + if (fetch instanceof RexLiteral) { + fetchStr = String.valueOf(((Number) ((RexLiteral) fetch).getValue()).intValue()); + } else if (fetch != null) { + fetchStr = fetch.toString(); + } + return "<" + offsetStr + ", " + fetchStr + ">"; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java index d203a8122790..d3ced787374a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java @@ -22,6 +22,7 @@ import com.alibaba.graphscope.common.ir.rel.type.TableConfig; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; +import com.google.common.collect.ImmutableList; import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.plan.RelTraitSet; @@ -30,6 +31,7 @@ import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexNode; import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; @@ -77,6 +79,27 @@ public static GraphLogicalExpand create( cluster, hints, input, opt, tableConfig, alias, startAlias, optional); } + public static GraphLogicalExpand create( + GraphOptCluster cluster, + List hints, + RelNode input, + GraphOpt.Expand opt, + TableConfig tableConfig, + @Nullable String alias, + AliasNameWithId startAlias, + boolean optional, + ImmutableList filters, + GraphSchemaType schemaType) { + GraphLogicalExpand expand = + GraphLogicalExpand.create( + cluster, hints, input, opt, tableConfig, alias, startAlias, optional); + if (ObjectUtils.isNotEmpty(filters)) { + expand.setFilters(filters); + } + expand.setSchemaType(schemaType); + return expand; + } + public GraphOpt.Expand getOpt() { return this.opt; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java index 7bb48bfe2d9c..52d6035a9d9d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java @@ -21,6 +21,7 @@ import com.alibaba.graphscope.common.ir.rel.type.AliasNameWithId; import com.alibaba.graphscope.common.ir.rel.type.TableConfig; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; +import com.google.common.collect.ImmutableList; import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.plan.RelTraitSet; @@ -29,6 +30,7 @@ import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexNode; import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; @@ -60,6 +62,23 @@ public static GraphLogicalGetV create( return new GraphLogicalGetV(cluster, hints, input, opt, tableConfig, alias, startAlias); } + public static GraphLogicalGetV create( + GraphOptCluster cluster, + List hints, + RelNode input, + GraphOpt.GetV opt, + TableConfig tableConfig, + @Nullable String alias, + AliasNameWithId startAlias, + ImmutableList filters) { + GraphLogicalGetV getV = + GraphLogicalGetV.create(cluster, hints, input, opt, tableConfig, alias, startAlias); + if (ObjectUtils.isNotEmpty(filters)) { + getV.setFilters(filters); + } + return getV; + } + public GraphOpt.GetV getOpt() { return this.opt; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java index 321b4109cc2f..ecfe0f948381 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java @@ -17,8 +17,10 @@ package com.alibaba.graphscope.common.ir.rel.graph; import com.alibaba.graphscope.common.ir.rel.GraphShuttle; +import com.alibaba.graphscope.common.ir.rel.QueryParams; import com.alibaba.graphscope.common.ir.rel.type.TableConfig; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; +import com.google.common.collect.ImmutableList; import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.rel.RelNode; @@ -26,6 +28,7 @@ import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rex.RexNode; +import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; @@ -34,15 +37,18 @@ public class GraphLogicalSource extends AbstractBindableTableScan { private final GraphOpt.Source opt; private @Nullable RexNode uniqueKeyFilters; + private final QueryParams params; protected GraphLogicalSource( GraphOptCluster cluster, List hints, GraphOpt.Source opt, TableConfig tableConfig, - @Nullable String alias) { + @Nullable String alias, + QueryParams params) { super(cluster, hints, tableConfig, alias); this.opt = opt; + this.params = Objects.requireNonNull(params); } public static GraphLogicalSource create( @@ -51,7 +57,27 @@ public static GraphLogicalSource create( GraphOpt.Source opt, TableConfig tableConfig, @Nullable String alias) { - return new GraphLogicalSource(cluster, hints, opt, tableConfig, alias); + return new GraphLogicalSource(cluster, hints, opt, tableConfig, alias, new QueryParams()); + } + + public static GraphLogicalSource create( + GraphOptCluster cluster, + List hints, + GraphOpt.Source opt, + TableConfig tableConfig, + @Nullable String alias, + QueryParams params, + RexNode uniqueKeyFilters, + ImmutableList filters) { + GraphLogicalSource source = + new GraphLogicalSource(cluster, hints, opt, tableConfig, alias, params); + if (uniqueKeyFilters != null) { + source.setUniqueKeyFilters(uniqueKeyFilters); + } + if (ObjectUtils.isNotEmpty(filters)) { + source.setFilters(filters); + } + return source; } public GraphOpt.Source getOpt() { @@ -62,7 +88,8 @@ public GraphOpt.Source getOpt() { public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw) .item("opt", getOpt()) - .itemIf("uniqueKeyFilters", uniqueKeyFilters, uniqueKeyFilters != null); + .itemIf("uniqueKeyFilters", uniqueKeyFilters, uniqueKeyFilters != null) + .itemIf("params", params.getParams(), !params.getParams().isEmpty()); } @Override @@ -80,4 +107,8 @@ public void setUniqueKeyFilters(RexNode uniqueKeyFilters) { public @Nullable RexNode getUniqueKeyFilters() { return uniqueKeyFilters; } + + public QueryParams getParams() { + return params; + } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/type/order/GraphRelCollations.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/type/order/GraphRelCollations.java index 85584ecab228..30a5c3a57616 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/type/order/GraphRelCollations.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/type/order/GraphRelCollations.java @@ -24,6 +24,7 @@ import org.apache.calcite.plan.RelTrait; import org.apache.calcite.plan.RelTraitDef; import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Util; @@ -62,7 +63,7 @@ public GraphRelCollationImpl(ImmutableList fieldCollations) { @Override public RelTraitDef getTraitDef() { - return null; + return RelCollationTraitDef.INSTANCE; } @Override diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java index e1e4524c2cc3..c523f22f902a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java @@ -56,11 +56,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; -import java.util.HashMap; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; public class GraphRelToProtoConverter extends GraphShuttle { @@ -1158,6 +1154,20 @@ private GraphAlgebra.QueryParams.Builder buildQueryParams(AbstractBindableTableS com.alibaba.graphscope.common.ir.tools.Utils.getGraphLabels(tableScan.getRowType()) .getLabelsEntry()); addQueryFilters(paramsBuilder, tableScan.getFilters()); + if (tableScan instanceof GraphLogicalSource) { + GraphLogicalSource source = (GraphLogicalSource) tableScan; + source.getParams() + .getParams() + .forEach( + (k, v) -> { + if (k.equalsIgnoreCase("range")) { + RangeParam rangeParam = (RangeParam) v; + GraphAlgebra.Range range = + buildRange(rangeParam.left, rangeParam.right); + paramsBuilder.setLimit(range); + } + }); + } return paramsBuilder; } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java new file mode 100644 index 000000000000..4997da466e11 --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java @@ -0,0 +1,102 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.planner.rbo; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.Utils; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; +import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; +import com.alibaba.graphscope.common.ir.tools.GraphBuilder; +import com.google.common.collect.ImmutableMap; + +import org.apache.calcite.rel.RelNode; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ScanEarlyStopTest { + private static Configs configs; + private static IrMeta irMeta; + private static GraphRelOptimizer optimizer; + + @BeforeClass + public static void beforeClass() { + configs = + new Configs( + ImmutableMap.of( + "graph.planner.is.on", + "true", + "graph.planner.opt", + "CBO", + "graph.planner.rules", + "FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule," + + " ScanExpandFusionRule, ExpandGetVFusionRule," + + " TopKPushDownRule, ScanEarlyStopRule")); + optimizer = new GraphRelOptimizer(configs); + irMeta = + Utils.mockIrMeta( + "schema/ldbc_schema_exp_hierarchy.json", + "statistics/ldbc30_hierarchy_statistics.json", + optimizer.getGlogueHolder()); + } + + @AfterClass + public static void afterClass() { + if (optimizer != null) { + optimizer.close(); + } + } + + @Test + public void scan_early_stop_0_test() { + GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); + RelNode before = + com.alibaba.graphscope.cypher.antlr4.Utils.eval( + "Match (n:PERSON) Where n.firstName = 'Li' Return n Limit 10", + builder) + .build(); + RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); + Assert.assertEquals( + "GraphLogicalProject(n=[n], isAppend=[false])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " alias=[n], fusedFilter=[[=(_.firstName, _UTF-8'Li')]], opt=[VERTEX]," + + " params=[{range=<0, 10>}])", + after.explain().trim()); + } + + @Test + public void scan_early_stop_1_test() { + GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); + RelNode before = + com.alibaba.graphscope.cypher.antlr4.Utils.eval( + "Match (:PERSON)-[n:KNOWS]->(b) Where n.creationDate > $date Return" + + " n Limit 10", + builder) + .build(); + RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); + Assert.assertEquals( + "GraphLogicalProject(n=[n], isAppend=[false])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[KNOWS]}], alias=[n]," + + " fusedFilter=[[>(_.creationDate, ?0)]], opt=[EDGE], params=[{range=<0," + + " 10>}])", + after.explain().trim()); + } +} From 1c60eb2e67fc5e4543cfbcea2034d2eaeef09193 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Tue, 14 Jan 2025 20:44:31 +0800 Subject: [PATCH 02/12] add comments --- .../common/ir/planner/rules/ScanEarlyStopRule.java | 3 ++- .../ir/planner/rules/ScanExpandFusionRule.java | 14 +++++++++++++- .../common/ir/planner/rules/TopKPushDownRule.java | 9 +++++++++ .../common/ir/planner/rbo/ScanEarlyStopTest.java | 8 ++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java index 50970bd65e48..8e5769e37ec3 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java @@ -30,7 +30,8 @@ import org.checkerframework.checker.nullness.qual.Nullable; /** - * fuse limit to source to stop scan at an early stage + * Pushes the limit operation down to the scan node. + * During the scan process, the scan stops as soon as the specified limit count is reached. */ public class ScanEarlyStopRule extends RelRule { protected ScanEarlyStopRule(RelRule.Config config) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java index 59164b13fdcc..bdd853152816 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java @@ -43,7 +43,19 @@ import java.util.stream.Collectors; /** - * transform scan(V) + expand(E) + getV + project(E) to scan(E) + project(E) + * This rule transforms edge expansion into edge scan wherever possible. + * For example, consider the following Cypher query: + * Match (a:PERSON)-[b:KNOWS]->(c:PERSON) Return b.name; + * + * Although the query involves Scan and GetV steps, their results are not directly utilized by subsequent + * project operations. The only effectively used data is the edge data produced by the Expand operation. + * In such cases, we can perform a fusion operation, transforming the pattern + * (a:PERSON)-[b:KNOWS]->(c:PERSON) into a scan operation on the KNOWS edge. + * + * It is important to note that whether fusion is feasible also depends on the label dependencies between + * nodes and edges. If the edge label is determined strictly by the triplet (src_label, edge_label, dst_label), + * fusion cannot be performed. For reference, consider the following query: + * Match (a:PERSON)-[b:LIKES]->(c:COMMENT) Return b.name; */ public class ScanExpandFusionRule extends RelRule { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java index 04460b06c1f4..bf1b318f9c23 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java @@ -26,6 +26,15 @@ import org.apache.calcite.tools.RelBuilderFactory; import org.checkerframework.checker.nullness.qual.Nullable; +/** + * This rule pushes down topK operations to the project node and is based on Calcite's {@code SortProjectTransposeRule}, + * leveraging the original rule wherever possible. + * However, in our more complex distributed scenario, deferring the execution of the project node + * can disrupt already sorted data. + * To address this, we modified the matching logic in {@code SortProjectTransposeRule}. + * Currently, the PushDown operation is applied only when the sort fields are empty, + * which means only the limit is pushed down to the project node. + */ public class TopKPushDownRule extends SortProjectTransposeRule { protected TopKPushDownRule(Config config) { super(config); diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java index 4997da466e11..096b0964a784 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java @@ -65,6 +65,9 @@ public static void afterClass() { } } + // the rules are applied by 2 steps: + // 1. TopKPushDownRule: push down the limit operation to the project node + // 2. ScanEarlyStopRule: fuse the limit operation to the source node @Test public void scan_early_stop_0_test() { GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); @@ -82,6 +85,11 @@ public void scan_early_stop_0_test() { after.explain().trim()); } + // the rules are applied by 3 steps: + // 1. ScanFusionRule: transform the edge expansion to edge scan, i.e. (:PERSON)-[n:KNOWS]->(b) + // => ScanE(KNOWS) + // 2. TopKPushDownRule: push down the limit operation to the project node + // 2. ScanEarlyStopRule: fuse the limit operation to the source node @Test public void scan_early_stop_1_test() { GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); From c55ec36f31445bb51775fb4c687105438b6aa9ea Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 15 Jan 2025 10:56:18 +0800 Subject: [PATCH 03/12] fix copyrights --- .../ir/planner/rules/ScanEarlyStopRule.java | 24 +++++++++---------- .../planner/rules/ScanExpandFusionRule.java | 24 +++++++++---------- .../ir/planner/rules/TopKPushDownRule.java | 24 +++++++++---------- .../graphscope/common/ir/rel/QueryParams.java | 24 +++++++++---------- .../graphscope/common/ir/rel/RangeParam.java | 24 +++++++++---------- .../proto/GraphRelToProtoConverter.java | 6 ++++- .../ir/planner/rbo/ScanEarlyStopTest.java | 24 +++++++++---------- 7 files changed, 71 insertions(+), 79 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java index 8e5769e37ec3..6a08d2088f5a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanEarlyStopRule.java @@ -1,19 +1,17 @@ /* + * Copyright 2020 Alibaba Group Holding Limited. * - * * Copyright 2020 Alibaba Group Holding Limited. - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.alibaba.graphscope.common.ir.planner.rules; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java index bdd853152816..c9a7d6dfaa43 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/ScanExpandFusionRule.java @@ -1,19 +1,17 @@ /* + * Copyright 2020 Alibaba Group Holding Limited. * - * * Copyright 2020 Alibaba Group Holding Limited. - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.alibaba.graphscope.common.ir.planner.rules; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java index bf1b318f9c23..7e3fb36fc5ab 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/TopKPushDownRule.java @@ -1,19 +1,17 @@ /* + * Copyright 2020 Alibaba Group Holding Limited. * - * * Copyright 2020 Alibaba Group Holding Limited. - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.alibaba.graphscope.common.ir.planner.rules; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/QueryParams.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/QueryParams.java index d5576d79fe5b..0ced1eae5285 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/QueryParams.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/QueryParams.java @@ -1,19 +1,17 @@ /* + * Copyright 2020 Alibaba Group Holding Limited. * - * * Copyright 2020 Alibaba Group Holding Limited. - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.alibaba.graphscope.common.ir.rel; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/RangeParam.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/RangeParam.java index 7a9a1a94c468..bab664c07e12 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/RangeParam.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/RangeParam.java @@ -1,19 +1,17 @@ /* + * Copyright 2020 Alibaba Group Holding Limited. * - * * Copyright 2020 Alibaba Group Holding Limited. - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.alibaba.graphscope.common.ir.rel; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java index c523f22f902a..86672df3bf75 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java @@ -56,7 +56,11 @@ import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; -import java.util.*; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; public class GraphRelToProtoConverter extends GraphShuttle { diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java index 096b0964a784..c5b0cb7ed558 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java @@ -1,19 +1,17 @@ /* + * Copyright 2020 Alibaba Group Holding Limited. * - * * Copyright 2020 Alibaba Group Holding Limited. - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.alibaba.graphscope.common.ir.planner.rbo; From 7650e4228e58e097fb21661232af86c572eca900 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 17 Jan 2025 17:14:50 +0800 Subject: [PATCH 04/12] fix bugs of type conversion --- .../common/ir/meta/schema/IrDataTypeConvertor.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java index 9cd921e6c975..ecb1ef8d193a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java @@ -135,10 +135,12 @@ public DataType convert(RelDataType dataFrom) { if (dataFrom.getPrecision() == 1) { return DataType.CHAR; } + break; case VARCHAR: if (dataFrom.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED) { return DataType.STRING; } + break; case SMALLINT: return DataType.SHORT; case INTEGER: @@ -158,7 +160,7 @@ public DataType convert(RelDataType dataFrom) { case MULTISET: case ARRAY: RelDataType componentType = dataFrom.getComponentType(); - // check the array or set is a unlimited size list of primitive type + // check the array or set is an unlimited size list of primitive type if (componentType != null && dataFrom.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED) { switch (componentType.getSqlTypeName()) { @@ -177,10 +179,11 @@ public DataType convert(RelDataType dataFrom) { } } } + break; case UNKNOWN: default: - return DataType.UNKNOWN; } + return DataType.UNKNOWN; } } From 971025a87bd432ea6a1879b31f83c78407451d4d Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 17 Jan 2025 17:38:24 +0800 Subject: [PATCH 05/12] add docs --- docs/interactive_engine/gopt.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/interactive_engine/gopt.md b/docs/interactive_engine/gopt.md index 391b76559f18..16e06a4c2c69 100644 --- a/docs/interactive_engine/gopt.md +++ b/docs/interactive_engine/gopt.md @@ -350,3 +350,19 @@ Design of GOpt ### Detailed Introduction A comprehensive introduction to GOpt will be provided in subsequent sections. Please stay tuned for detailed updates and information. + +#### Rules + +**ScanEarlyStopRule**: Pushes the limit operation down to the scan node. During the scan process, the scan stops as soon as the specified limit count is reached. + +**ScanExpandFusionRule**: This rule transforms edge expansion into edge scan wherever possible. For example, consider the following Cypher query: +```cypher +Match (a:PERSON)-[b:KNOWS]->(c:PERSON) Return b.name; +``` +Although the query involves Scan and GetV steps, their results are not directly utilized by subsequent project operations. The only effectively used data is the edge data produced by the Expand operation. In such cases, we can perform a fusion operation, transforming the pattern +`(a:PERSON)-[b:KNOWS]->(c:PERSON)` into a scan operation on the KNOWS edge. It is important to note that whether fusion is feasible also depends on the label dependencies between nodes and edges. If the edge label is determined strictly by the triplet (src_label, edge_label, dst_label), fusion cannot be performed. For example, consider the following query: +```cypher +Match (a:PERSON)-[b:LIKES]->(c:COMMENT) Return b.name; +``` + +**TopKPushDownRule**: This rule pushes down topK operations to the project node and is based on Calcite's [SortProjectTransposeRule](https://calcite.apache.org/javadocAggregate/org/apache/calcite/rel/rules/SortProjectTransposeRule.html), leveraging the original rule wherever possible. However, in our more complex distributed scenario, deferring the execution of the project node can disrupt already sorted data. To address this, we modified the matching logic in `SortProjectTransposeRule`. Currently, the PushDown operation is applied only when the sort fields are empty, which means only the limit is pushed down to the project node. \ No newline at end of file From 95f34ab0aa80d533cade698da349014328db7cbb Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 17 Jan 2025 17:43:41 +0800 Subject: [PATCH 06/12] refine docs --- docs/interactive_engine/gopt.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/interactive_engine/gopt.md b/docs/interactive_engine/gopt.md index 16e06a4c2c69..ef066bad9cd1 100644 --- a/docs/interactive_engine/gopt.md +++ b/docs/interactive_engine/gopt.md @@ -349,9 +349,10 @@ Design of GOpt ::: ### Detailed Introduction -A comprehensive introduction to GOpt will be provided in subsequent sections. Please stay tuned for detailed updates and information. -#### Rules +#### Rules + +Rules play a pivotal role in GOpt’s optimization framework, enabling efficient and effective query transformations. Below is an outline of some key rules implemented in GOpt: **ScanEarlyStopRule**: Pushes the limit operation down to the scan node. During the scan process, the scan stops as soon as the specified limit count is reached. @@ -365,4 +366,4 @@ Although the query involves Scan and GetV steps, their results are not directly Match (a:PERSON)-[b:LIKES]->(c:COMMENT) Return b.name; ``` -**TopKPushDownRule**: This rule pushes down topK operations to the project node and is based on Calcite's [SortProjectTransposeRule](https://calcite.apache.org/javadocAggregate/org/apache/calcite/rel/rules/SortProjectTransposeRule.html), leveraging the original rule wherever possible. However, in our more complex distributed scenario, deferring the execution of the project node can disrupt already sorted data. To address this, we modified the matching logic in `SortProjectTransposeRule`. Currently, the PushDown operation is applied only when the sort fields are empty, which means only the limit is pushed down to the project node. \ No newline at end of file +**TopKPushDownRule**: This rule pushes down topK operations to the project node and is based on Calcite's [SortProjectTransposeRule](https://calcite.apache.org/javadocAggregate/org/apache/calcite/rel/rules/SortProjectTransposeRule.html), leveraging the original rule wherever possible. However, in our more complex distributed scenario, deferring the execution of the project node can disrupt already sorted data. To address this, we modified the matching logic in `SortProjectTransposeRule`. Currently, the PushDown operation is applied only when the sort fields are empty, which means only the limit is pushed down to the project node. From 0498a1763509700071a89624d138d2abe5d5677e Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 21 Jan 2025 15:50:01 +0800 Subject: [PATCH 07/12] support scan+limit fuse in interactive Committed-by: xiaolei.zl from Dev container --- .../runtime/common/operators/retrieve/scan.cc | 60 +++++-------------- .../runtime/common/operators/retrieve/scan.h | 49 +++++++++++++-- .../runtime/execute/ops/retrieve/scan.cc | 14 ++++- .../gs_interactive/tests/test_robustness.py | 36 +++++++++++ .../hqps/interactive_config_test_cbo.yaml | 3 + 5 files changed, 109 insertions(+), 53 deletions(-) diff --git a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.cc b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.cc index dc24d1a1e34f..3ebdb24caed9 100644 --- a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.cc +++ b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.cc @@ -19,11 +19,14 @@ namespace gs { namespace runtime { Context Scan::find_vertex_with_oid(const GraphReadInterface& graph, - label_t label, const Any& oid, int alias) { + label_t label, const Any& oid, int32_t alias, + int32_t limit) { SLVertexColumnBuilder builder(label); vid_t vid; - if (graph.GetVertexIndex(label, oid, vid)) { - builder.push_back_opt(vid); + if (limit >= 1) { + if (graph.GetVertexIndex(label, oid, vid)) { + builder.push_back_opt(vid); + } } Context ctx; ctx.set(alias, builder.finish()); @@ -31,53 +34,20 @@ Context Scan::find_vertex_with_oid(const GraphReadInterface& graph, } Context Scan::find_vertex_with_gid(const GraphReadInterface& graph, - label_t label, int64_t gid, int alias) { + label_t label, int64_t gid, int32_t alias, + int32_t limit) { SLVertexColumnBuilder builder(label); - if (GlobalId::get_label_id(gid) == label) { - builder.push_back_opt(GlobalId::get_vid(gid)); - } else { - LOG(ERROR) << "Invalid label id: " - << static_cast(GlobalId::get_label_id(gid)); - } - Context ctx; - ctx.set(alias, builder.finish()); - return ctx; -} - -Context Scan::find_vertex_with_id(const GraphReadInterface& graph, - label_t label, const Any& pk, int alias, - bool scan_oid) { - if (scan_oid) { - SLVertexColumnBuilder builder(label); - vid_t vid; - if (graph.GetVertexIndex(label, pk, vid)) { - builder.push_back_opt(vid); - } - Context ctx; - ctx.set(alias, builder.finish()); - return ctx; - } else { - SLVertexColumnBuilder builder(label); - vid_t vid{}; - int64_t gid{}; - if (pk.type == PropertyType::kInt64) { - gid = pk.AsInt64(); - } else if (pk.type == PropertyType::kInt32) { - gid = pk.AsInt32(); - } else { - LOG(FATAL) << "Unsupported primary key type"; - } + if (limit >= 1) { if (GlobalId::get_label_id(gid) == label) { - vid = GlobalId::get_vid(gid); + builder.push_back_opt(GlobalId::get_vid(gid)); } else { - LOG(ERROR) << "Global id " << gid << " does not match label " << label; - return Context(); + LOG(ERROR) << "Invalid label id: " + << static_cast(GlobalId::get_label_id(gid)); } - builder.push_back_opt(vid); - Context ctx; - ctx.set(alias, builder.finish()); - return ctx; } + Context ctx; + ctx.set(alias, builder.finish()); + return ctx; } template diff --git a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h index 915b41ed9753..9a930a618050 100644 --- a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h +++ b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h @@ -26,6 +26,9 @@ namespace runtime { struct ScanParams { int alias; std::vector tables; + int32_t limit; // -1 means no limit + + ScanParams() : alias(-1), limit(std::numeric_limits::max()) {} }; class Scan { public: @@ -34,13 +37,18 @@ class Scan { const ScanParams& params, const PRED_T& predicate) { Context ctx; + int32_t cur_limit = params.limit; if (params.tables.size() == 1) { label_t label = params.tables[0]; SLVertexColumnBuilder builder(label); auto vertices = graph.GetVertexSet(label); for (auto vid : vertices) { + if (cur_limit <= 0) { + break; + } if (predicate(label, vid)) { builder.push_back_opt(vid); + cur_limit--; } } ctx.set(params.alias, builder.finish()); @@ -48,11 +56,18 @@ class Scan { MSVertexColumnBuilder builder; for (auto label : params.tables) { + if (cur_limit <= 0) { + break; + } auto vertices = graph.GetVertexSet(label); builder.start_label(label); for (auto vid : vertices) { + if (cur_limit <= 0) { + break; + } if (predicate(label, vid)) { builder.push_back_opt(vid); + cur_limit--; } } } @@ -70,13 +85,18 @@ class Scan { const ScanParams& params, const PRED_T& predicate, const std::vector& gids) { Context ctx; + int32_t cur_limit = params.limit; if (params.tables.size() == 1) { label_t label = params.tables[0]; SLVertexColumnBuilder builder(label); for (auto gid : gids) { + if (cur_limit <= 0) { + break; + } vid_t vid = GlobalId::get_vid(gid); if (GlobalId::get_label_id(gid) == label && predicate(label, vid)) { builder.push_back_opt(vid); + cur_limit--; } } ctx.set(params.alias, builder.finish()); @@ -84,10 +104,17 @@ class Scan { MLVertexColumnBuilder builder; for (auto label : params.tables) { + if (cur_limit <= 0) { + break; + } for (auto gid : gids) { + if (cur_limit <= 0) { + break; + } vid_t vid = GlobalId::get_vid(gid); if (GlobalId::get_label_id(gid) == label && predicate(label, vid)) { builder.push_back_vertex({label, vid}); + cur_limit--; } } } @@ -105,14 +132,19 @@ class Scan { const ScanParams& params, const PRED_T& predicate, const std::vector& oids) { Context ctx; + auto limit = params.limit; if (params.tables.size() == 1) { label_t label = params.tables[0]; SLVertexColumnBuilder builder(label); for (auto oid : oids) { + if (limit <= 0) { + break; + } vid_t vid; if (graph.GetVertexIndex(label, oid, vid)) { if (predicate(label, vid)) { builder.push_back_opt(vid); + --limit; } } } @@ -121,11 +153,18 @@ class Scan { std::vector> vids; for (auto label : params.tables) { + if (limit <= 0) { + break; + } for (auto oid : oids) { + if (limit <= 0) { + break; + } vid_t vid; if (graph.GetVertexIndex(label, oid, vid)) { if (predicate(label, vid)) { vids.emplace_back(label, vid); + --limit; } } } @@ -149,15 +188,13 @@ class Scan { const GraphReadInterface& graph, const ScanParams& params, const SPVertexPredicate& predicate, const std::vector& oids); - static Context find_vertex_with_id(const GraphReadInterface& graph, - label_t label, const Any& pk, int alias, - bool scan_oid); - static Context find_vertex_with_oid(const GraphReadInterface& graph, - label_t label, const Any& pk, int alias); + label_t label, const Any& pk, + int32_t alias, int32_t limit); static Context find_vertex_with_gid(const GraphReadInterface& graph, - label_t label, int64_t pk, int alias); + label_t label, int64_t pk, int32_t alias, + int32_t limit); }; } // namespace runtime diff --git a/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc b/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc index 4ff11ed2751e..92c4243bd400 100644 --- a/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc +++ b/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc @@ -176,7 +176,7 @@ class FilterOidsWithoutPredOpr : public IReadOperator { std::vector oids = oids_(params); if (params_.tables.size() == 1 && oids.size() == 1) { return Scan::find_vertex_with_oid(graph, params_.tables[0], oids[0], - params_.alias); + params_.alias, params_.limit); } return Scan::filter_oids( graph, params_, [](label_t, vid_t) { return true; }, oids); @@ -230,7 +230,7 @@ class FilterGidsWithoutPredOpr : public IReadOperator { } if (params_.tables.size() == 1 && gids.size() == 1) { return Scan::find_vertex_with_gid(graph, params_.tables[0], gids[0], - params_.alias); + params_.alias, params_.limit); } return Scan::filter_gids( graph, params_, [](label_t, vid_t) { return true; }, gids); @@ -566,6 +566,16 @@ std::pair, ContextMeta> ScanOprBuilder::Build( ScanParams scan_params; scan_params.alias = scan_opr.has_alias() ? scan_opr.alias().value() : -1; + if (scan_opr.params().has_limit()) { + auto& limit_range = scan_opr.params().limit(); + if (limit_range.lower() != 0) { + LOG(FATAL) << "Scan with lower limit expect 0, but got " + << limit_range.lower(); + } + scan_params.limit = limit_range.upper(); + } else { + scan_params.limit = std::numeric_limits::max(); + } for (auto& table : scan_opr.params().tables()) { // bug here, exclude invalid vertex label id if (schema.vertex_label_num() <= table.id()) { diff --git a/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py b/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py index 95e1c9db1807..3a21997c20e1 100644 --- a/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py +++ b/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py @@ -308,6 +308,42 @@ def test_call_proc_in_cypher(interactive_session, neo4j_session, create_modern_g for record in result: cnt += 1 assert cnt == 8 + +@pytest.mark.skipif( + os.environ.get("RUN_ON_PROTO", None) != "ON", reason="Scan+Limit fuse only works on proto" +) +def test_call_proc_in_cypher(interactive_session, neo4j_session, create_modern_graph): + print("[Test call procedure in cypher]") + import_data_to_full_modern_graph(interactive_session, create_modern_graph) + start_service_on_graph(interactive_session, create_modern_graph) + ensure_compiler_schema_ready( + interactive_session, neo4j_session, create_modern_graph + ) + result = neo4j_session.run( + 'MATCH(p: person) with p.id as oid CALL k_neighbors("person", oid, 1) return label_name, vertex_oid;' + ) + cnt = 0 + for record in result: + cnt += 1 + assert cnt == 8 + + # Q: Why we could use this query to verify whether Scan+Limit fuse works? + # A: If Scan+Limit fuse works, the result of this query should be 2, otherwise it should be 6 + result = neo4j_session.run( + 'MATCH(n) return n.id limit 2' + ) + cnt = 0 + for record in result: + cnt += 1 + assert cnt == 2 + + result = neo4j_session.run( + 'MATCH(n) return n.id limit 0' + ) + cnt = 0 + for record in result: + cnt += 1 + assert cnt == 0 def test_custom_pk_name( diff --git a/flex/tests/hqps/interactive_config_test_cbo.yaml b/flex/tests/hqps/interactive_config_test_cbo.yaml index fcb285cd5388..8803a0f9e751 100644 --- a/flex/tests/hqps/interactive_config_test_cbo.yaml +++ b/flex/tests/hqps/interactive_config_test_cbo.yaml @@ -20,6 +20,9 @@ compiler: - NotMatchToAntiJoinRule - ExtendIntersectRule - ExpandGetVFusionRule + - ScanExpandFusionRule + - TopKPushDownRule + - ScanEarlyStopRule # This rule must be placed after TopKPushDownRule and ScanExpandFusionRule meta: reader: schema: From 374cae1cd780f23bea2139d1f4ded32a4e320278 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Tue, 21 Jan 2025 16:49:19 +0800 Subject: [PATCH 08/12] format Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- .../gs_interactive/tests/test_robustness.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py b/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py index 3a21997c20e1..8fb70a4abf95 100644 --- a/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py +++ b/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py @@ -308,11 +308,13 @@ def test_call_proc_in_cypher(interactive_session, neo4j_session, create_modern_g for record in result: cnt += 1 assert cnt == 8 - + + @pytest.mark.skipif( - os.environ.get("RUN_ON_PROTO", None) != "ON", reason="Scan+Limit fuse only works on proto" + os.environ.get("RUN_ON_PROTO", None) != "ON", + reason="Scan+Limit fuse only works on proto", ) -def test_call_proc_in_cypher(interactive_session, neo4j_session, create_modern_graph): +def test_scan_limit_fuse(interactive_session, neo4j_session, create_modern_graph): print("[Test call procedure in cypher]") import_data_to_full_modern_graph(interactive_session, create_modern_graph) start_service_on_graph(interactive_session, create_modern_graph) @@ -326,20 +328,16 @@ def test_call_proc_in_cypher(interactive_session, neo4j_session, create_modern_g for record in result: cnt += 1 assert cnt == 8 - + # Q: Why we could use this query to verify whether Scan+Limit fuse works? # A: If Scan+Limit fuse works, the result of this query should be 2, otherwise it should be 6 - result = neo4j_session.run( - 'MATCH(n) return n.id limit 2' - ) + result = neo4j_session.run("MATCH(n) return n.id limit 2") cnt = 0 for record in result: cnt += 1 assert cnt == 2 - - result = neo4j_session.run( - 'MATCH(n) return n.id limit 0' - ) + + result = neo4j_session.run("MATCH(n) return n.id limit 0") cnt = 0 for record in result: cnt += 1 From 86fe5020f3b3522cdd6c6f35e6214f58b43ed83a Mon Sep 17 00:00:00 2001 From: shirly121 Date: Tue, 21 Jan 2025 20:27:09 +0800 Subject: [PATCH 09/12] fix bugs of empty results --- .../cypher/executor/GraphQueryExecutor.java | 24 ++++++++++++++----- .../ir/planner/rbo/ScanEarlyStopTest.java | 15 ++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/executor/GraphQueryExecutor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/executor/GraphQueryExecutor.java index 8db5c94c5109..7fad028a8603 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/executor/GraphQueryExecutor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/executor/GraphQueryExecutor.java @@ -145,15 +145,27 @@ public StatementResult run( statusCallback .getQueryLogger() .info("logical IR plan \n\n {} \n\n", planSummary.getLogicalPlan().explain()); - statusCallback - .getQueryLogger() - .debug("physical IR plan {}", planSummary.getPhysicalPlan().explain()); - if (planSummary.getLogicalPlan().isReturnEmpty()) { - return StatementResults.initial(); + boolean returnEmpty = planSummary.getLogicalPlan().isReturnEmpty(); + if (!returnEmpty) { + statusCallback + .getQueryLogger() + .debug("physical IR plan {}", planSummary.getPhysicalPlan().explain()); } QueryTimeoutConfig timeoutConfig = getQueryTimeoutConfig(); GraphPlanExecutor executor; - if (cacheValue.result != null && cacheValue.result.isCompleted) { + if (returnEmpty) { + executor = + new GraphPlanExecutor() { + @Override + public void execute( + GraphPlanner.Summary summary, + IrMeta irMeta, + ExecutionResponseListener listener) + throws Exception { + listener.onCompleted(); + } + }; + } else if (cacheValue.result != null && cacheValue.result.isCompleted) { executor = new GraphPlanExecutor() { @Override diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java index c5b0cb7ed558..6a27d4583276 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/ScanEarlyStopTest.java @@ -22,6 +22,7 @@ import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; import com.google.common.collect.ImmutableMap; import org.apache.calcite.rel.RelNode; @@ -105,4 +106,18 @@ public void scan_early_stop_1_test() { + " 10>}])", after.explain().trim()); } + + @Test + public void scan_early_stop_2_test() { + GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); + RelNode before = + com.alibaba.graphscope.cypher.antlr4.Utils.eval( + "Match (:PERSON)-[n:KNOWS]->(b) Where n.creationDate > $date Return" + + " n Limit 0", + builder) + .build(); + RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); + LogicalPlan plan = new LogicalPlan(after); + Assert.assertTrue(plan.isReturnEmpty()); + } } From 7b9842bab17921d3611add965057610e5b92b6c3 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Wed, 22 Jan 2025 12:05:06 +0800 Subject: [PATCH 10/12] fix Committed-by: xiaolei.zl@alibaba-inc.com from Dev container Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- .../runtime/common/operators/retrieve/scan.cc | 25 ++++++++----------- .../runtime/common/operators/retrieve/scan.h | 5 ++-- .../runtime/execute/ops/retrieve/scan.cc | 11 ++++---- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.cc b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.cc index 3ebdb24caed9..158308472e22 100644 --- a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.cc +++ b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.cc @@ -19,14 +19,12 @@ namespace gs { namespace runtime { Context Scan::find_vertex_with_oid(const GraphReadInterface& graph, - label_t label, const Any& oid, int32_t alias, - int32_t limit) { + label_t label, const Any& oid, + int32_t alias) { SLVertexColumnBuilder builder(label); vid_t vid; - if (limit >= 1) { - if (graph.GetVertexIndex(label, oid, vid)) { - builder.push_back_opt(vid); - } + if (graph.GetVertexIndex(label, oid, vid)) { + builder.push_back_opt(vid); } Context ctx; ctx.set(alias, builder.finish()); @@ -34,16 +32,13 @@ Context Scan::find_vertex_with_oid(const GraphReadInterface& graph, } Context Scan::find_vertex_with_gid(const GraphReadInterface& graph, - label_t label, int64_t gid, int32_t alias, - int32_t limit) { + label_t label, int64_t gid, int32_t alias) { SLVertexColumnBuilder builder(label); - if (limit >= 1) { - if (GlobalId::get_label_id(gid) == label) { - builder.push_back_opt(GlobalId::get_vid(gid)); - } else { - LOG(ERROR) << "Invalid label id: " - << static_cast(GlobalId::get_label_id(gid)); - } + if (GlobalId::get_label_id(gid) == label) { + builder.push_back_opt(GlobalId::get_vid(gid)); + } else { + LOG(ERROR) << "Invalid label id: " + << static_cast(GlobalId::get_label_id(gid)); } Context ctx; ctx.set(alias, builder.finish()); diff --git a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h index 9a930a618050..b35811c76afc 100644 --- a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h +++ b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h @@ -190,11 +190,10 @@ class Scan { static Context find_vertex_with_oid(const GraphReadInterface& graph, label_t label, const Any& pk, - int32_t alias, int32_t limit); + int32_t alias); static Context find_vertex_with_gid(const GraphReadInterface& graph, - label_t label, int64_t pk, int32_t alias, - int32_t limit); + label_t label, int64_t pk, int32_t alias); }; } // namespace runtime diff --git a/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc b/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc index 92c4243bd400..b055ff224f5a 100644 --- a/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc +++ b/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc @@ -176,7 +176,7 @@ class FilterOidsWithoutPredOpr : public IReadOperator { std::vector oids = oids_(params); if (params_.tables.size() == 1 && oids.size() == 1) { return Scan::find_vertex_with_oid(graph, params_.tables[0], oids[0], - params_.alias, params_.limit); + params_.alias); } return Scan::filter_oids( graph, params_, [](label_t, vid_t) { return true; }, oids); @@ -230,7 +230,7 @@ class FilterGidsWithoutPredOpr : public IReadOperator { } if (params_.tables.size() == 1 && gids.size() == 1) { return Scan::find_vertex_with_gid(graph, params_.tables[0], gids[0], - params_.alias, params_.limit); + params_.alias); } return Scan::filter_gids( graph, params_, [](label_t, vid_t) { return true; }, gids); @@ -566,15 +566,16 @@ std::pair, ContextMeta> ScanOprBuilder::Build( ScanParams scan_params; scan_params.alias = scan_opr.has_alias() ? scan_opr.alias().value() : -1; + scan_params.limit = std::numeric_limits::max(); if (scan_opr.params().has_limit()) { auto& limit_range = scan_opr.params().limit(); if (limit_range.lower() != 0) { LOG(FATAL) << "Scan with lower limit expect 0, but got " << limit_range.lower(); } - scan_params.limit = limit_range.upper(); - } else { - scan_params.limit = std::numeric_limits::max(); + if (limit_range.upper() > 0) { + scan_params.limit = limit_range.upper(); + } } for (auto& table : scan_opr.params().tables()) { // bug here, exclude invalid vertex label id From 88a48b1a7f9c8612406654a2c3db527af3c5b672 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Wed, 22 Jan 2025 12:07:12 +0800 Subject: [PATCH 11/12] fix Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- flex/engines/graph_db/runtime/common/operators/retrieve/scan.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h index b35811c76afc..74db5e6ad442 100644 --- a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h +++ b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h @@ -26,7 +26,7 @@ namespace runtime { struct ScanParams { int alias; std::vector tables; - int32_t limit; // -1 means no limit + int32_t limit; ScanParams() : alias(-1), limit(std::numeric_limits::max()) {} }; From d98c73a174ff29a782c5689e4b5acc0071321f4e Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Wed, 22 Jan 2025 13:41:44 +0800 Subject: [PATCH 12/12] add scan_with_limit --- .../runtime/common/operators/retrieve/scan.h | 32 ++++++++++++++ .../runtime/execute/ops/retrieve/scan.cc | 43 +++++++++++++------ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h index 74db5e6ad442..448f94f376bf 100644 --- a/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h +++ b/flex/engines/graph_db/runtime/common/operators/retrieve/scan.h @@ -37,6 +37,38 @@ class Scan { const ScanParams& params, const PRED_T& predicate) { Context ctx; + if (params.tables.size() == 1) { + label_t label = params.tables[0]; + SLVertexColumnBuilder builder(label); + auto vertices = graph.GetVertexSet(label); + for (auto vid : vertices) { + if (predicate(label, vid)) { + builder.push_back_opt(vid); + } + } + ctx.set(params.alias, builder.finish()); + } else if (params.tables.size() > 1) { + MSVertexColumnBuilder builder; + + for (auto label : params.tables) { + auto vertices = graph.GetVertexSet(label); + builder.start_label(label); + for (auto vid : vertices) { + if (predicate(label, vid)) { + builder.push_back_opt(vid); + } + } + } + ctx.set(params.alias, builder.finish()); + } + return ctx; + } + + template + static Context scan_vertex_with_limit(const GraphReadInterface& graph, + const ScanParams& params, + const PRED_T& predicate) { + Context ctx; int32_t cur_limit = params.limit; if (params.tables.size() == 1) { label_t label = params.tables[0]; diff --git a/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc b/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc index b055ff224f5a..d4a4bcd653fd 100644 --- a/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc +++ b/flex/engines/graph_db/runtime/execute/ops/retrieve/scan.cc @@ -495,17 +495,31 @@ class ScanWithGPredOpr : public IReadOperator { auto expr = parse_expression(graph, tmp, params, pred_, VarType::kVertexVar); if (expr->is_optional()) { - auto ret = Scan::scan_vertex( - graph, scan_params_, [&expr](label_t label, vid_t vid) { - return expr->eval_vertex(label, vid, 0, 0).as_bool(); - }); - return ret; + if (scan_params_.limit == std::numeric_limits::max()) { + return Scan::scan_vertex( + graph, scan_params_, [&expr](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0, 0).as_bool(); + }); + } else { + return Scan::scan_vertex_with_limit( + graph, scan_params_, [&expr](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0, 0).as_bool(); + }); + } } else { - auto ret = Scan::scan_vertex( - graph, scan_params_, [&expr](label_t label, vid_t vid) { - return expr->eval_vertex(label, vid, 0).as_bool(); - }); - return ret; + if (scan_params_.limit == std::numeric_limits::max()) { + auto ret = Scan::scan_vertex( + graph, scan_params_, [&expr](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0).as_bool(); + }); + return ret; + } else { + auto ret = Scan::scan_vertex_with_limit( + graph, scan_params_, [&expr](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0).as_bool(); + }); + return ret; + } } } @@ -523,8 +537,13 @@ class ScanWithoutPredOpr : public IReadOperator { const std::map& params, gs::runtime::Context&& ctx, gs::runtime::OprTimer& timer) override { - return Scan::scan_vertex(graph, scan_params_, - [](label_t, vid_t) { return true; }); + if (scan_params_.limit == std::numeric_limits::max()) { + return Scan::scan_vertex(graph, scan_params_, + [](label_t, vid_t) { return true; }); + } else { + return Scan::scan_vertex_with_limit(graph, scan_params_, + [](label_t, vid_t) { return true; }); + } } private: