Fix: vector::reserve takes up a lot of memory (#2764)

StarRocks · Jan 11, 2022 · f163676 · f163676
1 parent 29480b8
commit f163676
Show file tree

Hide file tree

Showing 7 changed files with 21 additions and 50 deletions.
diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
@@ -170,7 +170,6 @@ set(EXEC_FILES
     pipeline/aggregate/repeat/repeat_operator.cpp
     pipeline/analysis/analytic_sink_operator.cpp
     pipeline/analysis/analytic_source_operator.cpp
-    pipeline/analysis/analytor_factory.cpp
     pipeline/table_function_operator.cpp
     pipeline/assert_num_rows_operator.cpp
     pipeline/set/union_passthrough_operator.cpp

diff --git a/be/src/exec/pipeline/analysis/analytic_sink_operator.h b/be/src/exec/pipeline/analysis/analytic_sink_operator.h
@@ -2,7 +2,6 @@
 
 #pragma once
 
-#include "exec/pipeline/analysis/analytor_factory.h"
 #include "exec/pipeline/operator.h"
 #include "exec/vectorized/analytor.h"
 

diff --git a/be/src/exec/pipeline/analysis/analytic_source_operator.h b/be/src/exec/pipeline/analysis/analytic_source_operator.h
@@ -4,7 +4,6 @@
 
 #include <utility>
 
-#include "exec/pipeline/analysis/analytor_factory.h"
 #include "exec/pipeline/source_operator.h"
 #include "exec/vectorized/analytor.h"
 

diff --git a/be/src/exec/pipeline/analysis/analytor_factory.cpp b/be/src/exec/pipeline/analysis/analytor_factory.cpp
diff --git a/be/src/exec/pipeline/analysis/analytor_factory.h b/be/src/exec/pipeline/analysis/analytor_factory.h
diff --git a/be/src/exec/vectorized/analytor.cpp b/be/src/exec/vectorized/analytor.cpp
@@ -171,7 +171,6 @@ Status Analytor::prepare(RuntimeState* state, ObjectPool* pool, RuntimeProfile*
                         _agg_expr_ctxs[i][j]->root()->type(), _agg_expr_ctxs[i][j]->root()->is_nullable(),
                         _agg_expr_ctxs[i][j]->root()->is_constant(), 0);
             }
-            _agg_intput_columns[i][j]->reserve(state->chunk_size() * BUFFER_CHUNK_NUMBER);
         }
 
         DCHECK(_agg_functions[i] != nullptr);
@@ -203,7 +202,6 @@ Status Analytor::prepare(RuntimeState* state, ObjectPool* pool, RuntimeProfile*
         _partition_columns[i] = vectorized::ColumnHelper::create_column(
                 _partition_ctxs[i]->root()->type(), _partition_ctxs[i]->root()->is_nullable() | has_outer_join_child,
                 _partition_ctxs[i]->root()->is_constant(), 0);
-        _partition_columns[i]->reserve(state->chunk_size() * BUFFER_CHUNK_NUMBER);
     }
 
     RETURN_IF_ERROR(Expr::create_expr_trees(_pool, analytic_node.order_by_exprs, &_order_ctxs));
@@ -212,7 +210,6 @@ Status Analytor::prepare(RuntimeState* state, ObjectPool* pool, RuntimeProfile*
         _order_columns[i] = vectorized::ColumnHelper::create_column(
                 _order_ctxs[i]->root()->type(), _order_ctxs[i]->root()->is_nullable() | has_outer_join_child,
                 _order_ctxs[i]->root()->is_constant(), 0);
-        _order_columns[i]->reserve(state->chunk_size() * BUFFER_CHUNK_NUMBER);
     }
 
     SCOPED_TIMER(_runtime_profile->total_time_counter());
@@ -551,4 +548,10 @@ int64_t Analytor::_find_first_not_equal(vectorized::Column* column, int64_t star
     return end - 1;
 }
 
+AnalytorPtr AnalytorFactory::create(int i) {
+    if (!_analytors[i]) {
+        _analytors[i] = std::make_shared<Analytor>(_tnode, _child_row_desc, _result_tuple_desc);
+    }
+    return _analytors[i];
+}
 } // namespace starrocks
diff --git a/be/src/exec/vectorized/analytor.h b/be/src/exec/vectorized/analytor.h
@@ -120,12 +120,6 @@ class Analytor final : public pipeline::ContextWithDependency {
     static constexpr int32_t BUFFER_CHUNK_NUMBER = 1;
 #endif
 
-#ifdef NDEBUG
-    static constexpr size_t memory_check_batch_size = 65535;
-#else
-    static constexpr size_t memory_check_batch_size = 1;
-#endif
-
 private:
     RuntimeState* _state = nullptr;
     bool _is_closed = false;
@@ -233,4 +227,19 @@ class ManagedFunctionStates {
     Analytor* _agg_node;
 };
 
+class AnalytorFactory;
+using AnalytorFactoryPtr = std::shared_ptr<AnalytorFactory>;
+class AnalytorFactory {
+public:
+    AnalytorFactory(size_t dop, const TPlanNode& tnode, const RowDescriptor& child_row_desc,
+                    const TupleDescriptor* result_tuple_desc)
+            : _analytors(dop), _tnode(tnode), _child_row_desc(child_row_desc), _result_tuple_desc(result_tuple_desc) {}
+    AnalytorPtr create(int i);
+
+private:
+    Analytors _analytors;
+    const TPlanNode& _tnode;
+    const RowDescriptor& _child_row_desc;
+    const TupleDescriptor* _result_tuple_desc;
+};
 } // namespace starrocks