From dde429bfbace16612dc28a852a371fd31e32ebe2 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Wed, 26 Oct 2022 12:25:53 +0800 Subject: [PATCH] refactor(query): revert metrics #8444 --- Cargo.lock | 22 ------------------- src/query/datavalues/Cargo.toml | 1 - src/query/datavalues/src/data_value.rs | 3 +-- src/query/datavalues/src/variant_value.rs | 3 --- src/query/storages/fuse-meta/Cargo.toml | 1 - .../storages/fuse-meta/src/meta/common.rs | 7 +++--- .../storages/fuse-meta/src/meta/v0/segment.rs | 8 +++---- .../storages/fuse-meta/src/meta/v1/segment.rs | 6 ++--- src/query/storages/fuse/Cargo.toml | 1 - .../compact_mutator/full_compact_mutator.rs | 8 ++++--- 10 files changed, 14 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cdadd28d5d7c..ae9543b523776 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1327,7 +1327,6 @@ dependencies = [ "criterion", "dyn-clone", "enum_dispatch", - "get-size", "itertools", "lexical-core", "micromarshal", @@ -1495,7 +1494,6 @@ dependencies = [ "common-datavalues", "common-exception", "common-metrics", - "get-size", "once_cell", "parking_lot 0.12.1", "serde", @@ -2185,7 +2183,6 @@ dependencies = [ "common-storages-index", "futures", "futures-util", - "get-size", "itertools", "metrics", "opendal", @@ -3722,25 +3719,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "get-size" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffff7b577402f46b6e0069ae9c31a7859ab3f2abe7ff0fb614fe6441a1d6c306" -dependencies = [ - "get-size-derive", -] - -[[package]] -name = "get-size-derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "316da74ccd236d0915a5793f3cb37b1154cc41857373d9a7334053294c933f70" -dependencies = [ - "quote", - "syn", -] - [[package]] name = "gethostname" version = "0.2.3" diff --git a/src/query/datavalues/Cargo.toml b/src/query/datavalues/Cargo.toml index 8d812ee83e42c..ddb6dd8ee2341 100644 --- a/src/query/datavalues/Cargo.toml +++ b/src/query/datavalues/Cargo.toml @@ -25,7 +25,6 @@ chrono = { version = "0.4.22", features = ["serde"] } chrono-tz = "0.6.3" dyn-clone = "1.0.9" enum_dispatch = "0.3.8" -get-size = { version = "^0.1", features = ["derive"] } itertools = "0.10.5" lexical-core = "0.8.5" micromarshal = "0.2.0" diff --git a/src/query/datavalues/src/data_value.rs b/src/query/datavalues/src/data_value.rs index 213590bf431f2..8f6247e52b727 100644 --- a/src/query/datavalues/src/data_value.rs +++ b/src/query/datavalues/src/data_value.rs @@ -22,7 +22,6 @@ use std::sync::Arc; use common_exception::ErrorCode; use common_exception::Result; -use get_size::GetSize; use ordered_float::OrderedFloat; use serde_json::json; @@ -30,7 +29,7 @@ use crate::prelude::*; use crate::type_coercion::merge_types; /// A specific value of a data type. -#[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, GetSize)] +#[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq)] pub enum DataValue { /// Base type. Null, diff --git a/src/query/datavalues/src/variant_value.rs b/src/query/datavalues/src/variant_value.rs index 64b601c1cc976..2487666b4b3c9 100644 --- a/src/query/datavalues/src/variant_value.rs +++ b/src/query/datavalues/src/variant_value.rs @@ -22,7 +22,6 @@ use std::ops::Deref; use common_exception::ErrorCode; use common_exception::Result; -use get_size::GetSize; use itertools::EitherOrBoth::Both; use itertools::EitherOrBoth::Left; use itertools::EitherOrBoth::Right; @@ -32,8 +31,6 @@ use serde_json::Value; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] pub struct VariantValue(pub Value); -impl GetSize for VariantValue {} - impl From for VariantValue { fn from(val: Value) -> Self { VariantValue(val) diff --git a/src/query/storages/fuse-meta/Cargo.toml b/src/query/storages/fuse-meta/Cargo.toml index f8746b4d90ce6..a36af5dbe4841 100644 --- a/src/query/storages/fuse-meta/Cargo.toml +++ b/src/query/storages/fuse-meta/Cargo.toml @@ -18,7 +18,6 @@ common-datavalues = { path = "../../datavalues" } common-exception = { path = "../../../common/exception" } common-metrics = { path = "../../../common/metrics" } -get-size = { version = "^0.1", features = ["derive"] } once_cell = "1.15.0" parking_lot = "0.12.1" serde = { workspace = true } diff --git a/src/query/storages/fuse-meta/src/meta/common.rs b/src/query/storages/fuse-meta/src/meta/common.rs index 6d80d40921728..272fd9a102048 100644 --- a/src/query/storages/fuse-meta/src/meta/common.rs +++ b/src/query/storages/fuse-meta/src/meta/common.rs @@ -16,7 +16,6 @@ use std::collections::HashMap; use common_base::base::uuid::Uuid; use common_datavalues::DataValue; -use get_size::GetSize; use serde::Deserialize; use serde::Serialize; @@ -28,7 +27,7 @@ pub type ClusterKey = (u32, String); pub type StatisticsOfColumns = HashMap; -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq, GetSize)] +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] pub struct ColumnStatistics { pub min: DataValue, pub max: DataValue, @@ -40,7 +39,7 @@ pub struct ColumnStatistics { pub in_memory_size: u64, } -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, GetSize)] +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] pub struct ClusterStatistics { #[serde(default = "default_cluster_key_id")] pub cluster_key_id: u32, @@ -59,7 +58,7 @@ fn default_level() -> i32 { 0 } -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq, Default, GetSize)] +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq, Default)] pub struct Statistics { pub row_count: u64, pub block_count: u64, diff --git a/src/query/storages/fuse-meta/src/meta/v0/segment.rs b/src/query/storages/fuse-meta/src/meta/v0/segment.rs index 8e7d3f25fba56..5fa9775edbf4c 100644 --- a/src/query/storages/fuse-meta/src/meta/v0/segment.rs +++ b/src/query/storages/fuse-meta/src/meta/v0/segment.rs @@ -14,8 +14,6 @@ use std::collections::HashMap; -use get_size::GetSize; - use crate::meta::common::ColumnStatistics; use crate::meta::ColumnId; use crate::meta::Statistics; @@ -30,7 +28,7 @@ pub struct SegmentInfo { } /// Meta information of a block (currently, the parquet file) -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, GetSize)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct BlockMeta { /// Pointer of the data Block pub row_count: u64, @@ -42,7 +40,7 @@ pub struct BlockMeta { } // TODO move it to common -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, GetSize)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct ColumnMeta { /// where the data of column start pub offset: u64, @@ -52,7 +50,7 @@ pub struct ColumnMeta { pub num_values: u64, } -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, GetSize)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct BlockLocation { pub path: String, // for parquet, this filed can be used to fetch the meta data without seeking around diff --git a/src/query/storages/fuse-meta/src/meta/v1/segment.rs b/src/query/storages/fuse-meta/src/meta/v1/segment.rs index 360308510ed1f..c169e463c555a 100644 --- a/src/query/storages/fuse-meta/src/meta/v1/segment.rs +++ b/src/query/storages/fuse-meta/src/meta/v1/segment.rs @@ -15,7 +15,6 @@ use std::collections::HashMap; use common_datablocks::DataBlock; -use get_size::GetSize; use serde::Deserialize; use serde::Serialize; @@ -30,7 +29,7 @@ use crate::meta::Statistics; use crate::meta::Versioned; /// A segment comprises one or more blocks -#[derive(Serialize, Deserialize, Debug, GetSize)] +#[derive(Serialize, Deserialize, Debug)] pub struct SegmentInfo { /// format version format_version: FormatVersion, @@ -42,7 +41,7 @@ pub struct SegmentInfo { /// Meta information of a block /// Part of and kept inside the [SegmentInfo] -#[derive(Serialize, Deserialize, Clone, Debug, GetSize)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct BlockMeta { pub row_count: u64, pub block_size: u64, @@ -64,7 +63,6 @@ pub struct BlockMeta { /// `Lz4` is merely for backward compatibility, it will NO longer be /// used in the write path. #[serde(default = "Compression::legacy")] - #[get_size(ignore)] compression: Compression, } diff --git a/src/query/storages/fuse/Cargo.toml b/src/query/storages/fuse/Cargo.toml index 6497b3b949354..c4edd56f1e3c9 100644 --- a/src/query/storages/fuse/Cargo.toml +++ b/src/query/storages/fuse/Cargo.toml @@ -41,7 +41,6 @@ backon = "0.2" chrono = "0.4.22" futures = "0.3.24" futures-util = "0.3.24" -get-size = { version = "^0.1", features = ["derive"] } itertools = "0.10.5" metrics = "0.20.1" opendal = { version = "0.19", features = ["layers-retry"] } diff --git a/src/query/storages/fuse/src/operations/mutation/compact_mutator/full_compact_mutator.rs b/src/query/storages/fuse/src/operations/mutation/compact_mutator/full_compact_mutator.rs index 4aae19af36f24..d834d3fe85bb1 100644 --- a/src/query/storages/fuse/src/operations/mutation/compact_mutator/full_compact_mutator.rs +++ b/src/query/storages/fuse/src/operations/mutation/compact_mutator/full_compact_mutator.rs @@ -21,7 +21,6 @@ use common_fuse_meta::meta::Location; use common_fuse_meta::meta::SegmentInfo; use common_fuse_meta::meta::Statistics; use common_fuse_meta::meta::Versioned; -use get_size::GetSize; use opendal::Operator; use crate::io::BlockCompactor; @@ -109,7 +108,8 @@ impl TableMutator for FullCompactMutator { .into_iter() .collect::>>()?; - metrics_set_segments_memory_usage(segments.get_heap_size() as f64); + // todo: add real metrics + metrics_set_segments_memory_usage(0.0); let limit = self.compact_params.limit.unwrap_or(segments.len()); if limit < segments.len() { @@ -137,7 +137,9 @@ impl TableMutator for FullCompactMutator { need_merge = true; } }); - metrics_set_selected_blocks_memory_usage(self.selected_blocks.get_heap_size() as f64); + + // todo: add real metrics + metrics_set_selected_blocks_memory_usage(0.0); // If the number of blocks of segment meets block_per_seg, and the blocks in segments donot need to be compacted, // then record the segment information.