diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java index 8c3ea68e5fcd7..98a5a42278173 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java @@ -186,21 +186,21 @@ public static void checkTempPartitionConflict(List partitionList, Set tempSet = new HashSet<>(); Set> tempMultiSet = new HashSet<>(); for (Partition partition : tempPartitionList) { - if (!listMap.isEmpty()) { - listMap.get(partition.getId()).forEach(item -> { - tempSet.add(item.getRealObjectValue()); - }); - newListMap.remove(partition.getId()); - } - if (!multiListMap.isEmpty()) { - multiListMap.get(partition.getId()).forEach(itemList -> { - List cur = new ArrayList<>(); - itemList.forEach(item -> { - cur.add(item.getRealObjectValue()); - }); - tempMultiSet.add(cur); - }); + if (listPartitionInfo.isMultiColumnPartition()) { + tempMultiSet = multiListMap.get(partition.getId()) + .stream() + .map(literalExprs -> literalExprs + .stream() + .map(LiteralExpr::getRealObjectValue) + .collect(Collectors.toList())) + .collect(Collectors.toSet()); newMultiListMap.remove(partition.getId()); + } else { + tempSet = listMap.get(partition.getId()) + .stream() + .map(LiteralExpr::getRealObjectValue) + .collect(Collectors.toSet()); + newListMap.remove(partition.getId()); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java b/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java index cb77e1745d6ca..d96a89353c031 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java @@ -433,6 +433,7 @@ public void addPartition(long partitionId, DataProperty dataProperty, short repl } @Override +<<<<<<< HEAD public void createAutomaticShadowPartition(long partitionId, String replicateNum) { idToValues.put(partitionId, Collections.emptyList()); idToDataProperty.put(partitionId, new DataProperty(TStorageMedium.HDD)); @@ -440,6 +441,88 @@ public void createAutomaticShadowPartition(long partitionId, String replicateNum idToInMemory.put(partitionId, false); idToStorageCacheInfo.put(partitionId, new StorageCacheInfo(true, Config.lake_default_storage_cache_ttl_seconds, false)); +======= + public void createAutomaticShadowPartition(List schema, long partitionId, String replicateNum) { + if (isMultiColumnPartition()) { + idToMultiValues.put(partitionId, Collections.emptyList()); + idToMultiLiteralExprValues.put(partitionId, Collections.emptyList()); + } else { + idToValues.put(partitionId, Collections.emptyList()); + idToLiteralExprValues.put(partitionId, Collections.emptyList()); + } + + idToDataProperty.put(partitionId, new DataProperty(TStorageMedium.HDD)); + idToReplicationNum.put(partitionId, Short.valueOf(replicateNum)); + idToInMemory.put(partitionId, false); + idToStorageCacheInfo.put(partitionId, new DataCacheInfo(true, false)); + } + + public static int compareByValue(List> left, List> right) { + int valueSize = left.size(); + for (int i = 0; i < valueSize; i++) { + int partitionSize = left.get(i).size(); + for (int j = 0; j < partitionSize; j++) { + int compareResult = left.get(i).get(j).compareTo(right.get(i).get(j)); + if (compareResult != 0) { + return compareResult; + } + } + } + return 0; + } + + public void setStorageCacheInfo(long partitionId, DataCacheInfo dataCacheInfo) { + idToStorageCacheInfo.put(partitionId, dataCacheInfo); + } + + @Override + public List getSortedPartitions(boolean asc) { + if (MapUtils.isNotEmpty(idToLiteralExprValues)) { + return idToLiteralExprValues.entrySet().stream() + .filter(e -> !e.getValue().isEmpty()) + .sorted((x, y) -> compareRow(x.getValue(), y.getValue(), asc)) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } else if (MapUtils.isEmpty(idToMultiLiteralExprValues)) { + return idToMultiLiteralExprValues.entrySet().stream() + .filter(e -> !e.getValue().isEmpty()) + .sorted((x, y) -> compareMultiValueList(x.getValue(), y.getValue(), asc)) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } else { + throw new NotImplementedException("todo"); + } + } + + /** + * Compare based on the max/min value in the list + */ + private static int compareRow(List lhs, List rhs, boolean asc) { + ListPartitionValue lhsValue = + asc ? ListPartitionCell.single(lhs).minValue() : ListPartitionCell.single(lhs).maxValue(); + ListPartitionValue rhsValue = + asc ? ListPartitionCell.single(rhs).minValue() : ListPartitionCell.single(rhs).maxValue(); + return lhsValue.compareTo(rhsValue) * (asc ? 1 : -1); + } + + private static int compareColumns(List lhs, List rhs) { + assert lhs.size() == rhs.size(); + for (int i = 0; i < lhs.size(); i++) { + int x = lhs.get(i).compareTo(rhs.get(i)); + if (x != 0) { + return x; + } + } + return 0; + } + + private static int compareMultiValueList(List> lhs, List> rhs, boolean asc) { + ListPartitionValue lhsValue = + asc ? ListPartitionCell.multi(lhs).minValue() : ListPartitionCell.multi(lhs).maxValue(); + ListPartitionValue rhsValue = + asc ? ListPartitionCell.multi(rhs).minValue() : ListPartitionCell.multi(rhs).maxValue(); + return lhsValue.compareTo(rhsValue) * (asc ? 1 : -1); +>>>>>>> 4c585a7b93 ([BugFix] Fix replace temp partition bug for multi column partition table (#49764)) } @Override diff --git a/test/sql/test_automatic_partition/R/test_replace_temp_partition b/test/sql/test_automatic_partition/R/test_replace_temp_partition new file mode 100644 index 0000000000000..93577ffde9302 --- /dev/null +++ b/test/sql/test_automatic_partition/R/test_replace_temp_partition @@ -0,0 +1,137 @@ +-- name: test_replace_temp_partition +CREATE TABLE `test_replace_tmp_partition` ( + `a` varchar(64) NULL COMMENT "", + `b` varchar(10) NOT NULL COMMENT "", + `c` bigint(20) NOT NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`a`) +PARTITION BY (`b`,`c`) +DISTRIBUTED BY RANDOM; +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_1970 VALUES IN (("a025", "1970")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa019_9990 VALUES IN (("a019", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_9990 VALUES IN (("a025", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00002_9990 VALUES IN (("t00002", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00042_9990 VALUES IN (("ta00042", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00150_9990 VALUES IN (("t00150", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00045_9990 VALUES IN (("ta00045", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00043_9990 VALUES IN (("ta00043", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa024_9990 VALUES IN (("a024", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00040_9990 VALUES IN (("ta00040", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00028_9990 VALUES IN (("ta00028", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00027_9990 VALUES IN (("ta00027", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00154_9990 VALUES IN (("t00154", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00147_9990 VALUES IN (("t00147", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00144_9990 VALUES IN (("t00144", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00017_9990 VALUES IN (("ta00017", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00016_9990 VALUES IN (("ta00016", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00015_9990 VALUES IN (("ta00015", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00148_9990 VALUES IN (("t00148", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00149_9990 VALUES IN (("t00149", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00069_9990 VALUES IN (("t00069", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa019_9990 VALUES IN (("a019", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa025_9990 VALUES IN (("a025", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00002_9990 VALUES IN (("t00002", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00042_9990 VALUES IN (("ta00042", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00150_9990 VALUES IN (("t00150", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00045_9990 VALUES IN (("ta00045", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00043_9990 VALUES IN (("ta00043", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa024_9990 VALUES IN (("a024", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00040_9990 VALUES IN (("ta00040", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00028_9990 VALUES IN (("ta00028", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00027_9990 VALUES IN (("ta00027", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00154_9990 VALUES IN (("t00154", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00147_9990 VALUES IN (("t00147", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00144_9990 VALUES IN (("t00144", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00017_9990 VALUES IN (("ta00017", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00016_9990 VALUES IN (("ta00016", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00015_9990 VALUES IN (("ta00015", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00148_9990 VALUES IN (("t00148", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00149_9990 VALUES IN (("t00149", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00069_9990 VALUES IN (("t00069", "9990")); +-- result: +-- !result +ALTER TABLE test_replace_tmp_partition REPLACE PARTITION (pa019_9990,pa025_9990,pt00002_9990,pta00042_9990,pt00150_9990,pta00045_9990,pta00043_9990,pa024_9990,pta00040_9990,pta00028_9990,pta00027_9990,pt00154_9990,pt00147_9990,pt00144_9990,pta00017_9990,pta00016_9990,pta00015_9990,pt00148_9990,pt00149_9990,pt00069_9990) WITH TEMPORARY PARTITION (t_pa019_9990,t_pa025_9990,t_pt00002_9990,t_pta00042_9990,t_pt00150_9990,t_pta00045_9990,t_pta00043_9990,t_pa024_9990,t_pta00040_9990,t_pta00028_9990,t_pta00027_9990,t_pt00154_9990,t_pt00147_9990,t_pt00144_9990,t_pta00017_9990,t_pta00016_9990,t_pta00015_9990,t_pt00148_9990,t_pt00149_9990,t_pt00069_9990) PROPERTIES ("strict_range "="fasle" , "use_temp_partition_name"="false"); +-- result: +-- !result \ No newline at end of file diff --git a/test/sql/test_automatic_partition/T/test_replace_temp_partition b/test/sql/test_automatic_partition/T/test_replace_temp_partition new file mode 100644 index 0000000000000..56cd4650112b0 --- /dev/null +++ b/test/sql/test_automatic_partition/T/test_replace_temp_partition @@ -0,0 +1,55 @@ +-- name: test_replace_temp_partition + +CREATE TABLE `test_replace_tmp_partition` ( + `a` varchar(64) NULL COMMENT "", + `b` varchar(10) NOT NULL COMMENT "", + `c` bigint(20) NOT NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`a`) +PARTITION BY (`b`,`c`) +DISTRIBUTED BY RANDOM; + +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_1970 VALUES IN (("a025", "1970")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa019_9990 VALUES IN (("a019", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_9990 VALUES IN (("a025", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00002_9990 VALUES IN (("t00002", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00042_9990 VALUES IN (("ta00042", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00150_9990 VALUES IN (("t00150", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00045_9990 VALUES IN (("ta00045", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00043_9990 VALUES IN (("ta00043", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa024_9990 VALUES IN (("a024", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00040_9990 VALUES IN (("ta00040", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00028_9990 VALUES IN (("ta00028", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00027_9990 VALUES IN (("ta00027", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00154_9990 VALUES IN (("t00154", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00147_9990 VALUES IN (("t00147", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00144_9990 VALUES IN (("t00144", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00017_9990 VALUES IN (("ta00017", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00016_9990 VALUES IN (("ta00016", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00015_9990 VALUES IN (("ta00015", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00148_9990 VALUES IN (("t00148", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00149_9990 VALUES IN (("t00149", "9990")); +ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00069_9990 VALUES IN (("t00069", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa019_9990 VALUES IN (("a019", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa025_9990 VALUES IN (("a025", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00002_9990 VALUES IN (("t00002", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00042_9990 VALUES IN (("ta00042", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00150_9990 VALUES IN (("t00150", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00045_9990 VALUES IN (("ta00045", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00043_9990 VALUES IN (("ta00043", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa024_9990 VALUES IN (("a024", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00040_9990 VALUES IN (("ta00040", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00028_9990 VALUES IN (("ta00028", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00027_9990 VALUES IN (("ta00027", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00154_9990 VALUES IN (("t00154", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00147_9990 VALUES IN (("t00147", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00144_9990 VALUES IN (("t00144", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00017_9990 VALUES IN (("ta00017", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00016_9990 VALUES IN (("ta00016", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00015_9990 VALUES IN (("ta00015", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00148_9990 VALUES IN (("t00148", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00149_9990 VALUES IN (("t00149", "9990")); +ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00069_9990 VALUES IN (("t00069", "9990")); + +ALTER TABLE test_replace_tmp_partition REPLACE PARTITION (pa019_9990,pa025_9990,pt00002_9990,pta00042_9990,pt00150_9990,pta00045_9990,pta00043_9990,pa024_9990,pta00040_9990,pta00028_9990,pta00027_9990,pt00154_9990,pt00147_9990,pt00144_9990,pta00017_9990,pta00016_9990,pta00015_9990,pt00148_9990,pt00149_9990,pt00069_9990) WITH TEMPORARY PARTITION (t_pa019_9990,t_pa025_9990,t_pt00002_9990,t_pta00042_9990,t_pt00150_9990,t_pta00045_9990,t_pta00043_9990,t_pa024_9990,t_pta00040_9990,t_pta00028_9990,t_pta00027_9990,t_pt00154_9990,t_pt00147_9990,t_pt00144_9990,t_pta00017_9990,t_pta00016_9990,t_pta00015_9990,t_pt00148_9990,t_pt00149_9990,t_pt00069_9990) PROPERTIES ("strict_range "="fasle" , "use_temp_partition_name"="false"); +