Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[BugFix] Fix replace temp partition bug for multi column partition table #49764

Merged
merged 4 commits into from
Aug 16, 2024
Merged

[BugFix] Fix replace temp partition bug for multi column partition table #49764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7f06816
fix
gengjun-git Aug 13, 2024
14434f0
fix
gengjun-git Aug 13, 2024
2c39dd8
fix
gengjun-git Aug 14, 2024
863d998
add test
gengjun-git Aug 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,21 +213,21 @@ public static void checkTempPartitionConflict(List<Partition> partitionList,
Set<Object> tempSet = new HashSet<>();
Set<List<Object>> tempMultiSet = new HashSet<>();
for (Partition partition : tempPartitionList) {
if (!listMap.isEmpty()) {
listMap.get(partition.getId()).forEach(item -> {
tempSet.add(item.getRealObjectValue());
});
newListMap.remove(partition.getId());
}
if (!multiListMap.isEmpty()) {
multiListMap.get(partition.getId()).forEach(itemList -> {
List<Object> cur = new ArrayList<>();
itemList.forEach(item -> {
cur.add(item.getRealObjectValue());
});
tempMultiSet.add(cur);
});
if (listPartitionInfo.isMultiColumnPartition()) {
tempMultiSet = multiListMap.get(partition.getId())
.stream()
.map(literalExprs -> literalExprs
.stream()
.map(LiteralExpr::getRealObjectValue)
.collect(Collectors.toList()))
.collect(Collectors.toSet());
newMultiListMap.remove(partition.getId());
} else {
tempSet = listMap.get(partition.getId())
.stream()
.map(LiteralExpr::getRealObjectValue)
.collect(Collectors.toSet());
newListMap.remove(partition.getId());
}
}

Expand Down
11 changes: 10 additions & 1 deletion fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -570,7 +570,14 @@ public void addPartition(Map<ColumnId, Column> idToColumn, long partitionId, Dat

@Override
public void createAutomaticShadowPartition(List<Column> schema, long partitionId, String replicateNum) {
idToValues.put(partitionId, Collections.emptyList());
if (isMultiColumnPartition()) {
idToMultiValues.put(partitionId, Collections.emptyList());
idToMultiLiteralExprValues.put(partitionId, Collections.emptyList());
} else {
idToValues.put(partitionId, Collections.emptyList());
idToLiteralExprValues.put(partitionId, Collections.emptyList());
}

idToDataProperty.put(partitionId, new DataProperty(TStorageMedium.HDD));
idToReplicationNum.put(partitionId, Short.valueOf(replicateNum));
idToInMemory.put(partitionId, false);
Expand Down Expand Up @@ -599,11 +606,13 @@ public void setStorageCacheInfo(long partitionId, DataCacheInfo dataCacheInfo) {
public List<Long> getSortedPartitions(boolean asc) {
if (MapUtils.isNotEmpty(idToLiteralExprValues)) {
return idToLiteralExprValues.entrySet().stream()
.filter(e -> !e.getValue().isEmpty())
.sorted((x, y) -> compareRow(x.getValue(), y.getValue(), asc))
.map(Map.Entry::getKey)
.collect(Collectors.toList());
} else if (MapUtils.isEmpty(idToMultiLiteralExprValues)) {
return idToMultiLiteralExprValues.entrySet().stream()
.filter(e -> !e.getValue().isEmpty())
.sorted((x, y) -> compareMultiValueList(x.getValue(), y.getValue(), asc))
.map(Map.Entry::getKey)
.collect(Collectors.toList());
Expand Down
137 changes: 137 additions & 0 deletions test/sql/test_automatic_partition/R/test_replace_temp_partition
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
-- name: test_replace_temp_partition
CREATE TABLE `test_replace_tmp_partition` (
`a` varchar(64) NULL COMMENT "",
`b` varchar(10) NOT NULL COMMENT "",
`c` bigint(20) NOT NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`a`)
PARTITION BY (`b`,`c`)
DISTRIBUTED BY RANDOM;
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_1970 VALUES IN (("a025", "1970"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa019_9990 VALUES IN (("a019", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_9990 VALUES IN (("a025", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00002_9990 VALUES IN (("t00002", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00042_9990 VALUES IN (("ta00042", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00150_9990 VALUES IN (("t00150", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00045_9990 VALUES IN (("ta00045", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00043_9990 VALUES IN (("ta00043", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa024_9990 VALUES IN (("a024", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00040_9990 VALUES IN (("ta00040", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00028_9990 VALUES IN (("ta00028", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00027_9990 VALUES IN (("ta00027", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00154_9990 VALUES IN (("t00154", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00147_9990 VALUES IN (("t00147", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00144_9990 VALUES IN (("t00144", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00017_9990 VALUES IN (("ta00017", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00016_9990 VALUES IN (("ta00016", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00015_9990 VALUES IN (("ta00015", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00148_9990 VALUES IN (("t00148", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00149_9990 VALUES IN (("t00149", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00069_9990 VALUES IN (("t00069", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa019_9990 VALUES IN (("a019", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa025_9990 VALUES IN (("a025", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00002_9990 VALUES IN (("t00002", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00042_9990 VALUES IN (("ta00042", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00150_9990 VALUES IN (("t00150", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00045_9990 VALUES IN (("ta00045", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00043_9990 VALUES IN (("ta00043", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa024_9990 VALUES IN (("a024", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00040_9990 VALUES IN (("ta00040", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00028_9990 VALUES IN (("ta00028", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00027_9990 VALUES IN (("ta00027", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00154_9990 VALUES IN (("t00154", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00147_9990 VALUES IN (("t00147", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00144_9990 VALUES IN (("t00144", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00017_9990 VALUES IN (("ta00017", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00016_9990 VALUES IN (("ta00016", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00015_9990 VALUES IN (("ta00015", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00148_9990 VALUES IN (("t00148", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00149_9990 VALUES IN (("t00149", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00069_9990 VALUES IN (("t00069", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition REPLACE PARTITION (pa019_9990,pa025_9990,pt00002_9990,pta00042_9990,pt00150_9990,pta00045_9990,pta00043_9990,pa024_9990,pta00040_9990,pta00028_9990,pta00027_9990,pt00154_9990,pt00147_9990,pt00144_9990,pta00017_9990,pta00016_9990,pta00015_9990,pt00148_9990,pt00149_9990,pt00069_9990) WITH TEMPORARY PARTITION (t_pa019_9990,t_pa025_9990,t_pt00002_9990,t_pta00042_9990,t_pt00150_9990,t_pta00045_9990,t_pta00043_9990,t_pa024_9990,t_pta00040_9990,t_pta00028_9990,t_pta00027_9990,t_pt00154_9990,t_pt00147_9990,t_pt00144_9990,t_pta00017_9990,t_pta00016_9990,t_pta00015_9990,t_pt00148_9990,t_pt00149_9990,t_pt00069_9990) PROPERTIES ("strict_range "="fasle" , "use_temp_partition_name"="false");
-- result:
-- !result
55 changes: 55 additions & 0 deletions test/sql/test_automatic_partition/T/test_replace_temp_partition
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
-- name: test_replace_temp_partition

CREATE TABLE `test_replace_tmp_partition` (
`a` varchar(64) NULL COMMENT "",
`b` varchar(10) NOT NULL COMMENT "",
`c` bigint(20) NOT NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`a`)
PARTITION BY (`b`,`c`)
DISTRIBUTED BY RANDOM;

ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_1970 VALUES IN (("a025", "1970"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa019_9990 VALUES IN (("a019", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_9990 VALUES IN (("a025", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00002_9990 VALUES IN (("t00002", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00042_9990 VALUES IN (("ta00042", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00150_9990 VALUES IN (("t00150", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00045_9990 VALUES IN (("ta00045", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00043_9990 VALUES IN (("ta00043", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa024_9990 VALUES IN (("a024", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00040_9990 VALUES IN (("ta00040", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00028_9990 VALUES IN (("ta00028", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00027_9990 VALUES IN (("ta00027", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00154_9990 VALUES IN (("t00154", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00147_9990 VALUES IN (("t00147", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00144_9990 VALUES IN (("t00144", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00017_9990 VALUES IN (("ta00017", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00016_9990 VALUES IN (("ta00016", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00015_9990 VALUES IN (("ta00015", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00148_9990 VALUES IN (("t00148", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00149_9990 VALUES IN (("t00149", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00069_9990 VALUES IN (("t00069", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa019_9990 VALUES IN (("a019", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa025_9990 VALUES IN (("a025", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00002_9990 VALUES IN (("t00002", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00042_9990 VALUES IN (("ta00042", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00150_9990 VALUES IN (("t00150", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00045_9990 VALUES IN (("ta00045", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00043_9990 VALUES IN (("ta00043", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa024_9990 VALUES IN (("a024", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00040_9990 VALUES IN (("ta00040", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00028_9990 VALUES IN (("ta00028", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00027_9990 VALUES IN (("ta00027", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00154_9990 VALUES IN (("t00154", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00147_9990 VALUES IN (("t00147", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00144_9990 VALUES IN (("t00144", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00017_9990 VALUES IN (("ta00017", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00016_9990 VALUES IN (("ta00016", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00015_9990 VALUES IN (("ta00015", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00148_9990 VALUES IN (("t00148", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00149_9990 VALUES IN (("t00149", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00069_9990 VALUES IN (("t00069", "9990"));

ALTER TABLE test_replace_tmp_partition REPLACE PARTITION (pa019_9990,pa025_9990,pt00002_9990,pta00042_9990,pt00150_9990,pta00045_9990,pta00043_9990,pa024_9990,pta00040_9990,pta00028_9990,pta00027_9990,pt00154_9990,pt00147_9990,pt00144_9990,pta00017_9990,pta00016_9990,pta00015_9990,pt00148_9990,pt00149_9990,pt00069_9990) WITH TEMPORARY PARTITION (t_pa019_9990,t_pa025_9990,t_pt00002_9990,t_pta00042_9990,t_pt00150_9990,t_pta00045_9990,t_pta00043_9990,t_pa024_9990,t_pta00040_9990,t_pta00028_9990,t_pta00027_9990,t_pt00154_9990,t_pt00147_9990,t_pt00144_9990,t_pta00017_9990,t_pta00016_9990,t_pta00015_9990,t_pt00148_9990,t_pt00149_9990,t_pt00069_9990) PROPERTIES ("strict_range "="fasle" , "use_temp_partition_name"="false");

Loading