Skip to content

Commit

Permalink
[BugFix] Fix replace temp partition bug for multi column partition ta…
Browse files Browse the repository at this point in the history
…ble (#49764)

## Why I'm doing:
There will be a shadow partition for auto-create partition table, but for multi column partition table, it should be put into idToMultiValues.

## What I'm doing:
Create the shadow partition according to isMultiColumnPartition();

Fixes
```
 com.starrocks.journal.JournalInconsistentException: failed to load journal type 210
        at com.starrocks.persist.EditLog.loadJournal(EditLog.java:1195)
        at com.starrocks.server.GlobalStateMgr.replayJournalInner(GlobalStateMgr.java:1879)
        at com.starrocks.server.GlobalStateMgr.replayJournal(GlobalStateMgr.java:1828)
        at com.starrocks.server.GlobalStateMgr.transferToLeader(GlobalStateMgr.java:1190)
        at com.starrocks.server.GlobalStateMgr$1.transferToLeader(GlobalStateMgr.java:732)
        at com.starrocks.ha.StateChangeExecutor.runOneCycle(StateChangeExecutor.java:103)
        at com.starrocks.common.util.Daemon.run(Daemon.java:107)
Caused by: java.lang.NullPointerException
        at com.starrocks.catalog.CatalogUtils.checkTempPartitionConflict(CatalogUtils.java:217)
        at com.starrocks.catalog.OlapTable.replaceTempPartitions(OlapTable.java:2644)
        at com.starrocks.server.LocalMetastore.replayReplaceTempPartition(LocalMetastore.java:4724)
        at com.starrocks.persist.EditLog.loadJournal(EditLog.java:834)
```

Signed-off-by: gengjun-git <[email protected]>
(cherry picked from commit 4c585a7)

# Conflicts:
#	fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java
  • Loading branch information
gengjun-git authored and mergify[bot] committed Aug 16, 2024
1 parent e301b35 commit d16c95f
Show file tree
Hide file tree
Showing 4 changed files with 266 additions and 15 deletions.
28 changes: 14 additions & 14 deletions fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -213,21 +213,21 @@ public static void checkTempPartitionConflict(List<Partition> partitionList,
Set<Object> tempSet = new HashSet<>();
Set<List<Object>> tempMultiSet = new HashSet<>();
for (Partition partition : tempPartitionList) {
if (!listMap.isEmpty()) {
listMap.get(partition.getId()).forEach(item -> {
tempSet.add(item.getRealObjectValue());
});
newListMap.remove(partition.getId());
}
if (!multiListMap.isEmpty()) {
multiListMap.get(partition.getId()).forEach(itemList -> {
List<Object> cur = new ArrayList<>();
itemList.forEach(item -> {
cur.add(item.getRealObjectValue());
});
tempMultiSet.add(cur);
});
if (listPartitionInfo.isMultiColumnPartition()) {
tempMultiSet = multiListMap.get(partition.getId())
.stream()
.map(literalExprs -> literalExprs
.stream()
.map(LiteralExpr::getRealObjectValue)
.collect(Collectors.toList()))
.collect(Collectors.toSet());
newMultiListMap.remove(partition.getId());
} else {
tempSet = listMap.get(partition.getId())
.stream()
.map(LiteralExpr::getRealObjectValue)
.collect(Collectors.toSet());
newListMap.remove(partition.getId());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,14 @@ public void addPartition(Map<ColumnId, Column> idToColumn, long partitionId, Dat

@Override
public void createAutomaticShadowPartition(List<Column> schema, long partitionId, String replicateNum) {
idToValues.put(partitionId, Collections.emptyList());
if (isMultiColumnPartition()) {
idToMultiValues.put(partitionId, Collections.emptyList());
idToMultiLiteralExprValues.put(partitionId, Collections.emptyList());
} else {
idToValues.put(partitionId, Collections.emptyList());
idToLiteralExprValues.put(partitionId, Collections.emptyList());
}

idToDataProperty.put(partitionId, new DataProperty(TStorageMedium.HDD));
idToReplicationNum.put(partitionId, Short.valueOf(replicateNum));
idToInMemory.put(partitionId, false);
Expand All @@ -498,6 +505,58 @@ public void setStorageCacheInfo(long partitionId, DataCacheInfo dataCacheInfo) {
}

@Override
<<<<<<< HEAD
=======
public List<Long> getSortedPartitions(boolean asc) {
if (MapUtils.isNotEmpty(idToLiteralExprValues)) {
return idToLiteralExprValues.entrySet().stream()
.filter(e -> !e.getValue().isEmpty())
.sorted((x, y) -> compareRow(x.getValue(), y.getValue(), asc))
.map(Map.Entry::getKey)
.collect(Collectors.toList());
} else if (MapUtils.isEmpty(idToMultiLiteralExprValues)) {
return idToMultiLiteralExprValues.entrySet().stream()
.filter(e -> !e.getValue().isEmpty())
.sorted((x, y) -> compareMultiValueList(x.getValue(), y.getValue(), asc))
.map(Map.Entry::getKey)
.collect(Collectors.toList());
} else {
throw new NotImplementedException("todo");
}
}

/**
* Compare based on the max/min value in the list
*/
private static int compareRow(List<LiteralExpr> lhs, List<LiteralExpr> rhs, boolean asc) {
ListPartitionValue lhsValue =
asc ? ListPartitionCell.single(lhs).minValue() : ListPartitionCell.single(lhs).maxValue();
ListPartitionValue rhsValue =
asc ? ListPartitionCell.single(rhs).minValue() : ListPartitionCell.single(rhs).maxValue();
return lhsValue.compareTo(rhsValue) * (asc ? 1 : -1);
}

private static int compareColumns(List<LiteralExpr> lhs, List<LiteralExpr> rhs) {
assert lhs.size() == rhs.size();
for (int i = 0; i < lhs.size(); i++) {
int x = lhs.get(i).compareTo(rhs.get(i));
if (x != 0) {
return x;
}
}
return 0;
}

private static int compareMultiValueList(List<List<LiteralExpr>> lhs, List<List<LiteralExpr>> rhs, boolean asc) {
ListPartitionValue lhsValue =
asc ? ListPartitionCell.multi(lhs).minValue() : ListPartitionCell.multi(lhs).maxValue();
ListPartitionValue rhsValue =
asc ? ListPartitionCell.multi(rhs).minValue() : ListPartitionCell.multi(rhs).maxValue();
return lhsValue.compareTo(rhsValue) * (asc ? 1 : -1);
}

@Override
>>>>>>> 4c585a7b93 ([BugFix] Fix replace temp partition bug for multi column partition table (#49764))
public Object clone() {
ListPartitionInfo info = (ListPartitionInfo) super.clone();
info.deprecatedColumns = Lists.newArrayList(this.deprecatedColumns);
Expand Down
137 changes: 137 additions & 0 deletions test/sql/test_automatic_partition/R/test_replace_temp_partition
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
-- name: test_replace_temp_partition
CREATE TABLE `test_replace_tmp_partition` (
`a` varchar(64) NULL COMMENT "",
`b` varchar(10) NOT NULL COMMENT "",
`c` bigint(20) NOT NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`a`)
PARTITION BY (`b`,`c`)
DISTRIBUTED BY RANDOM;
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_1970 VALUES IN (("a025", "1970"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa019_9990 VALUES IN (("a019", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_9990 VALUES IN (("a025", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00002_9990 VALUES IN (("t00002", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00042_9990 VALUES IN (("ta00042", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00150_9990 VALUES IN (("t00150", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00045_9990 VALUES IN (("ta00045", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00043_9990 VALUES IN (("ta00043", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa024_9990 VALUES IN (("a024", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00040_9990 VALUES IN (("ta00040", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00028_9990 VALUES IN (("ta00028", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00027_9990 VALUES IN (("ta00027", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00154_9990 VALUES IN (("t00154", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00147_9990 VALUES IN (("t00147", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00144_9990 VALUES IN (("t00144", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00017_9990 VALUES IN (("ta00017", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00016_9990 VALUES IN (("ta00016", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00015_9990 VALUES IN (("ta00015", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00148_9990 VALUES IN (("t00148", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00149_9990 VALUES IN (("t00149", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00069_9990 VALUES IN (("t00069", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa019_9990 VALUES IN (("a019", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa025_9990 VALUES IN (("a025", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00002_9990 VALUES IN (("t00002", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00042_9990 VALUES IN (("ta00042", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00150_9990 VALUES IN (("t00150", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00045_9990 VALUES IN (("ta00045", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00043_9990 VALUES IN (("ta00043", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa024_9990 VALUES IN (("a024", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00040_9990 VALUES IN (("ta00040", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00028_9990 VALUES IN (("ta00028", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00027_9990 VALUES IN (("ta00027", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00154_9990 VALUES IN (("t00154", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00147_9990 VALUES IN (("t00147", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00144_9990 VALUES IN (("t00144", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00017_9990 VALUES IN (("ta00017", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00016_9990 VALUES IN (("ta00016", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00015_9990 VALUES IN (("ta00015", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00148_9990 VALUES IN (("t00148", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00149_9990 VALUES IN (("t00149", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00069_9990 VALUES IN (("t00069", "9990"));
-- result:
-- !result
ALTER TABLE test_replace_tmp_partition REPLACE PARTITION (pa019_9990,pa025_9990,pt00002_9990,pta00042_9990,pt00150_9990,pta00045_9990,pta00043_9990,pa024_9990,pta00040_9990,pta00028_9990,pta00027_9990,pt00154_9990,pt00147_9990,pt00144_9990,pta00017_9990,pta00016_9990,pta00015_9990,pt00148_9990,pt00149_9990,pt00069_9990) WITH TEMPORARY PARTITION (t_pa019_9990,t_pa025_9990,t_pt00002_9990,t_pta00042_9990,t_pt00150_9990,t_pta00045_9990,t_pta00043_9990,t_pa024_9990,t_pta00040_9990,t_pta00028_9990,t_pta00027_9990,t_pt00154_9990,t_pt00147_9990,t_pt00144_9990,t_pta00017_9990,t_pta00016_9990,t_pta00015_9990,t_pt00148_9990,t_pt00149_9990,t_pt00069_9990) PROPERTIES ("strict_range "="fasle" , "use_temp_partition_name"="false");
-- result:
-- !result
55 changes: 55 additions & 0 deletions test/sql/test_automatic_partition/T/test_replace_temp_partition
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
-- name: test_replace_temp_partition

CREATE TABLE `test_replace_tmp_partition` (
`a` varchar(64) NULL COMMENT "",
`b` varchar(10) NOT NULL COMMENT "",
`c` bigint(20) NOT NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`a`)
PARTITION BY (`b`,`c`)
DISTRIBUTED BY RANDOM;

ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_1970 VALUES IN (("a025", "1970"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa019_9990 VALUES IN (("a019", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa025_9990 VALUES IN (("a025", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00002_9990 VALUES IN (("t00002", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00042_9990 VALUES IN (("ta00042", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00150_9990 VALUES IN (("t00150", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00045_9990 VALUES IN (("ta00045", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00043_9990 VALUES IN (("ta00043", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pa024_9990 VALUES IN (("a024", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00040_9990 VALUES IN (("ta00040", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00028_9990 VALUES IN (("ta00028", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00027_9990 VALUES IN (("ta00027", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00154_9990 VALUES IN (("t00154", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00147_9990 VALUES IN (("t00147", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00144_9990 VALUES IN (("t00144", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00017_9990 VALUES IN (("ta00017", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00016_9990 VALUES IN (("ta00016", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pta00015_9990 VALUES IN (("ta00015", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00148_9990 VALUES IN (("t00148", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00149_9990 VALUES IN (("t00149", "9990"));
ALTER TABLE test_replace_tmp_partition ADD PARTITION IF NOT EXISTS pt00069_9990 VALUES IN (("t00069", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa019_9990 VALUES IN (("a019", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa025_9990 VALUES IN (("a025", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00002_9990 VALUES IN (("t00002", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00042_9990 VALUES IN (("ta00042", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00150_9990 VALUES IN (("t00150", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00045_9990 VALUES IN (("ta00045", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00043_9990 VALUES IN (("ta00043", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pa024_9990 VALUES IN (("a024", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00040_9990 VALUES IN (("ta00040", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00028_9990 VALUES IN (("ta00028", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00027_9990 VALUES IN (("ta00027", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00154_9990 VALUES IN (("t00154", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00147_9990 VALUES IN (("t00147", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00144_9990 VALUES IN (("t00144", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00017_9990 VALUES IN (("ta00017", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00016_9990 VALUES IN (("ta00016", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pta00015_9990 VALUES IN (("ta00015", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00148_9990 VALUES IN (("t00148", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00149_9990 VALUES IN (("t00149", "9990"));
ALTER TABLE test_replace_tmp_partition ADD TEMPORARY PARTITION IF NOT EXISTS t_pt00069_9990 VALUES IN (("t00069", "9990"));

ALTER TABLE test_replace_tmp_partition REPLACE PARTITION (pa019_9990,pa025_9990,pt00002_9990,pta00042_9990,pt00150_9990,pta00045_9990,pta00043_9990,pa024_9990,pta00040_9990,pta00028_9990,pta00027_9990,pt00154_9990,pt00147_9990,pt00144_9990,pta00017_9990,pta00016_9990,pta00015_9990,pt00148_9990,pt00149_9990,pt00069_9990) WITH TEMPORARY PARTITION (t_pa019_9990,t_pa025_9990,t_pt00002_9990,t_pta00042_9990,t_pt00150_9990,t_pta00045_9990,t_pta00043_9990,t_pa024_9990,t_pta00040_9990,t_pta00028_9990,t_pta00027_9990,t_pt00154_9990,t_pt00147_9990,t_pt00144_9990,t_pta00017_9990,t_pta00016_9990,t_pta00015_9990,t_pt00148_9990,t_pt00149_9990,t_pt00069_9990) PROPERTIES ("strict_range "="fasle" , "use_temp_partition_name"="false");

0 comments on commit d16c95f

Please sign in to comment.