From b174cba61b4618baf9014fa4eb6b708fce1e4a30 Mon Sep 17 00:00:00 2001 From: srlch Date: Thu, 14 Nov 2024 18:57:02 +0800 Subject: [PATCH 1/6] [Feature] Support Backup/Restore for external catalog Signed-off-by: srlch --- .../com/starrocks/backup/BackupHandler.java | 137 ++++++++++++++---- .../java/com/starrocks/backup/BackupJob.java | 15 ++ .../java/com/starrocks/backup/BackupMeta.java | 11 ++ .../java/com/starrocks/backup/RestoreJob.java | 38 ++++- .../java/com/starrocks/catalog/Catalog.java | 4 + .../java/com/starrocks/qe/ShowExecutor.java | 18 +++ .../java/com/starrocks/server/CatalogMgr.java | 15 ++ .../sql/analyzer/AuthorizerStmtVisitor.java | 83 ++++++++--- .../sql/analyzer/BackupRestoreAnalyzer.java | 35 ++++- .../starrocks/sql/ast/AbstractBackupStmt.java | 25 +++- .../com/starrocks/sql/ast/BackupStmt.java | 14 +- .../starrocks/sql/ast/CancelBackupStmt.java | 15 +- .../com/starrocks/sql/ast/CatalogRef.java | 80 ++++++++++ .../com/starrocks/sql/ast/RestoreStmt.java | 10 +- .../starrocks/sql/common/ParserErrorMsg.java | 9 ++ .../com/starrocks/sql/parser/AstBuilder.java | 60 +++++++- .../com/starrocks/sql/parser/StarRocks.g4 | 15 +- .../starrocks/backup/BackupHandlerTest.java | 14 +- .../analyzer/AnalyzeBackupRestoreTest.java | 44 ++++++ 19 files changed, 560 insertions(+), 82 deletions(-) create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/ast/CatalogRef.java diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java index e0f17b66929ab..87e46d7d333be 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java @@ -46,8 +46,10 @@ import com.starrocks.backup.BackupJob.BackupJobState; import com.starrocks.backup.BackupJobInfo.BackupTableInfo; import com.starrocks.backup.mv.MvRestoreContext; +import com.starrocks.catalog.Catalog; import com.starrocks.catalog.Database; import com.starrocks.catalog.Function; +import com.starrocks.catalog.InternalCatalog; import com.starrocks.catalog.MaterializedIndex.IndexExtState; import com.starrocks.catalog.MaterializedView; import com.starrocks.catalog.OlapTable; @@ -74,6 +76,7 @@ import com.starrocks.sql.ast.BackupStmt; import com.starrocks.sql.ast.BackupStmt.BackupType; import com.starrocks.sql.ast.CancelBackupStmt; +import com.starrocks.sql.ast.CatalogRef; import com.starrocks.sql.ast.CreateRepositoryStmt; import com.starrocks.sql.ast.DropRepositoryStmt; import com.starrocks.sql.ast.FunctionRef; @@ -98,6 +101,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @@ -124,6 +128,8 @@ public class BackupHandler extends FrontendDaemon implements Writable, MemoryTra // If the last job is finished, user can get the job info from repository. If the last job is cancelled, // user can get the error message before submitting the next one. // Use ConcurrentMap to get rid of locks. + // Backup/Restore job for external catalog using -1 to identify the job in dbIdToBackupOrRestoreJob + // which means that only one external catalog backup/restore job can be run in entire cluster protected Map dbIdToBackupOrRestoreJob = Maps.newConcurrentMap(); protected MvRestoreContext mvRestoreContext = new MvRestoreContext(); @@ -261,11 +267,12 @@ public void process(AbstractBackupStmt stmt) throws DdlException { // check if repo exist String repoName = stmt.getRepoName(); Repository repository = repoMgr.getRepo(repoName); + Database db = null; + BackupJobInfo jobInfo = null; if (repository == null) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Repository " + repoName + " does not exist"); } - BackupJobInfo jobInfo = null; if (stmt instanceof RestoreStmt) { // Check if snapshot exist in repository, if existed, get jobInfo for restore process List infos = Lists.newArrayList(); @@ -277,30 +284,49 @@ public void process(AbstractBackupStmt stmt) throws DdlException { } Preconditions.checkState(infos.size() == 1); jobInfo = infos.get(0); - } - // check if db exist - String dbName = stmt.getDbName(); - if (dbName == null) { - // if target dbName if null, use dbName in snapshot - dbName = jobInfo.dbName; - } + if (jobInfo.dbName == null || jobInfo.dbName.isEmpty()) { + // if jobInfo.dbName == null, means that this snapshot only contains external catalog info + if ((stmt.getOriginDbName() != null && !stmt.getOriginDbName().isEmpty()) || + stmt.getDbName() != null && !stmt.getDbName().isEmpty()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Can not specify database for external catalog snapshot"); + } - Database db = globalStateMgr.getLocalMetastore().getDb(dbName); - if (db == null) { - if (stmt instanceof RestoreStmt) { - try { - globalStateMgr.getLocalMetastore().createDb(dbName, null); - db = globalStateMgr.getLocalMetastore().getDb(dbName); - if (db == null) { - ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + if (!stmt.containsExternalCatalog()) { + // set `ALL` flag for external catalog restore if no `CATALOG(s)` set in restore stmt + stmt.setAllExternalCatalog(); + } + } else if (stmt.containsExternalCatalog()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "This is not a snapshot for external catalog restore, snapshot: " + stmt.getLabel()); + } + } + + if (!stmt.containsExternalCatalog()) { + // check if db exist + String dbName = stmt.getDbName(); + if (dbName == null) { + // if target dbName if null, use dbName in snapshot + dbName = jobInfo.dbName; + } + + db = globalStateMgr.getLocalMetastore().getDb(dbName); + if (db == null) { + if (stmt instanceof RestoreStmt) { + try { + globalStateMgr.getLocalMetastore().createDb(dbName, null); + db = globalStateMgr.getLocalMetastore().getDb(dbName); + if (db == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + } + } catch (Exception e) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Can not create database: " + dbName + " in restore process"); } - } catch (Exception e) { - ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Can not create database: " + dbName + " in restore process"); + } else { + ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } - } else { - ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } } @@ -311,7 +337,7 @@ public void process(AbstractBackupStmt stmt) throws DdlException { tryLock(); try { // Check if there is backup or restore job running on this database - AbstractJob currentJob = dbIdToBackupOrRestoreJob.get(db.getId()); + AbstractJob currentJob = dbIdToBackupOrRestoreJob.get(stmt.containsExternalCatalog() ? -1 : db.getId()); if (currentJob != null && !currentJob.isDone()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Can only run one backup or restore job of a database at same time"); @@ -351,7 +377,9 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws List tblRefs = stmt.getTableRefs(); BackupMeta curBackupMeta = null; Locker locker = new Locker(); - locker.lockDatabase(db.getId(), LockType.READ); + if (!stmt.containsExternalCatalog()) { + locker.lockDatabase(db.getId(), LockType.READ); + } try { List backupTbls = Lists.newArrayList(); for (TableRef tblRef : tblRefs) { @@ -410,7 +438,9 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws } curBackupMeta = new BackupMeta(backupTbls); } finally { - locker.unLockDatabase(db.getId(), LockType.READ); + if (!stmt.containsExternalCatalog()) { + locker.unLockDatabase(db.getId(), LockType.READ); + } } // Check if label already be used @@ -448,31 +478,36 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws } // Create a backup job - BackupJob backupJob = new BackupJob(stmt.getLabel(), db.getId(), db.getOriginName(), tblRefs, + long dbId = stmt.containsExternalCatalog() ? -1 : db.getId(); + String dbName = stmt.containsExternalCatalog() ? "" : db.getOriginName(); + + BackupJob backupJob = new BackupJob(stmt.getLabel(), dbId, dbName, tblRefs, stmt.getTimeoutMs(), globalStateMgr, repository.getId()); List allFunctions = Lists.newArrayList(); for (FunctionRef fnRef : stmt.getFnRefs()) { allFunctions.addAll(fnRef.getFunctions()); } backupJob.setBackupFunctions(allFunctions); + backupJob.setBackupCatalogs(stmt.getExternalCatalogRefs().stream() + .map(CatalogRef::getCatalog).collect(Collectors.toList())); // write log globalStateMgr.getEditLog().logBackupJob(backupJob); // must put to dbIdToBackupOrRestoreJob after edit log, otherwise the state of job may be changed. - dbIdToBackupOrRestoreJob.put(db.getId(), backupJob); + dbIdToBackupOrRestoreJob.put(dbId, backupJob); LOG.info("finished to submit backup job: {}", backupJob); } private void restore(Repository repository, Database db, RestoreStmt stmt, BackupJobInfo jobInfo) throws DdlException { + BackupMeta backupMeta = downloadAndDeserializeMetaInfo(jobInfo, repository, stmt); + // check the original dbName existed in snapshot or not if (!stmt.getOriginDbName().isEmpty() && !stmt.getOriginDbName().equals(jobInfo.dbName)) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "target database: " + stmt.getOriginDbName() + " is not existed in snapshot"); } - BackupMeta backupMeta = downloadAndDeserializeMetaInfo(jobInfo, repository, stmt); - // If restore statement contains `ON` clause, filter the specified backup objects which are needed through infomation // provide in stmt and BackupMeta. if (stmt.withOnClause()) { @@ -485,6 +520,10 @@ private void restore(Repository repository, Database db, RestoreStmt stmt, Backu checkAndFilterRestoreFunctionsInBackupMeta(stmt, backupMeta); } + if (stmt.containsExternalCatalog()) { + checkAndFilterRestoreCatalogsInBackupMeta(stmt, backupMeta); + } + // Create a restore job RestoreJob restoreJob = null; if (backupMeta != null) { @@ -498,13 +537,17 @@ private void restore(Repository repository, Database db, RestoreStmt stmt, Backu mvRestoreContext.addIntoMvBaseTableBackupInfoIfNeeded(db.getOriginName(), remoteTbl, jobInfo, tblInfo); } } + + long dbId = stmt.containsExternalCatalog() ? -1 : db.getId(); + String dbName = stmt.containsExternalCatalog() ? "" : db.getOriginName(); + restoreJob = new RestoreJob(stmt.getLabel(), stmt.getBackupTimestamp(), - db.getId(), db.getOriginName(), jobInfo, stmt.allowLoad(), stmt.getReplicationNum(), + dbId, dbName, jobInfo, stmt.allowLoad(), stmt.getReplicationNum(), stmt.getTimeoutMs(), globalStateMgr, repository.getId(), backupMeta, mvRestoreContext); globalStateMgr.getEditLog().logRestoreJob(restoreJob); // must put to dbIdToBackupOrRestoreJob after edit log, otherwise the state of job may be changed. - dbIdToBackupOrRestoreJob.put(db.getId(), restoreJob); + dbIdToBackupOrRestoreJob.put(dbId, restoreJob); LOG.info("finished to submit restore job: {}", restoreJob); } @@ -528,6 +571,33 @@ protected BackupMeta downloadAndDeserializeMetaInfo(BackupJobInfo jobInfo, Repos return backupMetas.get(0); } + protected void checkAndFilterRestoreCatalogsInBackupMeta(RestoreStmt stmt, BackupMeta backupMeta) throws DdlException { + List catalogsInBackupMeta = backupMeta.getCatalogs(); + List restoredCatalogs = Lists.newArrayList(); + for (CatalogRef catalogRef : stmt.getExternalCatalogRefs()) { + Optional hitCatalog = catalogsInBackupMeta.stream().filter(x -> catalogRef.getCatalogName() + .equalsIgnoreCase(x.getName())).findFirst(); + if (!hitCatalog.isPresent()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Can not find restore catalog: " + catalogRef.getCatalogName()); + } + + if (catalogRef.getAlias() != null && !catalogRef.getAlias().isEmpty()) { + if (catalogRef.getAlias().equalsIgnoreCase(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME)) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Do not support set alias as default catalog for external catalog restore"); + } + hitCatalog.get().setName(catalogRef.getAlias()); + } + + restoredCatalogs.add(hitCatalog.get()); + } + + if (!restoredCatalogs.isEmpty()) { + backupMeta.setCatalogs(restoredCatalogs); + } + } + protected void checkAndFilterRestoreFunctionsInBackupMeta(RestoreStmt stmt, BackupMeta backupMeta) throws DdlException { List functionsInBackupMeta = backupMeta.getFunctions(); List restoredFunctions = Lists.newArrayList(); @@ -630,7 +700,12 @@ public AbstractJob getAbstractJobByDbName(String dbName) throws DdlException { } public void cancel(CancelBackupStmt stmt) throws DdlException { - AbstractJob job = getAbstractJobByDbName(stmt.getDbName()); + AbstractJob job = null; + if (!stmt.isExternalCatalog()) { + job = getAbstractJobByDbName(stmt.getDbName()); + } else { + job = dbIdToBackupOrRestoreJob.get(-1L); + } if (job == null || (job instanceof BackupJob && stmt.isRestore()) || (job instanceof RestoreJob && !stmt.isRestore())) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "No " diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java index 622badc7a6dff..7b2886e269fba 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java @@ -46,6 +46,7 @@ import com.starrocks.analysis.BrokerDesc; import com.starrocks.analysis.TableRef; import com.starrocks.backup.Status.ErrCode; +import com.starrocks.catalog.Catalog; import com.starrocks.catalog.Database; import com.starrocks.catalog.FsBroker; import com.starrocks.catalog.Function; @@ -158,6 +159,8 @@ public enum BackupJobState { @SerializedName(value = "backupFunctions") private List backupFunctions = Lists.newArrayList(); + @SerializedName(value = "backupCatalogs") + private List backupCatalogs = Lists.newArrayList(); public BackupJob() { super(JobType.BACKUP); @@ -206,6 +209,10 @@ public void setBackupFunctions(List functions) { this.backupFunctions = functions; } + public void setBackupCatalogs(List backupCatalogs) { + this.backupCatalogs = backupCatalogs; + } + public synchronized boolean finishTabletSnapshotTask(SnapshotTask task, TFinishTaskRequest request) { Preconditions.checkState(task.getJobId() == jobId); @@ -463,6 +470,14 @@ protected void sendSnapshotRequests() { private void prepareAndSendSnapshotTask() { MetricRepo.COUNTER_UNFINISHED_BACKUP_JOB.increase(1L); + if (!backupCatalogs.isEmpty()) { + // short cut for external catalogs backup + backupMeta = new BackupMeta(Lists.newArrayList()); + backupMeta.setCatalogs(backupCatalogs); + state = BackupJobState.SAVE_META; + + return; + } Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " does not exist"); diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupMeta.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupMeta.java index 4c7b60c27a322..3805b01ccc0f5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupMeta.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupMeta.java @@ -37,6 +37,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; +import com.starrocks.catalog.Catalog; import com.starrocks.catalog.Function; import com.starrocks.catalog.Table; import com.starrocks.common.io.Text; @@ -67,6 +68,8 @@ public class BackupMeta implements Writable, GsonPostProcessable { private Map tblIdMap = Maps.newHashMap(); @SerializedName(value = "functions") private List functions = Lists.newArrayList(); + @SerializedName(value = "catalogs") + private List catalogs = Lists.newArrayList(); private BackupMeta() { @@ -99,6 +102,14 @@ public List getFunctions() { return functions; } + public void setCatalogs(List catalogs) { + this.catalogs = catalogs; + } + + public List getCatalogs() { + return catalogs; + } + public static BackupMeta fromFile(String filePath, int starrocksMetaVersion) throws IOException { File file = new File(filePath); try (DataInputStream dis = new DataInputStream(new FileInputStream(file))) { diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java index 77d2b8b8eb676..a17752261f8a7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java @@ -55,6 +55,7 @@ import com.starrocks.backup.RestoreFileMapping.IdChain; import com.starrocks.backup.Status.ErrCode; import com.starrocks.backup.mv.MvRestoreContext; +import com.starrocks.catalog.Catalog; import com.starrocks.catalog.ColocateTableIndex; import com.starrocks.catalog.Column; import com.starrocks.catalog.ColumnId; @@ -80,6 +81,7 @@ import com.starrocks.catalog.Tablet; import com.starrocks.catalog.TabletMeta; import com.starrocks.common.Config; +import com.starrocks.common.DdlException; import com.starrocks.common.Pair; import com.starrocks.common.UserException; import com.starrocks.common.io.Text; @@ -410,7 +412,8 @@ public void run() { * return true if some restored objs have been dropped. */ private void checkIfNeedCancel() { - if (state == RestoreJobState.PENDING) { + if (state == RestoreJobState.PENDING || + (backupMeta != null && !backupMeta.getCatalogs().isEmpty())) { return; } @@ -470,6 +473,22 @@ private void checkIfNeedCancel() { */ private void checkAndPrepareMeta() { MetricRepo.COUNTER_UNFINISHED_RESTORE_JOB.increase(1L); + if (backupMeta != null && !backupMeta.getCatalogs().isEmpty()) { + // short cut for external catalog restore + try { + for (Catalog catalog : backupMeta.getCatalogs()) { + globalStateMgr.getCatalogMgr().createCatalogForRestore(catalog); + } + } catch (DdlException e) { + status = new Status(ErrCode.COMMON_ERROR, + "Failed to restore external catalog, errmsg: " + e.getMessage()); + return; + } + state = RestoreJobState.COMMITTING; + globalStateMgr.getEditLog().logRestoreJob(this); + return; + } + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " does not exist"); @@ -1469,6 +1488,17 @@ protected void waitingAllTabletsCommitted() { } private Status allTabletCommitted(boolean isReplay) { + if (backupMeta != null && !backupMeta.getCatalogs().isEmpty()) { + if (!isReplay) { + finishedTime = System.currentTimeMillis(); + state = RestoreJobState.FINISHED; + + globalStateMgr.getEditLog().logRestoreJob(this); + } + LOG.info("job is finished. is replay: {}. {}", isReplay, this); + return Status.OK; + } + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { return new Status(ErrCode.NOT_FOUND, "database " + dbId + " does not exist"); @@ -1742,6 +1772,12 @@ public void cancelInternal(boolean isReplay) { } } + if (backupMeta != null && !backupMeta.getCatalogs().isEmpty()) { + for (Catalog catalog : backupMeta.getCatalogs()) { + globalStateMgr.getCatalogMgr().dropCatalogForRestore(catalog, isReplay); + } + } + for (ColocatePersistInfo colocatePersistInfo : colocatePersistInfos) { for (Table restoreTbl : restoredTbls) { if (restoreTbl instanceof OlapTable && restoreTbl.getId() == colocatePersistInfo.getTableId()) { diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Catalog.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Catalog.java index a172bc8575ab7..cc04ae7dff0b1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Catalog.java @@ -70,6 +70,10 @@ public String getName() { return name; } + public void setName(String name) { + this.name = name; + } + public String getType() { return config.get(CATALOG_TYPE); } diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java index 227a3235e6908..ae20b132227c2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java @@ -1812,6 +1812,15 @@ public ShowResultSet visitShowBackupStatement(ShowBackupStmt statement, ConnectC List info = backupJob.getInfo(); infos.add(info); } + + // backup info for external catalog + AbstractJob jobI = GlobalStateMgr.getCurrentState().getBackupHandler().getJob(-1L); + if (jobI != null && jobI instanceof BackupJob) { + BackupJob backupJob = (BackupJob) jobI; + List info = backupJob.getInfo(); + infos.add(info); + } + return new ShowResultSet(statement.getMetaData(), infos); } @@ -1841,6 +1850,15 @@ public ShowResultSet visitShowRestoreStatement(ShowRestoreStmt statement, Connec List info = restoreJob.getInfo(); infos.add(info); } + + // restore info for external catalog + AbstractJob jobI = GlobalStateMgr.getCurrentState().getBackupHandler().getJob(-1L); + if (jobI != null && jobI instanceof RestoreJob) { + RestoreJob restoreJob = (RestoreJob) jobI; + List info = restoreJob.getInfo(); + infos.add(info); + } + return new ShowResultSet(statement.getMetaData(), infos); } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/CatalogMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/CatalogMgr.java index d17da4b924405..32e88ee64bd2a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/CatalogMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/CatalogMgr.java @@ -95,6 +95,11 @@ public void createCatalog(CreateCatalogStmt stmt) throws DdlException { createCatalog(stmt.getCatalogType(), stmt.getCatalogName(), stmt.getComment(), stmt.getProperties()); } + public void createCatalogForRestore(Catalog catalog) throws DdlException { + dropCatalogForRestore(catalog, false); + createCatalog(catalog.getType(), catalog.getName(), catalog.getComment(), catalog.getConfig()); + } + // please keep connector and catalog create together, they need keep in consistent asap. public void createCatalog(String type, String catalogName, String comment, Map properties) throws DdlException { @@ -155,6 +160,16 @@ public void createCatalog(String type, String catalogName, String comment, Map tableRefs = statement.getTableRefs(); - if (statement.withOnClause() && tableRefs.isEmpty() && statement.getFnRefs().isEmpty()) { - String dBName = statement.getDbName(); - throw new SemanticException("Database: %s is empty", dBName); - } - tableRefs.forEach(tableRef -> { - TableName tableName = tableRef.getName(); - try { - Authorizer.checkTableAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), tableName, - PrivilegeType.EXPORT); - } catch (AccessDeniedException e) { - AccessDeniedException.reportAccessDenied( - tableName.getCatalog(), - context.getCurrentUserIdentity(), context.getCurrentRoleIds(), - PrivilegeType.EXPORT.name(), ObjectType.TABLE.name(), tableName.getTbl()); + if (!statement.containsExternalCatalog()) { + List tableRefs = statement.getTableRefs(); + List functionRefs = statement.getFnRefs(); + if (tableRefs.isEmpty() && functionRefs.isEmpty()) { + String dBName = statement.getDbName(); + throw new SemanticException("Database: %s is empty", dBName); } - }); + + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(statement.getDbName()); + tableRefs.forEach(tableRef -> { + TableName tableName = tableRef.getName(); + try { + Authorizer.checkTableAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), tableName, + PrivilegeType.EXPORT); + } catch (AccessDeniedException e) { + AccessDeniedException.reportAccessDenied( + tableName.getCatalog(), + context.getCurrentUserIdentity(), context.getCurrentRoleIds(), + PrivilegeType.EXPORT.name(), ObjectType.TABLE.name(), tableName.getTbl()); + } + }); + + functionRefs.forEach(functionRef -> { + List fns = functionRef.getFunctions(); + for (Function fn : fns) { + try { + Authorizer.checkFunctionAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), + db, fn, PrivilegeType.EXPORT); + } catch (AccessDeniedException e) { + AccessDeniedException.reportAccessDenied( + InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, + context.getCurrentUserIdentity(), context.getCurrentRoleIds(), + PrivilegeType.DROP.name(), ObjectType.FUNCTION.name(), fn.getSignature()); + } + } + }); + } else { + List externalCatalogs = statement.getExternalCatalogRefs(); + externalCatalogs.forEach(externalCatalog -> { + try { + Authorizer.checkCatalogAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), + externalCatalog.getCatalogName(), PrivilegeType.CREATE_DATABASE); + } catch (AccessDeniedException e) { + AccessDeniedException.reportAccessDenied( + externalCatalog.getCatalogName(), + context.getCurrentUserIdentity(), context.getCurrentRoleIds(), + PrivilegeType.CREATE_DATABASE.name(), ObjectType.CATALOG.name(), externalCatalog.getCatalogName()); + } + }); + } + return null; } @@ -2198,9 +2234,22 @@ public Void visitRestoreStatement(RestoreStmt statement, ConnectContext context) PrivilegeType.REPOSITORY.name(), ObjectType.SYSTEM.name(), null); } + if (statement.containsExternalCatalog()) { + try { + Authorizer.checkSystemAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), + PrivilegeType.CREATE_EXTERNAL_CATALOG); + } catch (AccessDeniedException e) { + AccessDeniedException.reportAccessDenied( + InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, + context.getCurrentUserIdentity(), context.getCurrentRoleIds(), + PrivilegeType.CREATE_EXTERNAL_CATALOG.name(), ObjectType.SYSTEM.name(), null); + } + return null; + } + List tableRefs = statement.getTableRefs(); // check create_database on current catalog if we're going to restore the whole database - if (!statement.withOnClause() && (tableRefs == null || tableRefs.isEmpty())) { + if (!statement.withOnClause()) { try { Authorizer.checkCatalogAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), context.getCurrentCatalog(), PrivilegeType.CREATE_DATABASE); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/BackupRestoreAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/BackupRestoreAnalyzer.java index e01bf889a6bc0..2eb0ce081506a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/BackupRestoreAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/BackupRestoreAnalyzer.java @@ -23,6 +23,7 @@ import com.starrocks.catalog.BaseTableInfo; import com.starrocks.catalog.Database; import com.starrocks.catalog.Function; +import com.starrocks.catalog.InternalCatalog; import com.starrocks.catalog.MaterializedView; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Partition; @@ -36,6 +37,7 @@ import com.starrocks.sql.ast.AstVisitor; import com.starrocks.sql.ast.BackupStmt; import com.starrocks.sql.ast.CancelBackupStmt; +import com.starrocks.sql.ast.CatalogRef; import com.starrocks.sql.ast.FunctionRef; import com.starrocks.sql.ast.PartitionNames; import com.starrocks.sql.ast.RestoreStmt; @@ -52,6 +54,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; public class BackupRestoreAnalyzer { private static final Logger LOG = LogManager.getLogger(BackupRestoreAnalyzer.class); @@ -77,6 +80,27 @@ public void analyze(StatementBase statement, ConnectContext session) { @Override public Void visitBackupStatement(BackupStmt backupStmt, ConnectContext context) { + if (backupStmt.containsExternalCatalog()) { + analyzeBackupProperties(backupStmt); + + if (backupStmt.allExternalCatalog()) { + backupStmt.getExternalCatalogRefs().clear(); + // get all catalog from CatalogMgr + backupStmt.getExternalCatalogRefs().addAll( + GlobalStateMgr.getCurrentState().getCatalogMgr().getCatalogs().keySet() + .stream().filter(x -> !x.equalsIgnoreCase(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME)) + .map(x -> new CatalogRef(x)).collect(Collectors.toList())); + } + + if (backupStmt.getExternalCatalogRefs().isEmpty()) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "No external catalog can be backed up"); + } + + backupStmt.getExternalCatalogRefs().stream().forEach(x -> x.analyzeForBackup()); + return null; + } + String dbName = getDbName(backupStmt.getDbName(), context); Database database = getDatabase(dbName, context); analyzeLabelAndRepo(backupStmt.getLabel(), backupStmt.getRepoName()); @@ -176,6 +200,12 @@ public Void visitBackupStatement(BackupStmt backupStmt, ConnectContext context) backupStmt.getFnRefs().stream().forEach(x -> x.analyzeForBackup(database)); } + analyzeBackupProperties(backupStmt); + + return null; + } + + private void analyzeBackupProperties(BackupStmt backupStmt) { Map properties = backupStmt.getProperties(); long timeoutMs = Config.backup_job_default_timeout_ms; Iterator> iterator = properties.entrySet().iterator(); @@ -211,8 +241,6 @@ public Void visitBackupStatement(BackupStmt backupStmt, ConnectContext context) ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, "Unknown backup job properties: " + copiedProperties.keySet()); } - - return null; } private Map reorderTableRefsWithMaterializedView(Database database, @@ -277,6 +305,9 @@ private void collectTableRefAndDependencies(Database database, @Override public Void visitCancelBackupStatement(CancelBackupStmt cancelBackupStmt, ConnectContext context) { + if (cancelBackupStmt.isExternalCatalog()) { + return null; + } String dbName = getDbName(cancelBackupStmt.getDbName(), context); cancelBackupStmt.setDbName(dbName); return null; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AbstractBackupStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AbstractBackupStmt.java index 32921453ddc7b..18e87cf7c9d44 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AbstractBackupStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AbstractBackupStmt.java @@ -20,6 +20,7 @@ import com.google.common.collect.Sets; import com.starrocks.analysis.LabelName; import com.starrocks.analysis.TableRef; +import com.starrocks.sql.ast.CatalogRef; import com.starrocks.sql.ast.FunctionRef; import com.starrocks.sql.parser.NodePosition; @@ -33,12 +34,14 @@ public enum BackupObjectType { MV, VIEW, FUNCTION, + EXTERNAL_CATALOG, } protected LabelName labelName; protected String repoName; protected List tblRefs; protected List fnRefs; + protected List externalCatalogRefs; protected Set allMarker; @@ -53,7 +56,7 @@ public enum BackupObjectType { protected long timeoutMs; public AbstractBackupStmt(LabelName labelName, String repoName, List tableRefs, - List fnRefs, Set allMarker, + List fnRefs, List externalCatalogRefs, Set allMarker, boolean withOnClause, String originDbName, Map properties, NodePosition pos) { super(pos); this.labelName = labelName; @@ -66,6 +69,10 @@ public AbstractBackupStmt(LabelName labelName, String repoName, List t if (this.fnRefs == null) { this.fnRefs = Lists.newArrayList(); } + this.externalCatalogRefs = externalCatalogRefs; + if (this.externalCatalogRefs == null) { + this.externalCatalogRefs = Lists.newArrayList(); + } this.allMarker = allMarker; if (this.allMarker == null) { this.allMarker = Sets.newHashSet(); @@ -100,6 +107,10 @@ public List getFnRefs() { return fnRefs; } + public List getExternalCatalogRefs() { + return externalCatalogRefs; + } + public Map getProperties() { return properties; } @@ -124,6 +135,14 @@ public boolean allView() { return allMarker.contains(BackupObjectType.VIEW); } + public boolean allExternalCatalog() { + return allMarker.contains(BackupObjectType.EXTERNAL_CATALOG); + } + + public void setAllExternalCatalog() { + allMarker.add(BackupObjectType.EXTERNAL_CATALOG); + } + public long getTimeoutMs() { return timeoutMs; } @@ -131,5 +150,9 @@ public long getTimeoutMs() { public String getOriginDbName() { return this.originDbName; } + + public boolean containsExternalCatalog() { + return allExternalCatalog() || !externalCatalogRefs.isEmpty(); + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/BackupStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/BackupStmt.java index 4ebeffaaf0beb..678f6013851ea 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/BackupStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/BackupStmt.java @@ -32,15 +32,17 @@ public enum BackupType { private BackupType type = BackupType.FULL; public BackupStmt(LabelName labelName, String repoName, List tblRefs, List fnRefs, - Set allMarker, boolean withOnClause, String originDbName, - Map properties) { - super(labelName, repoName, tblRefs, fnRefs, allMarker, withOnClause, originDbName, properties, NodePosition.ZERO); + List externalCatalogRefs, Set allMarker, + boolean withOnClause, String originDbName, Map properties) { + super(labelName, repoName, tblRefs, fnRefs, externalCatalogRefs, + allMarker, withOnClause, originDbName, properties, NodePosition.ZERO); } public BackupStmt(LabelName labelName, String repoName, List tblRefs, List fnRefs, - Set allMarker, boolean withOnClause, String originDbName, - Map properties, NodePosition pos) { - super(labelName, repoName, tblRefs, fnRefs, allMarker, withOnClause, originDbName, properties, pos); + List externalCatalogRefs, Set allMarker, + boolean withOnClause, String originDbName, Map properties, NodePosition pos) { + super(labelName, repoName, tblRefs, fnRefs, externalCatalogRefs, + allMarker, withOnClause, originDbName, properties, pos); } public long getTimeoutMs() { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CancelBackupStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CancelBackupStmt.java index 6361575ca3a51..c70bd2fd17c79 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CancelBackupStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CancelBackupStmt.java @@ -21,15 +21,22 @@ public class CancelBackupStmt extends CancelStmt { private String dbName; private final boolean isRestore; + private final boolean isExternalCatalog; public CancelBackupStmt(String dbName, boolean isRestore) { - this(dbName, isRestore, NodePosition.ZERO); + this(dbName, isRestore, false, NodePosition.ZERO); } - public CancelBackupStmt(String dbName, boolean isRestore, NodePosition pos) { + public CancelBackupStmt(String dbName, boolean isRestore, boolean isExternalCatalog) { + this(dbName, isRestore, isExternalCatalog, NodePosition.ZERO); + } + + public CancelBackupStmt(String dbName, boolean isRestore, boolean isExternalCatalog, + NodePosition pos) { super(pos); this.dbName = dbName; this.isRestore = isRestore; + this.isExternalCatalog = isExternalCatalog; } public String getDbName() { @@ -44,6 +51,10 @@ public boolean isRestore() { return isRestore; } + public boolean isExternalCatalog() { + return isExternalCatalog; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitCancelBackupStatement(this, context); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CatalogRef.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CatalogRef.java new file mode 100644 index 0000000000000..9ac59e6352cf8 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CatalogRef.java @@ -0,0 +1,80 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package com.starrocks.sql.ast; + +import com.google.common.base.Preconditions; +import com.starrocks.analysis.ParseNode; +import com.starrocks.catalog.Catalog; +import com.starrocks.catalog.InternalCatalog; +import com.starrocks.common.ErrorCode; +import com.starrocks.common.ErrorReport; +import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.parser.NodePosition; + +/** + * CatalogRef is used to represent Catalog used in Backup/Restore + */ +public class CatalogRef implements ParseNode { + private final NodePosition pos; + private String catalogName; + private String alias; + private Catalog catalog; + private boolean isAnalyzed; + + public CatalogRef(String catalogName) { + this(catalogName, ""); + } + + public CatalogRef(String catalogName, String alias) { + this.catalogName = catalogName; + this.alias = alias; + this.isAnalyzed = false; + this.pos = NodePosition.ZERO; + } + + public String getCatalogName() { + return this.catalogName; + } + + public String getAlias() { + return this.alias; + } + + public Catalog getCatalog() { + Preconditions.checkState(isAnalyzed); + return this.catalog; + } + + public void analyzeForBackup() { + if (catalogName.equalsIgnoreCase(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME)) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "Do not support Backup/Restore the entire default catalog"); + } + + if (!GlobalStateMgr.getCurrentState().getCatalogMgr().catalogExists(catalogName)) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "external catalog " + catalogName + " does not existed."); + } + + catalog = GlobalStateMgr.getCurrentState().getCatalogMgr().getCatalogByName(catalogName); + isAnalyzed = true; + } + + @Override + public NodePosition getPos() { + return pos; + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/RestoreStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/RestoreStmt.java index d833405a5566a..5163b2050ab09 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/RestoreStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/RestoreStmt.java @@ -33,16 +33,18 @@ public class RestoreStmt extends AbstractBackupStmt { private int starrocksMetaVersion = -1; public RestoreStmt(LabelName labelName, String repoName, List tblRefs, - List fnRefs, Set allMarker, + List fnRefs, List externalCatalogRefs, Set allMarker, boolean withOnClause, String originDbName, Map properties) { - this(labelName, repoName, tblRefs, fnRefs, allMarker, withOnClause, originDbName, properties, NodePosition.ZERO); + this(labelName, repoName, tblRefs, fnRefs, externalCatalogRefs, + allMarker, withOnClause, originDbName, properties, NodePosition.ZERO); } public RestoreStmt(LabelName labelName, String repoName, List tblRefs, - List fnRefs, Set allMarker, + List fnRefs, List externalCatalogRefs, Set allMarker, boolean withOnClause, String originDbName, Map properties, NodePosition pos) { - super(labelName, repoName, tblRefs, fnRefs, allMarker, withOnClause, originDbName, properties, pos); + super(labelName, repoName, tblRefs, fnRefs, externalCatalogRefs, + allMarker, withOnClause, originDbName, properties, pos); } public boolean allowLoad() { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/ParserErrorMsg.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/ParserErrorMsg.java index 06d2cc02689a0..39bdc8c5ec1ec 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/ParserErrorMsg.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/ParserErrorMsg.java @@ -231,4 +231,13 @@ public interface ParserErrorMsg { @BaseMessage("`ON` clause is forbidden if no Database explicitly specified in Restore stmt") String unsupportedOnClauseWithoutAnyDbNameInRestoreStmt(); + + @BaseMessage("Can not sepcify database name for external catalog Backup/Restore") + String unsupportedSepcifyDbForExternalCatalog(); + + @BaseMessage("Can not sepcify `ON` clause for external catalog Backup/Restore") + String unsupportedOnForExternalCatalog(); + + @BaseMessage("Can not sepcify database for cancel EXTERNAL CATALOG BACKUP/RESTORE") + String unsupportedSepcifyDatabaseForExternalCatalog(); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index 621104d3e197e..d2090d9f9ff51 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -187,6 +187,7 @@ import com.starrocks.sql.ast.CancelLoadStmt; import com.starrocks.sql.ast.CancelRefreshDictionaryStmt; import com.starrocks.sql.ast.CancelRefreshMaterializedViewStmt; +import com.starrocks.sql.ast.CatalogRef; import com.starrocks.sql.ast.CleanTabletSchedQClause; import com.starrocks.sql.ast.CleanTemporaryTableStmt; import com.starrocks.sql.ast.ClearDataCacheRulesStmt; @@ -3439,9 +3440,36 @@ private ParseNode parseBackupRestoreStatement(ParserRuleContext context) { backupContext = (StarRocksParser.BackupStatementContext) context; } + List externalCatalogRefs = new ArrayList<>(); + boolean allExternalCatalog = backupContext != null ? + (backupContext.CATALOGS() != null) : (restoreContext.CATALOGS() != null); + if (!allExternalCatalog && (backupContext != null ? + (backupContext.CATALOG() != null) : (restoreContext.CATALOG() != null))) { + if (backupContext != null) { + StarRocksParser.IdentifierListContext identifierListContext = backupContext.identifierList(); + externalCatalogRefs = visit(identifierListContext.identifier(), Identifier.class) + .stream().map(Identifier::getValue) + .map(x -> new CatalogRef(x)).collect(Collectors.toList()); + } else { + List identifierWithAliasList = + restoreContext.identifierWithAliasList().identifierWithAlias(); + for (StarRocksParser.IdentifierWithAliasContext identifierWithAliasContext : identifierWithAliasList) { + String originalName = getIdentifierName(identifierWithAliasContext.originalName); + String alias = identifierWithAliasContext.AS() != null ? + getIdentifierName(identifierWithAliasContext.alias) : ""; + externalCatalogRefs.add(new CatalogRef(originalName, alias)); + } + } + } + boolean containsExternalCatalog = allExternalCatalog || !externalCatalogRefs.isEmpty(); + boolean specifyDbExplicitly = backupContext != null ? (backupContext.DATABASE() != null) : (restoreContext.DATABASE() != null); + if (specifyDbExplicitly && containsExternalCatalog) { + throw new ParsingException(PARSER_ERROR_MSG.unsupportedSepcifyDbForExternalCatalog()); + } + LabelName labelName = null; String repoName = null; // db which the snapshot should be restored in @@ -3458,6 +3486,10 @@ private ParseNode parseBackupRestoreStatement(ParserRuleContext context) { List fnRefs = new ArrayList<>(); Set allMarker = Sets.newHashSet(); + if (allExternalCatalog) { + allMarker.add(BackupObjectType.EXTERNAL_CATALOG); + } + labelName = qualifiedNameToLabelName(getQualifiedName(backupContext != null ? backupContext.qualifiedName() : restoreContext.qualifiedName())); if (specifyDbExplicitly) { @@ -3471,6 +3503,8 @@ private ParseNode parseBackupRestoreStatement(ParserRuleContext context) { } labelName.setDbName(dbAlias != null ? dbAlias : originDb); + } else if (containsExternalCatalog && labelName.getDbName() != null) { + throw new ParsingException(PARSER_ERROR_MSG.unsupportedSepcifyDbForExternalCatalog()); } repoName = getIdentifierName(backupContext != null ? backupContext.repoName : restoreContext.repoName); @@ -3548,6 +3582,10 @@ private ParseNode parseBackupRestoreStatement(ParserRuleContext context) { throw new ParsingException(PARSER_ERROR_MSG.unsupportedOnClauseWithoutAnyDbNameInRestoreStmt()); } + if (withOnClause && containsExternalCatalog) { + throw new ParsingException(PARSER_ERROR_MSG.unsupportedOnForExternalCatalog()); + } + // merge mv, view, table mixTblRefs.addAll(mvRefs); mixTblRefs.addAll(viewRefs); @@ -3566,10 +3604,10 @@ private ParseNode parseBackupRestoreStatement(ParserRuleContext context) { AbstractBackupStmt stmt = null; if (backupContext != null) { - stmt = new BackupStmt(labelName, repoName, mixTblRefs, fnRefs, allMarker, withOnClause, + stmt = new BackupStmt(labelName, repoName, mixTblRefs, fnRefs, externalCatalogRefs, allMarker, withOnClause, originDb != null ? originDb : "", properties, createPos(backupContext)); } else { - stmt = new RestoreStmt(labelName, repoName, mixTblRefs, fnRefs, allMarker, withOnClause, + stmt = new RestoreStmt(labelName, repoName, mixTblRefs, fnRefs, externalCatalogRefs, allMarker, withOnClause, originDb != null ? originDb : "", properties, createPos(restoreContext)); } @@ -3583,11 +3621,14 @@ public ParseNode visitBackupStatement(StarRocksParser.BackupStatementContext con @Override public ParseNode visitCancelBackupStatement(StarRocksParser.CancelBackupStatementContext context) { - if (context.identifier() == null) { + if (context.CATALOG() != null && context.identifier() != null) { + throw new ParsingException(PARSER_ERROR_MSG.unsupportedSepcifyDatabaseForExternalCatalog()); + } + if (context.CATALOG() == null && context.identifier() == null) { throw new ParsingException(PARSER_ERROR_MSG.nullIdentifierCancelBackupRestore()); } - return new CancelBackupStmt(((Identifier) visit(context.identifier())).getValue(), - false, createPos(context)); + return new CancelBackupStmt(context.CATALOG() != null ? "" : ((Identifier) visit(context.identifier())).getValue(), + false, context.CATALOG() != null, createPos(context)); } @Override @@ -3606,11 +3647,14 @@ public ParseNode visitRestoreStatement(StarRocksParser.RestoreStatementContext c @Override public ParseNode visitCancelRestoreStatement(StarRocksParser.CancelRestoreStatementContext context) { - if (context.identifier() == null) { + if (context.CATALOG() != null && context.identifier() != null) { + throw new ParsingException(PARSER_ERROR_MSG.unsupportedSepcifyDatabaseForExternalCatalog()); + } + if (context.CATALOG() == null && context.identifier() == null) { throw new ParsingException(PARSER_ERROR_MSG.nullIdentifierCancelBackupRestore()); } - return new CancelBackupStmt(((Identifier) visit(context.identifier())).getValue(), true, - createPos(context)); + return new CancelBackupStmt(context.CATALOG() != null ? "" : ((Identifier) visit(context.identifier())).getValue(), + true, context.CATALOG() != null, createPos(context)); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 index f134e9af3f8b5..5fc6c581d10c9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 @@ -1731,14 +1731,14 @@ privObjectTypePlural // ---------------------------------------- Backup Restore Statement --------------------------------------------------- backupStatement - : BACKUP (DATABASE dbName=identifier)? + : BACKUP (ALL EXTERNAL CATALOGS | EXTERNAL CATALOG identifierList)? (DATABASE dbName=identifier)? SNAPSHOT qualifiedName TO repoName=identifier (ON '(' backupRestoreObjectDesc (',' backupRestoreObjectDesc) * ')')? (PROPERTIES propertyList)? ; cancelBackupStatement - : CANCEL BACKUP ((FROM | IN) identifier)? + : CANCEL (EXTERNAL CATALOG)? BACKUP ((FROM | IN) identifier)? ; showBackupStatement @@ -1748,13 +1748,14 @@ showBackupStatement restoreStatement : RESTORE SNAPSHOT qualifiedName FROM repoName=identifier + (ALL EXTERNAL CATALOGS | EXTERNAL CATALOG identifierWithAliasList)? (DATABASE dbName=identifier (AS dbAlias=identifier)?)? (ON '(' backupRestoreObjectDesc (',' backupRestoreObjectDesc) * ')')? (PROPERTIES propertyList)? ; cancelRestoreStatement - : CANCEL RESTORE ((FROM | IN) identifier)? + : CANCEL (EXTERNAL CATALOG)? RESTORE ((FROM | IN) identifier)? ; showRestoreStatement @@ -2815,6 +2816,14 @@ identifier | BACKQUOTED_IDENTIFIER #backQuotedIdentifier ; +identifierWithAlias + : originalName=identifier (AS alias=identifier)? + ; + +identifierWithAliasList + : '(' identifierWithAlias (',' identifierWithAlias)* ')' + ; + identifierList : '(' identifier (',' identifier)* ')' ; diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java index 2fd0f97fd3ef6..4d2d0b9878446 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java @@ -313,7 +313,7 @@ public Table getTable(String dbName, String tblName) { List tblRefs = Lists.newArrayList(); tblRefs.add(new TableRef(new TableName(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME), null)); BackupStmt backupStmt = new BackupStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label1"), "repo", tblRefs, - Lists.newArrayList(), null, false, "", null); + Lists.newArrayList(), null, null, false, "", null); try { handler.process(backupStmt); } catch (DdlException e1) { @@ -372,7 +372,7 @@ public Table getTable(String dbName, String tblName) { tblRefs1.add(new TableRef(new TableName(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL3_NAME), null)); BackupStmt backupStmt1 = new BackupStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label2"), "repo", tblRefs1, Lists.newArrayList(), - null, false, "", null); + null, null, false, "", null); try { handler.process(backupStmt1); } catch (DdlException e1) { @@ -432,7 +432,7 @@ public Table getTable(String dbName, String tblName) { Map properties = Maps.newHashMap(); properties.put("backup_timestamp", "2018-08-08-08-08-08"); RestoreStmt restoreStmt = new RestoreStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "ss2"), "repo", tblRefs2, - Lists.newArrayList(), null, false, "", properties); + Lists.newArrayList(), null, null, false, "", properties); try { BackupRestoreAnalyzer.analyze(restoreStmt, new ConnectContext()); } catch (SemanticException e2) { @@ -510,7 +510,7 @@ BackupHandler getBackupHandler() { Map properties1 = Maps.newHashMap(); properties1.put("backup_timestamp", "2018-08-08-08-08-08"); RestoreStmt restoreStmt1 = new RestoreStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label2"), "repo", tblRefs3, - Lists.newArrayList(), null, false, "", properties1); + Lists.newArrayList(), null, null, false, "", properties1); try { BackupRestoreAnalyzer.analyze(restoreStmt1, new ConnectContext()); } catch (SemanticException e2) { @@ -585,7 +585,7 @@ BackupHandler getBackupHandler() { Map properties2 = Maps.newHashMap(); properties2.put("backup_timestamp", "2018-08-08-08-08-08"); RestoreStmt restoreStmt2 = new RestoreStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label2"), "repo", emptyTableRef, - fnRefs, null, false, "", properties2); + fnRefs, null, null, false, "", properties2); BackupMeta backupMeta = new BackupMeta(Lists.newArrayList()); List fns = Lists.newArrayList(); Function f1 = new Function(new FunctionName(db.getFullName(), "wrong_name"), @@ -816,7 +816,7 @@ protected BackupMeta downloadAndDeserializeMetaInfo(BackupJobInfo jobInfo, Repos Map properties = Maps.newHashMap(); properties.put("backup_timestamp", "2018-08-08-08-08-08"); RestoreStmt restoreStmt = new RestoreStmt(new LabelName(null, "ss2"), "repo", tblRefs2, - Lists.newArrayList(), null, false, "", properties); + Lists.newArrayList(), null, null, false, "", properties); try { BackupRestoreAnalyzer.analyze(restoreStmt, new ConnectContext()); } catch (SemanticException e2) { @@ -834,7 +834,7 @@ protected BackupMeta downloadAndDeserializeMetaInfo(BackupJobInfo jobInfo, Repos allMarker.add(AbstractBackupStmt.BackupObjectType.MV); allMarker.add(AbstractBackupStmt.BackupObjectType.VIEW); restoreStmt = new RestoreStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "ss2"), "repo", tblRefs2, - Lists.newArrayList(), allMarker, true, "", properties); + Lists.newArrayList(), null, allMarker, true, "", properties); try { BackupRestoreAnalyzer.analyze(restoreStmt, new ConnectContext()); } catch (SemanticException e2) { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeBackupRestoreTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeBackupRestoreTest.java index 92dfc33a4f273..9e4fd244f6093 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeBackupRestoreTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeBackupRestoreTest.java @@ -61,6 +61,9 @@ public Status initRepository() { AnalyzeTestUtil.getStarRocksAssert().withView("CREATE VIEW v1 AS select 1;"); AnalyzeTestUtil.getStarRocksAssert().withView("CREATE VIEW v2 AS select 2;"); + + String sqlCatalog = "CREATE EXTERNAL CATALOG catalog1 PROPERTIES(\"type\"=\"hive\", \"hive.metastore.uris\"=\"thrift://127.0.0.1:9083\")"; + AnalyzeTestUtil.getStarRocksAssert().withCatalog(sqlCatalog); } @Test @@ -98,6 +101,10 @@ public void testBackup() { analyzeSuccess("BACKUP DATABASE test SNAPSHOT snapshot_label2 TO `repo` ON " + "( ALL FUNCTIONS, ALL TABLES, ALL VIEWS, ALL MATERIALIZED VIEWS ) " + "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); + analyzeSuccess("BACKUP ALL EXTERNAL CATALOGS SNAPSHOT snapshot_label2 TO `repo` " + + "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); + analyzeSuccess("BACKUP EXTERNAL CATALOG (catalog1) SNAPSHOT snapshot_label2 TO `repo` " + + "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); analyzeFail("BACKUP SNAPSHOT test.snapshot_label2 TO `repo` ON ( t0, t0 ) " + "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); analyzeFail("BACKUP SNAPSHOT test.snapshot_label2 TO `repo` ON ( t0, t1 ) " + @@ -124,6 +131,20 @@ public void testBackup() { "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); analyzeFail("BACKUP SNAPSHOT test.snapshot_label2 TO `repo` ON (v1 AS newV1)" + "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); + analyzeFail("BACKUP ALL EXTERNAL CATALOGS DATABASE test SNAPSHOT snapshot_label2 TO `repo` " + + "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); + analyzeFail("BACKUP EXTERNAL CATALOG (catalog1) DATABASE test SNAPSHOT snapshot_label2 TO `repo` " + + "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); + analyzeFail("BACKUP ALL EXTERNAL CATALOGS SNAPSHOT test.snapshot_label2 TO `repo` " + + "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); + analyzeFail("BACKUP EXTERNAL CATALOG (catalog1) SNAPSHOT test.snapshot_label2 TO `repo` " + + "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); + analyzeFail("BACKUP ALL EXTERNAL CATALOGS SNAPSHOT snapshot_label2 TO `repo` " + + " ON (`t0`) PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); + analyzeFail("BACKUP EXTERNAL CATALOG (catalog1) SNAPSHOT snapshot_label2 TO `repo` " + + "ON (`t0`) PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); + analyzeFail("BACKUP EXTERNAL CATALOG (catalog2) SNAPSHOT snapshot_label2 TO `repo` " + + "PROPERTIES (\"type\" = \"full\",\"timeout\" = \"3600\");"); } @Test @@ -192,6 +213,13 @@ public void testRestore() { "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); analyzeSuccess("RESTORE SNAPSHOT `snapshot_2` FROM `repo` " + "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); + analyzeSuccess("RESTORE SNAPSHOT `snapshot_2` FROM `repo` " + "ALL EXTERNAL CATALOGS " + + "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); + analyzeSuccess("RESTORE SNAPSHOT `snapshot_2` FROM `repo` " + "EXTERNAL CATALOG (catalog1) " + + "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); + analyzeSuccess("RESTORE SNAPSHOT `snapshot_2` FROM `repo` " + "EXTERNAL CATALOG (catalog1 AS catalog2) " + + "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); + analyzeSuccess("RESTORE SNAPSHOT `snapshot_2` FROM `repo` PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); analyzeFail("RESTORE SNAPSHOT test.`snapshot_2` FROM `repo` ON ( `t0` , `t1` AS `new_tbl` ) " + "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\",\"allow_load\"=\"a\" );"); analyzeFail("RESTORE SNAPSHOT test.`snapshot_2` FROM `repo` ON ( `t0` , `t1` AS `new_tbl` ) " + @@ -220,6 +248,18 @@ public void testRestore() { "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\",\"timeout\"=\"a\" );"); analyzeFail("RESTORE SNAPSHOT `snapshot_2` FROM `repo1` ON (`t0`)" + "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\",\"timeout\"=\"a\" );"); + analyzeFail("RESTORE SNAPSHOT .`snapshot_2` FROM `repo` " + "ALL EXTERNAL CATALOGS " + + "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); + analyzeFail("RESTORE SNAPSHOT `test`.`snapshot_2` FROM `repo` " + "EXTERNAL CATALOG (catalog1) " + + "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); + analyzeFail("RESTORE SNAPSHOT `snapshot_2` FROM `repo` " + "ALL EXTERNAL CATALOGS DATABASE test " + + "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); + analyzeFail("RESTORE SNAPSHOT `snapshot_2` FROM `repo` " + "EXTERNAL CATALOG (catalog1) DATABASE test " + + "PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); + analyzeFail("RESTORE SNAPSHOT `snapshot_2` FROM `repo` " + "EXTERNAL CATALOG (catalog1) " + + "ON (`t0`) PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); + analyzeFail("RESTORE SNAPSHOT `snapshot_2` FROM `repo` " + "ALL EXTERNAL CATALOGS " + + "ON (`t0`) PROPERTIES ( \"backup_timestamp\"=\"2018-05-04-17-11-01\");"); } @Test @@ -236,6 +276,10 @@ public void testShowRestore() { public void testCancelRestore() { analyzeFail("CANCEL BACKUP;"); analyzeFail("CANCEL RESTORE;"); + analyzeFail("CANCEL EXTERNAL CATALOG BACKUP FROM test;"); + analyzeFail("CANCEL EXTERNAL CATALOG RESTORE FROM test;"); + analyzeSuccess("CANCEL EXTERNAL CATALOG BACKUP;"); + analyzeSuccess("CANCEL EXTERNAL CATALOG BACKUP;"); } } From 22562a90977fb5c97e76e1ab47a33d3072a2a290 Mon Sep 17 00:00:00 2001 From: srlch Date: Thu, 14 Nov 2024 19:43:36 +0800 Subject: [PATCH 2/6] fix Signed-off-by: srlch --- .../starrocks/privilege/DefaultAuthorizationProvider.java | 6 ++++-- .../com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/privilege/DefaultAuthorizationProvider.java b/fe/fe-core/src/main/java/com/starrocks/privilege/DefaultAuthorizationProvider.java index 6dd2d9b59df22..2b6690a646cc9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/privilege/DefaultAuthorizationProvider.java +++ b/fe/fe-core/src/main/java/com/starrocks/privilege/DefaultAuthorizationProvider.java @@ -83,7 +83,8 @@ public DefaultAuthorizationProvider() { PrivilegeType.USAGE, PrivilegeType.CREATE_DATABASE, PrivilegeType.DROP, - PrivilegeType.ALTER)); + PrivilegeType.ALTER, + PrivilegeType.EXPORT)); typeToActionList.put(ObjectType.MATERIALIZED_VIEW, Lists.newArrayList( PrivilegeType.ALTER, @@ -93,7 +94,8 @@ public DefaultAuthorizationProvider() { typeToActionList.put(ObjectType.FUNCTION, Lists.newArrayList( PrivilegeType.USAGE, - PrivilegeType.DROP)); + PrivilegeType.DROP, + PrivilegeType.EXPORT)); typeToActionList.put(ObjectType.RESOURCE_GROUP, Lists.newArrayList( PrivilegeType.ALTER, diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java index 829aee49071e7..e18e60d749ac6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java @@ -2153,7 +2153,7 @@ public Void visitBackupStatement(BackupStmt statement, ConnectContext context) { externalCatalogs.forEach(externalCatalog -> { try { Authorizer.checkCatalogAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), - externalCatalog.getCatalogName(), PrivilegeType.CREATE_DATABASE); + externalCatalog.getCatalogName(), PrivilegeType.EXPORT); } catch (AccessDeniedException e) { AccessDeniedException.reportAccessDenied( externalCatalog.getCatalogName(), From 942c7e4efeab5bd8ea4f3ff99940dc54830fc536 Mon Sep 17 00:00:00 2001 From: srlch Date: Thu, 14 Nov 2024 21:40:43 +0800 Subject: [PATCH 3/6] fix Signed-off-by: srlch --- .../starrocks/privilege/DefaultAuthorizationProvider.java | 6 ++---- .../com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/privilege/DefaultAuthorizationProvider.java b/fe/fe-core/src/main/java/com/starrocks/privilege/DefaultAuthorizationProvider.java index 2b6690a646cc9..6dd2d9b59df22 100644 --- a/fe/fe-core/src/main/java/com/starrocks/privilege/DefaultAuthorizationProvider.java +++ b/fe/fe-core/src/main/java/com/starrocks/privilege/DefaultAuthorizationProvider.java @@ -83,8 +83,7 @@ public DefaultAuthorizationProvider() { PrivilegeType.USAGE, PrivilegeType.CREATE_DATABASE, PrivilegeType.DROP, - PrivilegeType.ALTER, - PrivilegeType.EXPORT)); + PrivilegeType.ALTER)); typeToActionList.put(ObjectType.MATERIALIZED_VIEW, Lists.newArrayList( PrivilegeType.ALTER, @@ -94,8 +93,7 @@ public DefaultAuthorizationProvider() { typeToActionList.put(ObjectType.FUNCTION, Lists.newArrayList( PrivilegeType.USAGE, - PrivilegeType.DROP, - PrivilegeType.EXPORT)); + PrivilegeType.DROP)); typeToActionList.put(ObjectType.RESOURCE_GROUP, Lists.newArrayList( PrivilegeType.ALTER, diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java index e18e60d749ac6..ed34d22ea4c37 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java @@ -2139,7 +2139,7 @@ public Void visitBackupStatement(BackupStmt statement, ConnectContext context) { for (Function fn : fns) { try { Authorizer.checkFunctionAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), - db, fn, PrivilegeType.EXPORT); + db, fn, PrivilegeType.USAGE); } catch (AccessDeniedException e) { AccessDeniedException.reportAccessDenied( InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, @@ -2153,7 +2153,7 @@ public Void visitBackupStatement(BackupStmt statement, ConnectContext context) { externalCatalogs.forEach(externalCatalog -> { try { Authorizer.checkCatalogAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), - externalCatalog.getCatalogName(), PrivilegeType.EXPORT); + externalCatalog.getCatalogName(), PrivilegeType.USAGE); } catch (AccessDeniedException e) { AccessDeniedException.reportAccessDenied( externalCatalog.getCatalogName(), From 62c2452f1d3423b3e14ebe07ff829b239c2b5e01 Mon Sep 17 00:00:00 2001 From: srlch Date: Fri, 15 Nov 2024 00:08:10 +0800 Subject: [PATCH 4/6] fix Signed-off-by: srlch --- .../src/main/java/com/starrocks/sql/parser/AstBuilder.java | 5 +++-- .../src/main/java/com/starrocks/sql/parser/StarRocks.g4 | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index d2090d9f9ff51..f6bbb89cef944 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -3442,9 +3442,10 @@ private ParseNode parseBackupRestoreStatement(ParserRuleContext context) { List externalCatalogRefs = new ArrayList<>(); boolean allExternalCatalog = backupContext != null ? - (backupContext.CATALOGS() != null) : (restoreContext.CATALOGS() != null); + (backupContext.ALL() != null) : (restoreContext.ALL() != null); if (!allExternalCatalog && (backupContext != null ? - (backupContext.CATALOG() != null) : (restoreContext.CATALOG() != null))) { + (backupContext.CATALOG() != null || backupContext.CATALOGS() != null) : + (restoreContext.CATALOG() != null || restoreContext.CATALOGS() != null))) { if (backupContext != null) { StarRocksParser.IdentifierListContext identifierListContext = backupContext.identifierList(); externalCatalogRefs = visit(identifierListContext.identifier(), Identifier.class) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 index 5fc6c581d10c9..907a51579f8ee 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 @@ -1731,7 +1731,7 @@ privObjectTypePlural // ---------------------------------------- Backup Restore Statement --------------------------------------------------- backupStatement - : BACKUP (ALL EXTERNAL CATALOGS | EXTERNAL CATALOG identifierList)? (DATABASE dbName=identifier)? + : BACKUP (ALL EXTERNAL CATALOGS | EXTERNAL (CATALOG | CATALOGS) identifierList)? (DATABASE dbName=identifier)? SNAPSHOT qualifiedName TO repoName=identifier (ON '(' backupRestoreObjectDesc (',' backupRestoreObjectDesc) * ')')? (PROPERTIES propertyList)? @@ -1748,7 +1748,7 @@ showBackupStatement restoreStatement : RESTORE SNAPSHOT qualifiedName FROM repoName=identifier - (ALL EXTERNAL CATALOGS | EXTERNAL CATALOG identifierWithAliasList)? + (ALL EXTERNAL CATALOGS | EXTERNAL (CATALOG | CATALOGS) identifierWithAliasList)? (DATABASE dbName=identifier (AS dbAlias=identifier)?)? (ON '(' backupRestoreObjectDesc (',' backupRestoreObjectDesc) * ')')? (PROPERTIES propertyList)? From 8008217249b5eb5b6c8910c548364476d779d17e Mon Sep 17 00:00:00 2001 From: srlch Date: Fri, 15 Nov 2024 10:32:09 +0800 Subject: [PATCH 5/6] fix comment Signed-off-by: srlch --- .../com/starrocks/sql/common/ParserErrorMsg.java | 3 --- .../com/starrocks/sql/parser/AstBuilder.java | 6 ------ .../java/com/starrocks/sql/parser/StarRocks.g4 | 4 ++-- .../com/starrocks/backup/BackupHandlerTest.java | 11 +++++++++++ .../com/starrocks/backup/RestoreJobTest.java | 16 ++++++++++++++++ .../sql/analyzer/AnalyzeBackupRestoreTest.java | 6 ++---- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/ParserErrorMsg.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/ParserErrorMsg.java index 39bdc8c5ec1ec..154f7b79653cf 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/ParserErrorMsg.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/ParserErrorMsg.java @@ -237,7 +237,4 @@ public interface ParserErrorMsg { @BaseMessage("Can not sepcify `ON` clause for external catalog Backup/Restore") String unsupportedOnForExternalCatalog(); - - @BaseMessage("Can not sepcify database for cancel EXTERNAL CATALOG BACKUP/RESTORE") - String unsupportedSepcifyDatabaseForExternalCatalog(); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index f6bbb89cef944..ff29f966bf265 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -3622,9 +3622,6 @@ public ParseNode visitBackupStatement(StarRocksParser.BackupStatementContext con @Override public ParseNode visitCancelBackupStatement(StarRocksParser.CancelBackupStatementContext context) { - if (context.CATALOG() != null && context.identifier() != null) { - throw new ParsingException(PARSER_ERROR_MSG.unsupportedSepcifyDatabaseForExternalCatalog()); - } if (context.CATALOG() == null && context.identifier() == null) { throw new ParsingException(PARSER_ERROR_MSG.nullIdentifierCancelBackupRestore()); } @@ -3648,9 +3645,6 @@ public ParseNode visitRestoreStatement(StarRocksParser.RestoreStatementContext c @Override public ParseNode visitCancelRestoreStatement(StarRocksParser.CancelRestoreStatementContext context) { - if (context.CATALOG() != null && context.identifier() != null) { - throw new ParsingException(PARSER_ERROR_MSG.unsupportedSepcifyDatabaseForExternalCatalog()); - } if (context.CATALOG() == null && context.identifier() == null) { throw new ParsingException(PARSER_ERROR_MSG.nullIdentifierCancelBackupRestore()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 index 907a51579f8ee..ba1f85648944d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 @@ -1738,7 +1738,7 @@ backupStatement ; cancelBackupStatement - : CANCEL (EXTERNAL CATALOG)? BACKUP ((FROM | IN) identifier)? + : CANCEL BACKUP ((FROM | IN) identifier | FOR EXTERNAL CATALOG)? ; showBackupStatement @@ -1755,7 +1755,7 @@ restoreStatement ; cancelRestoreStatement - : CANCEL (EXTERNAL CATALOG)? RESTORE ((FROM | IN) identifier)? + : CANCEL RESTORE ((FROM | IN) identifier | FOR EXTERNAL CATALOG)? ; showRestoreStatement diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java index 4d2d0b9878446..cc103487b5201 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java @@ -42,6 +42,7 @@ import com.starrocks.analysis.TableName; import com.starrocks.analysis.TableRef; import com.starrocks.catalog.BrokerMgr; +import com.starrocks.catalog.Catalog; import com.starrocks.catalog.Database; import com.starrocks.catalog.Function; import com.starrocks.catalog.MaterializedIndex; @@ -603,6 +604,16 @@ BackupHandler getBackupHandler() { backupMeta.setFunctions(fns); handler.checkAndFilterRestoreFunctionsInBackupMeta(restoreStmt2, backupMeta); + // process EXTERNAL CATALOG restore + Map properties3 = Maps.newHashMap(); + properties3.put("backup_timestamp", "2018-08-08-08-08-08"); + RestoreStmt restoreStmt3 = new RestoreStmt(new LabelName(null, "label2"), "repo", Lists.newArrayList(), + Lists.newArrayList(), null, null, false, "", properties3); + BackupMeta newBackupMeta = new BackupMeta(Lists.newArrayList()); + Catalog catalog = new Catalog(1111111, "test_catalog", Maps.newHashMap(), ""); + newBackupMeta.setCatalogs(Lists.newArrayList(catalog)); + handler.checkAndFilterRestoreCatalogsInBackupMeta(restoreStmt3, newBackupMeta); + // drop repo DDLStmtExecutor ddlStmtExecutor = new DDLStmtExecutor(DDLStmtExecutor.StmtExecutorVisitor.getInstance()); new Expectations() { diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java index fc88f90876262..5eb1814968bc6 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java @@ -45,6 +45,7 @@ import com.starrocks.backup.BackupJobInfo.BackupTabletInfo; import com.starrocks.backup.RestoreJob.RestoreJobState; import com.starrocks.backup.mv.MvRestoreContext; +import com.starrocks.catalog.Catalog; import com.starrocks.catalog.Database; import com.starrocks.catalog.FakeEditLog; import com.starrocks.catalog.Function; @@ -929,4 +930,19 @@ public void testRestoreAddFunction() { job.addRestoredFunctions(db); } + + @Test + public void testRestoreAddCatalog() { + backupMeta = new BackupMeta(Lists.newArrayList()); + Catalog catalog = new Catalog(1111111, "test_catalog", Maps.newHashMap(), ""); + + backupMeta.setCatalogs(Lists.newArrayList(catalog)); + job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), + new BackupJobInfo(), false, 3, 100000, + globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); + job.setRepo(repo); + job.addRestoredFunctions(db); + job.run(); + job.run(); + } } \ No newline at end of file diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeBackupRestoreTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeBackupRestoreTest.java index 9e4fd244f6093..c46eca70cf907 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeBackupRestoreTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeBackupRestoreTest.java @@ -276,10 +276,8 @@ public void testShowRestore() { public void testCancelRestore() { analyzeFail("CANCEL BACKUP;"); analyzeFail("CANCEL RESTORE;"); - analyzeFail("CANCEL EXTERNAL CATALOG BACKUP FROM test;"); - analyzeFail("CANCEL EXTERNAL CATALOG RESTORE FROM test;"); - analyzeSuccess("CANCEL EXTERNAL CATALOG BACKUP;"); - analyzeSuccess("CANCEL EXTERNAL CATALOG BACKUP;"); + analyzeSuccess("CANCEL BACKUP FOR EXTERNAL CATALOG;"); + analyzeSuccess("CANCEL RESTORE FOR EXTERNAL CATALOG;"); } } From dbd351b07c65e4a069d84537585d1af28b24598b Mon Sep 17 00:00:00 2001 From: srlch Date: Fri, 15 Nov 2024 11:05:45 +0800 Subject: [PATCH 6/6] fix Signed-off-by: srlch --- .../main/java/com/starrocks/backup/BackupHandler.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java index 87e46d7d333be..a427144ee813c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java @@ -112,6 +112,8 @@ public class BackupHandler extends FrontendDaemon implements Writable, MemoryTrackable { private static final Logger LOG = LogManager.getLogger(BackupHandler.class); + + private static final long FAKE_DB_ID = -1; public static final int SIGNATURE_VERSION = 1; public static final Path BACKUP_ROOT_DIR = Paths.get(Config.tmp_dir, "backup").normalize(); @@ -337,7 +339,7 @@ public void process(AbstractBackupStmt stmt) throws DdlException { tryLock(); try { // Check if there is backup or restore job running on this database - AbstractJob currentJob = dbIdToBackupOrRestoreJob.get(stmt.containsExternalCatalog() ? -1 : db.getId()); + AbstractJob currentJob = dbIdToBackupOrRestoreJob.get(stmt.containsExternalCatalog() ? FAKE_DB_ID : db.getId()); if (currentJob != null && !currentJob.isDone()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Can only run one backup or restore job of a database at same time"); @@ -478,7 +480,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws } // Create a backup job - long dbId = stmt.containsExternalCatalog() ? -1 : db.getId(); + long dbId = stmt.containsExternalCatalog() ? FAKE_DB_ID : db.getId(); String dbName = stmt.containsExternalCatalog() ? "" : db.getOriginName(); BackupJob backupJob = new BackupJob(stmt.getLabel(), dbId, dbName, tblRefs, @@ -538,7 +540,7 @@ private void restore(Repository repository, Database db, RestoreStmt stmt, Backu } } - long dbId = stmt.containsExternalCatalog() ? -1 : db.getId(); + long dbId = stmt.containsExternalCatalog() ? FAKE_DB_ID : db.getId(); String dbName = stmt.containsExternalCatalog() ? "" : db.getOriginName(); restoreJob = new RestoreJob(stmt.getLabel(), stmt.getBackupTimestamp(), @@ -704,7 +706,7 @@ public void cancel(CancelBackupStmt stmt) throws DdlException { if (!stmt.isExternalCatalog()) { job = getAbstractJobByDbName(stmt.getDbName()); } else { - job = dbIdToBackupOrRestoreJob.get(-1L); + job = dbIdToBackupOrRestoreJob.get(FAKE_DB_ID); } if (job == null || (job instanceof BackupJob && stmt.isRestore()) || (job instanceof RestoreJob && !stmt.isRestore())) {