This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch revert-3523-fix_codestyle in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 00882c5df85939072b2e61a490561d4bdaf4cd83 Author: Mingyu Chen <[email protected]> AuthorDate: Mon May 18 12:51:52 2020 +0800 Revert "Refactor some redunant code && Replace some UT by UtFrameUtils" --- .../main/java/org/apache/doris/alter/AlterJob.java | 18 +- .../java/org/apache/doris/backup/RestoreJob.java | 175 +++--- .../java/org/apache/doris/catalog/Database.java | 2 +- .../apache/doris/catalog/MaterializedIndex.java | 8 +- .../java/org/apache/doris/catalog/OlapTable.java | 6 +- .../java/org/apache/doris/catalog/Partition.java | 7 +- .../org/apache/doris/catalog/PartitionInfo.java | 8 +- .../java/org/apache/doris/catalog/Replica.java | 1 - .../main/java/org/apache/doris/catalog/Tablet.java | 19 +- .../java/org/apache/doris/catalog/TabletMeta.java | 12 +- .../apache/doris/clone/BackendLoadStatistic.java | 25 +- .../apache/doris/clone/RootPathLoadStatistic.java | 8 +- .../org/apache/doris/clone/TabletScheduler.java | 89 +-- .../common/proc/ClusterLoadStatisticProcDir.java | 6 +- .../org/apache/doris/system/BackendHbResponse.java | 1 - .../org/apache/doris/system/BrokerHbResponse.java | 1 - .../doris/catalog/DynamicPartitionTableTest.java | 684 +++++++++++++-------- .../apache/doris/common/PropertyAnalyzerTest.java | 10 +- 18 files changed, 654 insertions(+), 426 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJob.java b/fe/src/main/java/org/apache/doris/alter/AlterJob.java index 7b10fd5..ab927cc 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJob.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJob.java @@ -189,14 +189,18 @@ public abstract class AlterJob implements Writable { return false; } else if (!backend.isAlive()) { long currentTime = System.currentTimeMillis(); - // If this backend is done for a long time and not restart automatically. - // we consider it as dead and return false. - return backend.getLastUpdateMs() <= 0 - || currentTime - backend.getLastUpdateMs() <= Config.max_backend_down_time_second * 1000; - } else { - return !backend.isDecommissioned(); + if (backend.getLastUpdateMs() > 0 + && currentTime - backend.getLastUpdateMs() > Config.max_backend_down_time_second * 1000) { + // this backend is done for a long time and not restart automatically. + // we consider it as dead + return false; + } + return true; + } else if (backend.isDecommissioned()) { + return false; } - + + return true; } public static AlterJob read(DataInput in) throws IOException { diff --git a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java index 335ac17..ef576b4 100644 --- a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -172,7 +172,11 @@ public class RestoreJob extends AbstractJob { } public synchronized boolean finishTabletSnapshotTask(SnapshotTask task, TFinishTaskRequest request) { - if (checkTaskStatus(task, task.getJobId(), request)) { + Preconditions.checkState(task.getJobId() == jobId); + Preconditions.checkState(task.getDbId() == dbId); + + if (request.getTask_status().getStatus_code() != TStatusCode.OK) { + taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); return false; } @@ -194,15 +198,19 @@ public class RestoreJob extends AbstractJob { taskErrMsg.remove(task.getSignature()); Preconditions.checkState(task.getTabletId() == removedTabletId, removedTabletId); LOG.debug("get finished snapshot info: {}, unfinished tasks num: {}, remove result: {}. {}", - info, unfinishedSignatureToId.size(), this, removedTabletId); + info, unfinishedSignatureToId.size(), this); return true; } + return false; } - public synchronized boolean finishTabletDownloadTask(DownloadTask task, TFinishTaskRequest request) { - if (checkTaskStatus(task, task.getJobId(), request)) { + Preconditions.checkState(task.getJobId() == jobId); + Preconditions.checkState(task.getDbId() == dbId); + + if (request.getTask_status().getStatus_code() != TStatusCode.OK) { + taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); return false; } @@ -229,7 +237,11 @@ public class RestoreJob extends AbstractJob { } public synchronized boolean finishDirMoveTask(DirMoveTask task, TFinishTaskRequest request) { - if (checkTaskStatus(task, task.getJobId(), request)) { + Preconditions.checkState(task.getJobId() == jobId); + Preconditions.checkState(task.getDbId() == dbId); + + if (request.getTask_status().getStatus_code() != TStatusCode.OK) { + taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); return false; } @@ -244,17 +256,6 @@ public class RestoreJob extends AbstractJob { return true; } - private boolean checkTaskStatus(AgentTask task, long jobId, TFinishTaskRequest request) { - Preconditions.checkState(jobId == this.jobId); - Preconditions.checkState(dbId == task.getDbId()); - - if (request.getTask_status().getStatus_code() != TStatusCode.OK) { - taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); - return true; - } - return false; - } - @Override public synchronized void replayRun() { LOG.info("replay run restore job: {}", this); @@ -341,7 +342,7 @@ public class RestoreJob extends AbstractJob { } } - /** + /* * return true if some restored objs have been dropped. */ private void checkIfNeedCancel() { @@ -352,7 +353,6 @@ public class RestoreJob extends AbstractJob { Database db = catalog.getDb(dbId); if (db == null) { status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " has been dropped"); - return; } db.readLock(); @@ -381,7 +381,7 @@ public class RestoreJob extends AbstractJob { } } - /** + /* * Restore rules as follow: * A. Table already exist * A1. Partition already exist, generate file mapping @@ -505,9 +505,19 @@ public class RestoreJob extends AbstractJob { Range<PartitionKey> remoteRange = remoteRangePartInfo.getRange(backupPartInfo.id); if (localRange.equals(remoteRange)) { // Same partition, same range - if (genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl, backupPartInfo, tblInfo)) { + if (localRangePartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) { + status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name + + " in table " + localTbl.getName() + + " has different replication num '" + + localRangePartInfo.getReplicationNum(localPartition.getId()) + + "' with partition in repository, which is " + restoreReplicationNum); return; } + genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo, + true /* overwrite when commit */); + restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(), + Pair.create(backupPartInfo.version, + backupPartInfo.versionHash)); } else { // Same partition name, different range status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name @@ -517,9 +527,21 @@ public class RestoreJob extends AbstractJob { } } else { // If this is a single partitioned table. - if (genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl, backupPartInfo, tblInfo)) { + if (localPartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) { + status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name + + " in table " + localTbl.getName() + + " has different replication num '" + + localPartInfo.getReplicationNum(localPartition.getId()) + + "' with partition in repository, which is " + restoreReplicationNum); return; } + + // No need to check range, just generate file mapping + genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo, + true /* overwrite when commit */); + restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(), + Pair.create(backupPartInfo.version, + backupPartInfo.versionHash)); } } else { // partitions does not exist @@ -594,20 +616,65 @@ public class RestoreJob extends AbstractJob { BackupPartitionInfo backupPartitionInfo = jobInfo.getTableInfo(entry.first).getPartInfo(restorePart.getName()); - createReplicas(db, batchTask, localTbl, restorePart); + Set<String> bfColumns = localTbl.getCopiedBfColumns(); + double bfFpp = localTbl.getBfFpp(); + for (MaterializedIndex restoredIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { + MaterializedIndexMeta indexMeta = localTbl.getIndexMetaByIndexId(restoredIdx.getId()); + TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(), + restoredIdx.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD); + for (Tablet restoreTablet : restoredIdx.getTablets()) { + Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); + for (Replica restoreReplica : restoreTablet.getReplicas()) { + Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica); + CreateReplicaTask task = new CreateReplicaTask(restoreReplica.getBackendId(), dbId, + localTbl.getId(), restorePart.getId(), restoredIdx.getId(), + restoreTablet.getId(), indexMeta.getShortKeyColumnCount(), + indexMeta.getSchemaHash(), restoreReplica.getVersion(), + restoreReplica.getVersionHash(), indexMeta.getKeysType(), TStorageType.COLUMN, + TStorageMedium.HDD /* all restored replicas will be saved to HDD */, + indexMeta.getSchema(), bfColumns, bfFpp, null, + localTbl.getCopiedIndexes(), + localTbl.isInMemory()); + task.setInRestoreMode(true); + batchTask.addTask(task); + } + } + } genFileMapping(localTbl, restorePart, remoteTbl.getId(), backupPartitionInfo, - !allowLoad /* if allow load, do not overwrite when commit */); + allowLoad ? false : true /* if allow load, do not overwrite when commit */); } // generate create replica task for all restored tables for (OlapTable restoreTbl : restoredTbls) { for (Partition restorePart : restoreTbl.getPartitions()) { - createReplicas(db, batchTask, restoreTbl, restorePart); + Set<String> bfColumns = restoreTbl.getCopiedBfColumns(); + double bfFpp = restoreTbl.getBfFpp(); + for (MaterializedIndex index : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { + MaterializedIndexMeta indexMeta = restoreTbl.getIndexMetaByIndexId(index.getId()); + TabletMeta tabletMeta = new TabletMeta(db.getId(), restoreTbl.getId(), restorePart.getId(), + index.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD); + for (Tablet tablet : index.getTablets()) { + Catalog.getCurrentInvertedIndex().addTablet(tablet.getId(), tabletMeta); + for (Replica replica : tablet.getReplicas()) { + Catalog.getCurrentInvertedIndex().addReplica(tablet.getId(), replica); + CreateReplicaTask task = new CreateReplicaTask(replica.getBackendId(), dbId, + restoreTbl.getId(), restorePart.getId(), index.getId(), tablet.getId(), + indexMeta.getShortKeyColumnCount(), indexMeta.getSchemaHash(), + replica.getVersion(), replica.getVersionHash(), + indexMeta.getKeysType(), TStorageType.COLUMN, TStorageMedium.HDD, + indexMeta.getSchema(), bfColumns, bfFpp, null, + restoreTbl.getCopiedIndexes(), + restoreTbl.isInMemory()); + task.setInRestoreMode(true); + batchTask.addTask(task); + } + } + } BackupTableInfo backupTableInfo = jobInfo.getTableInfo(restoreTbl.getName()); genFileMapping(restoreTbl, restorePart, backupTableInfo.id, backupTableInfo.getPartInfo(restorePart.getName()), - !allowLoad /* if allow load, do not overwrite when commit */); + allowLoad ? false : true /* if allow load, do not overwrite when commit */); } // set restored table's new name after all 'genFileMapping' restoreTbl.setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName())); @@ -622,7 +689,7 @@ public class RestoreJob extends AbstractJob { if (batchTask.getTaskNum() > 0) { MarkedCountDownLatch<Long, Long> latch = new MarkedCountDownLatch<Long, Long>(batchTask.getTaskNum()); for (AgentTask task : batchTask.getAllTasks()) { - latch.addMark(task.getBackendId(), task.getTabletId()); + latch.addMark(((CreateReplicaTask) task).getBackendId(), ((CreateReplicaTask) task).getTabletId()); ((CreateReplicaTask) task).setLatch(latch); AgentTaskQueue.addTask(task); } @@ -739,54 +806,7 @@ public class RestoreJob extends AbstractJob { // No log here, PENDING state restore job will redo this method LOG.info("finished to prepare meta and send snapshot tasks, num: {}. {}", batchTask.getTaskNum(), this); - } - - private boolean genFileMappingWhenBackupReplicasEqual(PartitionInfo localPartInfo, Partition localPartition, Table localTbl, - BackupPartitionInfo backupPartInfo, BackupTableInfo tblInfo) { - if (localPartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) { - status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name - + " in table " + localTbl.getName() - + " has different replication num '" - + localPartInfo.getReplicationNum(localPartition.getId()) - + "' with partition in repository, which is " + restoreReplicationNum); - return true; - } - - // No need to check range, just generate file mapping - OlapTable localOlapTbl = (OlapTable) localTbl; - genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo, - true /* overwrite when commit */); - restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(), - Pair.create(backupPartInfo.version, - backupPartInfo.versionHash)); - return false; - } - - private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable localTbl, Partition restorePart) { - Set<String> bfColumns = localTbl.getCopiedBfColumns(); - double bfFpp = localTbl.getBfFpp(); - for (MaterializedIndex restoredIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { - MaterializedIndexMeta indexMeta = localTbl.getIndexMetaByIndexId(restoredIdx.getId()); - TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(), - restoredIdx.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD); - for (Tablet restoreTablet : restoredIdx.getTablets()) { - Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); - for (Replica restoreReplica : restoreTablet.getReplicas()) { - Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica); - CreateReplicaTask task = new CreateReplicaTask(restoreReplica.getBackendId(), dbId, - localTbl.getId(), restorePart.getId(), restoredIdx.getId(), - restoreTablet.getId(), indexMeta.getShortKeyColumnCount(), - indexMeta.getSchemaHash(), restoreReplica.getVersion(), - restoreReplica.getVersionHash(), indexMeta.getKeysType(), TStorageType.COLUMN, - TStorageMedium.HDD /* all restored replicas will be saved to HDD */, - indexMeta.getSchema(), bfColumns, bfFpp, null, - localTbl.getCopiedIndexes(), - localTbl.isInMemory()); - task.setInRestoreMode(true); - batchTask.addTask(task); - } - } - } + return; } // reset remote partition. @@ -805,11 +825,11 @@ public class RestoreJob extends AbstractJob { // indexes Map<String, Long> localIdxNameToId = localTbl.getIndexNameToId(); - for (String localIdxName : localIdxNameToId.keySet()) { + for (String localidxName : localIdxNameToId.keySet()) { // set ids of indexes in remote partition to the local index ids - long remoteIdxId = remoteTbl.getIndexIdByName(localIdxName); + long remoteIdxId = remoteTbl.getIndexIdByName(localidxName); MaterializedIndex remoteIdx = remotePart.getIndex(remoteIdxId); - long localIdxId = localIdxNameToId.get(localIdxName); + long localIdxId = localIdxNameToId.get(localidxName); remoteIdx.setIdForRestore(localIdxId); if (localIdxId != localTbl.getBaseIndexId()) { // not base table, reset @@ -1490,7 +1510,6 @@ public class RestoreJob extends AbstractJob { } } - @Override public void readFields(DataInput in) throws IOException { super.readFields(in); diff --git a/fe/src/main/java/org/apache/doris/catalog/Database.java b/fe/src/main/java/org/apache/doris/catalog/Database.java index 0d46c7a..fa53e14 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/src/main/java/org/apache/doris/catalog/Database.java @@ -113,7 +113,7 @@ public class Database extends MetaObject implements Writable { } this.rwLock = new ReentrantReadWriteLock(true); this.idToTable = new ConcurrentHashMap<>(); - this.nameToTable = new HashMap<>(); + this.nameToTable = new HashMap<String, Table>(); this.dataQuotaBytes = FeConstants.default_db_data_quota_bytes; this.replicaQuotaSize = FeConstants.default_db_replica_quota_size; this.dbState = DbState.NORMAL; diff --git a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java index a921758..6f8abd9 100644 --- a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java +++ b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java @@ -76,8 +76,8 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP public MaterializedIndex() { this.state = IndexState.NORMAL; - this.idToTablets = new HashMap<>(); - this.tablets = new ArrayList<>(); + this.idToTablets = new HashMap<Long, Tablet>(); + this.tablets = new ArrayList<Tablet>(); } public MaterializedIndex(long id, IndexState state) { @@ -88,8 +88,8 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP this.state = IndexState.NORMAL; } - this.idToTablets = new HashMap<>(); - this.tablets = new ArrayList<>(); + this.idToTablets = new HashMap<Long, Tablet>(); + this.tablets = new ArrayList<Tablet>(); this.rowCount = 0; diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index f307a5f..fe252dd 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -643,7 +643,6 @@ public class OlapTable extends Table { */ // get partition by name, not including temp partitions - @Override public Partition getPartition(String partitionName) { return getPartition(partitionName, false); } @@ -733,7 +732,6 @@ public class OlapTable extends Table { return false; } - @Override public TTableDescriptor toThrift() { TOlapTable tOlapTable = new TOlapTable(getName()); TTableDescriptor tTableDescriptor = new TTableDescriptor(id, TTableType.OLAP_TABLE, @@ -932,7 +930,6 @@ public class OlapTable extends Table { tempPartitions.write(out); } - @Override public void readFields(DataInput in) throws IOException { super.readFields(in); @@ -950,7 +947,7 @@ public class OlapTable extends Table { if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_75) { // schema int colCount = in.readInt(); - List<Column> schema = new LinkedList<>(); + List<Column> schema = new LinkedList<Column>(); for (int j = 0; j < colCount; j++) { Column column = Column.read(in); schema.add(column); @@ -1076,7 +1073,6 @@ public class OlapTable extends Table { rebuildFullSchema(); } - @Override public boolean equals(Table table) { if (this == table) { return true; diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index 0730bed..447555a 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -67,13 +67,13 @@ public class Partition extends MetaObject implements Writable { private PartitionState state; @SerializedName(value = "baseIndex") private MaterializedIndex baseIndex; - /** + /* * Visible rollup indexes are indexes which are visible to user. * User can do query on them, show them in related 'show' stmt. */ @SerializedName(value = "idToVisibleRollupIndex") private Map<Long, MaterializedIndex> idToVisibleRollupIndex = Maps.newHashMap(); - /** + /* * Shadow indexes are indexes which are not visible to user. * Query will not run on these shadow indexes, and user can not see them neither. * But load process will load data into these shadow indexes. @@ -81,7 +81,7 @@ public class Partition extends MetaObject implements Writable { @SerializedName(value = "idToShadowIndex") private Map<Long, MaterializedIndex> idToShadowIndex = Maps.newHashMap(); - /** + /* * committed version(hash): after txn is committed, set committed version(hash) * visible version(hash): after txn is published, set visible version * next version(hash): next version is set after finished committing, it should equals to committed version + 1 @@ -354,7 +354,6 @@ public class Partition extends MetaObject implements Writable { distributionInfo.write(out); } - @Override public void readFields(DataInput in) throws IOException { super.readFields(in); diff --git a/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java index f41f595..1359f0c 100644 --- a/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -51,15 +51,15 @@ public class PartitionInfo implements Writable { protected Map<Long, Boolean> idToInMemory; public PartitionInfo() { - this.idToDataProperty = new HashMap<>(); - this.idToReplicationNum = new HashMap<>(); + this.idToDataProperty = new HashMap<Long, DataProperty>(); + this.idToReplicationNum = new HashMap<Long, Short>(); this.idToInMemory = new HashMap<>(); } public PartitionInfo(PartitionType type) { this.type = type; - this.idToDataProperty = new HashMap<>(); - this.idToReplicationNum = new HashMap<>(); + this.idToDataProperty = new HashMap<Long, DataProperty>(); + this.idToReplicationNum = new HashMap<Long, Short>(); this.idToInMemory = new HashMap<>(); } diff --git a/fe/src/main/java/org/apache/doris/catalog/Replica.java b/fe/src/main/java/org/apache/doris/catalog/Replica.java index c9cab33..ad76292 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/src/main/java/org/apache/doris/catalog/Replica.java @@ -505,7 +505,6 @@ public class Replica implements Writable { return replica; } - @Override public boolean equals(Object obj) { if (this == obj) { return true; diff --git a/fe/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/src/main/java/org/apache/doris/catalog/Tablet.java index d8eb691..1b1cb35 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/src/main/java/org/apache/doris/catalog/Tablet.java @@ -39,6 +39,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -79,18 +80,18 @@ public class Tablet extends MetaObject implements Writable { private long lastStatusCheckTime = -1; public Tablet() { - this(0L, new ArrayList<>()); + this(0L, new ArrayList<Replica>()); } public Tablet(long tabletId) { - this(tabletId, new ArrayList<>()); + this(tabletId, new ArrayList<Replica>()); } public Tablet(long tabletId, List<Replica> replicas) { this.id = tabletId; this.replicas = replicas; if (this.replicas == null) { - this.replicas = new ArrayList<>(); + this.replicas = new ArrayList<Replica>(); } checkedVersion = -1L; @@ -299,15 +300,13 @@ public class Tablet extends MetaObject implements Writable { public static void sortReplicaByVersionDesc(List<Replica> replicas) { // sort replicas by version. higher version in the tops - replicas.sort(Replica.VERSION_DESC_COMPARATOR); + Collections.sort(replicas, Replica.VERSION_DESC_COMPARATOR); } - @Override public String toString() { return "tabletId=" + this.id; } - @Override public void write(DataOutput out) throws IOException { super.write(out); @@ -322,7 +321,6 @@ public class Tablet extends MetaObject implements Writable { out.writeLong(checkedVersionHash); out.writeBoolean(isConsistent); } - @Override public void readFields(DataInput in) throws IOException { super.readFields(in); @@ -348,7 +346,6 @@ public class Tablet extends MetaObject implements Writable { return tablet; } - @Override public boolean equals(Object obj) { if (this == obj) { return true; @@ -395,7 +392,7 @@ public class Tablet extends MetaObject implements Writable { return dataSize; } - /** + /* * A replica is healthy only if * 1. the backend is available * 2. replica version is caught up, and last failed version is -1 @@ -505,7 +502,7 @@ public class Tablet extends MetaObject implements Writable { return Pair.create(TabletStatus.HEALTHY, TabletSchedCtx.Priority.NORMAL); } - /** + /* * Check colocate table's tablet health * 1. Mismatch: * backends set: 1,2,3 @@ -554,7 +551,7 @@ public class Tablet extends MetaObject implements Writable { return TabletStatus.HEALTHY; } - /** + /* * check if this tablet is ready to be repaired, based on priority. * VERY_HIGH: repair immediately * HIGH: delay Config.tablet_repair_delay_factor_second * 1; diff --git a/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java b/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java index 0c695ca..4cb2b44 100644 --- a/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java +++ b/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java @@ -137,12 +137,12 @@ public class TabletMeta { lock.readLock().lock(); try { StringBuilder sb = new StringBuilder(); - sb.append("dbId=").append(dbId); - sb.append(" tableId=").append(tableId); - sb.append(" partitionId=").append(partitionId); - sb.append(" indexId=").append(indexId); - sb.append(" oldSchemaHash=").append(oldSchemaHash); - sb.append(" newSchemaHash=").append(newSchemaHash); + sb.append("dbId=" + dbId); + sb.append(" tableId=" + tableId); + sb.append(" partitionId=" + partitionId); + sb.append(" indexId=" + indexId); + sb.append(" oldSchemaHash=" + oldSchemaHash); + sb.append(" newSchemaHash=" + newSchemaHash); return sb.toString(); } finally { diff --git a/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java b/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java index 5dd6e60..5479a16 100644 --- a/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java +++ b/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java @@ -55,7 +55,13 @@ public class BackendLoadStatistic { public int compare(BackendLoadStatistic o1, BackendLoadStatistic o2) { double score1 = o1.getLoadScore(medium); double score2 = o2.getLoadScore(medium); - return Double.compare(score1, score2); + if (score1 > score2) { + return 1; + } else if (score1 == score2) { + return 0; + } else { + return -1; + } } } @@ -64,7 +70,14 @@ public class BackendLoadStatistic { public int compare(BackendLoadStatistic o1, BackendLoadStatistic o2) { Double score1 = o1.getMixLoadScore(); Double score2 = o2.getMixLoadScore(); - return score1.compareTo(score2); + + if (score1 > score2) { + return 1; + } else if (score1 == score2) { + return 0; + } else { + return -1; + } } } @@ -167,7 +180,11 @@ public class BackendLoadStatistic { throw new LoadBalanceException("backend " + beId + " does not exist"); } - isAvailable = be.isAvailable(); + if (!be.isAvailable()) { + isAvailable = false; + } else { + isAvailable = true; + } ImmutableMap<String, DiskInfo> disks = be.getDisks(); for (DiskInfo diskInfo : disks.values()) { @@ -316,7 +333,7 @@ public class BackendLoadStatistic { return false; } - /** + /* * Classify the paths into 'low', 'mid' and 'high', * and skip offline path, and path with different storage medium */ diff --git a/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java b/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java index 3aeb406..2b5fee9 100644 --- a/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java +++ b/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java @@ -115,7 +115,13 @@ public class RootPathLoadStatistic implements Comparable<RootPathLoadStatistic> public int compareTo(RootPathLoadStatistic o) { double myPercent = getUsedPercent(); double otherPercent = o.getUsedPercent(); - return Double.compare(myPercent, otherPercent); + if (myPercent < otherPercent) { + return -1; + } else if (myPercent > otherPercent) { + return 1; + } else { + return 0; + } } @Override diff --git a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java index a127255..036fe6e 100644 --- a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -21,7 +21,6 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.ColocateTableIndex; import org.apache.doris.catalog.ColocateTableIndex.GroupId; import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.DiskInfo; import org.apache.doris.catalog.DiskInfo.DiskState; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; @@ -61,6 +60,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.PriorityQueue; @@ -68,7 +68,7 @@ import java.util.Queue; import java.util.Set; import java.util.stream.Collectors; -/** +/* * TabletScheduler saved the tablets produced by TabletChecker and try to schedule them. * It also try to balance the cluster load. * @@ -170,7 +170,7 @@ public class TabletScheduler extends MasterDaemon { if (backends.containsKey(beId)) { List<Long> pathHashes = backends.get(beId).getDisks().values().stream() .filter(v -> v.getState()==DiskState.ONLINE) - .map(DiskInfo::getPathHash).collect(Collectors.toList()); + .map(v -> v.getPathHash()).collect(Collectors.toList()); backendsWorkingSlots.get(beId).updatePaths(pathHashes); } else { deletedBeIds.add(beId); @@ -186,7 +186,7 @@ public class TabletScheduler extends MasterDaemon { // add new backends for (Backend be : backends.values()) { if (!backendsWorkingSlots.containsKey(be.getId())) { - List<Long> pathHashes = be.getDisks().values().stream().map(DiskInfo::getPathHash).collect(Collectors.toList()); + List<Long> pathHashes = be.getDisks().values().stream().map(v -> v.getPathHash()).collect(Collectors.toList()); PathSlot slot = new PathSlot(pathHashes, Config.schedule_slot_num_per_path); backendsWorkingSlots.put(be.getId(), slot); LOG.info("add new backend {} with slots num: {}", be.getId(), be.getDisks().size()); @@ -200,7 +200,7 @@ public class TabletScheduler extends MasterDaemon { return backendsWorkingSlots; } - /** + /* * add a ready-to-be-scheduled tablet to pendingTablets, if it has not being added before. * if force is true, do not check if tablet is already added before. */ @@ -227,7 +227,7 @@ public class TabletScheduler extends MasterDaemon { return allTabletIds.contains(tabletId); } - /** + /* * Iterate current tablets, change their priority to VERY_HIGH if necessary. */ public synchronized void changeTabletsPriorityToVeryHigh(long dbId, long tblId, List<Long> partitionIds) { @@ -242,10 +242,10 @@ public class TabletScheduler extends MasterDaemon { pendingTablets = newPendingTablets; } - /** + /* * TabletScheduler will run as a daemon thread at a very short interval(default 5 sec) - * Firstly, it will try to update cluster load statistic and check if priority need to be adjusted. - * Then, it will schedule the tablets in pendingTablets. + * Firstly, it will try to update cluster load statistic and check if priority need to be adjuested. + * Than, it will schedule the tablets in pendingTablets. * Thirdly, it will check the current running tasks. * Finally, it try to balance the cluster if possible. * @@ -287,7 +287,7 @@ public class TabletScheduler extends MasterDaemon { lastStatUpdateTime = System.currentTimeMillis(); } - /** + /* * Here is the only place we update the cluster load statistic info. * We will not update this info dynamically along with the clone job's running. * Although it will cause a little bit inaccurate, but is within a controllable range, @@ -311,13 +311,13 @@ public class TabletScheduler extends MasterDaemon { return statisticMap; } - /** + /* * adjust priorities of all tablet infos */ private synchronized void adjustPriorities() { int size = pendingTablets.size(); int changedNum = 0; - TabletSchedCtx tabletCtx; + TabletSchedCtx tabletCtx = null; for (int i = 0; i < size; i++) { tabletCtx = pendingTablets.poll(); if (tabletCtx == null) { @@ -333,7 +333,7 @@ public class TabletScheduler extends MasterDaemon { LOG.info("adjust priority for all tablets. changed: {}, total: {}", changedNum, size); } - /** + /* * get at most BATCH_NUM tablets from queue, and try to schedule them. * After handle, the tablet info should be * 1. in runningTablets with state RUNNING, if being scheduled success. @@ -423,7 +423,7 @@ public class TabletScheduler extends MasterDaemon { runningTablets.put(tabletCtx.getTabletId(), tabletCtx); } - /** + /* * we take the tablet out of the runningTablets and than handle it, * avoid other threads see it. * Whoever takes this tablet, make sure to put it to the schedHistory or back to runningTablets. @@ -432,7 +432,7 @@ public class TabletScheduler extends MasterDaemon { return runningTablets.remove(tabletId); } - /** + /* * Try to schedule a single tablet. */ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) throws SchedException { @@ -448,7 +448,7 @@ public class TabletScheduler extends MasterDaemon { throw new SchedException(Status.UNRECOVERABLE, "db does not exist"); } - Pair<TabletStatus, TabletSchedCtx.Priority> statusPair; + Pair<TabletStatus, TabletSchedCtx.Priority> statusPair = null; db.writeLock(); try { OlapTable tbl = (OlapTable) db.getTable(tabletCtx.getTblId()); @@ -586,7 +586,7 @@ public class TabletScheduler extends MasterDaemon { } } - /** + /* * Replica is missing, which means there is no enough alive replicas. * So we need to find a destination backend to clone a new replica as possible as we can. * 1. find an available path in a backend as destination: @@ -615,7 +615,7 @@ public class TabletScheduler extends MasterDaemon { batchTask.addTask(tabletCtx.createCloneReplicaAndTask()); } - /** + /* * Replica version is incomplete, which means this replica is missing some version, * and need to be cloned from a healthy replica, in-place. * @@ -649,7 +649,7 @@ public class TabletScheduler extends MasterDaemon { handleReplicaMissing(tabletCtx, batchTask); } - /** + /* * replica is redundant, which means there are more replicas than we expected, which need to be dropped. * we just drop one redundant replica at a time, for safety reason. * choosing a replica to drop base on following priority: @@ -843,7 +843,7 @@ public class TabletScheduler extends MasterDaemon { return false; } - /** + /* * Just delete replica which does not located in colocate backends set. * return true if delete one replica, otherwise, return false. */ @@ -920,7 +920,7 @@ public class TabletScheduler extends MasterDaemon { LOG.info("send delete replica task for tablet {} in backend {}", tabletId, backendId); } - /** + /* * Cluster migration, which means the tablet has enough healthy replicas, * but some replicas are not in right cluster. * It is just same as 'replica missing'. @@ -933,7 +933,7 @@ public class TabletScheduler extends MasterDaemon { handleReplicaMissing(tabletCtx, batchTask); } - /** + /* * Replicas of colocate table's tablet does not locate on right backends set. * backends set: 1,2,3 * tablet replicas: 1,2,5 @@ -960,7 +960,7 @@ public class TabletScheduler extends MasterDaemon { batchTask.addTask(tabletCtx.createCloneReplicaAndTask()); } - /** + /* * Try to select some alternative tablets for balance. Add them to pendingTablets with priority LOW, * and waiting to be scheduled. */ @@ -984,7 +984,7 @@ public class TabletScheduler extends MasterDaemon { } } - /** + /* * Try to create a balance task for a tablet. */ private void doBalance(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) throws SchedException { @@ -1005,7 +1005,8 @@ public class TabletScheduler extends MasterDaemon { // get all available paths which this tablet can fit in. // beStatistics is sorted by mix load score in ascend order, so select from first to last. List<RootPathLoadStatistic> allFitPaths = Lists.newArrayList(); - for (BackendLoadStatistic bes : beStatistics) { + for (int i = 0; i < beStatistics.size(); i++) { + BackendLoadStatistic bes = beStatistics.get(i); if (!bes.isAvailable()) { continue; } @@ -1070,7 +1071,7 @@ public class TabletScheduler extends MasterDaemon { throw new SchedException(Status.SCHEDULE_FAILED, "unable to find dest path which can be fit in"); } - /** + /* * For some reason, a tablet info failed to be scheduled this time, * So we dynamically change its priority and add back to queue, waiting for next round. */ @@ -1126,7 +1127,7 @@ public class TabletScheduler extends MasterDaemon { return total; } - /** + /* * return true if we want to remove the clone task from AgentTaskQueue */ public boolean finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request) { @@ -1173,7 +1174,7 @@ public class TabletScheduler extends MasterDaemon { return true; } - /** + /* * Gather the running statistic of the task. * It will be evaluated for future strategy. * This should only be called when the tablet is down with state FINISHED. @@ -1207,7 +1208,7 @@ public class TabletScheduler extends MasterDaemon { lastSlotAdjustTime = System.currentTimeMillis(); } - /** + /* * handle tablets which are running. * We should finished the task if * 1. Tablet is already healthy @@ -1290,7 +1291,7 @@ public class TabletScheduler extends MasterDaemon { + runningTablets.values().stream().filter(t -> t.getType() == Type.BALANCE).count(); } - /** + /* * PathSlot keeps track of slot num per path of a Backend. * Each path on a Backend has several slot. * If a path's available slot num become 0, no task should be assigned to this path. @@ -1308,7 +1309,13 @@ public class TabletScheduler extends MasterDaemon { // update the path public synchronized void updatePaths(List<Long> paths) { // delete non exist path - pathSlots.entrySet().removeIf(entry -> !paths.contains(entry.getKey())); + Iterator<Map.Entry<Long, Slot>> iter = pathSlots.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry<Long, Slot> entry = iter.next(); + if (!paths.contains(entry.getKey())) { + iter.remove(); + } + } // add new path for (Long pathHash : paths) { @@ -1332,7 +1339,7 @@ public class TabletScheduler extends MasterDaemon { } } - /** + /* * Update the statistic of specified path */ public synchronized void updateStatistic(long pathHash, long copySize, long copyTimeMs) { @@ -1344,7 +1351,7 @@ public class TabletScheduler extends MasterDaemon { slot.totalCopyTimeMs += copyTimeMs; } - /** + /* * If the specified 'pathHash' has available slot, decrease the slot number and return this path hash */ public synchronized long takeSlot(long pathHash) throws SchedException { @@ -1393,7 +1400,7 @@ public class TabletScheduler extends MasterDaemon { return total; } - /** + /* * get path whose balance slot num is larger than 0 */ public synchronized Set<Long> getAvailPathsForBalance() { @@ -1416,15 +1423,15 @@ public class TabletScheduler extends MasterDaemon { public synchronized List<List<String>> getSlotInfo(long beId) { List<List<String>> results = Lists.newArrayList(); - pathSlots.forEach((key, value) -> { - value.rectify(); + pathSlots.entrySet().stream().forEach(t -> { + t.getValue().rectify(); List<String> result = Lists.newArrayList(); result.add(String.valueOf(beId)); - result.add(String.valueOf(key)); - result.add(String.valueOf(value.available)); - result.add(String.valueOf(value.total)); - result.add(String.valueOf(value.balanceSlot)); - result.add(String.valueOf(value.getAvgRate())); + result.add(String.valueOf(t.getKey())); + result.add(String.valueOf(t.getValue().available)); + result.add(String.valueOf(t.getValue().total)); + result.add(String.valueOf(t.getValue().balanceSlot)); + result.add(String.valueOf(t.getValue().getAvgRate())); results.add(result); }); return results; diff --git a/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java index df0c182..47121af 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java @@ -50,9 +50,11 @@ public class ClusterLoadStatisticProcDir implements ProcDirInterface { statMap = Catalog.getCurrentCatalog().getTabletScheduler().getStatisticMap(); - statMap.values().forEach(t -> { + statMap.values().stream().forEach(t -> { List<List<String>> statistics = t.getClusterStatistic(medium); - statistics.forEach(result::addRow); + statistics.stream().forEach(v -> { + result.addRow(v); + }); }); return result; diff --git a/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java b/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java index 8b0c39c..1559a4a 100644 --- a/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java +++ b/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java @@ -90,7 +90,6 @@ public class BackendHbResponse extends HeartbeatResponse implements Writable { out.writeInt(brpcPort); } - @Override public void readFields(DataInput in) throws IOException { super.readFields(in); beId = in.readLong(); diff --git a/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java b/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java index e6ae49b..dda679a 100644 --- a/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java +++ b/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java @@ -81,7 +81,6 @@ public class BrokerHbResponse extends HeartbeatResponse implements Writable { out.writeInt(port); } - @Override public void readFields(DataInput in) throws IOException { super.readFields(in); name = Text.readString(in); diff --git a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index b744cf2..70ce4c0 100644 --- a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -17,294 +17,484 @@ package org.apache.doris.catalog; -import org.apache.doris.analysis.CreateDbStmt; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.ColumnDef; import org.apache.doris.analysis.CreateTableStmt; +import org.apache.doris.analysis.HashDistributionDesc; +import org.apache.doris.analysis.KeysDesc; +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.RangePartitionDesc; +import org.apache.doris.analysis.SingleRangePartitionDesc; +import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.TypeDef; import org.apache.doris.common.DdlException; -import org.apache.doris.common.FeConstants; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.PaloAuth; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.task.AgentBatchTask; -import org.apache.doris.utframe.UtFrameUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import com.google.common.collect.Lists; + +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import java.util.UUID; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import mockit.Expectations; +import mockit.Injectable; +import mockit.Mock; +import mockit.MockUp; public class DynamicPartitionTableTest { - private static String runningDir = "fe/mocked/DynamicPartitionTableTest/" + UUID.randomUUID().toString() + "/"; + private TableName dbTableName; + private String dbName = "testDb"; + private String tableName = "testTable"; + private String clusterName = "default"; + private List<Long> beIds = Lists.newArrayList(); + private List<String> columnNames = Lists.newArrayList(); + private List<ColumnDef> columnDefs = Lists.newArrayList(); + + private Catalog catalog = Catalog.getInstance(); + private Database db = new Database(); + private Analyzer analyzer; - private static ConnectContext connectContext; + private Map<String, String> properties; + private List<SingleRangePartitionDesc> singleRangePartitionDescs; + + @Injectable + ConnectContext connectContext; @Rule - public ExpectedException expectedException = ExpectedException.none(); + public ExpectedException expectedEx = ExpectedException.none(); - @BeforeClass - public static void beforeClass() throws Exception { - FeConstants.default_scheduler_interval_millisecond = 1000; - FeConstants.runningUnitTest = true; + @Before + public void setUp() throws Exception { + dbTableName = new TableName(dbName, tableName); - UtFrameUtils.createMinDorisCluster(runningDir); + beIds.add(1L); + beIds.add(2L); + beIds.add(3L); - // create connect context - connectContext = UtFrameUtils.createDefaultCtx(); - // create database - String createDbStmtStr = "create database test;"; - CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext); - Catalog.getCurrentCatalog().createDb(createDbStmt); - } + columnNames.add("key1"); + columnNames.add("key2"); + columnNames.add("key3"); - @AfterClass - public static void TearDown() { - UtFrameUtils.cleanDorisFeDir(runningDir); - } + columnDefs.add(new ColumnDef("key1", new TypeDef(ScalarType.createType(PrimitiveType.INT)))); + columnDefs.add(new ColumnDef("key2", new TypeDef(ScalarType.createType(PrimitiveType.INT)))); + columnDefs.add(new ColumnDef("key3", new TypeDef(ScalarType.createVarchar(10)))); + + analyzer = new Analyzer(catalog, connectContext); + + properties = new HashMap<>(); + properties.put(DynamicPartitionProperty.ENABLE, "true"); + properties.put(DynamicPartitionProperty.PREFIX, "p"); + properties.put(DynamicPartitionProperty.TIME_UNIT, "day"); + properties.put(DynamicPartitionProperty.START, "-3"); + properties.put(DynamicPartitionProperty.END, "3"); + properties.put(DynamicPartitionProperty.BUCKETS, "30"); + + singleRangePartitionDescs = new LinkedList<>(); + singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", + new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-128"))), null)); + + new MockUp<AgentBatchTask>() { + @Mock + void run() { + return; + } + }; - private static void createTable(String sql) throws Exception { - CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); - Catalog.getCurrentCatalog().createTable(createTableStmt); + new MockUp<CountDownLatch>() { + @Mock + boolean await(long timeout, TimeUnit unit) { + return true; + } + }; + + new Expectations(analyzer, catalog) {{ + analyzer.getClusterName(); + minTimes = 0; + result = clusterName; + }}; + + dbTableName.analyze(analyzer); } @Test - public void testNormal() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; - createTable(createOlapTblStmt); + public void testNormal(@Injectable SystemInfoService systemInfoService, + @Injectable PaloAuth paloAuth, + @Injectable EditLog editLog) throws UserException { + new Expectations(catalog) { + { + catalog.getDb(dbTableName.getDb()); + minTimes = 0; + result = db; + + Catalog.getCurrentSystemInfo(); + minTimes = 0; + result = systemInfoService; + + systemInfoService.checkClusterCapacity(anyString); + minTimes = 0; + systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); + minTimes = 0; + result = beIds; + + catalog.getAuth(); + minTimes = 0; + result = paloAuth; + paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); + minTimes = 0; + result = true; + + catalog.getEditLog(); + minTimes = 0; + result = editLog; + } + }; + + CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", + new KeysDesc(KeysType.AGG_KEYS, columnNames), + new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), + new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); + stmt.analyze(analyzer); + + catalog.createTable(stmt); } @Test - public void testMissPrefix() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_prefix` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; - expectedException.expect(DdlException.class); - expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.prefix properties"); - createTable(createOlapTblStmt); + public void testMissPrefix(@Injectable SystemInfoService systemInfoService, + @Injectable PaloAuth paloAuth, + @Injectable EditLog editLog) throws UserException { + new Expectations(catalog) { + { + catalog.getDb(dbTableName.getDb()); + minTimes = 0; + result = db; + + Catalog.getCurrentSystemInfo(); + minTimes = 0; + result = systemInfoService; + + systemInfoService.checkClusterCapacity(anyString); + minTimes = 0; + systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); + minTimes = 0; + result = beIds; + + catalog.getAuth(); + minTimes = 0; + result = paloAuth; + paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); + minTimes = 0; + result = true; + + catalog.getEditLog(); + minTimes = 0; + result = editLog; + } + }; + + properties.remove(DynamicPartitionProperty.PREFIX); + + CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", + new KeysDesc(KeysType.AGG_KEYS, columnNames), + new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), + new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); + stmt.analyze(analyzer); + + expectedEx.expect(DdlException.class); + expectedEx.expectMessage("Must assign dynamic_partition.prefix properties"); + + catalog.createTable(stmt); } @Test - public void testMissTimeUnit() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_time_unit` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; - expectedException.expect(DdlException.class); - expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.time_unit properties"); - createTable(createOlapTblStmt); + public void testMissTimeUnit(@Injectable SystemInfoService systemInfoService, + @Injectable PaloAuth paloAuth, + @Injectable EditLog editLog) throws UserException { + new Expectations(catalog) { + { + catalog.getDb(dbTableName.getDb()); + minTimes = 0; + result = db; + + Catalog.getCurrentSystemInfo(); + minTimes = 0; + result = systemInfoService; + + systemInfoService.checkClusterCapacity(anyString); + minTimes = 0; + systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); + minTimes = 0; + result = beIds; + + catalog.getAuth(); + minTimes = 0; + result = paloAuth; + paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); + minTimes = 0; + result = true; + + catalog.getEditLog(); + minTimes = 0; + result = editLog; + } + }; + + properties.remove(DynamicPartitionProperty.TIME_UNIT); + + CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", + new KeysDesc(KeysType.AGG_KEYS, columnNames), + new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), + new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); + stmt.analyze(analyzer); + + expectedEx.expect(DdlException.class); + expectedEx.expectMessage("Must assign dynamic_partition.time_unit properties"); + + catalog.createTable(stmt); } @Test - public void testMissStart() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_start` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; - createTable(createOlapTblStmt); - } + public void testMissSTART(@Injectable SystemInfoService systemInfoService, + @Injectable PaloAuth paloAuth, + @Injectable EditLog editLog) throws UserException { + new Expectations(catalog) { + { + catalog.getDb(dbTableName.getDb()); + minTimes = 0; + result = db; + Catalog.getCurrentSystemInfo(); + minTimes = 0; + result = systemInfoService; - @Test - public void testMissEnd() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_end` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; - expectedException.expect(DdlException.class); - expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.end properties"); - createTable(createOlapTblStmt); + systemInfoService.checkClusterCapacity(anyString); + minTimes = 0; + systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); + minTimes = 0; + result = beIds; + + catalog.getAuth(); + minTimes = 0; + result = paloAuth; + paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); + minTimes = 0; + result = true; + + catalog.getEditLog(); + minTimes = 0; + result = editLog; + } + }; + + properties.remove(DynamicPartitionProperty.START); + + CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", + new KeysDesc(KeysType.AGG_KEYS, columnNames), + new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), + new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); + stmt.analyze(analyzer); + + catalog.createTable(stmt); } + @Test + public void testMissEnd(@Injectable SystemInfoService systemInfoService, + @Injectable PaloAuth paloAuth, + @Injectable EditLog editLog) throws UserException { + new Expectations(catalog) { + { + catalog.getDb(dbTableName.getDb()); + minTimes = 0; + result = db; + + Catalog.getCurrentSystemInfo(); + minTimes = 0; + result = systemInfoService; + + systemInfoService.checkClusterCapacity(anyString); + minTimes = 0; + systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); + minTimes = 0; + result = beIds; + + catalog.getAuth(); + minTimes = 0; + result = paloAuth; + paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); + minTimes = 0; + result = true; + + catalog.getEditLog(); + minTimes = 0; + result = editLog; + } + }; + + properties.remove(DynamicPartitionProperty.END); + + CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", + new KeysDesc(KeysType.AGG_KEYS, columnNames), + new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), + new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); + stmt.analyze(analyzer); + + expectedEx.expect(DdlException.class); + expectedEx.expectMessage("Must assign dynamic_partition.end properties"); + + catalog.createTable(stmt); + } @Test - public void testMissBuckets() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\"\n" + - ");"; - expectedException.expect(DdlException.class); - expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.buckets properties"); - createTable(createOlapTblStmt); + public void testMissBuckets(@Injectable SystemInfoService systemInfoService, + @Injectable PaloAuth paloAuth, + @Injectable EditLog editLog) throws UserException { + new Expectations(catalog) { + { + catalog.getDb(dbTableName.getDb()); + minTimes = 0; + result = db; + + Catalog.getCurrentSystemInfo(); + minTimes = 0; + result = systemInfoService; + + systemInfoService.checkClusterCapacity(anyString); + minTimes = 0; + systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); + minTimes = 0; + result = beIds; + + catalog.getAuth(); + minTimes = 0; + result = paloAuth; + paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); + minTimes = 0; + result = true; + + catalog.getEditLog(); + minTimes = 0; + result = editLog; + } + }; + + properties.remove(DynamicPartitionProperty.BUCKETS); + + CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", + new KeysDesc(KeysType.AGG_KEYS, columnNames), + new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), + new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); + stmt.analyze(analyzer); + + expectedEx.expect(DdlException.class); + expectedEx.expectMessage("Must assign dynamic_partition.buckets properties"); + + catalog.createTable(stmt); } @Test - public void testNotAllowed() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; - expectedException.expect(DdlException.class); - expectedException.expectMessage("errCode = 2, detailMessage = Only support dynamic partition properties on range partition table"); - createTable(createOlapTblStmt); + public void testNotAllowed(@Injectable SystemInfoService systemInfoService, + @Injectable PaloAuth paloAuth, + @Injectable EditLog editLog) throws UserException { + new Expectations(catalog) { + { + catalog.getDb(dbTableName.getDb()); + minTimes = 0; + result = db; + + Catalog.getCurrentSystemInfo(); + minTimes = 0; + result = systemInfoService; + + systemInfoService.checkClusterCapacity(anyString); + minTimes = 0; + systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); + minTimes = 0; + result = beIds; + + catalog.getAuth(); + minTimes = 0; + result = paloAuth; + paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); + minTimes = 0; + result = true; + + catalog.getEditLog(); + minTimes = 0; + result = editLog; + } + }; + + CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", + new KeysDesc(KeysType.AGG_KEYS, columnNames), null, + new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); + stmt.analyze(analyzer); + + expectedEx.expect(DdlException.class); + expectedEx.expectMessage("Only support dynamic partition properties on range partition table"); + + catalog.createTable(stmt); } @Test - public void testNotAllowedInMultiPartitions() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1, k2)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\", \"100\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\", \"200\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\", \"300\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; - expectedException.expect(DdlException.class); - expectedException.expectMessage("errCode = 2, detailMessage = Dynamic partition only support single-column range partition"); - createTable(createOlapTblStmt); + public void testNotAllowedInMultiPartitions(@Injectable SystemInfoService systemInfoService, + @Injectable PaloAuth paloAuth, + @Injectable EditLog editLog) throws UserException { + new Expectations(catalog) { + { + catalog.getDb(dbTableName.getDb()); + minTimes = 0; + result = db; + + Catalog.getCurrentSystemInfo(); + minTimes = 0; + result = systemInfoService; + + systemInfoService.checkClusterCapacity(anyString); + minTimes = 0; + systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); + minTimes = 0; + result = beIds; + + catalog.getAuth(); + minTimes = 0; + result = paloAuth; + paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); + minTimes = 0; + result = true; + + catalog.getEditLog(); + minTimes = 0; + result = editLog; + } + }; + + List<SingleRangePartitionDesc> rangePartitionDescs = new LinkedList<>(); + rangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", + new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-128"), new PartitionValue("100"))), null)); + + CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", + new KeysDesc(KeysType.AGG_KEYS, columnNames), + new RangePartitionDesc(Lists.newArrayList("key1", "key2"), singleRangePartitionDescs), + new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); + stmt.analyze(analyzer); + + expectedEx.expect(DdlException.class); + expectedEx.expectMessage("Dynamic partition only support single-column range partition"); + + catalog.createTable(stmt); } } diff --git a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java index 2f88ecd..81c883e 100644 --- a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java +++ b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java @@ -17,15 +17,12 @@ package org.apache.doris.common; -import org.apache.doris.analysis.DateLiteral; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DataProperty; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; -import org.apache.doris.catalog.Type; import org.apache.doris.common.util.PropertyAnalyzer; -import org.apache.doris.common.util.TimeUtils; import org.apache.doris.thrift.TStorageMedium; import com.google.common.collect.Lists; @@ -125,14 +122,11 @@ public class PropertyAnalyzerTest { @Test public void testStorageMedium() throws AnalysisException { long tomorrowTs = System.currentTimeMillis() / 1000 + 86400; - String tomorrowTimeStr = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tomorrowTs * 1000); Map<String, String> properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, "SSD"); - properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME, tomorrowTimeStr); + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tomorrowTs * 1000)); DataProperty dataProperty = PropertyAnalyzer.analyzeDataProperty(properties, new DataProperty(TStorageMedium.SSD)); - // avoid UT fail because time zone different - DateLiteral dateLiteral = new DateLiteral(tomorrowTimeStr, Type.DATETIME); - Assert.assertEquals(dateLiteral.unixTimestamp(TimeUtils.getTimeZone()), dataProperty.getCooldownTimeMs()); + Assert.assertEquals(tomorrowTs, dataProperty.getCooldownTimeMs() / 1000); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
