This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9d72d1b [Refactor] Refactor some redundant code && Replace some UT by
UtFrameUtils
9d72d1b is described below
commit 9d72d1bb8726d3f67caf679c180d1abec5864d2e
Author: Mingyu Chen <[email protected]>
AuthorDate: Mon May 18 14:53:59 2020 +0800
[Refactor] Refactor some redundant code && Replace some UT by UtFrameUtils
This CL have no logic changed, just do some code refactor and use new
UtFrameWork to replace some old UT.
NOTICE(#3622):
This is a "revert of revert pull request".
This pr is mainly used to synthesize the PRs whose commits were
scattered and submitted due to the wrong merge method into a complete
single commit.
---
.../main/java/org/apache/doris/alter/AlterJob.java | 18 +-
.../java/org/apache/doris/backup/RestoreJob.java | 175 +++---
.../java/org/apache/doris/catalog/Database.java | 2 +-
.../apache/doris/catalog/MaterializedIndex.java | 8 +-
.../java/org/apache/doris/catalog/OlapTable.java | 6 +-
.../java/org/apache/doris/catalog/Partition.java | 7 +-
.../org/apache/doris/catalog/PartitionInfo.java | 8 +-
.../java/org/apache/doris/catalog/Replica.java | 1 +
.../main/java/org/apache/doris/catalog/Tablet.java | 19 +-
.../java/org/apache/doris/catalog/TabletMeta.java | 12 +-
.../apache/doris/clone/BackendLoadStatistic.java | 25 +-
.../apache/doris/clone/RootPathLoadStatistic.java | 8 +-
.../org/apache/doris/clone/TabletScheduler.java | 89 ++-
.../common/proc/ClusterLoadStatisticProcDir.java | 6 +-
.../org/apache/doris/system/BackendHbResponse.java | 1 +
.../org/apache/doris/system/BrokerHbResponse.java | 1 +
.../doris/catalog/DynamicPartitionTableTest.java | 684 ++++++++-------------
.../apache/doris/common/PropertyAnalyzerTest.java | 10 +-
18 files changed, 426 insertions(+), 654 deletions(-)
diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJob.java
b/fe/src/main/java/org/apache/doris/alter/AlterJob.java
index ab927cc..7b10fd5 100644
--- a/fe/src/main/java/org/apache/doris/alter/AlterJob.java
+++ b/fe/src/main/java/org/apache/doris/alter/AlterJob.java
@@ -189,18 +189,14 @@ public abstract class AlterJob implements Writable {
return false;
} else if (!backend.isAlive()) {
long currentTime = System.currentTimeMillis();
- if (backend.getLastUpdateMs() > 0
- && currentTime - backend.getLastUpdateMs() >
Config.max_backend_down_time_second * 1000) {
- // this backend is done for a long time and not restart
automatically.
- // we consider it as dead
- return false;
- }
- return true;
- } else if (backend.isDecommissioned()) {
- return false;
+ // If this backend is done for a long time and not restart
automatically.
+ // we consider it as dead and return false.
+ return backend.getLastUpdateMs() <= 0
+ || currentTime - backend.getLastUpdateMs() <=
Config.max_backend_down_time_second * 1000;
+ } else {
+ return !backend.isDecommissioned();
}
-
- return true;
+
}
public static AlterJob read(DataInput in) throws IOException {
diff --git a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
index ef576b4..335ac17 100644
--- a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -172,11 +172,7 @@ public class RestoreJob extends AbstractJob {
}
public synchronized boolean finishTabletSnapshotTask(SnapshotTask task,
TFinishTaskRequest request) {
- Preconditions.checkState(task.getJobId() == jobId);
- Preconditions.checkState(task.getDbId() == dbId);
-
- if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
- taskErrMsg.put(task.getSignature(),
Joiner.on(",").join(request.getTask_status().getError_msgs()));
+ if (checkTaskStatus(task, task.getJobId(), request)) {
return false;
}
@@ -198,19 +194,15 @@ public class RestoreJob extends AbstractJob {
taskErrMsg.remove(task.getSignature());
Preconditions.checkState(task.getTabletId() == removedTabletId,
removedTabletId);
LOG.debug("get finished snapshot info: {}, unfinished tasks num:
{}, remove result: {}. {}",
- info, unfinishedSignatureToId.size(), this);
+ info, unfinishedSignatureToId.size(), this,
removedTabletId);
return true;
}
-
return false;
}
- public synchronized boolean finishTabletDownloadTask(DownloadTask task,
TFinishTaskRequest request) {
- Preconditions.checkState(task.getJobId() == jobId);
- Preconditions.checkState(task.getDbId() == dbId);
- if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
- taskErrMsg.put(task.getSignature(),
Joiner.on(",").join(request.getTask_status().getError_msgs()));
+ public synchronized boolean finishTabletDownloadTask(DownloadTask task,
TFinishTaskRequest request) {
+ if (checkTaskStatus(task, task.getJobId(), request)) {
return false;
}
@@ -237,11 +229,7 @@ public class RestoreJob extends AbstractJob {
}
public synchronized boolean finishDirMoveTask(DirMoveTask task,
TFinishTaskRequest request) {
- Preconditions.checkState(task.getJobId() == jobId);
- Preconditions.checkState(task.getDbId() == dbId);
-
- if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
- taskErrMsg.put(task.getSignature(),
Joiner.on(",").join(request.getTask_status().getError_msgs()));
+ if (checkTaskStatus(task, task.getJobId(), request)) {
return false;
}
@@ -256,6 +244,17 @@ public class RestoreJob extends AbstractJob {
return true;
}
+ private boolean checkTaskStatus(AgentTask task, long jobId,
TFinishTaskRequest request) {
+ Preconditions.checkState(jobId == this.jobId);
+ Preconditions.checkState(dbId == task.getDbId());
+
+ if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
+ taskErrMsg.put(task.getSignature(),
Joiner.on(",").join(request.getTask_status().getError_msgs()));
+ return true;
+ }
+ return false;
+ }
+
@Override
public synchronized void replayRun() {
LOG.info("replay run restore job: {}", this);
@@ -342,7 +341,7 @@ public class RestoreJob extends AbstractJob {
}
}
- /*
+ /**
* return true if some restored objs have been dropped.
*/
private void checkIfNeedCancel() {
@@ -353,6 +352,7 @@ public class RestoreJob extends AbstractJob {
Database db = catalog.getDb(dbId);
if (db == null) {
status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " has
been dropped");
+ return;
}
db.readLock();
@@ -381,7 +381,7 @@ public class RestoreJob extends AbstractJob {
}
}
- /*
+ /**
* Restore rules as follow:
* A. Table already exist
* A1. Partition already exist, generate file mapping
@@ -505,19 +505,9 @@ public class RestoreJob extends AbstractJob {
Range<PartitionKey> remoteRange =
remoteRangePartInfo.getRange(backupPartInfo.id);
if (localRange.equals(remoteRange)) {
// Same partition, same range
- if
(localRangePartInfo.getReplicationNum(localPartition.getId()) !=
restoreReplicationNum) {
- status = new
Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name
- + " in table " +
localTbl.getName()
- + " has different replication
num '"
- +
localRangePartInfo.getReplicationNum(localPartition.getId())
- + "' with partition in
repository, which is " + restoreReplicationNum);
+ if
(genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl,
backupPartInfo, tblInfo)) {
return;
}
- genFileMapping(localOlapTbl,
localPartition, tblInfo.id, backupPartInfo,
- true /* overwrite when
commit */);
-
restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(),
-
Pair.create(backupPartInfo.version,
-
backupPartInfo.versionHash));
} else {
// Same partition name, different range
status = new Status(ErrCode.COMMON_ERROR,
"Partition " + backupPartInfo.name
@@ -527,21 +517,9 @@ public class RestoreJob extends AbstractJob {
}
} else {
// If this is a single partitioned table.
- if
(localPartInfo.getReplicationNum(localPartition.getId()) !=
restoreReplicationNum) {
- status = new Status(ErrCode.COMMON_ERROR,
"Partition " + backupPartInfo.name
- + " in table " + localTbl.getName()
- + " has different replication num
'"
- +
localPartInfo.getReplicationNum(localPartition.getId())
- + "' with partition in repository,
which is " + restoreReplicationNum);
+ if
(genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl,
backupPartInfo, tblInfo)) {
return;
}
-
- // No need to check range, just generate file
mapping
- genFileMapping(localOlapTbl, localPartition,
tblInfo.id, backupPartInfo,
- true /* overwrite when commit
*/);
- restoredVersionInfo.put(localOlapTbl.getId(),
localPartition.getId(),
-
Pair.create(backupPartInfo.version,
-
backupPartInfo.versionHash));
}
} else {
// partitions does not exist
@@ -616,65 +594,20 @@ public class RestoreJob extends AbstractJob {
BackupPartitionInfo backupPartitionInfo
=
jobInfo.getTableInfo(entry.first).getPartInfo(restorePart.getName());
- Set<String> bfColumns = localTbl.getCopiedBfColumns();
- double bfFpp = localTbl.getBfFpp();
- for (MaterializedIndex restoredIdx :
restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
- MaterializedIndexMeta indexMeta =
localTbl.getIndexMetaByIndexId(restoredIdx.getId());
- TabletMeta tabletMeta = new TabletMeta(db.getId(),
localTbl.getId(), restorePart.getId(),
- restoredIdx.getId(), indexMeta.getSchemaHash(),
TStorageMedium.HDD);
- for (Tablet restoreTablet : restoredIdx.getTablets()) {
-
Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta);
- for (Replica restoreReplica :
restoreTablet.getReplicas()) {
-
Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(),
restoreReplica);
- CreateReplicaTask task = new
CreateReplicaTask(restoreReplica.getBackendId(), dbId,
- localTbl.getId(), restorePart.getId(),
restoredIdx.getId(),
- restoreTablet.getId(),
indexMeta.getShortKeyColumnCount(),
- indexMeta.getSchemaHash(),
restoreReplica.getVersion(),
- restoreReplica.getVersionHash(),
indexMeta.getKeysType(), TStorageType.COLUMN,
- TStorageMedium.HDD /* all restored
replicas will be saved to HDD */,
- indexMeta.getSchema(), bfColumns, bfFpp,
null,
- localTbl.getCopiedIndexes(),
- localTbl.isInMemory());
- task.setInRestoreMode(true);
- batchTask.addTask(task);
- }
- }
- }
+ createReplicas(db, batchTask, localTbl, restorePart);
genFileMapping(localTbl, restorePart, remoteTbl.getId(),
backupPartitionInfo,
- allowLoad ? false : true /* if allow load, do
not overwrite when commit */);
+ !allowLoad /* if allow load, do not overwrite
when commit */);
}
// generate create replica task for all restored tables
for (OlapTable restoreTbl : restoredTbls) {
for (Partition restorePart : restoreTbl.getPartitions()) {
- Set<String> bfColumns = restoreTbl.getCopiedBfColumns();
- double bfFpp = restoreTbl.getBfFpp();
- for (MaterializedIndex index :
restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
- MaterializedIndexMeta indexMeta =
restoreTbl.getIndexMetaByIndexId(index.getId());
- TabletMeta tabletMeta = new TabletMeta(db.getId(),
restoreTbl.getId(), restorePart.getId(),
- index.getId(), indexMeta.getSchemaHash(),
TStorageMedium.HDD);
- for (Tablet tablet : index.getTablets()) {
-
Catalog.getCurrentInvertedIndex().addTablet(tablet.getId(), tabletMeta);
- for (Replica replica : tablet.getReplicas()) {
-
Catalog.getCurrentInvertedIndex().addReplica(tablet.getId(), replica);
- CreateReplicaTask task = new
CreateReplicaTask(replica.getBackendId(), dbId,
- restoreTbl.getId(),
restorePart.getId(), index.getId(), tablet.getId(),
- indexMeta.getShortKeyColumnCount(),
indexMeta.getSchemaHash(),
- replica.getVersion(),
replica.getVersionHash(),
- indexMeta.getKeysType(),
TStorageType.COLUMN, TStorageMedium.HDD,
- indexMeta.getSchema(), bfColumns,
bfFpp, null,
- restoreTbl.getCopiedIndexes(),
- restoreTbl.isInMemory());
- task.setInRestoreMode(true);
- batchTask.addTask(task);
- }
- }
- }
+ createReplicas(db, batchTask, restoreTbl, restorePart);
BackupTableInfo backupTableInfo =
jobInfo.getTableInfo(restoreTbl.getName());
genFileMapping(restoreTbl, restorePart, backupTableInfo.id,
backupTableInfo.getPartInfo(restorePart.getName()),
- allowLoad ? false : true /* if allow load,
do not overwrite when commit */);
+ !allowLoad /* if allow load, do not
overwrite when commit */);
}
// set restored table's new name after all 'genFileMapping'
restoreTbl.setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName()));
@@ -689,7 +622,7 @@ public class RestoreJob extends AbstractJob {
if (batchTask.getTaskNum() > 0) {
MarkedCountDownLatch<Long, Long> latch = new
MarkedCountDownLatch<Long, Long>(batchTask.getTaskNum());
for (AgentTask task : batchTask.getAllTasks()) {
- latch.addMark(((CreateReplicaTask) task).getBackendId(),
((CreateReplicaTask) task).getTabletId());
+ latch.addMark(task.getBackendId(), task.getTabletId());
((CreateReplicaTask) task).setLatch(latch);
AgentTaskQueue.addTask(task);
}
@@ -806,7 +739,54 @@ public class RestoreJob extends AbstractJob {
// No log here, PENDING state restore job will redo this method
LOG.info("finished to prepare meta and send snapshot tasks, num: {}.
{}",
batchTask.getTaskNum(), this);
- return;
+ }
+
+ private boolean genFileMappingWhenBackupReplicasEqual(PartitionInfo
localPartInfo, Partition localPartition, Table localTbl,
+ BackupPartitionInfo
backupPartInfo, BackupTableInfo tblInfo) {
+ if (localPartInfo.getReplicationNum(localPartition.getId()) !=
restoreReplicationNum) {
+ status = new Status(ErrCode.COMMON_ERROR, "Partition " +
backupPartInfo.name
+ + " in table " + localTbl.getName()
+ + " has different replication num '"
+ + localPartInfo.getReplicationNum(localPartition.getId())
+ + "' with partition in repository, which is " +
restoreReplicationNum);
+ return true;
+ }
+
+ // No need to check range, just generate file mapping
+ OlapTable localOlapTbl = (OlapTable) localTbl;
+ genFileMapping(localOlapTbl, localPartition, tblInfo.id,
backupPartInfo,
+ true /* overwrite when commit */);
+ restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(),
+ Pair.create(backupPartInfo.version,
+ backupPartInfo.versionHash));
+ return false;
+ }
+
+ private void createReplicas(Database db, AgentBatchTask batchTask,
OlapTable localTbl, Partition restorePart) {
+ Set<String> bfColumns = localTbl.getCopiedBfColumns();
+ double bfFpp = localTbl.getBfFpp();
+ for (MaterializedIndex restoredIdx :
restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
+ MaterializedIndexMeta indexMeta =
localTbl.getIndexMetaByIndexId(restoredIdx.getId());
+ TabletMeta tabletMeta = new TabletMeta(db.getId(),
localTbl.getId(), restorePart.getId(),
+ restoredIdx.getId(), indexMeta.getSchemaHash(),
TStorageMedium.HDD);
+ for (Tablet restoreTablet : restoredIdx.getTablets()) {
+
Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta);
+ for (Replica restoreReplica : restoreTablet.getReplicas()) {
+
Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(),
restoreReplica);
+ CreateReplicaTask task = new
CreateReplicaTask(restoreReplica.getBackendId(), dbId,
+ localTbl.getId(), restorePart.getId(),
restoredIdx.getId(),
+ restoreTablet.getId(),
indexMeta.getShortKeyColumnCount(),
+ indexMeta.getSchemaHash(),
restoreReplica.getVersion(),
+ restoreReplica.getVersionHash(),
indexMeta.getKeysType(), TStorageType.COLUMN,
+ TStorageMedium.HDD /* all restored replicas will
be saved to HDD */,
+ indexMeta.getSchema(), bfColumns, bfFpp, null,
+ localTbl.getCopiedIndexes(),
+ localTbl.isInMemory());
+ task.setInRestoreMode(true);
+ batchTask.addTask(task);
+ }
+ }
+ }
}
// reset remote partition.
@@ -825,11 +805,11 @@ public class RestoreJob extends AbstractJob {
// indexes
Map<String, Long> localIdxNameToId = localTbl.getIndexNameToId();
- for (String localidxName : localIdxNameToId.keySet()) {
+ for (String localIdxName : localIdxNameToId.keySet()) {
// set ids of indexes in remote partition to the local index ids
- long remoteIdxId = remoteTbl.getIndexIdByName(localidxName);
+ long remoteIdxId = remoteTbl.getIndexIdByName(localIdxName);
MaterializedIndex remoteIdx = remotePart.getIndex(remoteIdxId);
- long localIdxId = localIdxNameToId.get(localidxName);
+ long localIdxId = localIdxNameToId.get(localIdxName);
remoteIdx.setIdForRestore(localIdxId);
if (localIdxId != localTbl.getBaseIndexId()) {
// not base table, reset
@@ -1510,6 +1490,7 @@ public class RestoreJob extends AbstractJob {
}
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
diff --git a/fe/src/main/java/org/apache/doris/catalog/Database.java
b/fe/src/main/java/org/apache/doris/catalog/Database.java
index fa53e14..0d46c7a 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Database.java
@@ -113,7 +113,7 @@ public class Database extends MetaObject implements
Writable {
}
this.rwLock = new ReentrantReadWriteLock(true);
this.idToTable = new ConcurrentHashMap<>();
- this.nameToTable = new HashMap<String, Table>();
+ this.nameToTable = new HashMap<>();
this.dataQuotaBytes = FeConstants.default_db_data_quota_bytes;
this.replicaQuotaSize = FeConstants.default_db_replica_quota_size;
this.dbState = DbState.NORMAL;
diff --git a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
index 6f8abd9..a921758 100644
--- a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
+++ b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
@@ -76,8 +76,8 @@ public class MaterializedIndex extends MetaObject implements
Writable, GsonPostP
public MaterializedIndex() {
this.state = IndexState.NORMAL;
- this.idToTablets = new HashMap<Long, Tablet>();
- this.tablets = new ArrayList<Tablet>();
+ this.idToTablets = new HashMap<>();
+ this.tablets = new ArrayList<>();
}
public MaterializedIndex(long id, IndexState state) {
@@ -88,8 +88,8 @@ public class MaterializedIndex extends MetaObject implements
Writable, GsonPostP
this.state = IndexState.NORMAL;
}
- this.idToTablets = new HashMap<Long, Tablet>();
- this.tablets = new ArrayList<Tablet>();
+ this.idToTablets = new HashMap<>();
+ this.tablets = new ArrayList<>();
this.rowCount = 0;
diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
index fe252dd..f307a5f 100644
--- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -643,6 +643,7 @@ public class OlapTable extends Table {
*/
// get partition by name, not including temp partitions
+ @Override
public Partition getPartition(String partitionName) {
return getPartition(partitionName, false);
}
@@ -732,6 +733,7 @@ public class OlapTable extends Table {
return false;
}
+ @Override
public TTableDescriptor toThrift() {
TOlapTable tOlapTable = new TOlapTable(getName());
TTableDescriptor tTableDescriptor = new TTableDescriptor(id,
TTableType.OLAP_TABLE,
@@ -930,6 +932,7 @@ public class OlapTable extends Table {
tempPartitions.write(out);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -947,7 +950,7 @@ public class OlapTable extends Table {
if (Catalog.getCurrentCatalogJournalVersion() <
FeMetaVersion.VERSION_75) {
// schema
int colCount = in.readInt();
- List<Column> schema = new LinkedList<Column>();
+ List<Column> schema = new LinkedList<>();
for (int j = 0; j < colCount; j++) {
Column column = Column.read(in);
schema.add(column);
@@ -1073,6 +1076,7 @@ public class OlapTable extends Table {
rebuildFullSchema();
}
+ @Override
public boolean equals(Table table) {
if (this == table) {
return true;
diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java
b/fe/src/main/java/org/apache/doris/catalog/Partition.java
index 447555a..0730bed 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Partition.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java
@@ -67,13 +67,13 @@ public class Partition extends MetaObject implements
Writable {
private PartitionState state;
@SerializedName(value = "baseIndex")
private MaterializedIndex baseIndex;
- /*
+ /**
* Visible rollup indexes are indexes which are visible to user.
* User can do query on them, show them in related 'show' stmt.
*/
@SerializedName(value = "idToVisibleRollupIndex")
private Map<Long, MaterializedIndex> idToVisibleRollupIndex =
Maps.newHashMap();
- /*
+ /**
* Shadow indexes are indexes which are not visible to user.
* Query will not run on these shadow indexes, and user can not see them
neither.
* But load process will load data into these shadow indexes.
@@ -81,7 +81,7 @@ public class Partition extends MetaObject implements Writable
{
@SerializedName(value = "idToShadowIndex")
private Map<Long, MaterializedIndex> idToShadowIndex = Maps.newHashMap();
- /*
+ /**
* committed version(hash): after txn is committed, set committed
version(hash)
* visible version(hash): after txn is published, set visible version
* next version(hash): next version is set after finished committing, it
should equals to committed version + 1
@@ -354,6 +354,7 @@ public class Partition extends MetaObject implements
Writable {
distributionInfo.write(out);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
diff --git a/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
b/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index 1359f0c..f41f595 100644
--- a/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -51,15 +51,15 @@ public class PartitionInfo implements Writable {
protected Map<Long, Boolean> idToInMemory;
public PartitionInfo() {
- this.idToDataProperty = new HashMap<Long, DataProperty>();
- this.idToReplicationNum = new HashMap<Long, Short>();
+ this.idToDataProperty = new HashMap<>();
+ this.idToReplicationNum = new HashMap<>();
this.idToInMemory = new HashMap<>();
}
public PartitionInfo(PartitionType type) {
this.type = type;
- this.idToDataProperty = new HashMap<Long, DataProperty>();
- this.idToReplicationNum = new HashMap<Long, Short>();
+ this.idToDataProperty = new HashMap<>();
+ this.idToReplicationNum = new HashMap<>();
this.idToInMemory = new HashMap<>();
}
diff --git a/fe/src/main/java/org/apache/doris/catalog/Replica.java
b/fe/src/main/java/org/apache/doris/catalog/Replica.java
index ad76292..c9cab33 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Replica.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Replica.java
@@ -505,6 +505,7 @@ public class Replica implements Writable {
return replica;
}
+ @Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
diff --git a/fe/src/main/java/org/apache/doris/catalog/Tablet.java
b/fe/src/main/java/org/apache/doris/catalog/Tablet.java
index 1b1cb35..d8eb691 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Tablet.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Tablet.java
@@ -39,7 +39,6 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -80,18 +79,18 @@ public class Tablet extends MetaObject implements Writable {
private long lastStatusCheckTime = -1;
public Tablet() {
- this(0L, new ArrayList<Replica>());
+ this(0L, new ArrayList<>());
}
public Tablet(long tabletId) {
- this(tabletId, new ArrayList<Replica>());
+ this(tabletId, new ArrayList<>());
}
public Tablet(long tabletId, List<Replica> replicas) {
this.id = tabletId;
this.replicas = replicas;
if (this.replicas == null) {
- this.replicas = new ArrayList<Replica>();
+ this.replicas = new ArrayList<>();
}
checkedVersion = -1L;
@@ -300,13 +299,15 @@ public class Tablet extends MetaObject implements
Writable {
public static void sortReplicaByVersionDesc(List<Replica> replicas) {
// sort replicas by version. higher version in the tops
- Collections.sort(replicas, Replica.VERSION_DESC_COMPARATOR);
+ replicas.sort(Replica.VERSION_DESC_COMPARATOR);
}
+ @Override
public String toString() {
return "tabletId=" + this.id;
}
+ @Override
public void write(DataOutput out) throws IOException {
super.write(out);
@@ -321,6 +322,7 @@ public class Tablet extends MetaObject implements Writable {
out.writeLong(checkedVersionHash);
out.writeBoolean(isConsistent);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -346,6 +348,7 @@ public class Tablet extends MetaObject implements Writable {
return tablet;
}
+ @Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
@@ -392,7 +395,7 @@ public class Tablet extends MetaObject implements Writable {
return dataSize;
}
- /*
+ /**
* A replica is healthy only if
* 1. the backend is available
* 2. replica version is caught up, and last failed version is -1
@@ -502,7 +505,7 @@ public class Tablet extends MetaObject implements Writable {
return Pair.create(TabletStatus.HEALTHY,
TabletSchedCtx.Priority.NORMAL);
}
- /*
+ /**
* Check colocate table's tablet health
* 1. Mismatch:
* backends set: 1,2,3
@@ -551,7 +554,7 @@ public class Tablet extends MetaObject implements Writable {
return TabletStatus.HEALTHY;
}
- /*
+ /**
* check if this tablet is ready to be repaired, based on priority.
* VERY_HIGH: repair immediately
* HIGH: delay Config.tablet_repair_delay_factor_second * 1;
diff --git a/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
b/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
index 4cb2b44..0c695ca 100644
--- a/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
+++ b/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
@@ -137,12 +137,12 @@ public class TabletMeta {
lock.readLock().lock();
try {
StringBuilder sb = new StringBuilder();
- sb.append("dbId=" + dbId);
- sb.append(" tableId=" + tableId);
- sb.append(" partitionId=" + partitionId);
- sb.append(" indexId=" + indexId);
- sb.append(" oldSchemaHash=" + oldSchemaHash);
- sb.append(" newSchemaHash=" + newSchemaHash);
+ sb.append("dbId=").append(dbId);
+ sb.append(" tableId=").append(tableId);
+ sb.append(" partitionId=").append(partitionId);
+ sb.append(" indexId=").append(indexId);
+ sb.append(" oldSchemaHash=").append(oldSchemaHash);
+ sb.append(" newSchemaHash=").append(newSchemaHash);
return sb.toString();
} finally {
diff --git a/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
b/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
index 5479a16..5dd6e60 100644
--- a/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
+++ b/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
@@ -55,13 +55,7 @@ public class BackendLoadStatistic {
public int compare(BackendLoadStatistic o1, BackendLoadStatistic o2) {
double score1 = o1.getLoadScore(medium);
double score2 = o2.getLoadScore(medium);
- if (score1 > score2) {
- return 1;
- } else if (score1 == score2) {
- return 0;
- } else {
- return -1;
- }
+ return Double.compare(score1, score2);
}
}
@@ -70,14 +64,7 @@ public class BackendLoadStatistic {
public int compare(BackendLoadStatistic o1, BackendLoadStatistic o2) {
Double score1 = o1.getMixLoadScore();
Double score2 = o2.getMixLoadScore();
-
- if (score1 > score2) {
- return 1;
- } else if (score1 == score2) {
- return 0;
- } else {
- return -1;
- }
+ return score1.compareTo(score2);
}
}
@@ -180,11 +167,7 @@ public class BackendLoadStatistic {
throw new LoadBalanceException("backend " + beId + " does not
exist");
}
- if (!be.isAvailable()) {
- isAvailable = false;
- } else {
- isAvailable = true;
- }
+ isAvailable = be.isAvailable();
ImmutableMap<String, DiskInfo> disks = be.getDisks();
for (DiskInfo diskInfo : disks.values()) {
@@ -333,7 +316,7 @@ public class BackendLoadStatistic {
return false;
}
- /*
+ /**
* Classify the paths into 'low', 'mid' and 'high',
* and skip offline path, and path with different storage medium
*/
diff --git a/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
b/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
index 2b5fee9..3aeb406 100644
--- a/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
+++ b/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
@@ -115,13 +115,7 @@ public class RootPathLoadStatistic implements
Comparable<RootPathLoadStatistic>
public int compareTo(RootPathLoadStatistic o) {
double myPercent = getUsedPercent();
double otherPercent = o.getUsedPercent();
- if (myPercent < otherPercent) {
- return -1;
- } else if (myPercent > otherPercent) {
- return 1;
- } else {
- return 0;
- }
+ return Double.compare(myPercent, otherPercent);
}
@Override
diff --git a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
index 036fe6e..a127255 100644
--- a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.ColocateTableIndex;
import org.apache.doris.catalog.ColocateTableIndex.GroupId;
import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.DiskInfo;
import org.apache.doris.catalog.DiskInfo.DiskState;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
@@ -60,7 +61,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
@@ -68,7 +68,7 @@ import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
-/*
+/**
* TabletScheduler saved the tablets produced by TabletChecker and try to
schedule them.
* It also try to balance the cluster load.
*
@@ -170,7 +170,7 @@ public class TabletScheduler extends MasterDaemon {
if (backends.containsKey(beId)) {
List<Long> pathHashes =
backends.get(beId).getDisks().values().stream()
.filter(v -> v.getState()==DiskState.ONLINE)
- .map(v ->
v.getPathHash()).collect(Collectors.toList());
+
.map(DiskInfo::getPathHash).collect(Collectors.toList());
backendsWorkingSlots.get(beId).updatePaths(pathHashes);
} else {
deletedBeIds.add(beId);
@@ -186,7 +186,7 @@ public class TabletScheduler extends MasterDaemon {
// add new backends
for (Backend be : backends.values()) {
if (!backendsWorkingSlots.containsKey(be.getId())) {
- List<Long> pathHashes = be.getDisks().values().stream().map(v
-> v.getPathHash()).collect(Collectors.toList());
+ List<Long> pathHashes =
be.getDisks().values().stream().map(DiskInfo::getPathHash).collect(Collectors.toList());
PathSlot slot = new PathSlot(pathHashes,
Config.schedule_slot_num_per_path);
backendsWorkingSlots.put(be.getId(), slot);
LOG.info("add new backend {} with slots num: {}", be.getId(),
be.getDisks().size());
@@ -200,7 +200,7 @@ public class TabletScheduler extends MasterDaemon {
return backendsWorkingSlots;
}
- /*
+ /**
* add a ready-to-be-scheduled tablet to pendingTablets, if it has not
being added before.
* if force is true, do not check if tablet is already added before.
*/
@@ -227,7 +227,7 @@ public class TabletScheduler extends MasterDaemon {
return allTabletIds.contains(tabletId);
}
- /*
+ /**
* Iterate current tablets, change their priority to VERY_HIGH if
necessary.
*/
public synchronized void changeTabletsPriorityToVeryHigh(long dbId, long
tblId, List<Long> partitionIds) {
@@ -242,10 +242,10 @@ public class TabletScheduler extends MasterDaemon {
pendingTablets = newPendingTablets;
}
- /*
+ /**
* TabletScheduler will run as a daemon thread at a very short
interval(default 5 sec)
- * Firstly, it will try to update cluster load statistic and check if
priority need to be adjuested.
- * Than, it will schedule the tablets in pendingTablets.
+ * Firstly, it will try to update cluster load statistic and check if
priority need to be adjusted.
+ * Then, it will schedule the tablets in pendingTablets.
* Thirdly, it will check the current running tasks.
* Finally, it try to balance the cluster if possible.
*
@@ -287,7 +287,7 @@ public class TabletScheduler extends MasterDaemon {
lastStatUpdateTime = System.currentTimeMillis();
}
- /*
+ /**
* Here is the only place we update the cluster load statistic info.
* We will not update this info dynamically along with the clone job's
running.
* Although it will cause a little bit inaccurate, but is within a
controllable range,
@@ -311,13 +311,13 @@ public class TabletScheduler extends MasterDaemon {
return statisticMap;
}
- /*
+ /**
* adjust priorities of all tablet infos
*/
private synchronized void adjustPriorities() {
int size = pendingTablets.size();
int changedNum = 0;
- TabletSchedCtx tabletCtx = null;
+ TabletSchedCtx tabletCtx;
for (int i = 0; i < size; i++) {
tabletCtx = pendingTablets.poll();
if (tabletCtx == null) {
@@ -333,7 +333,7 @@ public class TabletScheduler extends MasterDaemon {
LOG.info("adjust priority for all tablets. changed: {}, total: {}",
changedNum, size);
}
- /*
+ /**
* get at most BATCH_NUM tablets from queue, and try to schedule them.
* After handle, the tablet info should be
* 1. in runningTablets with state RUNNING, if being scheduled success.
@@ -423,7 +423,7 @@ public class TabletScheduler extends MasterDaemon {
runningTablets.put(tabletCtx.getTabletId(), tabletCtx);
}
- /*
+ /**
* we take the tablet out of the runningTablets and than handle it,
* avoid other threads see it.
* Whoever takes this tablet, make sure to put it to the schedHistory or
back to runningTablets.
@@ -432,7 +432,7 @@ public class TabletScheduler extends MasterDaemon {
return runningTablets.remove(tabletId);
}
- /*
+ /**
* Try to schedule a single tablet.
*/
private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask
batchTask) throws SchedException {
@@ -448,7 +448,7 @@ public class TabletScheduler extends MasterDaemon {
throw new SchedException(Status.UNRECOVERABLE, "db does not
exist");
}
- Pair<TabletStatus, TabletSchedCtx.Priority> statusPair = null;
+ Pair<TabletStatus, TabletSchedCtx.Priority> statusPair;
db.writeLock();
try {
OlapTable tbl = (OlapTable) db.getTable(tabletCtx.getTblId());
@@ -586,7 +586,7 @@ public class TabletScheduler extends MasterDaemon {
}
}
- /*
+ /**
* Replica is missing, which means there is no enough alive replicas.
* So we need to find a destination backend to clone a new replica as
possible as we can.
* 1. find an available path in a backend as destination:
@@ -615,7 +615,7 @@ public class TabletScheduler extends MasterDaemon {
batchTask.addTask(tabletCtx.createCloneReplicaAndTask());
}
- /*
+ /**
* Replica version is incomplete, which means this replica is missing some
version,
* and need to be cloned from a healthy replica, in-place.
*
@@ -649,7 +649,7 @@ public class TabletScheduler extends MasterDaemon {
handleReplicaMissing(tabletCtx, batchTask);
}
- /*
+ /**
* replica is redundant, which means there are more replicas than we
expected, which need to be dropped.
* we just drop one redundant replica at a time, for safety reason.
* choosing a replica to drop base on following priority:
@@ -843,7 +843,7 @@ public class TabletScheduler extends MasterDaemon {
return false;
}
- /*
+ /**
* Just delete replica which does not located in colocate backends set.
* return true if delete one replica, otherwise, return false.
*/
@@ -920,7 +920,7 @@ public class TabletScheduler extends MasterDaemon {
LOG.info("send delete replica task for tablet {} in backend {}",
tabletId, backendId);
}
- /*
+ /**
* Cluster migration, which means the tablet has enough healthy replicas,
* but some replicas are not in right cluster.
* It is just same as 'replica missing'.
@@ -933,7 +933,7 @@ public class TabletScheduler extends MasterDaemon {
handleReplicaMissing(tabletCtx, batchTask);
}
- /*
+ /**
* Replicas of colocate table's tablet does not locate on right backends
set.
* backends set: 1,2,3
* tablet replicas: 1,2,5
@@ -960,7 +960,7 @@ public class TabletScheduler extends MasterDaemon {
batchTask.addTask(tabletCtx.createCloneReplicaAndTask());
}
- /*
+ /**
* Try to select some alternative tablets for balance. Add them to
pendingTablets with priority LOW,
* and waiting to be scheduled.
*/
@@ -984,7 +984,7 @@ public class TabletScheduler extends MasterDaemon {
}
}
- /*
+ /**
* Try to create a balance task for a tablet.
*/
private void doBalance(TabletSchedCtx tabletCtx, AgentBatchTask batchTask)
throws SchedException {
@@ -1005,8 +1005,7 @@ public class TabletScheduler extends MasterDaemon {
// get all available paths which this tablet can fit in.
// beStatistics is sorted by mix load score in ascend order, so select
from first to last.
List<RootPathLoadStatistic> allFitPaths = Lists.newArrayList();
- for (int i = 0; i < beStatistics.size(); i++) {
- BackendLoadStatistic bes = beStatistics.get(i);
+ for (BackendLoadStatistic bes : beStatistics) {
if (!bes.isAvailable()) {
continue;
}
@@ -1071,7 +1070,7 @@ public class TabletScheduler extends MasterDaemon {
throw new SchedException(Status.SCHEDULE_FAILED, "unable to find dest
path which can be fit in");
}
- /*
+ /**
* For some reason, a tablet info failed to be scheduled this time,
* So we dynamically change its priority and add back to queue, waiting
for next round.
*/
@@ -1127,7 +1126,7 @@ public class TabletScheduler extends MasterDaemon {
return total;
}
- /*
+ /**
* return true if we want to remove the clone task from AgentTaskQueue
*/
public boolean finishCloneTask(CloneTask cloneTask, TFinishTaskRequest
request) {
@@ -1174,7 +1173,7 @@ public class TabletScheduler extends MasterDaemon {
return true;
}
- /*
+ /**
* Gather the running statistic of the task.
* It will be evaluated for future strategy.
* This should only be called when the tablet is down with state FINISHED.
@@ -1208,7 +1207,7 @@ public class TabletScheduler extends MasterDaemon {
lastSlotAdjustTime = System.currentTimeMillis();
}
- /*
+ /**
* handle tablets which are running.
* We should finished the task if
* 1. Tablet is already healthy
@@ -1291,7 +1290,7 @@ public class TabletScheduler extends MasterDaemon {
+ runningTablets.values().stream().filter(t -> t.getType() ==
Type.BALANCE).count();
}
- /*
+ /**
* PathSlot keeps track of slot num per path of a Backend.
* Each path on a Backend has several slot.
* If a path's available slot num become 0, no task should be assigned to
this path.
@@ -1309,13 +1308,7 @@ public class TabletScheduler extends MasterDaemon {
// update the path
public synchronized void updatePaths(List<Long> paths) {
// delete non exist path
- Iterator<Map.Entry<Long, Slot>> iter =
pathSlots.entrySet().iterator();
- while (iter.hasNext()) {
- Map.Entry<Long, Slot> entry = iter.next();
- if (!paths.contains(entry.getKey())) {
- iter.remove();
- }
- }
+ pathSlots.entrySet().removeIf(entry ->
!paths.contains(entry.getKey()));
// add new path
for (Long pathHash : paths) {
@@ -1339,7 +1332,7 @@ public class TabletScheduler extends MasterDaemon {
}
}
- /*
+ /**
* Update the statistic of specified path
*/
public synchronized void updateStatistic(long pathHash, long copySize,
long copyTimeMs) {
@@ -1351,7 +1344,7 @@ public class TabletScheduler extends MasterDaemon {
slot.totalCopyTimeMs += copyTimeMs;
}
- /*
+ /**
* If the specified 'pathHash' has available slot, decrease the slot
number and return this path hash
*/
public synchronized long takeSlot(long pathHash) throws SchedException
{
@@ -1400,7 +1393,7 @@ public class TabletScheduler extends MasterDaemon {
return total;
}
- /*
+ /**
* get path whose balance slot num is larger than 0
*/
public synchronized Set<Long> getAvailPathsForBalance() {
@@ -1423,15 +1416,15 @@ public class TabletScheduler extends MasterDaemon {
public synchronized List<List<String>> getSlotInfo(long beId) {
List<List<String>> results = Lists.newArrayList();
- pathSlots.entrySet().stream().forEach(t -> {
- t.getValue().rectify();
+ pathSlots.forEach((key, value) -> {
+ value.rectify();
List<String> result = Lists.newArrayList();
result.add(String.valueOf(beId));
- result.add(String.valueOf(t.getKey()));
- result.add(String.valueOf(t.getValue().available));
- result.add(String.valueOf(t.getValue().total));
- result.add(String.valueOf(t.getValue().balanceSlot));
- result.add(String.valueOf(t.getValue().getAvgRate()));
+ result.add(String.valueOf(key));
+ result.add(String.valueOf(value.available));
+ result.add(String.valueOf(value.total));
+ result.add(String.valueOf(value.balanceSlot));
+ result.add(String.valueOf(value.getAvgRate()));
results.add(result);
});
return results;
diff --git
a/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
b/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
index 47121af..df0c182 100644
---
a/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
+++
b/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
@@ -50,11 +50,9 @@ public class ClusterLoadStatisticProcDir implements
ProcDirInterface {
statMap =
Catalog.getCurrentCatalog().getTabletScheduler().getStatisticMap();
- statMap.values().stream().forEach(t -> {
+ statMap.values().forEach(t -> {
List<List<String>> statistics = t.getClusterStatistic(medium);
- statistics.stream().forEach(v -> {
- result.addRow(v);
- });
+ statistics.forEach(result::addRow);
});
return result;
diff --git a/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
b/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
index 1559a4a..8b0c39c 100644
--- a/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
+++ b/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
@@ -90,6 +90,7 @@ public class BackendHbResponse extends HeartbeatResponse
implements Writable {
out.writeInt(brpcPort);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
beId = in.readLong();
diff --git a/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
b/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
index dda679a..e6ae49b 100644
--- a/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
+++ b/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
@@ -81,6 +81,7 @@ public class BrokerHbResponse extends HeartbeatResponse
implements Writable {
out.writeInt(port);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
name = Text.readString(in);
diff --git
a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index 70ce4c0..b744cf2 100644
--- a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++ b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -17,484 +17,294 @@
package org.apache.doris.catalog;
-import org.apache.doris.analysis.Analyzer;
-import org.apache.doris.analysis.ColumnDef;
+import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
-import org.apache.doris.analysis.HashDistributionDesc;
-import org.apache.doris.analysis.KeysDesc;
-import org.apache.doris.analysis.PartitionKeyDesc;
-import org.apache.doris.analysis.PartitionValue;
-import org.apache.doris.analysis.RangePartitionDesc;
-import org.apache.doris.analysis.SingleRangePartitionDesc;
-import org.apache.doris.analysis.TableName;
-import org.apache.doris.analysis.TypeDef;
import org.apache.doris.common.DdlException;
-import org.apache.doris.common.UserException;
-import org.apache.doris.mysql.privilege.PaloAuth;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.persist.EditLog;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.system.SystemInfoService;
-import org.apache.doris.task.AgentBatchTask;
-import com.google.common.collect.Lists;
-
-import org.junit.Before;
+import org.apache.doris.utframe.UtFrameUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import mockit.Expectations;
-import mockit.Injectable;
-import mockit.Mock;
-import mockit.MockUp;
+import java.util.UUID;
public class DynamicPartitionTableTest {
- private TableName dbTableName;
- private String dbName = "testDb";
- private String tableName = "testTable";
- private String clusterName = "default";
- private List<Long> beIds = Lists.newArrayList();
- private List<String> columnNames = Lists.newArrayList();
- private List<ColumnDef> columnDefs = Lists.newArrayList();
-
- private Catalog catalog = Catalog.getInstance();
- private Database db = new Database();
- private Analyzer analyzer;
+ private static String runningDir = "fe/mocked/DynamicPartitionTableTest/"
+ UUID.randomUUID().toString() + "/";
- private Map<String, String> properties;
- private List<SingleRangePartitionDesc> singleRangePartitionDescs;
-
- @Injectable
- ConnectContext connectContext;
+ private static ConnectContext connectContext;
@Rule
- public ExpectedException expectedEx = ExpectedException.none();
-
- @Before
- public void setUp() throws Exception {
- dbTableName = new TableName(dbName, tableName);
-
- beIds.add(1L);
- beIds.add(2L);
- beIds.add(3L);
-
- columnNames.add("key1");
- columnNames.add("key2");
- columnNames.add("key3");
-
- columnDefs.add(new ColumnDef("key1", new
TypeDef(ScalarType.createType(PrimitiveType.INT))));
- columnDefs.add(new ColumnDef("key2", new
TypeDef(ScalarType.createType(PrimitiveType.INT))));
- columnDefs.add(new ColumnDef("key3", new
TypeDef(ScalarType.createVarchar(10))));
-
- analyzer = new Analyzer(catalog, connectContext);
-
- properties = new HashMap<>();
- properties.put(DynamicPartitionProperty.ENABLE, "true");
- properties.put(DynamicPartitionProperty.PREFIX, "p");
- properties.put(DynamicPartitionProperty.TIME_UNIT, "day");
- properties.put(DynamicPartitionProperty.START, "-3");
- properties.put(DynamicPartitionProperty.END, "3");
- properties.put(DynamicPartitionProperty.BUCKETS, "30");
+ public ExpectedException expectedException = ExpectedException.none();
- singleRangePartitionDescs = new LinkedList<>();
- singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1",
- new PartitionKeyDesc(Lists.newArrayList(new
PartitionValue("-128"))), null));
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ FeConstants.default_scheduler_interval_millisecond = 1000;
+ FeConstants.runningUnitTest = true;
- new MockUp<AgentBatchTask>() {
- @Mock
- void run() {
- return;
- }
- };
+ UtFrameUtils.createMinDorisCluster(runningDir);
- new MockUp<CountDownLatch>() {
- @Mock
- boolean await(long timeout, TimeUnit unit) {
- return true;
- }
- };
-
- new Expectations(analyzer, catalog) {{
- analyzer.getClusterName();
- minTimes = 0;
- result = clusterName;
- }};
-
- dbTableName.analyze(analyzer);
+ // create connect context
+ connectContext = UtFrameUtils.createDefaultCtx();
+ // create database
+ String createDbStmtStr = "create database test;";
+ CreateDbStmt createDbStmt = (CreateDbStmt)
UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
+ Catalog.getCurrentCatalog().createDb(createDbStmt);
}
- @Test
- public void testNormal(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
- stmt.analyze(analyzer);
-
- catalog.createTable(stmt);
+ @AfterClass
+ public static void TearDown() {
+ UtFrameUtils.cleanDorisFeDir(runningDir);
}
- @Test
- public void testMissPrefix(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws
UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- properties.remove(DynamicPartitionProperty.PREFIX);
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
- stmt.analyze(analyzer);
-
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Must assign dynamic_partition.prefix
properties");
-
- catalog.createTable(stmt);
+ private static void createTable(String sql) throws Exception {
+ CreateTableStmt createTableStmt = (CreateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Catalog.getCurrentCatalog().createTable(createTableStmt);
}
@Test
- public void testMissTimeUnit(@Injectable SystemInfoService
systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws
UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- properties.remove(DynamicPartitionProperty.TIME_UNIT);
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
- stmt.analyze(analyzer);
-
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Must assign dynamic_partition.time_unit
properties");
-
- catalog.createTable(stmt);
+ public void testNormal() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_normal` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
}
@Test
- public void testMissSTART(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws
UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- properties.remove(DynamicPartitionProperty.START);
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
- stmt.analyze(analyzer);
-
- catalog.createTable(stmt);
+ public void testMissPrefix() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_prefix` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Must
assign dynamic_partition.prefix properties");
+ createTable(createOlapTblStmt);
}
@Test
- public void testMissEnd(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- properties.remove(DynamicPartitionProperty.END);
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
- stmt.analyze(analyzer);
-
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Must assign dynamic_partition.end
properties");
-
- catalog.createTable(stmt);
+ public void testMissTimeUnit() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_time_unit` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Must
assign dynamic_partition.time_unit properties");
+ createTable(createOlapTblStmt);
}
@Test
- public void testMissBuckets(@Injectable SystemInfoService
systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws
UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
+ public void testMissStart() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_start` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ }
- properties.remove(DynamicPartitionProperty.BUCKETS);
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
- stmt.analyze(analyzer);
+ @Test
+ public void testMissEnd() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_end`
(\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Must
assign dynamic_partition.end properties");
+ createTable(createOlapTblStmt);
+ }
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Must assign dynamic_partition.buckets
properties");
- catalog.createTable(stmt);
+ @Test
+ public void testMissBuckets() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_buckets` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Must
assign dynamic_partition.buckets properties");
+ createTable(createOlapTblStmt);
}
@Test
- public void testNotAllowed(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws
UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames), null,
- new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
- stmt.analyze(analyzer);
-
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Only support dynamic partition properties on
range partition table");
-
- catalog.createTable(stmt);
+ public void testNotAllowed() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_buckets` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Only
support dynamic partition properties on range partition table");
+ createTable(createOlapTblStmt);
}
@Test
- public void testNotAllowedInMultiPartitions(@Injectable SystemInfoService
systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog)
throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- List<SingleRangePartitionDesc> rangePartitionDescs = new
LinkedList<>();
- rangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1",
- new PartitionKeyDesc(Lists.newArrayList(new
PartitionValue("-128"), new PartitionValue("100"))), null));
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1", "key2"),
singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
- stmt.analyze(analyzer);
-
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Dynamic partition only support single-column
range partition");
-
- catalog.createTable(stmt);
+ public void testNotAllowedInMultiPartitions() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_normal` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1, k2)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\", \"100\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\", \"200\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\", \"300\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Dynamic
partition only support single-column range partition");
+ createTable(createOlapTblStmt);
}
}
diff --git a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 81c883e..2f88ecd 100644
--- a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -17,12 +17,15 @@
package org.apache.doris.common;
+import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.Type;
import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.thrift.TStorageMedium;
import com.google.common.collect.Lists;
@@ -122,11 +125,14 @@ public class PropertyAnalyzerTest {
@Test
public void testStorageMedium() throws AnalysisException {
long tomorrowTs = System.currentTimeMillis() / 1000 + 86400;
+ String tomorrowTimeStr = new SimpleDateFormat("yyyy-MM-dd
HH:mm:ss").format(tomorrowTs * 1000);
Map<String, String> properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, "SSD");
- properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME, new
SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tomorrowTs * 1000));
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME,
tomorrowTimeStr);
DataProperty dataProperty =
PropertyAnalyzer.analyzeDataProperty(properties, new
DataProperty(TStorageMedium.SSD));
- Assert.assertEquals(tomorrowTs, dataProperty.getCooldownTimeMs() /
1000);
+ // avoid UT fail because time zone different
+ DateLiteral dateLiteral = new DateLiteral(tomorrowTimeStr,
Type.DATETIME);
+
Assert.assertEquals(dateLiteral.unixTimestamp(TimeUtils.getTimeZone()),
dataProperty.getCooldownTimeMs());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
