This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 903592d Revert "Refactor some redunant code && Replace some UT by
UtFrameUtils" (#3613)
903592d is described below
commit 903592d82bf5f73e8856763bc2c36a1793780b10
Author: Mingyu Chen <[email protected]>
AuthorDate: Mon May 18 13:11:39 2020 +0800
Revert "Refactor some redunant code && Replace some UT by UtFrameUtils"
(#3613)
This revert is used to correct the mess of the commit
timeline caused by the wrong merge method.
---
.../main/java/org/apache/doris/alter/AlterJob.java | 18 +-
.../java/org/apache/doris/backup/RestoreJob.java | 175 +++---
.../java/org/apache/doris/catalog/Database.java | 2 +-
.../apache/doris/catalog/MaterializedIndex.java | 8 +-
.../java/org/apache/doris/catalog/OlapTable.java | 6 +-
.../java/org/apache/doris/catalog/Partition.java | 7 +-
.../org/apache/doris/catalog/PartitionInfo.java | 8 +-
.../java/org/apache/doris/catalog/Replica.java | 1 -
.../main/java/org/apache/doris/catalog/Tablet.java | 19 +-
.../java/org/apache/doris/catalog/TabletMeta.java | 12 +-
.../apache/doris/clone/BackendLoadStatistic.java | 25 +-
.../apache/doris/clone/RootPathLoadStatistic.java | 8 +-
.../org/apache/doris/clone/TabletScheduler.java | 89 +--
.../common/proc/ClusterLoadStatisticProcDir.java | 6 +-
.../org/apache/doris/system/BackendHbResponse.java | 1 -
.../org/apache/doris/system/BrokerHbResponse.java | 1 -
.../doris/catalog/DynamicPartitionTableTest.java | 684 +++++++++++++--------
.../apache/doris/common/PropertyAnalyzerTest.java | 10 +-
18 files changed, 654 insertions(+), 426 deletions(-)
diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJob.java
b/fe/src/main/java/org/apache/doris/alter/AlterJob.java
index 7b10fd5..ab927cc 100644
--- a/fe/src/main/java/org/apache/doris/alter/AlterJob.java
+++ b/fe/src/main/java/org/apache/doris/alter/AlterJob.java
@@ -189,14 +189,18 @@ public abstract class AlterJob implements Writable {
return false;
} else if (!backend.isAlive()) {
long currentTime = System.currentTimeMillis();
- // If this backend is done for a long time and not restart
automatically.
- // we consider it as dead and return false.
- return backend.getLastUpdateMs() <= 0
- || currentTime - backend.getLastUpdateMs() <=
Config.max_backend_down_time_second * 1000;
- } else {
- return !backend.isDecommissioned();
+ if (backend.getLastUpdateMs() > 0
+ && currentTime - backend.getLastUpdateMs() >
Config.max_backend_down_time_second * 1000) {
+ // this backend is done for a long time and not restart
automatically.
+ // we consider it as dead
+ return false;
+ }
+ return true;
+ } else if (backend.isDecommissioned()) {
+ return false;
}
-
+
+ return true;
}
public static AlterJob read(DataInput in) throws IOException {
diff --git a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
index 335ac17..ef576b4 100644
--- a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -172,7 +172,11 @@ public class RestoreJob extends AbstractJob {
}
public synchronized boolean finishTabletSnapshotTask(SnapshotTask task,
TFinishTaskRequest request) {
- if (checkTaskStatus(task, task.getJobId(), request)) {
+ Preconditions.checkState(task.getJobId() == jobId);
+ Preconditions.checkState(task.getDbId() == dbId);
+
+ if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
+ taskErrMsg.put(task.getSignature(),
Joiner.on(",").join(request.getTask_status().getError_msgs()));
return false;
}
@@ -194,15 +198,19 @@ public class RestoreJob extends AbstractJob {
taskErrMsg.remove(task.getSignature());
Preconditions.checkState(task.getTabletId() == removedTabletId,
removedTabletId);
LOG.debug("get finished snapshot info: {}, unfinished tasks num:
{}, remove result: {}. {}",
- info, unfinishedSignatureToId.size(), this,
removedTabletId);
+ info, unfinishedSignatureToId.size(), this);
return true;
}
+
return false;
}
-
public synchronized boolean finishTabletDownloadTask(DownloadTask task,
TFinishTaskRequest request) {
- if (checkTaskStatus(task, task.getJobId(), request)) {
+ Preconditions.checkState(task.getJobId() == jobId);
+ Preconditions.checkState(task.getDbId() == dbId);
+
+ if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
+ taskErrMsg.put(task.getSignature(),
Joiner.on(",").join(request.getTask_status().getError_msgs()));
return false;
}
@@ -229,7 +237,11 @@ public class RestoreJob extends AbstractJob {
}
public synchronized boolean finishDirMoveTask(DirMoveTask task,
TFinishTaskRequest request) {
- if (checkTaskStatus(task, task.getJobId(), request)) {
+ Preconditions.checkState(task.getJobId() == jobId);
+ Preconditions.checkState(task.getDbId() == dbId);
+
+ if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
+ taskErrMsg.put(task.getSignature(),
Joiner.on(",").join(request.getTask_status().getError_msgs()));
return false;
}
@@ -244,17 +256,6 @@ public class RestoreJob extends AbstractJob {
return true;
}
- private boolean checkTaskStatus(AgentTask task, long jobId,
TFinishTaskRequest request) {
- Preconditions.checkState(jobId == this.jobId);
- Preconditions.checkState(dbId == task.getDbId());
-
- if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
- taskErrMsg.put(task.getSignature(),
Joiner.on(",").join(request.getTask_status().getError_msgs()));
- return true;
- }
- return false;
- }
-
@Override
public synchronized void replayRun() {
LOG.info("replay run restore job: {}", this);
@@ -341,7 +342,7 @@ public class RestoreJob extends AbstractJob {
}
}
- /**
+ /*
* return true if some restored objs have been dropped.
*/
private void checkIfNeedCancel() {
@@ -352,7 +353,6 @@ public class RestoreJob extends AbstractJob {
Database db = catalog.getDb(dbId);
if (db == null) {
status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " has
been dropped");
- return;
}
db.readLock();
@@ -381,7 +381,7 @@ public class RestoreJob extends AbstractJob {
}
}
- /**
+ /*
* Restore rules as follow:
* A. Table already exist
* A1. Partition already exist, generate file mapping
@@ -505,9 +505,19 @@ public class RestoreJob extends AbstractJob {
Range<PartitionKey> remoteRange =
remoteRangePartInfo.getRange(backupPartInfo.id);
if (localRange.equals(remoteRange)) {
// Same partition, same range
- if
(genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl,
backupPartInfo, tblInfo)) {
+ if
(localRangePartInfo.getReplicationNum(localPartition.getId()) !=
restoreReplicationNum) {
+ status = new
Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name
+ + " in table " +
localTbl.getName()
+ + " has different replication
num '"
+ +
localRangePartInfo.getReplicationNum(localPartition.getId())
+ + "' with partition in
repository, which is " + restoreReplicationNum);
return;
}
+ genFileMapping(localOlapTbl,
localPartition, tblInfo.id, backupPartInfo,
+ true /* overwrite when
commit */);
+
restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(),
+
Pair.create(backupPartInfo.version,
+
backupPartInfo.versionHash));
} else {
// Same partition name, different range
status = new Status(ErrCode.COMMON_ERROR,
"Partition " + backupPartInfo.name
@@ -517,9 +527,21 @@ public class RestoreJob extends AbstractJob {
}
} else {
// If this is a single partitioned table.
- if
(genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl,
backupPartInfo, tblInfo)) {
+ if
(localPartInfo.getReplicationNum(localPartition.getId()) !=
restoreReplicationNum) {
+ status = new Status(ErrCode.COMMON_ERROR,
"Partition " + backupPartInfo.name
+ + " in table " + localTbl.getName()
+ + " has different replication num
'"
+ +
localPartInfo.getReplicationNum(localPartition.getId())
+ + "' with partition in repository,
which is " + restoreReplicationNum);
return;
}
+
+ // No need to check range, just generate file
mapping
+ genFileMapping(localOlapTbl, localPartition,
tblInfo.id, backupPartInfo,
+ true /* overwrite when commit
*/);
+ restoredVersionInfo.put(localOlapTbl.getId(),
localPartition.getId(),
+
Pair.create(backupPartInfo.version,
+
backupPartInfo.versionHash));
}
} else {
// partitions does not exist
@@ -594,20 +616,65 @@ public class RestoreJob extends AbstractJob {
BackupPartitionInfo backupPartitionInfo
=
jobInfo.getTableInfo(entry.first).getPartInfo(restorePart.getName());
- createReplicas(db, batchTask, localTbl, restorePart);
+ Set<String> bfColumns = localTbl.getCopiedBfColumns();
+ double bfFpp = localTbl.getBfFpp();
+ for (MaterializedIndex restoredIdx :
restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
+ MaterializedIndexMeta indexMeta =
localTbl.getIndexMetaByIndexId(restoredIdx.getId());
+ TabletMeta tabletMeta = new TabletMeta(db.getId(),
localTbl.getId(), restorePart.getId(),
+ restoredIdx.getId(), indexMeta.getSchemaHash(),
TStorageMedium.HDD);
+ for (Tablet restoreTablet : restoredIdx.getTablets()) {
+
Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta);
+ for (Replica restoreReplica :
restoreTablet.getReplicas()) {
+
Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(),
restoreReplica);
+ CreateReplicaTask task = new
CreateReplicaTask(restoreReplica.getBackendId(), dbId,
+ localTbl.getId(), restorePart.getId(),
restoredIdx.getId(),
+ restoreTablet.getId(),
indexMeta.getShortKeyColumnCount(),
+ indexMeta.getSchemaHash(),
restoreReplica.getVersion(),
+ restoreReplica.getVersionHash(),
indexMeta.getKeysType(), TStorageType.COLUMN,
+ TStorageMedium.HDD /* all restored
replicas will be saved to HDD */,
+ indexMeta.getSchema(), bfColumns, bfFpp,
null,
+ localTbl.getCopiedIndexes(),
+ localTbl.isInMemory());
+ task.setInRestoreMode(true);
+ batchTask.addTask(task);
+ }
+ }
+ }
genFileMapping(localTbl, restorePart, remoteTbl.getId(),
backupPartitionInfo,
- !allowLoad /* if allow load, do not overwrite
when commit */);
+ allowLoad ? false : true /* if allow load, do
not overwrite when commit */);
}
// generate create replica task for all restored tables
for (OlapTable restoreTbl : restoredTbls) {
for (Partition restorePart : restoreTbl.getPartitions()) {
- createReplicas(db, batchTask, restoreTbl, restorePart);
+ Set<String> bfColumns = restoreTbl.getCopiedBfColumns();
+ double bfFpp = restoreTbl.getBfFpp();
+ for (MaterializedIndex index :
restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
+ MaterializedIndexMeta indexMeta =
restoreTbl.getIndexMetaByIndexId(index.getId());
+ TabletMeta tabletMeta = new TabletMeta(db.getId(),
restoreTbl.getId(), restorePart.getId(),
+ index.getId(), indexMeta.getSchemaHash(),
TStorageMedium.HDD);
+ for (Tablet tablet : index.getTablets()) {
+
Catalog.getCurrentInvertedIndex().addTablet(tablet.getId(), tabletMeta);
+ for (Replica replica : tablet.getReplicas()) {
+
Catalog.getCurrentInvertedIndex().addReplica(tablet.getId(), replica);
+ CreateReplicaTask task = new
CreateReplicaTask(replica.getBackendId(), dbId,
+ restoreTbl.getId(),
restorePart.getId(), index.getId(), tablet.getId(),
+ indexMeta.getShortKeyColumnCount(),
indexMeta.getSchemaHash(),
+ replica.getVersion(),
replica.getVersionHash(),
+ indexMeta.getKeysType(),
TStorageType.COLUMN, TStorageMedium.HDD,
+ indexMeta.getSchema(), bfColumns,
bfFpp, null,
+ restoreTbl.getCopiedIndexes(),
+ restoreTbl.isInMemory());
+ task.setInRestoreMode(true);
+ batchTask.addTask(task);
+ }
+ }
+ }
BackupTableInfo backupTableInfo =
jobInfo.getTableInfo(restoreTbl.getName());
genFileMapping(restoreTbl, restorePart, backupTableInfo.id,
backupTableInfo.getPartInfo(restorePart.getName()),
- !allowLoad /* if allow load, do not
overwrite when commit */);
+ allowLoad ? false : true /* if allow load,
do not overwrite when commit */);
}
// set restored table's new name after all 'genFileMapping'
restoreTbl.setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName()));
@@ -622,7 +689,7 @@ public class RestoreJob extends AbstractJob {
if (batchTask.getTaskNum() > 0) {
MarkedCountDownLatch<Long, Long> latch = new
MarkedCountDownLatch<Long, Long>(batchTask.getTaskNum());
for (AgentTask task : batchTask.getAllTasks()) {
- latch.addMark(task.getBackendId(), task.getTabletId());
+ latch.addMark(((CreateReplicaTask) task).getBackendId(),
((CreateReplicaTask) task).getTabletId());
((CreateReplicaTask) task).setLatch(latch);
AgentTaskQueue.addTask(task);
}
@@ -739,54 +806,7 @@ public class RestoreJob extends AbstractJob {
// No log here, PENDING state restore job will redo this method
LOG.info("finished to prepare meta and send snapshot tasks, num: {}.
{}",
batchTask.getTaskNum(), this);
- }
-
- private boolean genFileMappingWhenBackupReplicasEqual(PartitionInfo
localPartInfo, Partition localPartition, Table localTbl,
- BackupPartitionInfo
backupPartInfo, BackupTableInfo tblInfo) {
- if (localPartInfo.getReplicationNum(localPartition.getId()) !=
restoreReplicationNum) {
- status = new Status(ErrCode.COMMON_ERROR, "Partition " +
backupPartInfo.name
- + " in table " + localTbl.getName()
- + " has different replication num '"
- + localPartInfo.getReplicationNum(localPartition.getId())
- + "' with partition in repository, which is " +
restoreReplicationNum);
- return true;
- }
-
- // No need to check range, just generate file mapping
- OlapTable localOlapTbl = (OlapTable) localTbl;
- genFileMapping(localOlapTbl, localPartition, tblInfo.id,
backupPartInfo,
- true /* overwrite when commit */);
- restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(),
- Pair.create(backupPartInfo.version,
- backupPartInfo.versionHash));
- return false;
- }
-
- private void createReplicas(Database db, AgentBatchTask batchTask,
OlapTable localTbl, Partition restorePart) {
- Set<String> bfColumns = localTbl.getCopiedBfColumns();
- double bfFpp = localTbl.getBfFpp();
- for (MaterializedIndex restoredIdx :
restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
- MaterializedIndexMeta indexMeta =
localTbl.getIndexMetaByIndexId(restoredIdx.getId());
- TabletMeta tabletMeta = new TabletMeta(db.getId(),
localTbl.getId(), restorePart.getId(),
- restoredIdx.getId(), indexMeta.getSchemaHash(),
TStorageMedium.HDD);
- for (Tablet restoreTablet : restoredIdx.getTablets()) {
-
Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta);
- for (Replica restoreReplica : restoreTablet.getReplicas()) {
-
Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(),
restoreReplica);
- CreateReplicaTask task = new
CreateReplicaTask(restoreReplica.getBackendId(), dbId,
- localTbl.getId(), restorePart.getId(),
restoredIdx.getId(),
- restoreTablet.getId(),
indexMeta.getShortKeyColumnCount(),
- indexMeta.getSchemaHash(),
restoreReplica.getVersion(),
- restoreReplica.getVersionHash(),
indexMeta.getKeysType(), TStorageType.COLUMN,
- TStorageMedium.HDD /* all restored replicas will
be saved to HDD */,
- indexMeta.getSchema(), bfColumns, bfFpp, null,
- localTbl.getCopiedIndexes(),
- localTbl.isInMemory());
- task.setInRestoreMode(true);
- batchTask.addTask(task);
- }
- }
- }
+ return;
}
// reset remote partition.
@@ -805,11 +825,11 @@ public class RestoreJob extends AbstractJob {
// indexes
Map<String, Long> localIdxNameToId = localTbl.getIndexNameToId();
- for (String localIdxName : localIdxNameToId.keySet()) {
+ for (String localidxName : localIdxNameToId.keySet()) {
// set ids of indexes in remote partition to the local index ids
- long remoteIdxId = remoteTbl.getIndexIdByName(localIdxName);
+ long remoteIdxId = remoteTbl.getIndexIdByName(localidxName);
MaterializedIndex remoteIdx = remotePart.getIndex(remoteIdxId);
- long localIdxId = localIdxNameToId.get(localIdxName);
+ long localIdxId = localIdxNameToId.get(localidxName);
remoteIdx.setIdForRestore(localIdxId);
if (localIdxId != localTbl.getBaseIndexId()) {
// not base table, reset
@@ -1490,7 +1510,6 @@ public class RestoreJob extends AbstractJob {
}
}
- @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
diff --git a/fe/src/main/java/org/apache/doris/catalog/Database.java
b/fe/src/main/java/org/apache/doris/catalog/Database.java
index 0d46c7a..fa53e14 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Database.java
@@ -113,7 +113,7 @@ public class Database extends MetaObject implements
Writable {
}
this.rwLock = new ReentrantReadWriteLock(true);
this.idToTable = new ConcurrentHashMap<>();
- this.nameToTable = new HashMap<>();
+ this.nameToTable = new HashMap<String, Table>();
this.dataQuotaBytes = FeConstants.default_db_data_quota_bytes;
this.replicaQuotaSize = FeConstants.default_db_replica_quota_size;
this.dbState = DbState.NORMAL;
diff --git a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
index a921758..6f8abd9 100644
--- a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
+++ b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
@@ -76,8 +76,8 @@ public class MaterializedIndex extends MetaObject implements
Writable, GsonPostP
public MaterializedIndex() {
this.state = IndexState.NORMAL;
- this.idToTablets = new HashMap<>();
- this.tablets = new ArrayList<>();
+ this.idToTablets = new HashMap<Long, Tablet>();
+ this.tablets = new ArrayList<Tablet>();
}
public MaterializedIndex(long id, IndexState state) {
@@ -88,8 +88,8 @@ public class MaterializedIndex extends MetaObject implements
Writable, GsonPostP
this.state = IndexState.NORMAL;
}
- this.idToTablets = new HashMap<>();
- this.tablets = new ArrayList<>();
+ this.idToTablets = new HashMap<Long, Tablet>();
+ this.tablets = new ArrayList<Tablet>();
this.rowCount = 0;
diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
index f307a5f..fe252dd 100644
--- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -643,7 +643,6 @@ public class OlapTable extends Table {
*/
// get partition by name, not including temp partitions
- @Override
public Partition getPartition(String partitionName) {
return getPartition(partitionName, false);
}
@@ -733,7 +732,6 @@ public class OlapTable extends Table {
return false;
}
- @Override
public TTableDescriptor toThrift() {
TOlapTable tOlapTable = new TOlapTable(getName());
TTableDescriptor tTableDescriptor = new TTableDescriptor(id,
TTableType.OLAP_TABLE,
@@ -932,7 +930,6 @@ public class OlapTable extends Table {
tempPartitions.write(out);
}
- @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -950,7 +947,7 @@ public class OlapTable extends Table {
if (Catalog.getCurrentCatalogJournalVersion() <
FeMetaVersion.VERSION_75) {
// schema
int colCount = in.readInt();
- List<Column> schema = new LinkedList<>();
+ List<Column> schema = new LinkedList<Column>();
for (int j = 0; j < colCount; j++) {
Column column = Column.read(in);
schema.add(column);
@@ -1076,7 +1073,6 @@ public class OlapTable extends Table {
rebuildFullSchema();
}
- @Override
public boolean equals(Table table) {
if (this == table) {
return true;
diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java
b/fe/src/main/java/org/apache/doris/catalog/Partition.java
index 0730bed..447555a 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Partition.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java
@@ -67,13 +67,13 @@ public class Partition extends MetaObject implements
Writable {
private PartitionState state;
@SerializedName(value = "baseIndex")
private MaterializedIndex baseIndex;
- /**
+ /*
* Visible rollup indexes are indexes which are visible to user.
* User can do query on them, show them in related 'show' stmt.
*/
@SerializedName(value = "idToVisibleRollupIndex")
private Map<Long, MaterializedIndex> idToVisibleRollupIndex =
Maps.newHashMap();
- /**
+ /*
* Shadow indexes are indexes which are not visible to user.
* Query will not run on these shadow indexes, and user can not see them
neither.
* But load process will load data into these shadow indexes.
@@ -81,7 +81,7 @@ public class Partition extends MetaObject implements Writable
{
@SerializedName(value = "idToShadowIndex")
private Map<Long, MaterializedIndex> idToShadowIndex = Maps.newHashMap();
- /**
+ /*
* committed version(hash): after txn is committed, set committed
version(hash)
* visible version(hash): after txn is published, set visible version
* next version(hash): next version is set after finished committing, it
should equals to committed version + 1
@@ -354,7 +354,6 @@ public class Partition extends MetaObject implements
Writable {
distributionInfo.write(out);
}
- @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
diff --git a/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
b/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index ee7d6f2..bde4815 100644
--- a/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -51,15 +51,15 @@ public class PartitionInfo implements Writable {
protected Map<Long, Boolean> idToInMemory;
public PartitionInfo() {
- this.idToDataProperty = new HashMap<>();
- this.idToReplicationNum = new HashMap<>();
+ this.idToDataProperty = new HashMap<Long, DataProperty>();
+ this.idToReplicationNum = new HashMap<Long, Short>();
this.idToInMemory = new HashMap<>();
}
public PartitionInfo(PartitionType type) {
this.type = type;
- this.idToDataProperty = new HashMap<>();
- this.idToReplicationNum = new HashMap<>();
+ this.idToDataProperty = new HashMap<Long, DataProperty>();
+ this.idToReplicationNum = new HashMap<Long, Short>();
this.idToInMemory = new HashMap<>();
}
diff --git a/fe/src/main/java/org/apache/doris/catalog/Replica.java
b/fe/src/main/java/org/apache/doris/catalog/Replica.java
index c9cab33..ad76292 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Replica.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Replica.java
@@ -505,7 +505,6 @@ public class Replica implements Writable {
return replica;
}
- @Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
diff --git a/fe/src/main/java/org/apache/doris/catalog/Tablet.java
b/fe/src/main/java/org/apache/doris/catalog/Tablet.java
index d8eb691..1b1cb35 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Tablet.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Tablet.java
@@ -39,6 +39,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -79,18 +80,18 @@ public class Tablet extends MetaObject implements Writable {
private long lastStatusCheckTime = -1;
public Tablet() {
- this(0L, new ArrayList<>());
+ this(0L, new ArrayList<Replica>());
}
public Tablet(long tabletId) {
- this(tabletId, new ArrayList<>());
+ this(tabletId, new ArrayList<Replica>());
}
public Tablet(long tabletId, List<Replica> replicas) {
this.id = tabletId;
this.replicas = replicas;
if (this.replicas == null) {
- this.replicas = new ArrayList<>();
+ this.replicas = new ArrayList<Replica>();
}
checkedVersion = -1L;
@@ -299,15 +300,13 @@ public class Tablet extends MetaObject implements
Writable {
public static void sortReplicaByVersionDesc(List<Replica> replicas) {
// sort replicas by version. higher version in the tops
- replicas.sort(Replica.VERSION_DESC_COMPARATOR);
+ Collections.sort(replicas, Replica.VERSION_DESC_COMPARATOR);
}
- @Override
public String toString() {
return "tabletId=" + this.id;
}
- @Override
public void write(DataOutput out) throws IOException {
super.write(out);
@@ -322,7 +321,6 @@ public class Tablet extends MetaObject implements Writable {
out.writeLong(checkedVersionHash);
out.writeBoolean(isConsistent);
}
- @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -348,7 +346,6 @@ public class Tablet extends MetaObject implements Writable {
return tablet;
}
- @Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
@@ -395,7 +392,7 @@ public class Tablet extends MetaObject implements Writable {
return dataSize;
}
- /**
+ /*
* A replica is healthy only if
* 1. the backend is available
* 2. replica version is caught up, and last failed version is -1
@@ -505,7 +502,7 @@ public class Tablet extends MetaObject implements Writable {
return Pair.create(TabletStatus.HEALTHY,
TabletSchedCtx.Priority.NORMAL);
}
- /**
+ /*
* Check colocate table's tablet health
* 1. Mismatch:
* backends set: 1,2,3
@@ -554,7 +551,7 @@ public class Tablet extends MetaObject implements Writable {
return TabletStatus.HEALTHY;
}
- /**
+ /*
* check if this tablet is ready to be repaired, based on priority.
* VERY_HIGH: repair immediately
* HIGH: delay Config.tablet_repair_delay_factor_second * 1;
diff --git a/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
b/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
index 0c695ca..4cb2b44 100644
--- a/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
+++ b/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
@@ -137,12 +137,12 @@ public class TabletMeta {
lock.readLock().lock();
try {
StringBuilder sb = new StringBuilder();
- sb.append("dbId=").append(dbId);
- sb.append(" tableId=").append(tableId);
- sb.append(" partitionId=").append(partitionId);
- sb.append(" indexId=").append(indexId);
- sb.append(" oldSchemaHash=").append(oldSchemaHash);
- sb.append(" newSchemaHash=").append(newSchemaHash);
+ sb.append("dbId=" + dbId);
+ sb.append(" tableId=" + tableId);
+ sb.append(" partitionId=" + partitionId);
+ sb.append(" indexId=" + indexId);
+ sb.append(" oldSchemaHash=" + oldSchemaHash);
+ sb.append(" newSchemaHash=" + newSchemaHash);
return sb.toString();
} finally {
diff --git a/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
b/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
index 5dd6e60..5479a16 100644
--- a/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
+++ b/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
@@ -55,7 +55,13 @@ public class BackendLoadStatistic {
public int compare(BackendLoadStatistic o1, BackendLoadStatistic o2) {
double score1 = o1.getLoadScore(medium);
double score2 = o2.getLoadScore(medium);
- return Double.compare(score1, score2);
+ if (score1 > score2) {
+ return 1;
+ } else if (score1 == score2) {
+ return 0;
+ } else {
+ return -1;
+ }
}
}
@@ -64,7 +70,14 @@ public class BackendLoadStatistic {
public int compare(BackendLoadStatistic o1, BackendLoadStatistic o2) {
Double score1 = o1.getMixLoadScore();
Double score2 = o2.getMixLoadScore();
- return score1.compareTo(score2);
+
+ if (score1 > score2) {
+ return 1;
+ } else if (score1 == score2) {
+ return 0;
+ } else {
+ return -1;
+ }
}
}
@@ -167,7 +180,11 @@ public class BackendLoadStatistic {
throw new LoadBalanceException("backend " + beId + " does not
exist");
}
- isAvailable = be.isAvailable();
+ if (!be.isAvailable()) {
+ isAvailable = false;
+ } else {
+ isAvailable = true;
+ }
ImmutableMap<String, DiskInfo> disks = be.getDisks();
for (DiskInfo diskInfo : disks.values()) {
@@ -316,7 +333,7 @@ public class BackendLoadStatistic {
return false;
}
- /**
+ /*
* Classify the paths into 'low', 'mid' and 'high',
* and skip offline path, and path with different storage medium
*/
diff --git a/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
b/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
index 3aeb406..2b5fee9 100644
--- a/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
+++ b/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
@@ -115,7 +115,13 @@ public class RootPathLoadStatistic implements
Comparable<RootPathLoadStatistic>
public int compareTo(RootPathLoadStatistic o) {
double myPercent = getUsedPercent();
double otherPercent = o.getUsedPercent();
- return Double.compare(myPercent, otherPercent);
+ if (myPercent < otherPercent) {
+ return -1;
+ } else if (myPercent > otherPercent) {
+ return 1;
+ } else {
+ return 0;
+ }
}
@Override
diff --git a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
index a127255..036fe6e 100644
--- a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -21,7 +21,6 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.ColocateTableIndex;
import org.apache.doris.catalog.ColocateTableIndex.GroupId;
import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.DiskInfo;
import org.apache.doris.catalog.DiskInfo.DiskState;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
@@ -61,6 +60,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Collection;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
@@ -68,7 +68,7 @@ import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
-/**
+/*
* TabletScheduler saved the tablets produced by TabletChecker and try to
schedule them.
* It also try to balance the cluster load.
*
@@ -170,7 +170,7 @@ public class TabletScheduler extends MasterDaemon {
if (backends.containsKey(beId)) {
List<Long> pathHashes =
backends.get(beId).getDisks().values().stream()
.filter(v -> v.getState()==DiskState.ONLINE)
-
.map(DiskInfo::getPathHash).collect(Collectors.toList());
+ .map(v ->
v.getPathHash()).collect(Collectors.toList());
backendsWorkingSlots.get(beId).updatePaths(pathHashes);
} else {
deletedBeIds.add(beId);
@@ -186,7 +186,7 @@ public class TabletScheduler extends MasterDaemon {
// add new backends
for (Backend be : backends.values()) {
if (!backendsWorkingSlots.containsKey(be.getId())) {
- List<Long> pathHashes =
be.getDisks().values().stream().map(DiskInfo::getPathHash).collect(Collectors.toList());
+ List<Long> pathHashes = be.getDisks().values().stream().map(v
-> v.getPathHash()).collect(Collectors.toList());
PathSlot slot = new PathSlot(pathHashes,
Config.schedule_slot_num_per_path);
backendsWorkingSlots.put(be.getId(), slot);
LOG.info("add new backend {} with slots num: {}", be.getId(),
be.getDisks().size());
@@ -200,7 +200,7 @@ public class TabletScheduler extends MasterDaemon {
return backendsWorkingSlots;
}
- /**
+ /*
* add a ready-to-be-scheduled tablet to pendingTablets, if it has not
being added before.
* if force is true, do not check if tablet is already added before.
*/
@@ -227,7 +227,7 @@ public class TabletScheduler extends MasterDaemon {
return allTabletIds.contains(tabletId);
}
- /**
+ /*
* Iterate current tablets, change their priority to VERY_HIGH if
necessary.
*/
public synchronized void changeTabletsPriorityToVeryHigh(long dbId, long
tblId, List<Long> partitionIds) {
@@ -242,10 +242,10 @@ public class TabletScheduler extends MasterDaemon {
pendingTablets = newPendingTablets;
}
- /**
+ /*
* TabletScheduler will run as a daemon thread at a very short
interval(default 5 sec)
- * Firstly, it will try to update cluster load statistic and check if
priority need to be adjusted.
- * Then, it will schedule the tablets in pendingTablets.
+ * Firstly, it will try to update cluster load statistic and check if
priority need to be adjuested.
+ * Than, it will schedule the tablets in pendingTablets.
* Thirdly, it will check the current running tasks.
* Finally, it try to balance the cluster if possible.
*
@@ -287,7 +287,7 @@ public class TabletScheduler extends MasterDaemon {
lastStatUpdateTime = System.currentTimeMillis();
}
- /**
+ /*
* Here is the only place we update the cluster load statistic info.
* We will not update this info dynamically along with the clone job's
running.
* Although it will cause a little bit inaccurate, but is within a
controllable range,
@@ -311,13 +311,13 @@ public class TabletScheduler extends MasterDaemon {
return statisticMap;
}
- /**
+ /*
* adjust priorities of all tablet infos
*/
private synchronized void adjustPriorities() {
int size = pendingTablets.size();
int changedNum = 0;
- TabletSchedCtx tabletCtx;
+ TabletSchedCtx tabletCtx = null;
for (int i = 0; i < size; i++) {
tabletCtx = pendingTablets.poll();
if (tabletCtx == null) {
@@ -333,7 +333,7 @@ public class TabletScheduler extends MasterDaemon {
LOG.info("adjust priority for all tablets. changed: {}, total: {}",
changedNum, size);
}
- /**
+ /*
* get at most BATCH_NUM tablets from queue, and try to schedule them.
* After handle, the tablet info should be
* 1. in runningTablets with state RUNNING, if being scheduled success.
@@ -423,7 +423,7 @@ public class TabletScheduler extends MasterDaemon {
runningTablets.put(tabletCtx.getTabletId(), tabletCtx);
}
- /**
+ /*
* we take the tablet out of the runningTablets and than handle it,
* avoid other threads see it.
* Whoever takes this tablet, make sure to put it to the schedHistory or
back to runningTablets.
@@ -432,7 +432,7 @@ public class TabletScheduler extends MasterDaemon {
return runningTablets.remove(tabletId);
}
- /**
+ /*
* Try to schedule a single tablet.
*/
private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask
batchTask) throws SchedException {
@@ -448,7 +448,7 @@ public class TabletScheduler extends MasterDaemon {
throw new SchedException(Status.UNRECOVERABLE, "db does not
exist");
}
- Pair<TabletStatus, TabletSchedCtx.Priority> statusPair;
+ Pair<TabletStatus, TabletSchedCtx.Priority> statusPair = null;
db.writeLock();
try {
OlapTable tbl = (OlapTable) db.getTable(tabletCtx.getTblId());
@@ -586,7 +586,7 @@ public class TabletScheduler extends MasterDaemon {
}
}
- /**
+ /*
* Replica is missing, which means there is no enough alive replicas.
* So we need to find a destination backend to clone a new replica as
possible as we can.
* 1. find an available path in a backend as destination:
@@ -615,7 +615,7 @@ public class TabletScheduler extends MasterDaemon {
batchTask.addTask(tabletCtx.createCloneReplicaAndTask());
}
- /**
+ /*
* Replica version is incomplete, which means this replica is missing some
version,
* and need to be cloned from a healthy replica, in-place.
*
@@ -649,7 +649,7 @@ public class TabletScheduler extends MasterDaemon {
handleReplicaMissing(tabletCtx, batchTask);
}
- /**
+ /*
* replica is redundant, which means there are more replicas than we
expected, which need to be dropped.
* we just drop one redundant replica at a time, for safety reason.
* choosing a replica to drop base on following priority:
@@ -843,7 +843,7 @@ public class TabletScheduler extends MasterDaemon {
return false;
}
- /**
+ /*
* Just delete replica which does not located in colocate backends set.
* return true if delete one replica, otherwise, return false.
*/
@@ -920,7 +920,7 @@ public class TabletScheduler extends MasterDaemon {
LOG.info("send delete replica task for tablet {} in backend {}",
tabletId, backendId);
}
- /**
+ /*
* Cluster migration, which means the tablet has enough healthy replicas,
* but some replicas are not in right cluster.
* It is just same as 'replica missing'.
@@ -933,7 +933,7 @@ public class TabletScheduler extends MasterDaemon {
handleReplicaMissing(tabletCtx, batchTask);
}
- /**
+ /*
* Replicas of colocate table's tablet does not locate on right backends
set.
* backends set: 1,2,3
* tablet replicas: 1,2,5
@@ -960,7 +960,7 @@ public class TabletScheduler extends MasterDaemon {
batchTask.addTask(tabletCtx.createCloneReplicaAndTask());
}
- /**
+ /*
* Try to select some alternative tablets for balance. Add them to
pendingTablets with priority LOW,
* and waiting to be scheduled.
*/
@@ -984,7 +984,7 @@ public class TabletScheduler extends MasterDaemon {
}
}
- /**
+ /*
* Try to create a balance task for a tablet.
*/
private void doBalance(TabletSchedCtx tabletCtx, AgentBatchTask batchTask)
throws SchedException {
@@ -1005,7 +1005,8 @@ public class TabletScheduler extends MasterDaemon {
// get all available paths which this tablet can fit in.
// beStatistics is sorted by mix load score in ascend order, so select
from first to last.
List<RootPathLoadStatistic> allFitPaths = Lists.newArrayList();
- for (BackendLoadStatistic bes : beStatistics) {
+ for (int i = 0; i < beStatistics.size(); i++) {
+ BackendLoadStatistic bes = beStatistics.get(i);
if (!bes.isAvailable()) {
continue;
}
@@ -1070,7 +1071,7 @@ public class TabletScheduler extends MasterDaemon {
throw new SchedException(Status.SCHEDULE_FAILED, "unable to find dest
path which can be fit in");
}
- /**
+ /*
* For some reason, a tablet info failed to be scheduled this time,
* So we dynamically change its priority and add back to queue, waiting
for next round.
*/
@@ -1126,7 +1127,7 @@ public class TabletScheduler extends MasterDaemon {
return total;
}
- /**
+ /*
* return true if we want to remove the clone task from AgentTaskQueue
*/
public boolean finishCloneTask(CloneTask cloneTask, TFinishTaskRequest
request) {
@@ -1173,7 +1174,7 @@ public class TabletScheduler extends MasterDaemon {
return true;
}
- /**
+ /*
* Gather the running statistic of the task.
* It will be evaluated for future strategy.
* This should only be called when the tablet is down with state FINISHED.
@@ -1207,7 +1208,7 @@ public class TabletScheduler extends MasterDaemon {
lastSlotAdjustTime = System.currentTimeMillis();
}
- /**
+ /*
* handle tablets which are running.
* We should finished the task if
* 1. Tablet is already healthy
@@ -1290,7 +1291,7 @@ public class TabletScheduler extends MasterDaemon {
+ runningTablets.values().stream().filter(t -> t.getType() ==
Type.BALANCE).count();
}
- /**
+ /*
* PathSlot keeps track of slot num per path of a Backend.
* Each path on a Backend has several slot.
* If a path's available slot num become 0, no task should be assigned to
this path.
@@ -1308,7 +1309,13 @@ public class TabletScheduler extends MasterDaemon {
// update the path
public synchronized void updatePaths(List<Long> paths) {
// delete non exist path
- pathSlots.entrySet().removeIf(entry ->
!paths.contains(entry.getKey()));
+ Iterator<Map.Entry<Long, Slot>> iter =
pathSlots.entrySet().iterator();
+ while (iter.hasNext()) {
+ Map.Entry<Long, Slot> entry = iter.next();
+ if (!paths.contains(entry.getKey())) {
+ iter.remove();
+ }
+ }
// add new path
for (Long pathHash : paths) {
@@ -1332,7 +1339,7 @@ public class TabletScheduler extends MasterDaemon {
}
}
- /**
+ /*
* Update the statistic of specified path
*/
public synchronized void updateStatistic(long pathHash, long copySize,
long copyTimeMs) {
@@ -1344,7 +1351,7 @@ public class TabletScheduler extends MasterDaemon {
slot.totalCopyTimeMs += copyTimeMs;
}
- /**
+ /*
* If the specified 'pathHash' has available slot, decrease the slot
number and return this path hash
*/
public synchronized long takeSlot(long pathHash) throws SchedException
{
@@ -1393,7 +1400,7 @@ public class TabletScheduler extends MasterDaemon {
return total;
}
- /**
+ /*
* get path whose balance slot num is larger than 0
*/
public synchronized Set<Long> getAvailPathsForBalance() {
@@ -1416,15 +1423,15 @@ public class TabletScheduler extends MasterDaemon {
public synchronized List<List<String>> getSlotInfo(long beId) {
List<List<String>> results = Lists.newArrayList();
- pathSlots.forEach((key, value) -> {
- value.rectify();
+ pathSlots.entrySet().stream().forEach(t -> {
+ t.getValue().rectify();
List<String> result = Lists.newArrayList();
result.add(String.valueOf(beId));
- result.add(String.valueOf(key));
- result.add(String.valueOf(value.available));
- result.add(String.valueOf(value.total));
- result.add(String.valueOf(value.balanceSlot));
- result.add(String.valueOf(value.getAvgRate()));
+ result.add(String.valueOf(t.getKey()));
+ result.add(String.valueOf(t.getValue().available));
+ result.add(String.valueOf(t.getValue().total));
+ result.add(String.valueOf(t.getValue().balanceSlot));
+ result.add(String.valueOf(t.getValue().getAvgRate()));
results.add(result);
});
return results;
diff --git
a/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
b/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
index df0c182..47121af 100644
---
a/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
+++
b/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
@@ -50,9 +50,11 @@ public class ClusterLoadStatisticProcDir implements
ProcDirInterface {
statMap =
Catalog.getCurrentCatalog().getTabletScheduler().getStatisticMap();
- statMap.values().forEach(t -> {
+ statMap.values().stream().forEach(t -> {
List<List<String>> statistics = t.getClusterStatistic(medium);
- statistics.forEach(result::addRow);
+ statistics.stream().forEach(v -> {
+ result.addRow(v);
+ });
});
return result;
diff --git a/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
b/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
index 8b0c39c..1559a4a 100644
--- a/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
+++ b/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
@@ -90,7 +90,6 @@ public class BackendHbResponse extends HeartbeatResponse
implements Writable {
out.writeInt(brpcPort);
}
- @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
beId = in.readLong();
diff --git a/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
b/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
index e6ae49b..dda679a 100644
--- a/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
+++ b/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
@@ -81,7 +81,6 @@ public class BrokerHbResponse extends HeartbeatResponse
implements Writable {
out.writeInt(port);
}
- @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
name = Text.readString(in);
diff --git
a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index b744cf2..70ce4c0 100644
--- a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++ b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -17,294 +17,484 @@
package org.apache.doris.catalog;
-import org.apache.doris.analysis.CreateDbStmt;
+import org.apache.doris.analysis.Analyzer;
+import org.apache.doris.analysis.ColumnDef;
import org.apache.doris.analysis.CreateTableStmt;
+import org.apache.doris.analysis.HashDistributionDesc;
+import org.apache.doris.analysis.KeysDesc;
+import org.apache.doris.analysis.PartitionKeyDesc;
+import org.apache.doris.analysis.PartitionValue;
+import org.apache.doris.analysis.RangePartitionDesc;
+import org.apache.doris.analysis.SingleRangePartitionDesc;
+import org.apache.doris.analysis.TableName;
+import org.apache.doris.analysis.TypeDef;
import org.apache.doris.common.DdlException;
-import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.UserException;
+import org.apache.doris.mysql.privilege.PaloAuth;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.persist.EditLog;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.system.SystemInfoService;
+import org.apache.doris.task.AgentBatchTask;
-import org.apache.doris.utframe.UtFrameUtils;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import com.google.common.collect.Lists;
+
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.util.UUID;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import mockit.Expectations;
+import mockit.Injectable;
+import mockit.Mock;
+import mockit.MockUp;
public class DynamicPartitionTableTest {
- private static String runningDir = "fe/mocked/DynamicPartitionTableTest/"
+ UUID.randomUUID().toString() + "/";
+ private TableName dbTableName;
+ private String dbName = "testDb";
+ private String tableName = "testTable";
+ private String clusterName = "default";
+ private List<Long> beIds = Lists.newArrayList();
+ private List<String> columnNames = Lists.newArrayList();
+ private List<ColumnDef> columnDefs = Lists.newArrayList();
+
+ private Catalog catalog = Catalog.getInstance();
+ private Database db = new Database();
+ private Analyzer analyzer;
- private static ConnectContext connectContext;
+ private Map<String, String> properties;
+ private List<SingleRangePartitionDesc> singleRangePartitionDescs;
+
+ @Injectable
+ ConnectContext connectContext;
@Rule
- public ExpectedException expectedException = ExpectedException.none();
+ public ExpectedException expectedEx = ExpectedException.none();
- @BeforeClass
- public static void beforeClass() throws Exception {
- FeConstants.default_scheduler_interval_millisecond = 1000;
- FeConstants.runningUnitTest = true;
+ @Before
+ public void setUp() throws Exception {
+ dbTableName = new TableName(dbName, tableName);
- UtFrameUtils.createMinDorisCluster(runningDir);
+ beIds.add(1L);
+ beIds.add(2L);
+ beIds.add(3L);
- // create connect context
- connectContext = UtFrameUtils.createDefaultCtx();
- // create database
- String createDbStmtStr = "create database test;";
- CreateDbStmt createDbStmt = (CreateDbStmt)
UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
- Catalog.getCurrentCatalog().createDb(createDbStmt);
- }
+ columnNames.add("key1");
+ columnNames.add("key2");
+ columnNames.add("key3");
- @AfterClass
- public static void TearDown() {
- UtFrameUtils.cleanDorisFeDir(runningDir);
- }
+ columnDefs.add(new ColumnDef("key1", new
TypeDef(ScalarType.createType(PrimitiveType.INT))));
+ columnDefs.add(new ColumnDef("key2", new
TypeDef(ScalarType.createType(PrimitiveType.INT))));
+ columnDefs.add(new ColumnDef("key3", new
TypeDef(ScalarType.createVarchar(10))));
+
+ analyzer = new Analyzer(catalog, connectContext);
+
+ properties = new HashMap<>();
+ properties.put(DynamicPartitionProperty.ENABLE, "true");
+ properties.put(DynamicPartitionProperty.PREFIX, "p");
+ properties.put(DynamicPartitionProperty.TIME_UNIT, "day");
+ properties.put(DynamicPartitionProperty.START, "-3");
+ properties.put(DynamicPartitionProperty.END, "3");
+ properties.put(DynamicPartitionProperty.BUCKETS, "30");
+
+ singleRangePartitionDescs = new LinkedList<>();
+ singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1",
+ new PartitionKeyDesc(Lists.newArrayList(new
PartitionValue("-128"))), null));
+
+ new MockUp<AgentBatchTask>() {
+ @Mock
+ void run() {
+ return;
+ }
+ };
- private static void createTable(String sql) throws Exception {
- CreateTableStmt createTableStmt = (CreateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
- Catalog.getCurrentCatalog().createTable(createTableStmt);
+ new MockUp<CountDownLatch>() {
+ @Mock
+ boolean await(long timeout, TimeUnit unit) {
+ return true;
+ }
+ };
+
+ new Expectations(analyzer, catalog) {{
+ analyzer.getClusterName();
+ minTimes = 0;
+ result = clusterName;
+ }};
+
+ dbTableName.analyze(analyzer);
}
@Test
- public void testNormal() throws Exception {
- String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_normal` (\n" +
- " `k1` date NULL COMMENT \"\",\n" +
- " `k2` int NULL COMMENT \"\",\n" +
- " `k3` smallint NULL COMMENT \"\",\n" +
- " `v1` varchar(2048) NULL COMMENT \"\",\n" +
- " `v2` datetime NULL COMMENT \"\"\n" +
- ") ENGINE=OLAP\n" +
- "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
- "COMMENT \"OLAP\"\n" +
- "PARTITION BY RANGE (k1)\n" +
- "(\n" +
- "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
- "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
- "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
- ")\n" +
- "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
- "PROPERTIES (\n" +
- "\"replication_num\" = \"1\",\n" +
- "\"dynamic_partition.enable\" = \"true\",\n" +
- "\"dynamic_partition.start\" = \"-3\",\n" +
- "\"dynamic_partition.end\" = \"3\",\n" +
- "\"dynamic_partition.time_unit\" = \"day\",\n" +
- "\"dynamic_partition.prefix\" = \"p\",\n" +
- "\"dynamic_partition.buckets\" = \"1\"\n" +
- ");";
- createTable(createOlapTblStmt);
+ public void testNormal(@Injectable SystemInfoService systemInfoService,
+ @Injectable PaloAuth paloAuth,
+ @Injectable EditLog editLog) throws UserException {
+ new Expectations(catalog) {
+ {
+ catalog.getDb(dbTableName.getDb());
+ minTimes = 0;
+ result = db;
+
+ Catalog.getCurrentSystemInfo();
+ minTimes = 0;
+ result = systemInfoService;
+
+ systemInfoService.checkClusterCapacity(anyString);
+ minTimes = 0;
+ systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
+ minTimes = 0;
+ result = beIds;
+
+ catalog.getAuth();
+ minTimes = 0;
+ result = paloAuth;
+ paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
+ minTimes = 0;
+ result = true;
+
+ catalog.getEditLog();
+ minTimes = 0;
+ result = editLog;
+ }
+ };
+
+ CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, columnNames),
+ new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
+ new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
+ stmt.analyze(analyzer);
+
+ catalog.createTable(stmt);
}
@Test
- public void testMissPrefix() throws Exception {
- String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_prefix` (\n" +
- " `k1` date NULL COMMENT \"\",\n" +
- " `k2` int NULL COMMENT \"\",\n" +
- " `k3` smallint NULL COMMENT \"\",\n" +
- " `v1` varchar(2048) NULL COMMENT \"\",\n" +
- " `v2` datetime NULL COMMENT \"\"\n" +
- ") ENGINE=OLAP\n" +
- "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
- "COMMENT \"OLAP\"\n" +
- "PARTITION BY RANGE (k1)\n" +
- "(\n" +
- "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
- "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
- "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
- ")\n" +
- "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
- "PROPERTIES (\n" +
- "\"replication_num\" = \"1\",\n" +
- "\"dynamic_partition.enable\" = \"true\",\n" +
- "\"dynamic_partition.start\" = \"-3\",\n" +
- "\"dynamic_partition.end\" = \"3\",\n" +
- "\"dynamic_partition.time_unit\" = \"day\",\n" +
- "\"dynamic_partition.buckets\" = \"1\"\n" +
- ");";
- expectedException.expect(DdlException.class);
- expectedException.expectMessage("errCode = 2, detailMessage = Must
assign dynamic_partition.prefix properties");
- createTable(createOlapTblStmt);
+ public void testMissPrefix(@Injectable SystemInfoService systemInfoService,
+ @Injectable PaloAuth paloAuth,
+ @Injectable EditLog editLog) throws
UserException {
+ new Expectations(catalog) {
+ {
+ catalog.getDb(dbTableName.getDb());
+ minTimes = 0;
+ result = db;
+
+ Catalog.getCurrentSystemInfo();
+ minTimes = 0;
+ result = systemInfoService;
+
+ systemInfoService.checkClusterCapacity(anyString);
+ minTimes = 0;
+ systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
+ minTimes = 0;
+ result = beIds;
+
+ catalog.getAuth();
+ minTimes = 0;
+ result = paloAuth;
+ paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
+ minTimes = 0;
+ result = true;
+
+ catalog.getEditLog();
+ minTimes = 0;
+ result = editLog;
+ }
+ };
+
+ properties.remove(DynamicPartitionProperty.PREFIX);
+
+ CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, columnNames),
+ new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
+ new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
+ stmt.analyze(analyzer);
+
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("Must assign dynamic_partition.prefix
properties");
+
+ catalog.createTable(stmt);
}
@Test
- public void testMissTimeUnit() throws Exception {
- String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_time_unit` (\n" +
- " `k1` date NULL COMMENT \"\",\n" +
- " `k2` int NULL COMMENT \"\",\n" +
- " `k3` smallint NULL COMMENT \"\",\n" +
- " `v1` varchar(2048) NULL COMMENT \"\",\n" +
- " `v2` datetime NULL COMMENT \"\"\n" +
- ") ENGINE=OLAP\n" +
- "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
- "COMMENT \"OLAP\"\n" +
- "PARTITION BY RANGE (k1)\n" +
- "(\n" +
- "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
- "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
- "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
- ")\n" +
- "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
- "PROPERTIES (\n" +
- "\"replication_num\" = \"1\",\n" +
- "\"dynamic_partition.enable\" = \"true\",\n" +
- "\"dynamic_partition.start\" = \"-3\",\n" +
- "\"dynamic_partition.end\" = \"3\",\n" +
- "\"dynamic_partition.prefix\" = \"p\",\n" +
- "\"dynamic_partition.buckets\" = \"1\"\n" +
- ");";
- expectedException.expect(DdlException.class);
- expectedException.expectMessage("errCode = 2, detailMessage = Must
assign dynamic_partition.time_unit properties");
- createTable(createOlapTblStmt);
+ public void testMissTimeUnit(@Injectable SystemInfoService
systemInfoService,
+ @Injectable PaloAuth paloAuth,
+ @Injectable EditLog editLog) throws
UserException {
+ new Expectations(catalog) {
+ {
+ catalog.getDb(dbTableName.getDb());
+ minTimes = 0;
+ result = db;
+
+ Catalog.getCurrentSystemInfo();
+ minTimes = 0;
+ result = systemInfoService;
+
+ systemInfoService.checkClusterCapacity(anyString);
+ minTimes = 0;
+ systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
+ minTimes = 0;
+ result = beIds;
+
+ catalog.getAuth();
+ minTimes = 0;
+ result = paloAuth;
+ paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
+ minTimes = 0;
+ result = true;
+
+ catalog.getEditLog();
+ minTimes = 0;
+ result = editLog;
+ }
+ };
+
+ properties.remove(DynamicPartitionProperty.TIME_UNIT);
+
+ CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, columnNames),
+ new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
+ new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
+ stmt.analyze(analyzer);
+
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("Must assign dynamic_partition.time_unit
properties");
+
+ catalog.createTable(stmt);
}
@Test
- public void testMissStart() throws Exception {
- String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_start` (\n" +
- " `k1` date NULL COMMENT \"\",\n" +
- " `k2` int NULL COMMENT \"\",\n" +
- " `k3` smallint NULL COMMENT \"\",\n" +
- " `v1` varchar(2048) NULL COMMENT \"\",\n" +
- " `v2` datetime NULL COMMENT \"\"\n" +
- ") ENGINE=OLAP\n" +
- "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
- "COMMENT \"OLAP\"\n" +
- "PARTITION BY RANGE (k1)\n" +
- "(\n" +
- "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
- "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
- "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
- ")\n" +
- "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
- "PROPERTIES (\n" +
- "\"replication_num\" = \"1\",\n" +
- "\"dynamic_partition.enable\" = \"true\",\n" +
- "\"dynamic_partition.end\" = \"3\",\n" +
- "\"dynamic_partition.time_unit\" = \"day\",\n" +
- "\"dynamic_partition.prefix\" = \"p\",\n" +
- "\"dynamic_partition.buckets\" = \"1\"\n" +
- ");";
- createTable(createOlapTblStmt);
- }
+ public void testMissSTART(@Injectable SystemInfoService systemInfoService,
+ @Injectable PaloAuth paloAuth,
+ @Injectable EditLog editLog) throws
UserException {
+ new Expectations(catalog) {
+ {
+ catalog.getDb(dbTableName.getDb());
+ minTimes = 0;
+ result = db;
+ Catalog.getCurrentSystemInfo();
+ minTimes = 0;
+ result = systemInfoService;
- @Test
- public void testMissEnd() throws Exception {
- String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_end`
(\n" +
- " `k1` date NULL COMMENT \"\",\n" +
- " `k2` int NULL COMMENT \"\",\n" +
- " `k3` smallint NULL COMMENT \"\",\n" +
- " `v1` varchar(2048) NULL COMMENT \"\",\n" +
- " `v2` datetime NULL COMMENT \"\"\n" +
- ") ENGINE=OLAP\n" +
- "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
- "COMMENT \"OLAP\"\n" +
- "PARTITION BY RANGE (k1)\n" +
- "(\n" +
- "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
- "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
- "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
- ")\n" +
- "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
- "PROPERTIES (\n" +
- "\"replication_num\" = \"1\",\n" +
- "\"dynamic_partition.enable\" = \"true\",\n" +
- "\"dynamic_partition.start\" = \"-3\",\n" +
- "\"dynamic_partition.time_unit\" = \"day\",\n" +
- "\"dynamic_partition.prefix\" = \"p\",\n" +
- "\"dynamic_partition.buckets\" = \"1\"\n" +
- ");";
- expectedException.expect(DdlException.class);
- expectedException.expectMessage("errCode = 2, detailMessage = Must
assign dynamic_partition.end properties");
- createTable(createOlapTblStmt);
+ systemInfoService.checkClusterCapacity(anyString);
+ minTimes = 0;
+ systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
+ minTimes = 0;
+ result = beIds;
+
+ catalog.getAuth();
+ minTimes = 0;
+ result = paloAuth;
+ paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
+ minTimes = 0;
+ result = true;
+
+ catalog.getEditLog();
+ minTimes = 0;
+ result = editLog;
+ }
+ };
+
+ properties.remove(DynamicPartitionProperty.START);
+
+ CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, columnNames),
+ new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
+ new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
+ stmt.analyze(analyzer);
+
+ catalog.createTable(stmt);
}
+ @Test
+ public void testMissEnd(@Injectable SystemInfoService systemInfoService,
+ @Injectable PaloAuth paloAuth,
+ @Injectable EditLog editLog) throws UserException {
+ new Expectations(catalog) {
+ {
+ catalog.getDb(dbTableName.getDb());
+ minTimes = 0;
+ result = db;
+
+ Catalog.getCurrentSystemInfo();
+ minTimes = 0;
+ result = systemInfoService;
+
+ systemInfoService.checkClusterCapacity(anyString);
+ minTimes = 0;
+ systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
+ minTimes = 0;
+ result = beIds;
+
+ catalog.getAuth();
+ minTimes = 0;
+ result = paloAuth;
+ paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
+ minTimes = 0;
+ result = true;
+
+ catalog.getEditLog();
+ minTimes = 0;
+ result = editLog;
+ }
+ };
+
+ properties.remove(DynamicPartitionProperty.END);
+
+ CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, columnNames),
+ new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
+ new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
+ stmt.analyze(analyzer);
+
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("Must assign dynamic_partition.end
properties");
+
+ catalog.createTable(stmt);
+ }
@Test
- public void testMissBuckets() throws Exception {
- String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_buckets` (\n" +
- " `k1` date NULL COMMENT \"\",\n" +
- " `k2` int NULL COMMENT \"\",\n" +
- " `k3` smallint NULL COMMENT \"\",\n" +
- " `v1` varchar(2048) NULL COMMENT \"\",\n" +
- " `v2` datetime NULL COMMENT \"\"\n" +
- ") ENGINE=OLAP\n" +
- "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
- "COMMENT \"OLAP\"\n" +
- "PARTITION BY RANGE (k1)\n" +
- "(\n" +
- "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
- "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
- "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
- ")\n" +
- "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
- "PROPERTIES (\n" +
- "\"replication_num\" = \"1\",\n" +
- "\"dynamic_partition.enable\" = \"true\",\n" +
- "\"dynamic_partition.start\" = \"-3\",\n" +
- "\"dynamic_partition.end\" = \"3\",\n" +
- "\"dynamic_partition.time_unit\" = \"day\",\n" +
- "\"dynamic_partition.prefix\" = \"p\"\n" +
- ");";
- expectedException.expect(DdlException.class);
- expectedException.expectMessage("errCode = 2, detailMessage = Must
assign dynamic_partition.buckets properties");
- createTable(createOlapTblStmt);
+ public void testMissBuckets(@Injectable SystemInfoService
systemInfoService,
+ @Injectable PaloAuth paloAuth,
+ @Injectable EditLog editLog) throws
UserException {
+ new Expectations(catalog) {
+ {
+ catalog.getDb(dbTableName.getDb());
+ minTimes = 0;
+ result = db;
+
+ Catalog.getCurrentSystemInfo();
+ minTimes = 0;
+ result = systemInfoService;
+
+ systemInfoService.checkClusterCapacity(anyString);
+ minTimes = 0;
+ systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
+ minTimes = 0;
+ result = beIds;
+
+ catalog.getAuth();
+ minTimes = 0;
+ result = paloAuth;
+ paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
+ minTimes = 0;
+ result = true;
+
+ catalog.getEditLog();
+ minTimes = 0;
+ result = editLog;
+ }
+ };
+
+ properties.remove(DynamicPartitionProperty.BUCKETS);
+
+ CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, columnNames),
+ new RangePartitionDesc(Lists.newArrayList("key1"),
singleRangePartitionDescs),
+ new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
+ stmt.analyze(analyzer);
+
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("Must assign dynamic_partition.buckets
properties");
+
+ catalog.createTable(stmt);
}
@Test
- public void testNotAllowed() throws Exception {
- String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_buckets` (\n" +
- " `k1` date NULL COMMENT \"\",\n" +
- " `k2` int NULL COMMENT \"\",\n" +
- " `k3` smallint NULL COMMENT \"\",\n" +
- " `v1` varchar(2048) NULL COMMENT \"\",\n" +
- " `v2` datetime NULL COMMENT \"\"\n" +
- ") ENGINE=OLAP\n" +
- "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
- "COMMENT \"OLAP\"\n" +
- "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
- "PROPERTIES (\n" +
- "\"replication_num\" = \"1\",\n" +
- "\"dynamic_partition.enable\" = \"true\",\n" +
- "\"dynamic_partition.start\" = \"-3\",\n" +
- "\"dynamic_partition.end\" = \"3\",\n" +
- "\"dynamic_partition.time_unit\" = \"day\",\n" +
- "\"dynamic_partition.prefix\" = \"p\",\n" +
- "\"dynamic_partition.buckets\" = \"1\"\n" +
- ");";
- expectedException.expect(DdlException.class);
- expectedException.expectMessage("errCode = 2, detailMessage = Only
support dynamic partition properties on range partition table");
- createTable(createOlapTblStmt);
+ public void testNotAllowed(@Injectable SystemInfoService systemInfoService,
+ @Injectable PaloAuth paloAuth,
+ @Injectable EditLog editLog) throws
UserException {
+ new Expectations(catalog) {
+ {
+ catalog.getDb(dbTableName.getDb());
+ minTimes = 0;
+ result = db;
+
+ Catalog.getCurrentSystemInfo();
+ minTimes = 0;
+ result = systemInfoService;
+
+ systemInfoService.checkClusterCapacity(anyString);
+ minTimes = 0;
+ systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
+ minTimes = 0;
+ result = beIds;
+
+ catalog.getAuth();
+ minTimes = 0;
+ result = paloAuth;
+ paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
+ minTimes = 0;
+ result = true;
+
+ catalog.getEditLog();
+ minTimes = 0;
+ result = editLog;
+ }
+ };
+
+ CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, columnNames), null,
+ new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
+ stmt.analyze(analyzer);
+
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("Only support dynamic partition properties on
range partition table");
+
+ catalog.createTable(stmt);
}
@Test
- public void testNotAllowedInMultiPartitions() throws Exception {
- String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_normal` (\n" +
- " `k1` date NULL COMMENT \"\",\n" +
- " `k2` int NULL COMMENT \"\",\n" +
- " `k3` smallint NULL COMMENT \"\",\n" +
- " `v1` varchar(2048) NULL COMMENT \"\",\n" +
- " `v2` datetime NULL COMMENT \"\"\n" +
- ") ENGINE=OLAP\n" +
- "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
- "COMMENT \"OLAP\"\n" +
- "PARTITION BY RANGE (k1, k2)\n" +
- "(\n" +
- "PARTITION p1 VALUES LESS THAN (\"2014-01-01\", \"100\"),\n" +
- "PARTITION p2 VALUES LESS THAN (\"2014-06-01\", \"200\"),\n" +
- "PARTITION p3 VALUES LESS THAN (\"2014-12-01\", \"300\")\n" +
- ")\n" +
- "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
- "PROPERTIES (\n" +
- "\"replication_num\" = \"1\",\n" +
- "\"dynamic_partition.enable\" = \"true\",\n" +
- "\"dynamic_partition.start\" = \"-3\",\n" +
- "\"dynamic_partition.end\" = \"3\",\n" +
- "\"dynamic_partition.time_unit\" = \"day\",\n" +
- "\"dynamic_partition.prefix\" = \"p\",\n" +
- "\"dynamic_partition.buckets\" = \"1\"\n" +
- ");";
- expectedException.expect(DdlException.class);
- expectedException.expectMessage("errCode = 2, detailMessage = Dynamic
partition only support single-column range partition");
- createTable(createOlapTblStmt);
+ public void testNotAllowedInMultiPartitions(@Injectable SystemInfoService
systemInfoService,
+ @Injectable PaloAuth paloAuth,
+ @Injectable EditLog editLog)
throws UserException {
+ new Expectations(catalog) {
+ {
+ catalog.getDb(dbTableName.getDb());
+ minTimes = 0;
+ result = db;
+
+ Catalog.getCurrentSystemInfo();
+ minTimes = 0;
+ result = systemInfoService;
+
+ systemInfoService.checkClusterCapacity(anyString);
+ minTimes = 0;
+ systemInfoService.seqChooseBackendIds(anyInt, true, true,
anyString);
+ minTimes = 0;
+ result = beIds;
+
+ catalog.getAuth();
+ minTimes = 0;
+ result = paloAuth;
+ paloAuth.checkTblPriv((ConnectContext) any, anyString,
anyString, PrivPredicate.CREATE);
+ minTimes = 0;
+ result = true;
+
+ catalog.getEditLog();
+ minTimes = 0;
+ result = editLog;
+ }
+ };
+
+ List<SingleRangePartitionDesc> rangePartitionDescs = new
LinkedList<>();
+ rangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1",
+ new PartitionKeyDesc(Lists.newArrayList(new
PartitionValue("-128"), new PartitionValue("100"))), null));
+
+ CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName,
columnDefs, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, columnNames),
+ new RangePartitionDesc(Lists.newArrayList("key1", "key2"),
singleRangePartitionDescs),
+ new HashDistributionDesc(1, Lists.newArrayList("key1")),
properties, null, "");
+ stmt.analyze(analyzer);
+
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("Dynamic partition only support single-column
range partition");
+
+ catalog.createTable(stmt);
}
}
diff --git a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 2f88ecd..81c883e 100644
--- a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -17,15 +17,12 @@
package org.apache.doris.common;
-import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
-import org.apache.doris.catalog.Type;
import org.apache.doris.common.util.PropertyAnalyzer;
-import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.thrift.TStorageMedium;
import com.google.common.collect.Lists;
@@ -125,14 +122,11 @@ public class PropertyAnalyzerTest {
@Test
public void testStorageMedium() throws AnalysisException {
long tomorrowTs = System.currentTimeMillis() / 1000 + 86400;
- String tomorrowTimeStr = new SimpleDateFormat("yyyy-MM-dd
HH:mm:ss").format(tomorrowTs * 1000);
Map<String, String> properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, "SSD");
- properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME,
tomorrowTimeStr);
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME, new
SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tomorrowTs * 1000));
DataProperty dataProperty =
PropertyAnalyzer.analyzeDataProperty(properties, new
DataProperty(TStorageMedium.SSD));
- // avoid UT fail because time zone different
- DateLiteral dateLiteral = new DateLiteral(tomorrowTimeStr,
Type.DATETIME);
-
Assert.assertEquals(dateLiteral.unixTimestamp(TimeUtils.getTimeZone()),
dataProperty.getCooldownTimeMs());
+ Assert.assertEquals(tomorrowTs, dataProperty.getCooldownTimeMs() /
1000);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
