This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 524607d003 [Fix](storage) Fix bug that cooldown time is error (#15444)
524607d003 is described below
commit 524607d003bb1d88e05c77c9678f43ab1560fd24
Author: weizuo93 <[email protected]>
AuthorDate: Thu Dec 29 21:01:36 2022 +0800
[Fix](storage) Fix bug that cooldown time is error (#15444)
Cooldown time is wrong for data in SSD, because cooldown time for all
`table/partitionis`
is only calculated once when class `DataProperty` loaded and that cannot be
updated later.
This patch is to ensure that cooldown time for each table/partition can be
calculated in real time
when table/partition is created.
Co-authored-by: weizuo <[email protected]>
---
.../src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java | 4 ++--
fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java | 5 ++---
.../src/main/java/org/apache/doris/datasource/InternalCatalog.java | 5 +++--
.../src/test/java/org/apache/doris/catalog/CatalogTestUtil.java | 4 ++--
.../src/test/java/org/apache/doris/catalog/DataPropertyTest.java | 2 +-
.../src/test/java/org/apache/doris/common/util/UnitTestUtil.java | 2 +-
.../src/test/java/org/apache/doris/http/DorisHttpTestCase.java | 5 +++--
.../java/org/apache/doris/persist/BatchModifyPartitionsInfoTest.java | 5 +++--
8 files changed, 17 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
index 06b4bcfa71..7ffec7c1d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
@@ -58,7 +58,7 @@ public class SinglePartitionDesc implements AllPartitionDesc {
this.partitionKeyDesc = partitionKeyDesc;
this.properties = properties;
- this.partitionDataProperty = DataProperty.DEFAULT_DATA_PROPERTY;
+ this.partitionDataProperty = new
DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM);
this.replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION;
this.storagePolicy = "";
}
@@ -127,7 +127,7 @@ public class SinglePartitionDesc implements
AllPartitionDesc {
// analyze data property
partitionDataProperty =
PropertyAnalyzer.analyzeDataProperty(properties,
- DataProperty.DEFAULT_DATA_PROPERTY);
+ new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
Preconditions.checkNotNull(partitionDataProperty);
// analyze replication num
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
index 50fcbf7e72..13b1fd6123 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
@@ -35,9 +35,8 @@ import java.io.IOException;
import java.util.Objects;
public class DataProperty implements Writable, GsonPostProcessable {
- public static final DataProperty DEFAULT_DATA_PROPERTY = new DataProperty(
- "SSD".equalsIgnoreCase(Config.default_storage_medium) ?
TStorageMedium.SSD : TStorageMedium.HDD
- );
+ public static final TStorageMedium DEFAULT_STORAGE_MEDIUM =
+ "SSD".equalsIgnoreCase(Config.default_storage_medium) ?
TStorageMedium.SSD : TStorageMedium.HDD;
public static final long MAX_COOLDOWN_TIME_MS = 253402271999000L; //
9999-12-31 23:59:59
@SerializedName(value = "storageMedium")
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 6466a29e12..0a264598d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1947,7 +1947,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
DataProperty dataProperty = null;
try {
dataProperty =
PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
- DataProperty.DEFAULT_DATA_PROPERTY);
+ new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
@@ -2104,7 +2104,8 @@ public class InternalCatalog implements
CatalogIf<Database> {
try {
// just for remove entries in stmt.getProperties(),
// and then check if there still has unknown properties
- PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
DataProperty.DEFAULT_DATA_PROPERTY);
+ PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
+ new
DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
if (partitionInfo.getType() == PartitionType.RANGE) {
DynamicPartitionUtil.checkAndSetDynamicPartitionProperty(olapTable, properties,
db);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
index b475e9a57b..61f5f0d967 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
@@ -225,7 +225,7 @@ public class CatalogTestUtil {
// table
PartitionInfo partitionInfo = new SinglePartitionInfo();
- partitionInfo.setDataProperty(partitionId,
DataProperty.DEFAULT_DATA_PROPERTY);
+ partitionInfo.setDataProperty(partitionId, new
DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
partitionInfo.setReplicaAllocation(partitionId, new
ReplicaAllocation((short) 3));
OlapTable table = new OlapTable(tableId, testTable1, columns,
KeysType.AGG_KEYS, partitionInfo,
distributionInfo);
@@ -284,7 +284,7 @@ public class CatalogTestUtil {
// table
PartitionInfo partitionInfo = new SinglePartitionInfo();
- partitionInfo.setDataProperty(testPartitionId2,
DataProperty.DEFAULT_DATA_PROPERTY);
+ partitionInfo.setDataProperty(testPartitionId2, new
DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
partitionInfo.setReplicaAllocation(testPartitionId2, new
ReplicaAllocation((short) 1));
OlapTable table = new OlapTable(testTableId2, testTable2, columns,
KeysType.DUP_KEYS, partitionInfo,
distributionInfo);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java
index 5114142569..b1b006067e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java
@@ -28,7 +28,7 @@ public class DataPropertyTest {
@Test
public void testCooldownTimeMs() throws Exception {
Config.default_storage_medium = "ssd";
- DataProperty dataProperty = DataProperty.DEFAULT_DATA_PROPERTY;
+ DataProperty dataProperty = new
DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM);
Assert.assertNotEquals(DataProperty.MAX_COOLDOWN_TIME_MS,
dataProperty.getCooldownTimeMs());
dataProperty = new DataProperty(TStorageMedium.SSD);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/UnitTestUtil.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/UnitTestUtil.java
index 66885b2b4b..4460fcb31f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/UnitTestUtil.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/UnitTestUtil.java
@@ -109,7 +109,7 @@ public class UnitTestUtil {
// table
PartitionInfo partitionInfo = new SinglePartitionInfo();
- partitionInfo.setDataProperty(partitionId,
DataProperty.DEFAULT_DATA_PROPERTY);
+ partitionInfo.setDataProperty(partitionId, new
DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
partitionInfo.setReplicaAllocation(partitionId, new
ReplicaAllocation((short) 3));
partitionInfo.setIsInMemory(partitionId, false);
partitionInfo.setTabletType(partitionId, TTabletType.TABLET_TYPE_DISK);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java
b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java
index fdc5df25a4..e4310243c4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java
@@ -171,7 +171,7 @@ public abstract class DorisHttpTestCase {
// table
PartitionInfo partitionInfo = new SinglePartitionInfo();
- partitionInfo.setDataProperty(testPartitionId,
DataProperty.DEFAULT_DATA_PROPERTY);
+ partitionInfo.setDataProperty(testPartitionId, new
DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
partitionInfo.setReplicaAllocation(testPartitionId, new
ReplicaAllocation((short) 3));
OlapTable table = new OlapTable(testTableId, name, columns,
KeysType.AGG_KEYS, partitionInfo,
distributionInfo);
@@ -189,7 +189,8 @@ public abstract class DorisHttpTestCase {
columns.add(k1);
columns.add(k2);
PartitionInfo partitionInfo = new SinglePartitionInfo();
- partitionInfo.setDataProperty(testPartitionId + 100,
DataProperty.DEFAULT_DATA_PROPERTY);
+ partitionInfo.setDataProperty(testPartitionId + 100,
+ new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
partitionInfo.setReplicaAllocation(testPartitionId + 100,
ReplicaAllocation.DEFAULT_ALLOCATION);
EsTable table = null;
Map<String, String> props = new HashMap<>();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/persist/BatchModifyPartitionsInfoTest.java
b/fe/fe-core/src/test/java/org/apache/doris/persist/BatchModifyPartitionsInfoTest.java
index 159ef9f2e0..82b0766c76 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/persist/BatchModifyPartitionsInfoTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/persist/BatchModifyPartitionsInfoTest.java
@@ -60,8 +60,9 @@ public class BatchModifyPartitionsInfoTest {
List<Long> partitionIds = Lists.newArrayList(PARTITION_ID_1,
PARTITION_ID_2, PARTITION_ID_3);
for (long partitionId : partitionIds) {
- modifyInfos.add(new ModifyPartitionInfo(DB_ID, TB_ID, partitionId,
DataProperty.DEFAULT_DATA_PROPERTY,
- ReplicaAllocation.DEFAULT_ALLOCATION, true, "",
Maps.newHashMap()));
+ modifyInfos.add(new ModifyPartitionInfo(DB_ID, TB_ID, partitionId,
+ new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM),
ReplicaAllocation.DEFAULT_ALLOCATION,
+ true, "", Maps.newHashMap()));
}
BatchModifyPartitionsInfo batchModifyPartitionsInfo = new
BatchModifyPartitionsInfo(modifyInfos);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
