This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ec72383d33 [fix](show-stmt) fix show create table missing
storage_medium info (#21757)
ec72383d33 is described below
commit ec72383d3372b519e7957f237fad456130230804
Author: zhengyu <[email protected]>
AuthorDate: Mon Jul 31 19:26:21 2023 +0800
[fix](show-stmt) fix show create table missing storage_medium info (#21757)
---
.../org/apache/doris/catalog/DataProperty.java | 16 ++++++
.../main/java/org/apache/doris/catalog/Env.java | 4 ++
.../java/org/apache/doris/catalog/OlapTable.java | 22 ++++++++
.../apache/doris/datasource/InternalCatalog.java | 5 +-
.../analysis/CreateTableAsSelectStmtTest.java | 60 ++++++++++++++--------
.../org/apache/doris/catalog/CreateTableTest.java | 23 +++++++++
6 files changed, 109 insertions(+), 21 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
index 7028e5e449..23b4508c94 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
@@ -55,6 +55,22 @@ public class DataProperty implements Writable,
GsonPostProcessable {
// for persist
}
+ public String getStorageMediumString() {
+ int val = storageMedium.getValue();
+ switch (val) {
+ case 0:
+ return "hdd";
+ case 1:
+ return "ssd";
+ case 2:
+ return "s3";
+ case 3:
+ return "remote_cache";
+ default:
+ return "unknown";
+ }
+ }
+
public DataProperty(TStorageMedium medium) {
this.storageMedium = medium;
this.cooldownTimeMs = MAX_COOLDOWN_TIME_MS;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 7903ebac44..7a2f542667 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3156,6 +3156,10 @@ public class Env {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION).append("\"
= \"");
sb.append(olapTable.enableSingleReplicaCompaction()).append("\"");
+ // storage medium
+
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM).append("\"
= \"");
+ sb.append(olapTable.getStorageMedium()).append("\"");
+
// enable duplicate without keys by default
if (olapTable.isDuplicateWithoutKey()) {
sb.append(",\n\"")
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 73baf96656..fe25c49c1c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -170,6 +170,8 @@ public class OlapTable extends Table {
private AutoIncrementGenerator autoIncrementGenerator;
+ private String storageMedium;
+
public OlapTable() {
// for persist
super(TableType.OLAP);
@@ -1314,6 +1316,13 @@ public class OlapTable extends Table {
}
tempPartitions.write(out);
+
+ if (storageMedium == null || storageMedium.length() == 0) {
+ out.writeBoolean(false);
+ } else {
+ out.writeBoolean(true);
+ Text.writeString(out, storageMedium);
+ }
}
@Override
@@ -1419,6 +1428,10 @@ public class OlapTable extends Table {
}
tempPartitions.unsetPartitionInfo();
+ if (in.readBoolean()) {
+ storageMedium = Text.readString(in);
+ }
+
// In the present, the fullSchema could be rebuilt by schema change
while the properties is changed by MV.
// After that, some properties of fullSchema and nameToColumn may be
not same as properties of base columns.
// So, here we need to rebuild the fullSchema to ensure the
correctness of the properties.
@@ -1840,6 +1853,15 @@ public class OlapTable extends Table {
return false;
}
+
+ public void setStorageMedium(String medium) {
+ storageMedium = medium;
+ }
+
+ public String getStorageMedium() {
+ return storageMedium;
+ }
+
public void setStoreRowColumn(boolean storeRowColumn) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index c4166870ad..2eaa26a093 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2162,6 +2162,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
try {
dataProperty =
PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
+
olapTable.setStorageMedium(dataProperty.getStorageMediumString());
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
@@ -2328,10 +2329,12 @@ public class InternalCatalog implements
CatalogIf<Database> {
} else if (partitionInfo.getType() == PartitionType.RANGE
|| partitionInfo.getType() == PartitionType.LIST) {
try {
+ DataProperty dataProperty = null;
// just for remove entries in stmt.getProperties(),
// and then check if there still has unknown properties
- PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
+ dataProperty =
PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
new
DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
+
olapTable.setStorageMedium(dataProperty.getStorageMediumString());
if (partitionInfo.getType() == PartitionType.RANGE) {
DynamicPartitionUtil.checkAndSetDynamicPartitionProperty(olapTable, properties,
db);
} else if (partitionInfo.getType() == PartitionType.LIST) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
index 2a5ff740d5..348c8571f6 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
@@ -96,7 +96,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showCreateTableByName("select_decimal_table").getResultRows().get(0).get(1));
String selectFromDecimal1 =
@@ -117,7 +118,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" =
\"false\"\n"
+ + "\"enable_single_replica_compaction\" =
\"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1));
} else {
@@ -134,7 +136,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" =
\"false\"\n"
+ + "\"enable_single_replica_compaction\" =
\"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1));
}
@@ -169,7 +172,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -193,7 +197,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet1.getResultRows().get(0).get(1));
@@ -219,7 +224,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet2.getResultRows().get(0).get(1));
}
@@ -242,7 +248,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");", showResultSet1.getResultRows().get(0).get(1));
String selectAlias2 = "create table `test`.`select_alias_2`
PROPERTIES(\"replication_num\" = \"1\") "
+ "as select userId as alias_name, username from
`test`.`varchar_table`";
@@ -261,7 +268,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet2.getResultRows().get(0).get(1));
}
@@ -287,7 +295,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
String selectFromJoin1 = "create table `test`.`select_join1`
PROPERTIES(\"replication_num\" = \"1\") "
@@ -310,7 +319,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet1.getResultRows().get(0).get(1));
}
@@ -337,7 +347,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -361,7 +372,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");", showResultSet.getResultRows().get(0).get(1));
}
@@ -384,7 +396,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
String selectFromCteAndUnion = "create table `test`.`select_cte_union`
PROPERTIES(\"replication_num\" = \"1\")"
@@ -403,7 +416,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");", showResultSet1.getResultRows().get(0).get(1));
}
@@ -429,7 +443,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -454,7 +469,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -478,7 +494,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -503,7 +520,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -552,7 +570,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" =
\"false\"\n"
+ + "\"enable_single_replica_compaction\" =
\"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
createTableStmts.get(0));
} else {
@@ -569,7 +588,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" =
\"false\"\n"
+ + "\"enable_single_replica_compaction\" =
\"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
createTableStmts.get(0));
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 3e98d8e285..ff173f4ff6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -146,6 +146,16 @@ public class CreateTableTest {
.expectThrowsNoException(() -> createTable("create table
test.tb7(key1 int, key2 varchar(10)) \n"
+ "distributed by hash(key1) buckets 1
properties('replication_num' = '1', 'storage_medium' = 'ssd');"));
+ ConfigBase.setMutableConfig("disable_storage_medium_check", "true");
+ ExceptionChecker
+ .expectThrowsNoException(() -> createTable("create table
test.tb7_1(key1 int, key2 varchar(10))\n"
+ + "PARTITION BY RANGE(`key1`) (\n"
+ + " PARTITION `p1` VALUES LESS THAN
(\"10\"),\n"
+ + " PARTITION `p2` VALUES LESS THAN
(\"20\"),\n"
+ + " PARTITION `p3` VALUES LESS THAN
(\"30\"))\n"
+ + "distributed by hash(key1)\n"
+ + "buckets 1 properties('replication_num' =
'1', 'storage_medium' = 'ssd');"));
+
ExceptionChecker
.expectThrowsNoException(() -> createTable("create table
test.compression1(key1 int, key2 varchar(10)) \n"
+ "distributed by hash(key1) buckets 1 \n"
@@ -301,6 +311,19 @@ public class CreateTableTest {
() -> createTable("create table test.tb7(key1 int,
key2 varchar(10)) distributed by hash(key1) \n"
+ "buckets 1 properties('replication_num' =
'1', 'storage_medium' = 'ssd');"));
+ ConfigBase.setMutableConfig("disable_storage_medium_check", "false");
+ ExceptionChecker
+ .expectThrowsWithMsg(DdlException.class, "Failed to find
enough backend, please check the replication num,replication tag and storage
medium.\n"
+ + "Create failed replications:\n"
+ + "replication tag: {\"location\" :
\"default\"}, replication num: 1, storage medium: SSD",
+ () -> createTable("create table test.tb7_1(key1 int,
key2 varchar(10))\n"
+ + "PARTITION BY RANGE(`key1`) (\n"
+ + " PARTITION `p1` VALUES LESS THAN
(\"10\"),\n"
+ + " PARTITION `p2` VALUES LESS THAN
(\"20\"),\n"
+ + " PARTITION `p3` VALUES LESS THAN
(\"30\"))\n"
+ + "distributed by hash(key1)\n"
+ + "buckets 1 properties('replication_num' =
'1', 'storage_medium' = 'ssd');"));
+
ExceptionChecker
.expectThrowsWithMsg(DdlException.class, "sequence column only
support UNIQUE_KEYS",
() -> createTable("create table test.atbl8\n" + "(k1
varchar(40), k2 int, v1 int sum)\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
