This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1b059a1e5 [hive] Support use partitioned by to create hive partition
table. (#1726)
1b059a1e5 is described below
commit 1b059a1e5f0db97ff2eb9ab857711ca0a287fdaf
Author: Kerwin <[email protected]>
AuthorDate: Fri Aug 4 12:06:35 2023 +0800
[hive] Support use partitioned by to create hive partition table. (#1726)
---
docs/content/how-to/creating-tables.md | 6 +--
.../org/apache/paimon/hive/PaimonMetaHook.java | 35 ++++++++++++----
.../org/apache/paimon/hive/CreateTableITCase.java | 49 +++++++++++++++-------
3 files changed, 64 insertions(+), 26 deletions(-)
diff --git a/docs/content/how-to/creating-tables.md
b/docs/content/how-to/creating-tables.md
index 57c74b83d..d1a18a015 100644
--- a/docs/content/how-to/creating-tables.md
+++ b/docs/content/how-to/creating-tables.md
@@ -168,14 +168,14 @@ SET hive.metastore.warehouse.dir=warehouse_path;
CREATE TABLE MyTable (
user_id BIGINT,
item_id BIGINT,
- behavior STRING,
+ behavior STRING
+) PARTITIONED BY (
dt STRING,
hh STRING
)
STORED BY 'org.apache.paimon.hive.PaimonStorageHandler'
TBLPROPERTIES (
- 'primary-key' = 'dt,hh,user_id',
- 'partition'='dt,hh'
+ 'primary-key' = 'dt,hh,user_id'
);
```
diff --git
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
index a0c20772e..69f24f73b 100644
---
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
+++
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
@@ -45,8 +45,10 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
+import java.util.stream.Collectors;
import static org.apache.hadoop.hive.metastore.Warehouse.getDnsPath;
+import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE;
import static org.apache.paimon.hive.HiveTypeUtils.toPaimonType;
/**
@@ -66,12 +68,6 @@ public class PaimonMetaHook implements HiveMetaHook {
@Override
public void preCreateTable(Table table) throws MetaException {
- if (table.getPartitionKeysSize() != 0) {
- throw new MetaException(
- "Paimon currently does not support creating partitioned
table "
- + "with PARTITIONED BY clause. If you want to
create a partitioned table, "
- + "please set partition fields in properties.");
- }
// hive ql parse cannot recognize input near '$' in table name, no
need to add paimon system
// table verification.
@@ -107,9 +103,7 @@ public class PaimonMetaHook implements HiveMetaHook {
// create paimon table
List<FieldSchema> cols = table.getSd().getCols();
Schema.Builder schemaBuilder =
- Schema.newBuilder()
- .options(context.options().toMap())
- .comment(table.getParameters().get(COMMENT));
+
Schema.newBuilder().comment(table.getParameters().get(COMMENT));
cols.iterator()
.forEachRemaining(
fieldSchema ->
@@ -117,6 +111,29 @@ public class PaimonMetaHook implements HiveMetaHook {
fieldSchema.getName().toLowerCase(),
toPaimonType(fieldSchema.getType()),
fieldSchema.getComment()));
+ // partition columns
+ if (table.getPartitionKeysSize() > 0) {
+ // set metastore.partitioned-table = true
+ context.options().set(METASTORE_PARTITIONED_TABLE, true);
+
+ table.getPartitionKeys()
+ .iterator()
+ .forEachRemaining(
+ fieldSchema ->
+ schemaBuilder.column(
+
fieldSchema.getName().toLowerCase(),
+
toPaimonType(fieldSchema.getType()),
+ fieldSchema.getComment()));
+
+ List<String> partitionKeys =
+ table.getPartitionKeys().stream()
+ .map(FieldSchema::getName)
+ .map(String::toLowerCase)
+ .collect(Collectors.toList());
+ schemaBuilder.partitionKeys(partitionKeys);
+ }
+ schemaBuilder.options(context.options().toMap());
+
try {
schemaManager.createTable(schemaBuilder.build());
} catch (Exception e) {
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java
index 28deafbae..fe5c8f845 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java
@@ -31,7 +31,6 @@ import org.apache.paimon.types.DataTypes;
import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
import org.apache.paimon.shade.guava30.com.google.common.collect.Maps;
-import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.junit.Test;
@@ -40,6 +39,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Optional;
+import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -100,26 +100,47 @@ public class CreateTableITCase extends HiveTestBase {
@Test
public void testCreateTableUsePartitionedBy() {
// Use `partitioned by` to create hive partition table
- String tableName = "not_support_partitioned_by_table";
+ String tableName = "support_partitioned_by_table";
+ hiveShell.execute("SET hive.metastore.warehouse.dir=" + path);
String hiveSql =
String.join(
"\n",
Arrays.asList(
"CREATE TABLE " + tableName + " (",
- "col1 "
- +
TypeInfoFactory.intTypeInfo.getTypeName()
- + " COMMENT 'The col1 field'",
+ "user_id "
+ +
TypeInfoFactory.longTypeInfo.getTypeName()
+ + " COMMENT 'The user_id field',",
+ "item_id "
+ +
TypeInfoFactory.longTypeInfo.getTypeName()
+ + " COMMENT 'The item_id field',",
+ "behavior "
+ +
TypeInfoFactory.stringTypeInfo.getTypeName()
+ + " COMMENT 'The behavior field'",
")",
- "PARTITIONED BY (dt "
+ "PARTITIONED BY ( ",
+ "dt "
+
TypeInfoFactory.stringTypeInfo.getTypeName()
- + ")",
- "STORED BY '" +
PaimonStorageHandler.class.getName() + "'"));
- assertThatThrownBy(() -> hiveShell.execute(hiveSql))
- .hasRootCauseInstanceOf(MetaException.class)
- .hasRootCauseMessage(
- "Paimon currently does not support creating
partitioned table "
- + "with PARTITIONED BY clause. If you want to
create a partitioned table, "
- + "please set partition fields in
properties.");
+ + " COMMENT 'The dt field',",
+ "hh "
+ +
TypeInfoFactory.stringTypeInfo.getTypeName()
+ + " COMMENT 'The hh field'",
+ ")",
+ "STORED BY '" +
PaimonStorageHandler.class.getName() + "'",
+ "TBLPROPERTIES (",
+ " 'primary-key'='dt,hh,user_id'",
+ ")"));
+ assertThatCode(() ->
hiveShell.execute(hiveSql)).doesNotThrowAnyException();
+
+ // check the paimon table schema
+ Identifier identifier = Identifier.create(DATABASE_TEST, tableName);
+ Path tablePath = AbstractCatalog.dataTableLocation(path, identifier);
+ Optional<TableSchema> tableSchema =
+ new SchemaManager(LocalFileIO.create(), tablePath).latest();
+ assertThat(tableSchema).isPresent();
+ assertThat(tableSchema.get().primaryKeys()).contains("dt", "hh",
"user_id");
+ assertThat(tableSchema.get().partitionKeys()).contains("dt", "hh");
+ assertThat(tableSchema.get().options())
+ .containsEntry(METASTORE_PARTITIONED_TABLE.key(), "true");
}
@Test
