This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 10a1b013b [hive] Hive create table force db name and table names to
be lowercase (#1787)
10a1b013b is described below
commit 10a1b013b74627cd6335d64c8776d8ee57f840e4
Author: HZY <[email protected]>
AuthorDate: Wed Aug 16 14:26:57 2023 +0800
[hive] Hive create table force db name and table names to be lowercase
(#1787)
---
.../org/apache/paimon/hive/PaimonMetaHook.java | 2 +
.../org/apache/paimon/hive/CreateTableITCase.java | 130 +++++++++++++++++++++
2 files changed, 132 insertions(+)
diff --git
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
index 64d359045..97e7bfcfd 100644
---
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
+++
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
@@ -78,6 +78,8 @@ public class PaimonMetaHook implements HiveMetaHook {
table.getSd().setInputFormat(PaimonInputFormat.class.getCanonicalName());
table.getSd().setOutputFormat(PaimonOutputFormat.class.getCanonicalName());
+ table.setDbName(table.getDbName().toLowerCase());
+ table.setTableName(table.getTableName().toLowerCase());
String location = LocationKeyExtractor.getPaimonLocation(conf, table);
Identifier identifier = Identifier.create(table.getDbName(),
table.getTableName());
if (location == null) {
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java
index 85506453d..df7ea6806 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java
@@ -19,12 +19,18 @@
package org.apache.paimon.hive;
import org.apache.paimon.catalog.AbstractCatalog;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.CatalogFactory;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.Table;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataTypes;
@@ -42,6 +48,7 @@ import org.junit.Test;
import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
import java.util.Optional;
import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE;
@@ -148,6 +155,129 @@ public class CreateTableITCase extends HiveTestBase {
.containsEntry(METASTORE_PARTITIONED_TABLE.key(), "true");
}
+ @Test
+ public void testLowerTableName() throws Catalog.TableNotExistException {
+ // Use `partitioned by` to create hive partition table
+ String tableName = "UPPER_NAME";
+ hiveShell.execute("SET hive.metastore.warehouse.dir=" + path);
+ String hiveSql =
+ String.join(
+ "\n",
+ Arrays.asList(
+ "CREATE TABLE " + tableName + " (",
+ "user_id "
+ +
TypeInfoFactory.longTypeInfo.getTypeName()
+ + " COMMENT 'The user_id field',",
+ "item_id "
+ +
TypeInfoFactory.longTypeInfo.getTypeName()
+ + " COMMENT 'The item_id field',",
+ "behavior "
+ +
TypeInfoFactory.stringTypeInfo.getTypeName()
+ + " COMMENT 'The behavior field'",
+ ")",
+ "PARTITIONED BY ( ",
+ "dt "
+ +
TypeInfoFactory.stringTypeInfo.getTypeName()
+ + " COMMENT 'The dt field',",
+ "hh "
+ +
TypeInfoFactory.stringTypeInfo.getTypeName()
+ + " COMMENT 'The hh field'",
+ ")",
+ "STORED BY '" +
PaimonStorageHandler.class.getName() + "'",
+ "TBLPROPERTIES (",
+ " 'primary-key'='dt,hh,user_id'",
+ ")"));
+ assertThatCode(() ->
hiveShell.execute(hiveSql)).doesNotThrowAnyException();
+
+ // check table path from hive metastore
+ List<String> queryResult =
+ hiveShell.executeQuery(String.format("describe formatted %s",
tableName));
+ for (String result : queryResult) {
+ if (result.contains("Location")) {
+ String location = result.split("\t")[1];
+ String tableNameFromPath =
location.substring(location.lastIndexOf("/") + 1);
+
assertThat(tableNameFromPath).isEqualTo(tableName.toLowerCase());
+ }
+ }
+ // check the paimon table name and schema
+ Identifier identifier = Identifier.create(DATABASE_TEST,
tableName.toLowerCase());
+ Path tablePath = AbstractCatalog.dataTableLocation(path, identifier);
+ Options conf = new Options();
+ conf.set(CatalogOptions.WAREHOUSE, path);
+ CatalogContext catalogContext = CatalogContext.create(conf);
+ Catalog catalog = CatalogFactory.createCatalog(catalogContext);
+ Table table = catalog.getTable(identifier);
+ assertThat(table.name()).isEqualTo(tableName.toLowerCase());
+ Optional<TableSchema> tableSchema =
+ new SchemaManager(LocalFileIO.create(), tablePath).latest();
+ assertThat(tableSchema).isPresent();
+ }
+
+ @Test
+ public void testLowerDBName() throws Catalog.TableNotExistException {
+ String upperDB = "UPPER_DB";
+
+ hiveShell.execute(String.format("create database %s", upperDB));
+
+ String tableName = "UPPER_NAME";
+ hiveShell.execute("SET hive.metastore.warehouse.dir=" + path);
+ String hiveSql =
+ String.join(
+ "\n",
+ Arrays.asList(
+ "CREATE TABLE " + upperDB + "." + tableName +
" (",
+ "user_id "
+ +
TypeInfoFactory.longTypeInfo.getTypeName()
+ + " COMMENT 'The user_id field',",
+ "item_id "
+ +
TypeInfoFactory.longTypeInfo.getTypeName()
+ + " COMMENT 'The item_id field',",
+ "behavior "
+ +
TypeInfoFactory.stringTypeInfo.getTypeName()
+ + " COMMENT 'The behavior field'",
+ ")",
+ "PARTITIONED BY ( ",
+ "dt "
+ +
TypeInfoFactory.stringTypeInfo.getTypeName()
+ + " COMMENT 'The dt field',",
+ "hh "
+ +
TypeInfoFactory.stringTypeInfo.getTypeName()
+ + " COMMENT 'The hh field'",
+ ")",
+ "STORED BY '" +
PaimonStorageHandler.class.getName() + "'",
+ "TBLPROPERTIES (",
+ " 'primary-key'='dt,hh,user_id'",
+ ")"));
+ assertThatCode(() ->
hiveShell.execute(hiveSql)).doesNotThrowAnyException();
+
+ // check table path from hive metastore
+ hiveShell.execute("USE " + upperDB);
+ List<String> queryResult =
+ hiveShell.executeQuery(String.format("describe formatted %s",
tableName));
+ for (String result : queryResult) {
+ if (result.contains("Location")) {
+ String location = result.split("\t")[1];
+ int index = location.lastIndexOf(upperDB.toLowerCase() +
".db");
+ assertThat(index).isGreaterThan(0);
+ String dbNameFromPath = location.substring(index,
location.lastIndexOf("/"));
+ assertThat(dbNameFromPath).isEqualTo(upperDB.toLowerCase() +
".db");
+ }
+ }
+
+ // check the paimon db namećtable name and schema
+ Identifier identifier = Identifier.create(upperDB.toLowerCase(),
tableName.toLowerCase());
+ Path tablePath = AbstractCatalog.dataTableLocation(path, identifier);
+ Options conf = new Options();
+ conf.set(CatalogOptions.WAREHOUSE, path);
+ CatalogContext catalogContext = CatalogContext.create(conf);
+ Catalog catalog = CatalogFactory.createCatalog(catalogContext);
+ Table table = catalog.getTable(identifier);
+ assertThat(table.name()).isEqualTo(tableName.toLowerCase());
+ Optional<TableSchema> tableSchema =
+ new SchemaManager(LocalFileIO.create(), tablePath).latest();
+ assertThat(tableSchema).isPresent();
+ }
+
@Test
public void testCreateTableWithPrimaryKey() {
String tableName = "primary_key_table";
