This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a0059ff HIVE-25517: Move table location logic from translation layer
into Warehouse APIs(Sourabh Goyal via Naveen Gangam)
a0059ff is described below
commit a0059ff4e8b957ab666e7a683210e3e1a61b5730
Author: Sourabh Goyal <[email protected]>
AuthorDate: Mon Sep 13 23:37:38 2021 -0700
HIVE-25517: Move table location logic from translation layer into Warehouse
APIs(Sourabh Goyal via Naveen Gangam)
Change-Id: I143c0ca19717e6fab5daffefc921004efa174883
---
.../metastore/TestHiveMetastoreTransformer.java | 40 ++++++++++++++++++----
.../apache/hadoop/hive/metastore/Warehouse.java | 3 +-
.../metastore/MetastoreDefaultTransformer.java | 8 +----
3 files changed, 36 insertions(+), 15 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
index 4eb55e0..f9d19b0 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Locale;
import org.apache.hadoop.hive.metastore.client.builder.GetTablesRequestBuilder;
import org.apache.hadoop.hive.metastore.api.Catalog;
@@ -44,6 +45,7 @@ import
org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_NONE;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READONLY;
@@ -154,11 +156,12 @@ public class TestHiveMetastoreTransformer {
/**
* EXTERNAL_TABLE
- * 1) Old table with no capabilities
- * 2a) New table with capabilities with no client requirements
- * 2b) New table with capabilities with no matching client requirements
- * 2c) New table with capabilities with partial match requirements
- * 2d) New table with capabilities with full match requirements
+ * 1) Old table (name in upper case) with no capabilities
+ * 2) Old table with no capabilities
+ * 3a) New table with capabilities with no client requirements
+ * 3b) New table with capabilities with no matching client requirements
+ * 3c) New table with capabilities with partial match requirements
+ * 3d) New table with capabilities with full match requirements
*/
@Test
public void testTransformerExternalTable() throws Exception {
@@ -168,9 +171,10 @@ public class TestHiveMetastoreTransformer {
Map<String, Object> tProps = new HashMap<>();
int buckets = 32;
- String tblName = basetblName;
+ // create external table with uppercase
+ String tblNameUpper = "TAB_EXT1";
tProps.put("DBNAME", dbName);
- tProps.put("TBLNAME", tblName);
+ tProps.put("TBLNAME", tblNameUpper);
tProps.put("TBLTYPE", TableType.EXTERNAL_TABLE);
tProps.put("BUCKETS", buckets);
StringBuilder properties = new StringBuilder();
@@ -178,6 +182,28 @@ public class TestHiveMetastoreTransformer {
tProps.put("PROPERTIES", properties.toString());
Table tbl = createTableWithCapabilities(tProps);
+ Table hiveTbl = client.getTable(dbName,
tblNameUpper.toLowerCase(Locale.ROOT));
+ Path actualPath = new Path(hiveTbl.getSd().getLocation());
+ Database db = client.getDatabase(dbName);
+ LOG.info("Table=" + tblNameUpper + ",Table Details=" + hiveTbl);
+ assertEquals("Created and retrieved tables do not match:" +
hiveTbl.getTableName() + ":" +
+ tblNameUpper.toLowerCase(Locale.ROOT), hiveTbl.getTableName(),
+ tblNameUpper.toLowerCase(Locale.ROOT));
+ Path expectedTablePath = new Path(db.getLocationUri(),
tblNameUpper.toLowerCase(Locale.ROOT));
+ assertEquals(String.format("Table location %s is not a subset of the
database location %s",
+ actualPath.toString(), db.getLocationUri()),
expectedTablePath.toString(), actualPath.toString());
+
+ String tblName = basetblName;
+ tProps = new HashMap<>();
+ tProps.put("DBNAME", dbName);
+ tProps.put("TBLNAME", tblName);
+ tProps.put("TBLTYPE", TableType.EXTERNAL_TABLE);
+ tProps.put("BUCKETS", buckets);
+ properties = new StringBuilder();
+ properties.append("EXTERNAL").append("=").append("TRUE");
+ tProps.put("PROPERTIES", properties.toString());
+ tbl = createTableWithCapabilities(tProps);
+
// retrieve the table
Table tbl2 = client.getTable(dbName, tblName);
LOG.info("Table=" + tblName + ",Access=" + tbl2.getAccessType());
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
index 97df316..46593aa 100755
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
@@ -355,7 +355,8 @@ public class Warehouse {
Path dbPath = null;
if (isExternal) {
dbPath = new Path(db.getLocationUri());
- if (FileUtils.isSubdirectory(getWhRoot().toString(), dbPath.toString() +
Path.SEPARATOR)) {
+ if (FileUtils.isSubdirectory(getWhRoot().toString(), dbPath.toString())
||
+ getWhRoot().equals(dbPath)) {
// db metadata incorrect, find new location based on external
warehouse root
dbPath = getDefaultExternalDatabasePath(db.getName());
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
index 74e69cc..99fc9e4 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
@@ -939,13 +939,7 @@ public class MetastoreDefaultTransformer implements
IMetaStoreMetadataTransforme
return table;
}
} else {
- dbLocation = Path.getPathWithoutSchemeAndAuthority(new
Path(db.getLocationUri()));
- Path tablePath = null;
- if (!FileUtils.isSubdirectory(whRootPath.toString(),
dbLocation.toString())) {
- tablePath = new Path(db.getLocationUri(),
table.getTableName().toLowerCase());
- } else {
- tablePath = hmsHandler.getWh().getDefaultTablePath(db,
table.getTableName(), true);
- }
+ Path tablePath = hmsHandler.getWh().getDefaultTablePath(db,
table.getTableName(), true);
table.getSd().setLocation(tablePath.toString());
}
}
