This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e570856 HIVE-25331: Create database query doesn't create
MANAGEDLOCATION directory (#2478) (Robbie Zhang reviewed by Laszlo Bodor and
Pravin Kumar Sinha)
e570856 is described below
commit e5708566884611dcbacc8d369982da3e4eeb809e
Author: Robbie Zhang <[email protected]>
AuthorDate: Mon Sep 27 22:00:11 2021 +1000
HIVE-25331: Create database query doesn't create MANAGEDLOCATION directory
(#2478) (Robbie Zhang reviewed by Laszlo Bodor and Pravin Kumar Sinha)
---
.../TestReplicationScenariosExclusiveReplica.java | 8 ++++----
.../security/TestMetastoreAuthorizationProvider.java | 4 ++++
.../ddl/database/create/CreateDatabaseOperation.java | 9 ++++++++-
ql/src/test/queries/clientpositive/create_database.q | 4 ++++
.../clientpositive/llap/alter_change_db_location.q.out | 2 +-
.../results/clientpositive/llap/create_database.q.out | 18 ++++++++++++++++++
.../clientpositive/llap/database_location.q.out | 2 +-
.../results/clientpositive/tez/explainanalyze_3.q.out | 2 +-
.../results/clientpositive/tez/explainuser_3.q.out | 2 +-
.../org/apache/hadoop/hive/metastore/HMSHandler.java | 2 +-
10 files changed, 43 insertions(+), 10 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExclusiveReplica.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExclusiveReplica.java
index 3506088..4e6a77f 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExclusiveReplica.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExclusiveReplica.java
@@ -700,18 +700,18 @@ public class TestReplicationScenariosExclusiveReplica
extends BaseReplicationAcr
private void verifyCustomDBLocations(String srcDb, List<String>
listOfTables, String managedCustLocOnSrc,
String externalCustLocOnSrc, boolean
replaceCustPath) throws Exception {
Database replDatabase = replica.getDatabase(replicatedDbName);
+ String managedCustLocOnTgt = new
Path(replDatabase.getManagedLocationUri()).toUri().getPath();
+ String externalCustLocOnTgt = new
Path(replDatabase.getLocationUri()).toUri().getPath();
if (replaceCustPath ) {
- String managedCustLocOnTgt = new
Path(replDatabase.getManagedLocationUri()).toUri().getPath();
Assert.assertEquals(managedCustLocOnSrc, managedCustLocOnTgt);
Assert.assertNotEquals(managedCustLocOnTgt,
replica.warehouseRoot.toUri().getPath());
- String externalCustLocOnTgt = new
Path(replDatabase.getLocationUri()).toUri().getPath();
Assert.assertEquals(externalCustLocOnSrc, externalCustLocOnTgt);
Assert.assertNotEquals(externalCustLocOnTgt, new
Path(replica.externalTableWarehouseRoot,
replicatedDbName.toLowerCase() + ".db").toUri().getPath());
} else {
Assert.assertNotEquals(managedCustLocOnSrc, null);
- Assert.assertEquals(replDatabase.getManagedLocationUri(), null);
- String externalCustLocOnTgt = new
Path(replDatabase.getLocationUri()).toUri().getPath();
+ Assert.assertEquals(managedCustLocOnTgt, new Path(replica.warehouseRoot,
+ replicatedDbName.toLowerCase() + ".db").toUri().getPath());
Assert.assertNotEquals(externalCustLocOnSrc, externalCustLocOnTgt);
Assert.assertEquals(externalCustLocOnTgt, new
Path(replica.externalTableWarehouseRoot,
replicatedDbName.toLowerCase() + ".db").toUri().getPath());
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
index ef9ea73..3fa1d0d 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
@@ -333,6 +333,10 @@ public class TestMetastoreAuthorizationProvider {
db = msc.getDatabase(dbName);
dbLocn = db.getLocationUri();
allowCreateInDb(dbName, userName, dbLocn);
+ dbLocn = db.getManagedLocationUri();
+ if (dbLocn != null) {
+ allowCreateInDb(dbName, userName, dbLocn);
+ }
tbl.setTableType("EXTERNAL_TABLE");
msc.createTable(tbl);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseOperation.java
index c4961b2..221d2e7 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseOperation.java
@@ -93,9 +93,16 @@ public class CreateDatabaseOperation extends
DDLOperation<CreateDatabaseDesc> {
}
if (database.isSetManagedLocationUri()) {
- // TODO should we enforce a location check here?
database.setManagedLocationUri(Utilities.getQualifiedPath(context.getConf(),
new Path(database.getManagedLocationUri())));
+ } else {
+ // ManagedLocation is not set we utilize WAREHOUSE together with
database name
+ String rootDir = MetastoreConf.getVar(context.getConf(),
MetastoreConf.ConfVars.WAREHOUSE);
+ Path path = new Path(rootDir, database.getName().toLowerCase() +
DATABASE_PATH_SUFFIX);
+ String qualifiedPath = Utilities.getQualifiedPath(context.getConf(),
path);
+ if (!qualifiedPath.equals(database.getLocationUri())) {
+ database.setManagedLocationUri(qualifiedPath);
+ }
}
}
}
diff --git a/ql/src/test/queries/clientpositive/create_database.q
b/ql/src/test/queries/clientpositive/create_database.q
new file mode 100644
index 0000000..d5fe869
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/create_database.q
@@ -0,0 +1,4 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+create database testdb location '/tmp/testdb.db';
+create table testdb.test as select 1;
diff --git
a/ql/src/test/results/clientpositive/llap/alter_change_db_location.q.out
b/ql/src/test/results/clientpositive/llap/alter_change_db_location.q.out
index d07eaa6..d47b388 100644
--- a/ql/src/test/results/clientpositive/llap/alter_change_db_location.q.out
+++ b/ql/src/test/results/clientpositive/llap/alter_change_db_location.q.out
@@ -11,7 +11,7 @@ PREHOOK: Input: database:newdb
POSTHOOK: query: describe database extended newDB
POSTHOOK: type: DESCDATABASE
POSTHOOK: Input: database:newdb
-newdb location/in/test hive_test_user USER
+#### A masked pattern was here ####
PREHOOK: query: use newDB
PREHOOK: type: SWITCHDATABASE
PREHOOK: Input: database:newdb
diff --git a/ql/src/test/results/clientpositive/llap/create_database.q.out
b/ql/src/test/results/clientpositive/llap/create_database.q.out
new file mode 100644
index 0000000..459a506
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/create_database.q.out
@@ -0,0 +1,18 @@
+#### A masked pattern was here ####
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:testdb
+#### A masked pattern was here ####
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:testdb
+#### A masked pattern was here ####
+PREHOOK: query: create table testdb.test as select 1
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: database:testdb
+PREHOOK: Output: testdb@test
+POSTHOOK: query: create table testdb.test as select 1
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: database:testdb
+POSTHOOK: Output: testdb@test
+POSTHOOK: Lineage: test._c0 SIMPLE []
diff --git a/ql/src/test/results/clientpositive/llap/database_location.q.out
b/ql/src/test/results/clientpositive/llap/database_location.q.out
index 1cb5127..d0bd47f 100644
--- a/ql/src/test/results/clientpositive/llap/database_location.q.out
+++ b/ql/src/test/results/clientpositive/llap/database_location.q.out
@@ -85,7 +85,7 @@ PREHOOK: Input: database:db2
POSTHOOK: query: DESCRIBE DATABASE EXTENDED db2
POSTHOOK: type: DESCDATABASE
POSTHOOK: Input: database:db2
-db2 database 2 location/in/test hive_test_user USER
+#### A masked pattern was here ####
PREHOOK: query: USE db2
PREHOOK: type: SWITCHDATABASE
PREHOOK: Input: database:db2
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
index 9add485..ad2550b 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
@@ -127,7 +127,7 @@ PREHOOK: Input: database:newdb
POSTHOOK: query: describe database extended newDB
POSTHOOK: type: DESCDATABASE
POSTHOOK: Input: database:newdb
-newdb location/in/test hive_test_user USER
+newdb location/in/test hdfs://### HDFS PATH ###
hive_test_user USER
PREHOOK: query: use newDB
PREHOOK: type: SWITCHDATABASE
PREHOOK: Input: database:newdb
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
index 990d1bd..a3e9ea1 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
@@ -145,7 +145,7 @@ PREHOOK: Input: database:newdb
POSTHOOK: query: describe database extended newDB
POSTHOOK: type: DESCDATABASE
POSTHOOK: Input: database:newdb
-newdb location/in/test hive_test_user USER
+newdb location/in/test hdfs://### HDFS PATH ###
hive_test_user USER
PREHOOK: query: explain use newDB
PREHOOK: type: SWITCHDATABASE
PREHOOK: Input: database:newdb
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index 1cca7bf..7c46857 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -1362,7 +1362,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
});
if (madeManagedDir) {
LOG.info("Created database path in managed directory " +
dbMgdPath);
- } else {
+ } else if (!isInTest || !isDbReplicationTarget(db)) { // Hive
replication tests doesn't drop the db after each test
throw new MetaException(
"Unable to create database managed directory " + dbMgdPath +
", failed to create database " + db.getName());
}
