Author: vgumashta
Date: Thu Sep 4 20:25:49 2014
New Revision: 1622556
URL: http://svn.apache.org/r1622556
Log:
HIVE-6847: Improve / fix bugs in Hive scratch dir setup (Vaibhav Gumashta
reviewed by Jason Dere, Szehon Ho, Thejas Nair, Venki Korukanti)
Removed:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestUtilitiesDfs.java
hive/trunk/service/src/test/org/apache/hive/service/cli/TestScratchDir.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java
hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java
(original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java
Thu Sep 4 20:25:49 2014
@@ -25,9 +25,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
/**
- *
- * ServerUtils.
- *
+ * ServerUtils (specific to HiveServer version 1)
*/
public class ServerUtils {
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu
Sep 4 20:25:49 2014
@@ -206,11 +206,18 @@ public class HiveConf extends Configurat
PLAN_SERIALIZATION("hive.plan.serialization.format", "kryo",
"Query plan format serialization between client and task nodes. \n" +
"Two supported values are : kryo and javaXML. Kryo is default."),
- SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive-${system:user.name}",
"Scratch space for Hive jobs"),
+ SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive",
+ "HDFS root scratch dir for Hive jobs which gets created with 777
permission. " +
+ "For each connecting user, an HDFS scratch dir:
${hive.exec.scratchdir}/<username> is created, " +
+ "with ${hive.scratch.dir.permission}."),
LOCALSCRATCHDIR("hive.exec.local.scratchdir",
"${system:java.io.tmpdir}" + File.separator + "${system:user.name}",
"Local scratch space for Hive jobs"),
- SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700", ""),
+ DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir",
+ "${system:java.io.tmpdir}" + File.separator +
"${hive.session.id}_resources",
+ "Temporary local directory for added resources in the remote file
system."),
+ SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700",
+ "The permission for the user specific scratch directories that get
created."),
SUBMITVIACHILD("hive.exec.submitviachild", false, ""),
SUBMITLOCALTASKVIACHILD("hive.exec.submit.local.task.via.child", true,
"Determines whether local tasks (typically mapjoin hashtable
generation phase) runs in \n" +
@@ -274,9 +281,6 @@ public class HiveConf extends Configurat
"Maximum number of dynamic partitions allowed to be created in each
mapper/reducer node."),
MAXCREATEDFILES("hive.exec.max.created.files", 100000L,
"Maximum number of HDFS files created by all mappers/reducers in a
MapReduce job."),
- DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir",
- "${system:java.io.tmpdir}" + File.separator +
"${hive.session.id}_resources",
- "Temporary local directory for added resources in the remote file
system."),
DEFAULTPARTITIONNAME("hive.exec.default.partition.name",
"__HIVE_DEFAULT_PARTITION__",
"The default partition name in case the dynamic partition column value
is null/empty string or any other values that cannot be escaped. \n" +
"This value must not contain any special character used in HDFS URI
(e.g., ':', '%', '/' etc). \n" +
Modified:
hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
---
hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
(original)
+++
hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
Thu Sep 4 20:25:49 2014
@@ -31,7 +31,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
import org.apache.hadoop.hive.shims.HadoopShims.MiniMrShim;
@@ -57,6 +56,7 @@ public class MiniHS2 extends AbstractHiv
private static final AtomicLong hs2Counter = new AtomicLong();
private MiniMrShim mr;
private MiniDFSShim dfs;
+ private FileSystem localFS;
private boolean useMiniMR = false;
private boolean useMiniKdc = false;
private final String serverPrincipal;
@@ -137,6 +137,10 @@ public class MiniHS2 extends AbstractHiv
this.dfs = dfs;
}
+ public FileSystem getLocalFS() {
+ return localFS;
+ }
+
public boolean isUseMiniMR() {
return useMiniMR;
}
@@ -157,7 +161,8 @@ public class MiniHS2 extends AbstractHiv
this.serverPrincipal = serverPrincipal;
this.serverKeytab = serverKeytab;
this.isMetastoreRemote = isMetastoreRemote;
- baseDir = Files.createTempDir();
+ baseDir = Files.createTempDir();
+ localFS = FileSystem.getLocal(hiveConf);
FileSystem fs;
if (useMiniMR) {
dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, 4, true, null);
@@ -371,7 +376,7 @@ public class MiniHS2 extends AbstractHiv
getMiniKdc().loginUser(getMiniKdc().getDefaultUserPrincipal());
sessionConf.put("principal", serverPrincipal);
}
- */
+ */
sessionHandle = hs2Client.openSession("foo", "bar", sessionConf);
} catch (Exception e) {
// service not started yet
Modified:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
---
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
(original)
+++
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
Thu Sep 4 20:25:49 2014
@@ -31,7 +31,9 @@ import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
@@ -41,255 +43,357 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
- public class TestJdbcWithMiniHS2 {
- private static MiniHS2 miniHS2 = null;
- private static Path dataFilePath;
-
- private Connection hs2Conn = null;
-
- @BeforeClass
- public static void beforeTest() throws Exception {
- Class.forName(MiniHS2.getJdbcDriverName());
- HiveConf conf = new HiveConf();
- conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- miniHS2 = new MiniHS2(conf);
- String dataFileDir = conf.get("test.data.files").replace('\\', '/')
- .replace("c:", "");
- dataFilePath = new Path(dataFileDir, "kv1.txt");
- Map<String, String> confOverlay = new HashMap<String, String>();
- miniHS2.start(confOverlay);
- }
+public class TestJdbcWithMiniHS2 {
+ private static MiniHS2 miniHS2 = null;
+ private static Path dataFilePath;
+
+ private Connection hs2Conn = null;
+
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ Class.forName(MiniHS2.getJdbcDriverName());
+ HiveConf conf = new HiveConf();
+ conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ miniHS2 = new MiniHS2(conf);
+ String dataFileDir = conf.get("test.data.files").replace('\\', '/')
+ .replace("c:", "");
+ dataFilePath = new Path(dataFileDir, "kv1.txt");
+ Map<String, String> confOverlay = new HashMap<String, String>();
+ miniHS2.start(confOverlay);
+ }
- @Before
- public void setUp() throws Exception {
- hs2Conn = getConnection(miniHS2.getJdbcURL(),
System.getProperty("user.name"), "bar");
- }
+ @Before
+ public void setUp() throws Exception {
+ hs2Conn = getConnection(miniHS2.getJdbcURL(),
System.getProperty("user.name"), "bar");
+ }
- private Connection getConnection(String jdbcURL, String user, String pwd)
throws SQLException {
- Connection conn = DriverManager.getConnection(jdbcURL, user, pwd);
- conn.createStatement().execute("set hive.support.concurrency = false");
- return conn;
- }
+ private Connection getConnection(String jdbcURL, String user, String pwd)
throws SQLException {
+ Connection conn = DriverManager.getConnection(jdbcURL, user, pwd);
+ conn.createStatement().execute("set hive.support.concurrency = false");
+ return conn;
+ }
- @After
- public void tearDown() throws Exception {
- hs2Conn.close();
+ @After
+ public void tearDown() throws Exception {
+ hs2Conn.close();
+ }
+
+ @AfterClass
+ public static void afterTest() throws Exception {
+ if (miniHS2.isStarted()) {
+ miniHS2.stop();
}
+ }
+
+ @Test
+ public void testConnection() throws Exception {
+ String tableName = "testTab1";
+ Statement stmt = hs2Conn.createStatement();
+
+ // create table
+ stmt.execute("DROP TABLE IF EXISTS " + tableName);
+ stmt.execute("CREATE TABLE " + tableName
+ + " (under_col INT COMMENT 'the under column', value STRING) COMMENT '
test table'");
+
+ // load data
+ stmt.execute("load data local inpath '"
+ + dataFilePath.toString() + "' into table " + tableName);
+
+ ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
+ assertTrue(res.next());
+ assertEquals("val_238", res.getString(2));
+ res.close();
+ stmt.close();
+ }
+
- @AfterClass
- public static void afterTest() throws Exception {
- if (miniHS2.isStarted())
- miniHS2.stop();
+ /** This test is to connect to any database without using the command "Use
<<DB>>"
+ * 1)connect to default database.
+ * 2) Create a new DB test_default.
+ * 3) Connect to test_default database.
+ * 4) Connect and create table under test_default_test.
+ * 5) Connect and display all tables.
+ * 6) Connect to default database and shouldn't find table
test_default_test.
+ * 7) Connect and drop test_default_test.
+ * 8) drop test_default database.
+ */
+
+ @Test
+ public void testURIDatabaseName() throws Exception{
+
+ String jdbcUri = miniHS2.getJdbcURL().substring(0,
miniHS2.getJdbcURL().indexOf("default"));
+
+ hs2Conn= getConnection(jdbcUri+"default",
System.getProperty("user.name"),"bar");
+ String dbName="test_connection_non_default_db";
+ String tableInNonDefaultSchema="table_in_non_default_schema";
+ Statement stmt = hs2Conn.createStatement();
+ stmt.execute("create database if not exists "+dbName);
+ stmt.close();
+ hs2Conn.close();
+
+ hs2Conn =
getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ boolean expected = stmt.execute(" create table "+tableInNonDefaultSchema
+" (x int)");
+ stmt.close();
+ hs2Conn .close();
+
+ hs2Conn =
getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ ResultSet res = stmt.executeQuery("show tables");
+ boolean testTableExists = false;
+ while (res.next()) {
+ assertNotNull("table name is null in result set", res.getString(1));
+ if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
+ testTableExists = true;
+ }
+ }
+ assertTrue("table name "+tableInNonDefaultSchema
+ + " found in SHOW TABLES result set", testTableExists);
+ stmt.close();
+ hs2Conn .close();
+
+ hs2Conn =
getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ res = stmt.executeQuery("show tables");
+ testTableExists = false;
+ while (res.next()) {
+ assertNotNull("table name is null in result set", res.getString(1));
+ if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
+ testTableExists = true;
+ }
}
- @Test
- public void testConnection() throws Exception {
- String tableName = "testTab1";
- Statement stmt = hs2Conn.createStatement();
-
- // create table
- stmt.execute("DROP TABLE IF EXISTS " + tableName);
- stmt.execute("CREATE TABLE " + tableName
- + " (under_col INT COMMENT 'the under column', value STRING) COMMENT
' test table'");
-
- // load data
- stmt.execute("load data local inpath '"
- + dataFilePath.toString() + "' into table " + tableName);
-
- ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
- assertTrue(res.next());
- assertEquals("val_238", res.getString(2));
- res.close();
- stmt.close();
+ assertFalse("table name "+tableInNonDefaultSchema
+ + " NOT found in SHOW TABLES result set", testTableExists);
+ stmt.close();
+ hs2Conn .close();
+
+ hs2Conn =
getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ stmt.execute("set hive.support.concurrency = false");
+ res = stmt.executeQuery("show tables");
+
+ stmt.execute(" drop table if exists table_in_non_default_schema");
+ expected = stmt.execute("DROP DATABASE "+ dbName);
+ stmt.close();
+
+ hs2Conn =
getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ res = stmt.executeQuery("show tables");
+ testTableExists = false;
+ while (res.next()) {
+ assertNotNull("table name is null in result set", res.getString(1));
+ if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
+ testTableExists = true;
+ }
}
+ // test URI with no dbName
+ hs2Conn = getConnection(jdbcUri, System.getProperty("user.name"),"bar");
+ verifyCurrentDB("default", hs2Conn);
+ hs2Conn.close();
+
+ hs2Conn = getConnection(jdbcUri + ";",
System.getProperty("user.name"),"bar");
+ verifyCurrentDB("default", hs2Conn);
+ hs2Conn.close();
+
+ hs2Conn = getConnection(jdbcUri + ";/foo=bar;foo1=bar1",
System.getProperty("user.name"),"bar");
+ verifyCurrentDB("default", hs2Conn);
+ hs2Conn.close();
+ }
- /** This test is to connect to any database without using the command
"Use <<DB>>"
- * 1)connect to default database.
- * 2) Create a new DB test_default.
- * 3) Connect to test_default database.
- * 4) Connect and create table under test_default_test.
- * 5) Connect and display all tables.
- * 6) Connect to default database and shouldn't find table
test_default_test.
- * 7) Connect and drop test_default_test.
- * 8) drop test_default database.
+ @Test
+ public void testConnectionSchemaAPIs() throws Exception {
+ String db1 = "DB1";
+ /**
+ * get/set Schema are new in JDK7 and not available in java.sql.Connection
in JDK6.
+ * Hence the test uses HiveConnection object to call these methods so that
test will run with older JDKs
*/
+ HiveConnection hiveConn = (HiveConnection)hs2Conn;
- @Test
- public void testURIDatabaseName() throws Exception{
+ assertEquals("default", hiveConn.getSchema());
+ Statement stmt = hs2Conn.createStatement();
+ stmt.execute("DROP DATABASE IF EXISTS " + db1 + " CASCADE");
+ stmt.execute("CREATE DATABASE " + db1);
+ assertEquals("default", hiveConn.getSchema());
+
+ stmt.execute("USE " + db1);
+ assertEquals(db1, hiveConn.getSchema());
+
+ stmt.execute("USE default");
+ assertEquals("default", hiveConn.getSchema());
+
+ hiveConn.setSchema(db1);
+ assertEquals(db1, hiveConn.getSchema());
+ hiveConn.setSchema("default");
+ assertEquals("default", hiveConn.getSchema());
+
+ assertTrue(hiveConn.getCatalog().isEmpty());
+ hiveConn.setCatalog("foo");
+ assertTrue(hiveConn.getCatalog().isEmpty());
+ }
- String jdbcUri = miniHS2.getJdbcURL().substring(0,
miniHS2.getJdbcURL().indexOf("default"));
+ /**
+ * verify that the current db is the one expected. first create table as
<db>.tab and then
+ * describe that table to check if <db> is the current database
+ * @param expectedDbName
+ * @param hs2Conn
+ * @throws Exception
+ */
+ private void verifyCurrentDB(String expectedDbName, Connection hs2Conn)
throws Exception {
+ String verifyTab = "miniHS2DbVerificationTable";
+ Statement stmt = hs2Conn.createStatement();
+ stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
+ stmt.execute("CREATE TABLE " + expectedDbName + "." + verifyTab + "(id
INT)");
+ stmt.execute("DESCRIBE " + verifyTab);
+ stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
+ stmt.close();
+ }
- hs2Conn=
getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
- String dbName="test_connection_non_default_db";
- String tableInNonDefaultSchema="table_in_non_default_schema";
- Statement stmt = hs2Conn.createStatement();
- stmt.execute("create database if not exists "+dbName);
- stmt.close();
- hs2Conn.close();
-
- hs2Conn =
getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
- stmt = hs2Conn .createStatement();
- boolean expected = stmt.execute(" create table "+tableInNonDefaultSchema
+" (x int)");
- stmt.close();
- hs2Conn .close();
-
- hs2Conn =
getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
- stmt = hs2Conn .createStatement();
- ResultSet res = stmt.executeQuery("show tables");
- boolean testTableExists = false;
- while (res.next()) {
- assertNotNull("table name is null in result set", res.getString(1));
- if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
- testTableExists = true;
- }
- }
- assertTrue("table name "+tableInNonDefaultSchema
- + " found in SHOW TABLES result set", testTableExists);
- stmt.close();
- hs2Conn .close();
-
- hs2Conn =
getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
- stmt = hs2Conn .createStatement();
- res = stmt.executeQuery("show tables");
- testTableExists = false;
- while (res.next()) {
- assertNotNull("table name is null in result set", res.getString(1));
- if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
- testTableExists = true;
- }
- }
-
- assertFalse("table name "+tableInNonDefaultSchema
- + " NOT found in SHOW TABLES result set", testTableExists);
- stmt.close();
- hs2Conn .close();
-
- hs2Conn =
getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
- stmt = hs2Conn .createStatement();
- stmt.execute("set hive.support.concurrency = false");
- res = stmt.executeQuery("show tables");
-
- stmt.execute(" drop table if exists table_in_non_default_schema");
- expected = stmt.execute("DROP DATABASE "+ dbName);
- stmt.close();
-
- hs2Conn =
getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
- stmt = hs2Conn .createStatement();
- res = stmt.executeQuery("show tables");
- testTableExists = false;
- while (res.next()) {
- assertNotNull("table name is null in result set", res.getString(1));
- if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
- testTableExists = true;
- }
- }
-
- // test URI with no dbName
- hs2Conn = getConnection(jdbcUri, System.getProperty("user.name"),"bar");
- verifyCurrentDB("default", hs2Conn);
- hs2Conn.close();
-
- hs2Conn = getConnection(jdbcUri + ";",
System.getProperty("user.name"),"bar");
- verifyCurrentDB("default", hs2Conn);
- hs2Conn.close();
-
- hs2Conn = getConnection(jdbcUri + ";/foo=bar;foo1=bar1",
System.getProperty("user.name"),"bar");
- verifyCurrentDB("default", hs2Conn);
- hs2Conn.close();
- }
-
- @Test
- public void testConnectionSchemaAPIs() throws Exception {
- String db1 = "DB1";
- /**
- * get/set Schema are new in JDK7 and not available in
java.sql.Connection in JDK6.
- * Hence the test uses HiveConnection object to call these methods so
that test will run with older JDKs
- */
- HiveConnection hiveConn = (HiveConnection)hs2Conn;
-
- assertEquals("default", hiveConn.getSchema());
- Statement stmt = hs2Conn.createStatement();
- stmt.execute("DROP DATABASE IF EXISTS " + db1 + " CASCADE");
- stmt.execute("CREATE DATABASE " + db1);
- assertEquals("default", hiveConn.getSchema());
-
- stmt.execute("USE " + db1);
- assertEquals(db1, hiveConn.getSchema());
-
- stmt.execute("USE default");
- assertEquals("default", hiveConn.getSchema());
-
- hiveConn.setSchema(db1);
- assertEquals(db1, hiveConn.getSchema());
- hiveConn.setSchema("default");
- assertEquals("default", hiveConn.getSchema());
-
- assertTrue(hiveConn.getCatalog().isEmpty());
- hiveConn.setCatalog("foo");
- assertTrue(hiveConn.getCatalog().isEmpty());
- }
-
- /**
- * verify that the current db is the one expected. first create table as
<db>.tab and then
- * describe that table to check if <db> is the current database
- * @param expectedDbName
- * @param hs2Conn
- * @throws Exception
- */
- private void verifyCurrentDB(String expectedDbName, Connection hs2Conn)
throws Exception {
- String verifyTab = "miniHS2DbVerificationTable";
- Statement stmt = hs2Conn.createStatement();
- stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
- stmt.execute("CREATE TABLE " + expectedDbName + "." + verifyTab + "(id
INT)");
- stmt.execute("DESCRIBE " + verifyTab);
- stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
- stmt.close();
- }
-
- /**
- * This method tests whether while creating a new connection, the config
- * variables specified in the JDBC URI are properly set for the connection.
- * This is a test for HiveConnection#configureConnection.
- *
- * @throws Exception
- */
- @Test
- public void testNewConnectionConfiguration() throws Exception {
-
- // Set some conf parameters
- String hiveConf =
"hive.cli.print.header=true;hive.server2.async.exec.shutdown.timeout=20;"
- +
"hive.server2.async.exec.threads=30;hive.server2.thrift.http.max.worker.threads=15";
- // Set some conf vars
- String hiveVar = "stab=salesTable;icol=customerID";
- String jdbcUri = miniHS2.getJdbcURL() + "?" + hiveConf + "#" + hiveVar;
-
- // Open a new connection with these conf & vars
- Connection con1 = DriverManager.getConnection(jdbcUri);
-
- // Execute "set" command and retrieve values for the conf & vars specified
- // above
- // Assert values retrieved
- Statement stmt = con1.createStatement();
-
- // Verify that the property has been properly set while creating the
- // connection above
- verifyConfProperty(stmt, "hive.cli.print.header", "true");
- verifyConfProperty(stmt, "hive.server2.async.exec.shutdown.timeout",
"20");
- verifyConfProperty(stmt, "hive.server2.async.exec.threads", "30");
- verifyConfProperty(stmt, "hive.server2.thrift.http.max.worker.threads",
- "15");
- verifyConfProperty(stmt, "stab", "salesTable");
- verifyConfProperty(stmt, "icol", "customerID");
- con1.close();
- }
-
- private void verifyConfProperty(Statement stmt, String property,
- String expectedValue) throws Exception {
- ResultSet res = stmt.executeQuery("set " + property);
- while (res.next()) {
- String resultValues[] = res.getString(1).split("=");
- assertEquals(resultValues[1], expectedValue);
- }
- }
+ /**
+ * This method tests whether while creating a new connection, the config
+ * variables specified in the JDBC URI are properly set for the connection.
+ * This is a test for HiveConnection#configureConnection.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testNewConnectionConfiguration() throws Exception {
+
+ // Set some conf parameters
+ String hiveConf =
"hive.cli.print.header=true;hive.server2.async.exec.shutdown.timeout=20;"
+ +
"hive.server2.async.exec.threads=30;hive.server2.thrift.http.max.worker.threads=15";
+ // Set some conf vars
+ String hiveVar = "stab=salesTable;icol=customerID";
+ String jdbcUri = miniHS2.getJdbcURL() + "?" + hiveConf + "#" + hiveVar;
+
+ // Open a new connection with these conf & vars
+ Connection con1 = DriverManager.getConnection(jdbcUri);
+
+ // Execute "set" command and retrieve values for the conf & vars specified
+ // above
+ // Assert values retrieved
+ Statement stmt = con1.createStatement();
+
+ // Verify that the property has been properly set while creating the
+ // connection above
+ verifyConfProperty(stmt, "hive.cli.print.header", "true");
+ verifyConfProperty(stmt, "hive.server2.async.exec.shutdown.timeout", "20");
+ verifyConfProperty(stmt, "hive.server2.async.exec.threads", "30");
+ verifyConfProperty(stmt, "hive.server2.thrift.http.max.worker.threads",
+ "15");
+ verifyConfProperty(stmt, "stab", "salesTable");
+ verifyConfProperty(stmt, "icol", "customerID");
+ con1.close();
+ }
+
+ private void verifyConfProperty(Statement stmt, String property,
+ String expectedValue) throws Exception {
+ ResultSet res = stmt.executeQuery("set " + property);
+ while (res.next()) {
+ String resultValues[] = res.getString(1).split("=");
+ assertEquals(resultValues[1], expectedValue);
+ }
+ }
+
+ /**
+ * Tests the creation of the 3 scratch dirs: hdfs, local, downloaded
resources (which is also local).
+ * 1. Test with doAs=false: open a new JDBC session and verify the presence
of directories/permissions
+ * 2. Test with doAs=true: open a new JDBC session and verify the presence
of directories/permissions
+ * @throws Exception
+ */
+ @Test
+ public void testScratchDirs() throws Exception {
+ // Stop HiveServer2
+ if (miniHS2.isStarted()) {
+ miniHS2.stop();
+ }
+ HiveConf conf = new HiveConf();
+ String userName;
+ Path scratchDirPath;
+ // 1. Test with doAs=false
+ conf.setBoolean("hive.server2.enable.doAs", false);
+ // Set a custom prefix for hdfs scratch dir path
+ conf.set("hive.exec.scratchdir", "/tmp/hs2");
+ // Set a scratch dir permission
+ String fsPermissionStr = "700";
+ conf.set("hive.scratch.dir.permission", fsPermissionStr);
+ // Start an instance of HiveServer2 which uses miniMR
+ miniHS2 = new MiniHS2(conf);
+ Map<String, String> confOverlay = new HashMap<String, String>();
+ miniHS2.start(confOverlay);
+ userName = System.getProperty("user.name");
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "password");
+ // FS
+ FileSystem fs = miniHS2.getLocalFS();
+
+ // Verify scratch dir paths and permission
+ // HDFS scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.SCRATCHDIR) + "/" + userName);
+ verifyScratchDir(conf, fs, scratchDirPath, userName, false);
+
+ // Local scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.LOCALSCRATCHDIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+ // Downloaded resources dir
+ scratchDirPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+ // 2. Test with doAs=true
+ // Restart HiveServer2 with doAs=true
+ if (miniHS2.isStarted()) {
+ miniHS2.stop();
+ }
+ conf.setBoolean("hive.server2.enable.doAs", true);
+ // Start HS2
+ miniHS2 = new MiniHS2(conf);
+ miniHS2.start(confOverlay);
+ // Test for user "neo"
+ userName = "neo";
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "the-one");
+
+ // Verify scratch dir paths and permission
+ // HDFS scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.SCRATCHDIR) + "/" + userName);
+ verifyScratchDir(conf, fs, scratchDirPath, userName, false);
+
+ // Local scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.LOCALSCRATCHDIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+ // Downloaded resources dir
+ scratchDirPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+ // Test for user "trinity"
+ userName = "trinity";
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "the-one");
+
+ // Verify scratch dir paths and permission
+ // HDFS scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.SCRATCHDIR) + "/" + userName);
+ verifyScratchDir(conf, fs, scratchDirPath, userName, false);
+
+ // Local scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.LOCALSCRATCHDIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+ // Downloaded resources dir
+ scratchDirPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+ }
+
+ private void verifyScratchDir(HiveConf conf, FileSystem fs, Path
scratchDirPath,
+ String userName, boolean isLocal) throws Exception {
+ String dirType = isLocal ? "Local" : "DFS";
+ FsPermission expectedFSPermission = new FsPermission(HiveConf.getVar(conf,
+ HiveConf.ConfVars.SCRATCHDIRPERMISSION));
+ assertTrue("The expected " + dirType + " scratch dir does not exist for
the user: " +
+ userName, fs.exists(scratchDirPath));
+ if (fs.exists(scratchDirPath) && !isLocal) {
+ assertEquals("DFS scratch dir permissions don't match",
expectedFSPermission,
+ fs.getFileStatus(scratchDirPath).getPermission());
+ }
}
+}
\ No newline at end of file
Modified:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
---
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
(original)
+++
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
Thu Sep 4 20:25:49 2014
@@ -19,16 +19,16 @@
package org.apache.hive.jdbc;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
-import java.util.Map;
-import java.util.HashMap;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -48,11 +48,11 @@ public class TestJdbcWithMiniMr {
public static final String TEST_TAG = "miniHS2.miniMr.tag";
public static final String TEST_TAG_VALUE = "miniHS2.miniMr.value";
public static class MiniMrTestSessionHook implements HiveSessionHook {
- @Override
- public void run(HiveSessionHookContext sessionHookContext) throws
HiveSQLException {
- sessionHookContext.getSessionConf().set(TEST_TAG, TEST_TAG_VALUE);
- }
- }
+ @Override
+ public void run(HiveSessionHookContext sessionHookContext) throws
HiveSQLException {
+ sessionHookContext.getSessionConf().set(TEST_TAG, TEST_TAG_VALUE);
+ }
+ }
private static MiniHS2 miniHS2 = null;
private static HiveConf conf;
@@ -93,7 +93,7 @@ public class TestJdbcWithMiniMr {
@Before
public void setUp() throws Exception {
hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
- System.getProperty("user.name"), "bar");
+ System.getProperty("user.name"), "bar");
stmt = hs2Conn.createStatement();
stmt.execute("USE " + dbName);
}
@@ -225,7 +225,7 @@ public class TestJdbcWithMiniMr {
String queryStr = "SELECT * FROM " + tempTableName +
" where value = '" + resultVal + "'";
verifyResult(queryStr, resultVal, 2);
-
+
// A second connection should not be able to see the table
Connection conn2 = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
System.getProperty("user.name"), "bar");
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java Thu Sep 4
20:25:49 2014
@@ -18,6 +18,18 @@
package org.apache.hadoop.hive.ql;
+import java.io.DataInput;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+
import org.antlr.runtime.TokenRewriteStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -30,7 +42,6 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.TaskRunner;
-import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager;
@@ -41,18 +52,6 @@ import org.apache.hadoop.hive.ql.session
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.util.StringUtils;
-import java.io.DataInput;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-
/**
* Context for Semantic Analyzers. Usage: not reusable - construct a new one
for
* each query should call clear() at end of use to remove temporary folders
@@ -191,7 +190,7 @@ public class Context {
* @param scratchDir path of tmp directory
*/
private Path getScratchDir(String scheme, String authority,
- boolean mkdir, String scratchDir) {
+ boolean mkdir, String scratchDir) {
String fileSystem = scheme + ":" + authority;
Path dir = fsScratchDirs.get(fileSystem + "-" +
TaskRunner.getTaskRunnerID());
@@ -203,11 +202,11 @@ public class Context {
try {
FileSystem fs = dirPath.getFileSystem(conf);
dirPath = new Path(fs.makeQualified(dirPath).toString());
- FsPermission fsPermission = new
FsPermission(Short.parseShort(scratchDirPermission.trim(), 8));
+ FsPermission fsPermission = new FsPermission(scratchDirPermission);
- if (!Utilities.createDirsWithPermission(conf, dirPath,
fsPermission)) {
+ if (!fs.mkdirs(dirPath, fsPermission)) {
throw new RuntimeException("Cannot make directory: "
- + dirPath.toString());
+ + dirPath.toString());
}
if (isHDFSCleanup) {
fs.deleteOnExit(dirPath);
@@ -233,7 +232,7 @@ public class Context {
FileSystem fs = FileSystem.getLocal(conf);
URI uri = fs.getUri();
return getScratchDir(uri.getScheme(), uri.getAuthority(),
- mkdir, localScratchDir);
+ mkdir, localScratchDir);
} catch (IOException e) {
throw new RuntimeException (e);
}
@@ -257,7 +256,7 @@ public class Context {
URI uri = dir.toUri();
Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(),
- !explain, uri.getPath());
+ !explain, uri.getPath());
LOG.info("New scratch dir is " + newScratchDir);
return newScratchDir;
} catch (IOException e) {
@@ -270,7 +269,7 @@ public class Context {
private Path getExternalScratchDir(URI extURI) {
return getScratchDir(extURI.getScheme(), extURI.getAuthority(),
- !explain, nonLocalScratchPath.toUri().getPath());
+ !explain, nonLocalScratchPath.toUri().getPath());
}
/**
@@ -283,7 +282,7 @@ public class Context {
p.getFileSystem(conf).delete(p, true);
} catch (Exception e) {
LOG.warn("Error Removing Scratch: "
- + StringUtils.stringifyException(e));
+ + StringUtils.stringifyException(e));
}
}
fsScratchDirs.clear();
@@ -305,7 +304,7 @@ public class Context {
*/
public boolean isMRTmpFileURI(String uriStr) {
return (uriStr.indexOf(executionId) != -1) &&
- (uriStr.indexOf(MR_PREFIX) != -1);
+ (uriStr.indexOf(MR_PREFIX) != -1);
}
/**
@@ -315,7 +314,7 @@ public class Context {
*/
public Path getMRTmpPath() {
return new Path(getMRScratchDir(), MR_PREFIX +
- nextPathId());
+ nextPathId());
}
/**
@@ -343,7 +342,7 @@ public class Context {
return getExtTmpPathRelTo(path.getParent());
}
return new Path(getExternalScratchDir(extURI), EXT_PREFIX +
- nextPathId());
+ nextPathId());
}
/**
@@ -353,8 +352,8 @@ public class Context {
*/
public Path getExtTmpPathRelTo(Path path) {
URI uri = path.toUri();
- return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(),
!explain,
- uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX +
nextPathId());
+ return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(),
!explain,
+ uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX +
nextPathId());
}
/**
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Thu
Sep 4 20:25:49 2014
@@ -92,7 +92,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.common.HiveInterruptCallback;
import org.apache.hadoop.hive.common.HiveInterruptUtils;
import org.apache.hadoop.hive.common.HiveStatsUtils;
@@ -3355,7 +3354,6 @@ public final class Utilities {
private static void createTmpDirs(Configuration conf,
List<Operator<? extends OperatorDesc>> ops) throws IOException {
- FsPermission fsPermission = new FsPermission((short)00777);
while (!ops.isEmpty()) {
Operator<? extends OperatorDesc> op = ops.remove(0);
@@ -3365,7 +3363,8 @@ public final class Utilities {
if (tempDir != null) {
Path tempPath = Utilities.toTempPath(tempDir);
- createDirsWithPermission(conf, tempPath, fsPermission);
+ FileSystem fs = tempPath.getFileSystem(conf);
+ fs.mkdirs(tempPath);
}
}
@@ -3501,76 +3500,6 @@ public final class Utilities {
}
/**
- * @param conf the configuration used to derive the filesystem to create the
path
- * @param mkdir the path to be created
- * @param fsPermission ignored if it is hive server session and doAs is
enabled
- * @return true if successfully created the directory else false
- * @throws IOException if hdfs experiences any error conditions
- */
- public static boolean createDirsWithPermission(Configuration conf, Path
mkdir,
- FsPermission fsPermission) throws IOException {
-
- boolean recursive = false;
- if (SessionState.get() != null) {
- recursive = SessionState.get().isHiveServerQuery() &&
- conf.getBoolean(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname,
- HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.defaultBoolVal);
- // we reset the permission in case of hive server and doAs enabled
because
- // currently scratch directory uses /tmp/hive-hive as the scratch
directory.
- // However, with doAs enabled, the first user to create this directory
would
- // own the directory and subsequent users cannot access the scratch
directory.
- // The right fix is to have scratch dir per user.
- fsPermission = new FsPermission((short)00777);
- }
-
- // if we made it so far without exception we are good!
- return createDirsWithPermission(conf, mkdir, fsPermission, recursive);
- }
-
- private static void resetConfAndCloseFS (Configuration conf, boolean
unsetUmask,
- String origUmask, FileSystem fs) throws IOException {
- if (unsetUmask) {
- if (origUmask != null) {
- conf.set(FsPermission.UMASK_LABEL, origUmask);
- } else {
- conf.unset(FsPermission.UMASK_LABEL);
- }
- }
-
- fs.close();
- }
-
- public static boolean createDirsWithPermission(Configuration conf, Path
mkdirPath,
- FsPermission fsPermission, boolean recursive) throws IOException {
- String origUmask = null;
- LOG.debug("Create dirs " + mkdirPath + " with permission " + fsPermission
+ " recursive " +
- recursive);
-
- if (recursive) {
- origUmask = conf.get(FsPermission.UMASK_LABEL);
- // this umask is required because by default the hdfs mask is 022
resulting in
- // all parents getting the fsPermission & !(022) permission instead of
fsPermission
- conf.set(FsPermission.UMASK_LABEL, "000");
- }
-
- FileSystem fs =
ShimLoader.getHadoopShims().getNonCachedFileSystem(mkdirPath.toUri(), conf);
- boolean retval = false;
- try {
- retval = fs.mkdirs(mkdirPath, fsPermission);
- resetConfAndCloseFS(conf, recursive, origUmask, fs);
- } catch (IOException ioe) {
- try {
- resetConfAndCloseFS(conf, recursive, origUmask, fs);
- }
- catch (IOException e) {
- // do nothing - double failure
- }
- }
- return retval;
- }
-
-
- /**
* Convert path to qualified path.
*
* @param conf
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
Thu Sep 4 20:25:49 2014
@@ -41,12 +41,9 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.ErrorMsg;
-import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -300,11 +297,11 @@ public class TezSessionState {
throws IOException {
// tez needs its own scratch dir (per session)
- Path tezDir = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.SCRATCHDIR), TEZ_DIR);
+ Path tezDir = new Path(SessionState.get().getHdfsScratchDirURIString(),
TEZ_DIR);
tezDir = new Path(tezDir, sessionId);
FileSystem fs = tezDir.getFileSystem(conf);
- FsPermission fsPermission = new FsPermission((short)00777);
- Utilities.createDirsWithPermission(conf, tezDir, fsPermission, true);
+ FsPermission fsPermission = new FsPermission(HiveConf.getVar(conf,
HiveConf.ConfVars.SCRATCHDIRPERMISSION));
+ fs.mkdirs(tezDir, fsPermission);
// Make sure the path is normalized (we expect validation to pass since we
just created it).
tezDir = DagUtils.validateTargetDir(tezDir, conf).getPath();
// don't keep the directory around on non-clean exit
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
Thu Sep 4 20:25:49 2014
@@ -45,7 +45,6 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.ql.MapRedStats;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -69,6 +68,7 @@ import org.apache.hadoop.hive.ql.securit
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactoryImpl;
import org.apache.hadoop.hive.ql.util.DosToUnix;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.base.Preconditions;
@@ -208,6 +208,8 @@ public class SessionState {
*/
private Path localSessionPath;
+ private String hdfsScratchDirURIString;
+
/**
* Get the lineage state stored in this session.
*
@@ -344,36 +346,39 @@ public class SessionState {
setCurrentSessionState(startSs);
- if(startSs.hiveHist == null){
+ if (startSs.hiveHist == null){
if
(startSs.getConf().getBoolVar(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED)) {
startSs.hiveHist = new HiveHistoryImpl(startSs);
- }else {
- //Hive history is disabled, create a no-op proxy
+ } else {
+ // Hive history is disabled, create a no-op proxy
startSs.hiveHist = HiveHistoryProxyHandler.getNoOpHiveHistoryProxy();
}
}
- if (startSs.getTmpOutputFile() == null) {
- // set temp file containing results to be sent to HiveClient
- try {
- startSs.setTmpOutputFile(createTempFile(startSs.getConf()));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
// Get the following out of the way when you start the session these take a
// while and should be done when we start up.
try {
- //Hive object instance should be created with a copy of the conf object.
If the conf is
+ // Hive object instance should be created with a copy of the conf
object. If the conf is
// shared with SessionState, other parts of the code might update the
config, but
// Hive.get(HiveConf) would not recognize the case when it needs
refreshing
Hive.get(new HiveConf(startSs.conf)).getMSC();
- ShimLoader.getHadoopShims().getUGIForConf(startSs.conf);
+ UserGroupInformation sessionUGI =
ShimLoader.getHadoopShims().getUGIForConf(startSs.conf);
FileSystem.get(startSs.conf);
- startSs.createSessionPaths(startSs.conf);
+
+ // Create scratch dirs for this session
+ startSs.createSessionDirs(sessionUGI.getShortUserName());
+
+ // Set temp file containing results to be sent to HiveClient
+ if (startSs.getTmpOutputFile() == null) {
+ try {
+ startSs.setTmpOutputFile(createTempFile(startSs.getConf()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
} catch (Exception e) {
- // catch-all due to some exec time dependencies on session state
+ // Catch-all due to some exec time dependencies on session state
// that would cause ClassNoFoundException otherwise
throw new RuntimeException(e);
}
@@ -396,6 +401,88 @@ public class SessionState {
return startSs;
}
+ /**
+ * Create dirs & session paths for this session:
+ * 1. HDFS scratch dir
+ * 2. Local scratch dir
+ * 3. Local downloaded resource dir
+ * 4. HDFS session path
+ * 5. Local session path
+ * 6. HDFS temp table space
+ * @param userName
+ * @throws IOException
+ */
+ private void createSessionDirs(String userName) throws IOException {
+ HiveConf conf = getConf();
+ // First create the root scratch dir on hdfs (if it doesn't already exist)
and make it writable
+ Path rootHDFSDirPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.SCRATCHDIR));
+ String rootHDFSDirPermission = "777";
+ createPath(conf, rootHDFSDirPath, rootHDFSDirPermission, false, false);
+ // Now create session specific dirs
+ String scratchDirPermission = HiveConf.getVar(conf,
HiveConf.ConfVars.SCRATCHDIRPERMISSION);
+ Path path;
+ // 1. HDFS scratch dir
+ path = new Path(rootHDFSDirPath, userName);
+ hdfsScratchDirURIString = path.toUri().toString();
+ createPath(conf, path, scratchDirPermission, false, false);
+ // 2. Local scratch dir
+ path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR));
+ createPath(conf, path, scratchDirPermission, true, false);
+ // 3. Download resources dir
+ path = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+ createPath(conf, path, scratchDirPermission, true, false);
+ // Finally, create session paths for this session
+ // Local & non-local tmp location is configurable. however it is the same
across
+ // all external file systems
+ String sessionId = getSessionId();
+ // 4. HDFS session path
+ hdfsSessionPath = new Path(hdfsScratchDirURIString, sessionId);
+ createPath(conf, hdfsSessionPath, scratchDirPermission, false, true);
+ conf.set(HDFS_SESSION_PATH_KEY, hdfsSessionPath.toUri().toString());
+ // 5. Local session path
+ localSessionPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.LOCALSCRATCHDIR), sessionId);
+ createPath(conf, localSessionPath, scratchDirPermission, true, true);
+ conf.set(LOCAL_SESSION_PATH_KEY, localSessionPath.toUri().toString());
+ // 6. HDFS temp table space
+ hdfsTmpTableSpace = new Path(hdfsSessionPath, TMP_PREFIX);
+ createPath(conf, hdfsTmpTableSpace, scratchDirPermission, false, true);
+ conf.set(TMP_TABLE_SPACE_KEY, hdfsTmpTableSpace.toUri().toString());
+ }
+
+ /**
+ * Create a given path if it doesn't exist.
+ *
+ * @param conf
+ * @param pathString
+ * @param permission
+ * @param isLocal
+ * @param isCleanUp
+ * @return
+ * @throws IOException
+ */
+ private void createPath(HiveConf conf, Path path, String permission, boolean
isLocal,
+ boolean isCleanUp) throws IOException {
+ FsPermission fsPermission = new FsPermission(permission);
+ FileSystem fs;
+ if (isLocal) {
+ fs = FileSystem.getLocal(conf);
+ } else {
+ fs = path.getFileSystem(conf);
+ }
+ if (!fs.exists(path)) {
+ fs.mkdirs(path, fsPermission);
+ String dirType = isLocal ? "local" : "HDFS";
+ LOG.info("Created " + dirType + " directory: " + path.toString());
+ }
+ if (isCleanUp) {
+ fs.deleteOnExit(path);
+ }
+ }
+
+ public String getHdfsScratchDirURIString() {
+ return hdfsScratchDirURIString;
+ }
+
public static Path getLocalSessionPath(Configuration conf) {
SessionState ss = SessionState.get();
if (ss == null) {
@@ -448,43 +535,6 @@ public class SessionState {
}
}
- private void createSessionPaths(Configuration conf) throws IOException {
-
- String scratchDirPermission = HiveConf.getVar(conf,
HiveConf.ConfVars.SCRATCHDIRPERMISSION);
- String sessionId = getSessionId();
-
- // local & non-local tmp location is configurable. however it is the same
across
- // all external file systems
- hdfsSessionPath =
- new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR),
- sessionId);
- createPath(conf, hdfsSessionPath, scratchDirPermission);
- conf.set(HDFS_SESSION_PATH_KEY, hdfsSessionPath.toUri().toString());
-
- localSessionPath = new Path(HiveConf.getVar(conf,
HiveConf.ConfVars.LOCALSCRATCHDIR),
- sessionId);
- createPath(conf, localSessionPath, scratchDirPermission);
- conf.set(LOCAL_SESSION_PATH_KEY, localSessionPath.toUri().toString());
- hdfsTmpTableSpace = new Path(hdfsSessionPath, TMP_PREFIX);
- createPath(conf, hdfsTmpTableSpace, scratchDirPermission);
- conf.set(TMP_TABLE_SPACE_KEY, hdfsTmpTableSpace.toUri().toString());
- }
-
- private void createPath(Configuration conf, Path p, String perm) throws
IOException {
- FileSystem fs = p.getFileSystem(conf);
- p = new Path(fs.makeQualified(p).toString());
- FsPermission fsPermission = new FsPermission(Short.parseShort(perm.trim(),
8));
-
- if (!Utilities.createDirsWithPermission(conf, p, fsPermission)) {
- throw new IOException("Cannot create directory: "
- + p.toString());
- }
-
- // best effort to clean up if we don't shut down properly
- fs.deleteOnExit(p);
- }
-
-
/**
* Setup authentication and authorization plugins for this session.
*/
Modified:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
(original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
Thu Sep 4 20:25:49 2014
@@ -20,15 +20,12 @@ package org.apache.hadoop.hive.ql.exec;
import static org.apache.hadoop.hive.ql.exec.Utilities.getFileExtension;
-import java.io.IOException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import junit.framework.TestCase;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -39,7 +36,6 @@ import org.apache.hadoop.hive.ql.session
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.mapred.JobConf;
-import org.junit.Test;
public class TestUtilities extends TestCase {
@@ -77,9 +73,9 @@ public class TestUtilities extends TestC
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(1);
children.add(constant);
ExprNodeGenericFuncDesc desc = new
ExprNodeGenericFuncDesc(TypeInfoFactory.timestampTypeInfo,
- new GenericUDFFromUtcTimestamp(), children);
+ new GenericUDFFromUtcTimestamp(), children);
assertEquals(desc.getExprString(), Utilities.deserializeExpression(
- Utilities.serializeExpression(desc)).getExprString());
+ Utilities.serializeExpression(desc)).getExprString());
}
public void testgetDbTableName() throws HiveException{
@@ -109,23 +105,4 @@ public class TestUtilities extends TestC
assertEquals("Invalid table name " + tablename, ex.getMessage());
}
}
-
- @Test
- public void testFSUmaskReset() throws Exception {
- // ensure that FS Umask is not reset (HIVE-7001)
- checkFSUMaskReset(true);
- checkFSUMaskReset(false);
- }
-
- private void checkFSUMaskReset(boolean recursiveArg) throws
IllegalArgumentException, IOException {
- final String FS_MASK_VAL = "055";
- HiveConf conf = new HiveConf();
- String dir = System.getProperty("test.tmp.dir") +
"/testUtilitiesUMaskReset";
- conf.set(FsPermission.UMASK_LABEL, FS_MASK_VAL);
- Utilities.createDirsWithPermission(conf, new Path(dir), new
FsPermission((short) 00777),
- recursiveArg);
- assertEquals(conf.get(FsPermission.UMASK_LABEL), FS_MASK_VAL);
- }
-
-
}
Modified:
hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java
URL:
http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java
(original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java Thu
Sep 4 20:25:49 2014
@@ -30,12 +30,8 @@ import javax.security.auth.login.LoginEx
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.conf.SystemVariables;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -118,15 +114,6 @@ public class CLIService extends Composit
@Override
public synchronized void start() {
super.start();
-
- try {
- // make sure that the base scratch directories exists and writable
- setupStagingDir(hiveConf.getVar(HiveConf.ConfVars.SCRATCHDIR), false);
- setupStagingDir(hiveConf.getVar(HiveConf.ConfVars.LOCALSCRATCHDIR),
true);
-
setupStagingDir(hiveConf.getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR),
true);
- } catch (IOException eIO) {
- throw new ServiceException("Error setting stage directories", eIO);
- }
// Initialize and test a connection to the metastore
IMetaStoreClient metastoreClient = null;
try {
@@ -460,25 +447,6 @@ public class CLIService extends Composit
}
}
- // create the give Path if doesn't exists and make it writable
- private void setupStagingDir(String dirPath, boolean isLocal) throws
IOException {
- Path scratchDir = getStaticPath(new Path(dirPath));
- if (scratchDir == null) {
- return;
- }
- FileSystem fs;
- if (isLocal) {
- fs = FileSystem.getLocal(hiveConf);
- } else {
- fs = scratchDir.getFileSystem(hiveConf);
- }
- if (!fs.exists(scratchDir)) {
- fs.mkdirs(scratchDir);
- }
- FsPermission fsPermission = new FsPermission((short)0777);
- fs.setPermission(scratchDir, fsPermission);
- }
-
@Override
public String getDelegationToken(SessionHandle sessionHandle,
HiveAuthFactory authFactory,
String owner, String renewer) throws HiveSQLException {
@@ -502,16 +470,4 @@ public class CLIService extends Composit
sessionManager.getSession(sessionHandle).renewDelegationToken(authFactory,
tokenStr);
LOG.info(sessionHandle + ": renewDelegationToken()");
}
-
- // DOWNLOADED_RESOURCES_DIR for example, which is by default
${system:java.io.tmpdir}/${hive.session.id}_resources,
- // {system:java.io.tmpdir} would be already evaluated but ${hive.session.id}
would be not in here.
- // for that case, this returns evaluatd parts only, in this case, "/tmp"
- // what for ${hive.session.id}_resources/${system:java.io.tmpdir}? just
don't do that.
- private Path getStaticPath(Path path) {
- Path current = path;
- for (; current != null && SystemVariables.containsVar(current.getName());
- current = current.getParent()) {
- }
- return current;
- }
}
Modified:
hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
URL:
http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
---
hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
(original)
+++
hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
Thu Sep 4 20:25:49 2014
@@ -18,6 +18,8 @@
package org.apache.hive.service.cli.session;
+import java.util.Map;
+
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.cli.SessionHandle;
Modified:
hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL:
http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
---
hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
(original)
+++
hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
Thu Sep 4 20:25:49 2014
@@ -69,7 +69,7 @@ public class HiveSessionImpl implements
private String username;
private final String password;
- private final HiveConf hiveConf;
+ private HiveConf hiveConf;
private final SessionState sessionState;
private String ipAddress;
@@ -240,6 +240,12 @@ public class HiveSessionImpl implements
}
@Override
+ /**
+ * Opens a new HiveServer2 session for the client connection.
+ * Note that if doAs is true, this call goes through a proxy object,
+ * which wraps the method logic in a UserGroupInformation#doAs.
+ * That is why it is important to call SessionState#start here rather than
the constructor.
+ */
public void open() {
SessionState.start(sessionState);
}
Modified:
hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
URL:
http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java?rev=1622556&r1=1622555&r2=1622556&view=diff
==============================================================================
---
hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
(original)
+++
hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
Thu Sep 4 20:25:49 2014
@@ -233,11 +233,13 @@ public class SessionManager extends Comp
Map<String, String> sessionConf, boolean withImpersonation, String
delegationToken)
throws HiveSQLException {
HiveSession session;
+ // If doAs is set to true for HiveServer2, we will create a proxy object
for the session impl.
+ // Within the proxy object, we wrap the method call in a
UserGroupInformation#doAs
if (withImpersonation) {
- HiveSessionImplwithUGI hiveSessionUgi = new
HiveSessionImplwithUGI(protocol, username, password,
- hiveConf, ipAddress, delegationToken);
- session = HiveSessionProxy.getProxy(hiveSessionUgi,
hiveSessionUgi.getSessionUgi());
- hiveSessionUgi.setProxySession(session);
+ HiveSessionImplwithUGI sessionWithUGI = new
HiveSessionImplwithUGI(protocol, username, password,
+ hiveConf, ipAddress, delegationToken);
+ session = HiveSessionProxy.getProxy(sessionWithUGI,
sessionWithUGI.getSessionUgi());
+ sessionWithUGI.setProxySession(session);
} else {
session = new HiveSessionImpl(protocol, username, password, hiveConf,
ipAddress);
}
