This is an automated email from the ASF dual-hosted git repository.
asalamon74 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new ca9eee9 OOZIE-3326 [action] Sqoop Action should support tez
delegation tokens for hive-import (bgoerlitz, dionusos via asalamon74)
ca9eee9 is described below
commit ca9eee998c81012b122b66a05465a930c66195e2
Author: Andras Salamon <[email protected]>
AuthorDate: Fri Feb 22 13:35:01 2019 +0100
OOZIE-3326 [action] Sqoop Action should support tez delegation tokens for
hive-import (bgoerlitz, dionusos via asalamon74)
---
release-log.txt | 1 +
.../org/apache/oozie/action/hadoop/LauncherAM.java | 7 +++--
.../apache/oozie/action/hadoop/LauncherMain.java | 5 +++-
.../oozie/action/hadoop/SystemEnvironment.java | 33 ++++++++++++++++++++++
.../org/apache/oozie/action/hadoop/SqoopMain.java | 13 ++++++---
.../apache/oozie/action/hadoop/TestSqoopMain.java | 32 +++++++++++++++++++++
6 files changed, 84 insertions(+), 7 deletions(-)
diff --git a/release-log.txt b/release-log.txt
index b65afac..ca89f56 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.2.0 release (trunk - unreleased)
+OOZIE-3326 [action] Sqoop Action should support tez delegation tokens for
hive-import (bgoerlitz, dionusos via asalamon74)
OOZIE-3395 [build] Migration from FindBugs to SpotBugs (kmarton via asalamon74)
OOZIE-3438 Copy only apache-jsp server dependencies to distro (asalamon74 via
kmarton)
OOZIE-3437 Missing README from distro (nobigo via asalamon74)
diff --git
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
index 63afd91..6df151c 100644
---
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
+++
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
@@ -91,6 +91,9 @@ public class LauncherAM {
public static final String ACTION_DATA_FINAL_STATUS = "final.status";
public static final String OOZIE_SUBMITTER_USER = "oozie.submitter.user";
+ @VisibleForTesting
+ private static final SystemEnvironment sysenv = new SystemEnvironment();
+
private final AMRMClientAsyncFactory amrmClientAsyncFactory;
private final HdfsOperations hdfsOperations;
private final LocalFsOperations localFsOperations;
@@ -148,7 +151,7 @@ public class LauncherAM {
new PrepareActionsHandler(new
LauncherURIHandlerFactory(null)),
new LauncherAMCallbackNotifierFactory(),
new LauncherSecurityManager(),
-
System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name()),
+
sysenv.getenv(ApplicationConstants.Environment.CONTAINER_ID.name()),
launcherConf);
launcher.run();
return null;
@@ -321,7 +324,7 @@ public class LauncherAM {
System.out.println();
System.out.println("Environment variables");
- Map<String, String> env = System.getenv();
+ Map<String, String> env = sysenv.getenv();
System.out.println("------------------------");
for (Map.Entry<String, String> entry : env.entrySet()) {
System.out.println(entry.getKey() + "=" + entry.getValue());
diff --git
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
index b6599f7..5658ea5 100644
---
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
+++
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
@@ -95,6 +95,9 @@ public abstract class LauncherMain {
*/
private static final String DEFAULT_LOG4J_LOCATION =
"default-log4j.properties";
+ @VisibleForTesting
+ static SystemEnvironment sysenv = new SystemEnvironment();
+
protected Properties log4jProperties = new Properties();
protected static void run(Class<? extends LauncherMain> klass, String[]
args) throws Exception {
@@ -333,7 +336,7 @@ public abstract class LauncherMain {
* @return path given environment returns file path
*/
protected static String getFilePathFromEnv(String env) {
- String path = System.getenv(env);
+ String path = sysenv.getenv(env);
if (path != null && Shell.WINDOWS) {
// In Windows, file paths are enclosed in \" so remove them here
// to avoid path errors
diff --git
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/SystemEnvironment.java
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/SystemEnvironment.java
new file mode 100644
index 0000000..8527414
--- /dev/null
+++
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/SystemEnvironment.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.util.Map;
+
+/**
+ * Purpose of this class is to avoid static System method calls so unit
testing is much easier
+ */
+public class SystemEnvironment {
+ public String getenv(final String name) {
+ return System.getenv(name);
+ }
+
+ public Map<String, String> getenv() {
+ return System.getenv();
+ }
+}
diff --git
a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
index 27f9306..f4f4123 100644
--- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
+++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
@@ -20,8 +20,6 @@ package org.apache.oozie.action.hadoop;
import java.io.File;
import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Map;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
@@ -43,6 +41,7 @@ public class SqoopMain extends LauncherMain {
};
private static final String SQOOP_LOG4J_PROPS = "sqoop-log4j.properties";
+ public static final String TEZ_CREDENTIALS_PATH = "tez.credentials.path";
public static void main(String[] args) throws Exception {
run(SqoopMain.class, args);
@@ -52,10 +51,11 @@ public class SqoopMain extends LauncherMain {
// loading action conf prepared by Oozie
Configuration sqoopConf = new Configuration(false);
- String actionXml = System.getProperty("oozie.action.conf.xml");
+ String actionXml =
System.getProperty(LauncherAM.OOZIE_ACTION_CONF_XML);
if (actionXml == null) {
- throw new RuntimeException("Missing Java System Property
[oozie.action.conf.xml]");
+ throw new RuntimeException(String.format("Missing Java System
Property [%s]",
+ LauncherAM.OOZIE_ACTION_CONF_XML));
}
if (!new File(actionXml).exists()) {
throw new RuntimeException("Action Configuration XML file [" +
actionXml + "] does not exist");
@@ -68,10 +68,15 @@ public class SqoopMain extends LauncherMain {
if (delegationToken != null) {
sqoopConf.setBoolean("sqoop.hbase.security.token.skip", true);
sqoopConf.set("mapreduce.job.credentials.binary", delegationToken);
+ sqoopConf.set(TEZ_CREDENTIALS_PATH, delegationToken);
System.out.println("------------------------");
System.out.println("Setting env property for
mapreduce.job.credentials.binary to: " + delegationToken);
System.out.println("------------------------");
System.setProperty("mapreduce.job.credentials.binary",
delegationToken);
+ System.out.println("------------------------");
+ System.out.println("Setting env property for tez.credentials.path
to: " + delegationToken);
+ System.out.println("------------------------");
+ System.setProperty(TEZ_CREDENTIALS_PATH, delegationToken);
} else {
System.out.println("Non-Kerberos execution");
}
diff --git
a/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java
b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java
index d6f96d5..64aaf62 100644
---
a/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java
+++
b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java
@@ -17,12 +17,16 @@
*/
package org.apache.oozie.action.hadoop;
+import java.io.File;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.oozie.test.XTestCase;
+import org.junit.Assert;
public class TestSqoopMain extends XTestCase {
@@ -46,4 +50,32 @@ public class TestSqoopMain extends XTestCase {
expected.add("job_003");
assertEquals(expected, jobIds);
}
+
+ public void testIfDelegationTokenForTezAdded() throws Exception {
+ final File actionXml = new File(LauncherAM.ACTION_CONF_XML);
+ LauncherMain.sysenv = new MockedSystemEnvironment();
+ setSystemProperty(LauncherAM.OOZIE_ACTION_CONF_XML,
LauncherAM.ACTION_CONF_XML);
+
+ try {
+ actionXml.createNewFile();
+ final Configuration conf = SqoopMain.setUpSqoopSite();
+ Assert.assertNotNull(
+ String.format("Property [%s] shall be part of
configuration", SqoopMain.TEZ_CREDENTIALS_PATH),
+ conf.get(SqoopMain.TEZ_CREDENTIALS_PATH));
+ } finally {
+ actionXml.delete();
+ // sqoop-site.xml stays there after test run so that shall be
cleaned up explicitly
+ new File(SqoopMain.SQOOP_SITE_CONF).delete();
+ }
+ }
+
+ class MockedSystemEnvironment extends SystemEnvironment {
+ @Override
+ public String getenv(final String name) {
+ if(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION.equals(name)) {
+ return UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
+ }
+ return super.getenv(name);
+ }
+ }
}
\ No newline at end of file
