Repository: oozie Updated Branches: refs/heads/master 3637874e9 -> b6769f4dd
OOZIE-1724 Make it easier to specify the HCat hive-site.xml for the Oozie Server (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/83af85c1 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/83af85c1 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/83af85c1 Branch: refs/heads/master Commit: 83af85c19133b2a9d5a482ca2f61949c8327d348 Parents: 3637874 Author: Robert Kanter <[email protected]> Authored: Sat May 31 23:00:54 2014 -0700 Committer: Robert Kanter <[email protected]> Committed: Sat May 31 23:00:54 2014 -0700 ---------------------------------------------------------------------- .../apache/oozie/dependency/HCatURIHandler.java | 4 ++ .../oozie/service/HCatAccessorService.java | 70 ++++++++++++++++++++ .../hadoop/TestLauncherHCatURIHandler.java | 2 + .../apache/oozie/coord/TestHCatELFunctions.java | 2 + .../oozie/dependency/TestHCatURIHandler.java | 2 + .../oozie/service/TestHCatAccessorService.java | 51 +++++++++++++- docs/src/site/twiki/AG_Install.twiki | 11 ++- release-log.txt | 1 + 8 files changed, 139 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java b/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java index fecd783..da78105 100644 --- a/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java +++ b/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java @@ -177,6 +177,10 @@ public class HCatURIHandler implements URIHandler { } private HCatClient getHCatClient(URI uri, Configuration conf, String user) throws HCatAccessorException { + HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class); + if (hcatService.getHCatConf() != null) { + conf = hcatService.getHCatConf(); + } final HiveConf hiveConf = new HiveConf(conf, this.getClass()); String serverURI = getMetastoreConnectURI(uri); if (!serverURI.equals("")) { http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java b/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java index a4b23de..ebef79b 100644 --- a/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java @@ -17,6 +17,10 @@ */ package org.apache.oozie.service; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -27,16 +31,22 @@ import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.ErrorCode; import org.apache.oozie.dependency.hcat.HCatMessageHandler; import org.apache.oozie.jms.JMSConnectionInfo; import org.apache.oozie.util.HCatURI; import org.apache.oozie.util.MappingRule; +import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XLog; public class HCatAccessorService implements Service { public static final String CONF_PREFIX = Service.CONF_PREFIX + "HCatAccessorService."; public static final String JMS_CONNECTIONS_PROPERTIES = CONF_PREFIX + "jmsconnections"; + public static final String HCAT_CONFIGURATION = CONF_PREFIX + "hcat.configuration"; private static XLog LOG; private static String DELIMITER = "#"; @@ -44,6 +54,7 @@ public class HCatAccessorService implements Service { private JMSAccessorService jmsService; private List<MappingRule> mappingRules; private JMSConnectionInfo defaultJMSConnInfo; + private Configuration hcatConf; /** * Map of publisher(host:port) to JMS connection info */ @@ -66,6 +77,65 @@ public class HCatAccessorService implements Service { this.nonJMSPublishers = new HashSet<String>(); this.publisherJMSConnInfoMap = new HashMap<String, JMSConnectionInfo>(); this.registeredTopicsMap = new HashMap<String, String>(); + try { + loadHCatConf(services); + } catch(IOException ioe) { + throw new ServiceException(ErrorCode.E0100, HCatAccessorService.class.getName(), "An exception occured while attempting" + + "to load the HCat Configuration", ioe); + } + } + + private void loadHCatConf(Services services) throws IOException { + String path = conf.get(HCAT_CONFIGURATION); + if (path != null) { + if (path.startsWith("hdfs")) { + Path p = new Path(path); + HadoopAccessorService has = services.get(HadoopAccessorService.class); + try { + FileSystem fs = has.createFileSystem( + System.getProperty("user.name"), p.toUri(), has.createJobConf(p.toUri().getAuthority())); + if (fs.exists(p)) { + FSDataInputStream is = null; + try { + is = fs.open(p); + hcatConf = new XConfiguration(is); + } finally { + if (is != null) { + is.close(); + } + } + LOG.info("Loaded HCat Configuration: " + path); + } else { + LOG.warn("HCat Configuration could not be found at [" + path + "]"); + } + } catch (HadoopAccessorException hae) { + throw new IOException(hae); + } + } else { + File f = new File(path); + if (f.exists()) { + InputStream is = null; + try { + is = new FileInputStream(f); + hcatConf = new XConfiguration(is); + } finally { + if (is != null) { + is.close(); + } + } + LOG.info("Loaded HCat Configuration: " + path); + } else { + LOG.warn("HCat Configuration could not be found at [" + path + "]"); + } + } + } + else { + LOG.info("HCat Configuration not specified"); + } + } + + public Configuration getHCatConf() { + return hcatConf; } private void initializeMappingRules() { http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java index 47d766d..7a8c00b 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java @@ -24,6 +24,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.dependency.FSURIHandler; import org.apache.oozie.dependency.HCatURIHandler; import org.apache.oozie.dependency.URIHandler; +import org.apache.oozie.service.HCatAccessorService; import org.apache.oozie.service.Services; import org.apache.oozie.service.URIHandlerService; import org.apache.oozie.test.XHCatTestCase; @@ -43,6 +44,7 @@ public class TestLauncherHCatURIHandler extends XHCatTestCase { services = new Services(); services.getConf().set(URIHandlerService.URI_HANDLERS, FSURIHandler.class.getName() + "," + HCatURIHandler.class.getName()); + services.setService(HCatAccessorService.class); services.init(); conf = createJobConf(); uriService = Services.get().get(URIHandlerService.class); http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java b/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java index a689cb1..f63b682 100644 --- a/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java +++ b/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java @@ -27,6 +27,7 @@ import org.apache.oozie.client.OozieClient; import org.apache.oozie.dependency.FSURIHandler; import org.apache.oozie.dependency.HCatURIHandler; import org.apache.oozie.service.ELService; +import org.apache.oozie.service.HCatAccessorService; import org.apache.oozie.service.LiteWorkflowStoreService; import org.apache.oozie.service.Services; import org.apache.oozie.service.URIHandlerService; @@ -52,6 +53,7 @@ public class TestHCatELFunctions extends XHCatTestCase { services = new Services(); services.getConf().set(URIHandlerService.URI_HANDLERS, FSURIHandler.class.getName() + "," + HCatURIHandler.class.getName()); + services.setService(HCatAccessorService.class); services.init(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java b/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java index a23ed11..a2383c6 100644 --- a/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java +++ b/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java @@ -20,6 +20,7 @@ package org.apache.oozie.dependency; import java.net.URI; import org.apache.hadoop.mapred.JobConf; +import org.apache.oozie.service.HCatAccessorService; import org.apache.oozie.service.Services; import org.apache.oozie.service.URIHandlerService; import org.apache.oozie.test.XHCatTestCase; @@ -39,6 +40,7 @@ public class TestHCatURIHandler extends XHCatTestCase { services = new Services(); services.getConf().set(URIHandlerService.URI_HANDLERS, FSURIHandler.class.getName() + "," + HCatURIHandler.class.getName()); + services.setService(HCatAccessorService.class); services.init(); conf = createJobConf(); uriService = Services.get().get(URIHandlerService.class); http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/test/java/org/apache/oozie/service/TestHCatAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestHCatAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHCatAccessorService.java index 978c72c..43ac3db 100644 --- a/core/src/test/java/org/apache/oozie/service/TestHCatAccessorService.java +++ b/core/src/test/java/org/apache/oozie/service/TestHCatAccessorService.java @@ -17,14 +17,18 @@ */ package org.apache.oozie.service; +import java.io.File; +import java.io.FileOutputStream; import java.net.URI; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; import org.apache.oozie.jms.JMSConnectionInfo; -import org.apache.oozie.test.XTestCase; +import org.apache.oozie.test.XHCatTestCase; import org.junit.Test; -public class TestHCatAccessorService extends XTestCase { +public class TestHCatAccessorService extends XHCatTestCase { private Services services; @Override @@ -36,7 +40,9 @@ public class TestHCatAccessorService extends XTestCase { @Override protected void tearDown() throws Exception { - services.destroy(); + if (services != null) { + services.destroy(); + } super.tearDown(); } @@ -100,4 +106,43 @@ public class TestHCatAccessorService extends XTestCase { connInfo.getJNDIPropertiesString()); } + @Test + public void testGetHCatConfLocal() throws Exception { + File hcatConfFile = new File(getTestCaseConfDir(), "hive-site.xml"); + assertFalse(hcatConfFile.exists()); + assertNull(services.get(HCatAccessorService.class).getHCatConf()); + + Configuration hcatConf = new Configuration(false); + hcatConf.set("A", "a"); + hcatConf.writeXml(new FileOutputStream(hcatConfFile)); + assertTrue(hcatConfFile.exists()); + services.destroy(); + services = super.setupServicesForHCatalog(); + Configuration conf = services.getConf(); + conf.set("oozie.service.HCatAccessorService.hcat.configuration", hcatConfFile.getAbsolutePath()); + services.init(); + Configuration hcatConfLoaded = services.get(HCatAccessorService.class).getHCatConf(); + assertEquals("a", hcatConfLoaded.get("A")); + } + + @Test + public void testGetHCatConfHDFS() throws Exception { + Path hcatConfPath = new Path(getFsTestCaseDir(), "hive-site.xml"); + assertFalse(getFileSystem().exists(hcatConfPath)); + assertNull(services.get(HCatAccessorService.class).getHCatConf()); + + Configuration hcatConf = new Configuration(false); + hcatConf.set("A", "a"); + FSDataOutputStream out = getFileSystem().create(hcatConfPath); + hcatConf.writeXml(out); + out.close(); + assertTrue(getFileSystem().exists(hcatConfPath)); + services.destroy(); + services = super.setupServicesForHCatalog(); + Configuration conf = services.getConf(); + conf.set("oozie.service.HCatAccessorService.hcat.configuration", hcatConfPath.toUri().toString()); + services.init(); + Configuration hcatConfLoaded = services.get(HCatAccessorService.class).getHCatConf(); + assertEquals("a", hcatConfLoaded.get("A")); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/docs/src/site/twiki/AG_Install.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki index e343d7e..89f7407 100644 --- a/docs/src/site/twiki/AG_Install.twiki +++ b/docs/src/site/twiki/AG_Install.twiki @@ -422,7 +422,16 @@ the Oozie server to enable Oozie to work with HCatalog. *Adding HCatalog jars to Oozie war:* For Oozie server to talk to HCatalog server, HCatalog and hive jars need to be in the server classpath. -hive-site.xml which has the configuration to talk to the HCatalog server also needs to be in the classpath. +hive-site.xml which has the configuration to talk to the HCatalog server also needs to be in the classpath or specified by the +following configuration property in oozie-site.xml: +<verbatim> + <property> + <name>oozie.service.HCatAccessorService.hcat.configuration</name> + <value>/local/filesystem/path/to/hive-site.xml</value> + </property> +</verbatim> +The hive-site.xml can also be placed in a location on HDFS and the above property can have a value +of =hdfs://HOST:PORT/path/to/hive-site.xml= to point there instead of the local file system. The oozie-[version]-hcataloglibs.tar.gz in the oozie distribution bundles the required hcatalog and hive jars that needs to be placed in the Oozie server classpath. If using a version of HCatalog bundled in http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index fbd7bea..e7eaa1e 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1724 Make it easier to specify the HCat hive-site.xml for the Oozie Server (rkanter) OOZIE-1812 Bundle status is always in RUNNING if one of the action status is in PREP (puru via rohini) OOZIE-1848 Pig actions fail due to missing joda-time jar from pig sharelib (bzhang) OOZIE-1319 "LAST_ONLY" in execution control for coordinator job still runs all the actions (rkanter)
