Repository: oozie Updated Branches: refs/heads/master 878cb0462 -> 58e510f3b
OOZIE-2617 Read properties files in action configurations (wutaklon via jaydeepvishwakarma) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/58e510f3 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/58e510f3 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/58e510f3 Branch: refs/heads/master Commit: 58e510f3b858f9c08d0c9b6304985634678bf16e Parents: 878cb04 Author: jvishwakarma <[email protected]> Authored: Mon Sep 12 21:35:36 2016 +0530 Committer: jvishwakarma <[email protected]> Committed: Mon Sep 12 21:35:36 2016 +0530 ---------------------------------------------------------------------- .../oozie/service/ActionConfFileType.java | 56 +++++++++++ .../oozie/service/HadoopAccessorService.java | 98 +++++++++++--------- .../oozie/service/TestActionConfFileType.java | 52 +++++++++++ .../service/TestHadoopAccessorService.java | 8 ++ .../src/site/twiki/AG_ActionConfiguration.twiki | 6 +- release-log.txt | 1 + 6 files changed, 176 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/58e510f3/core/src/main/java/org/apache/oozie/service/ActionConfFileType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ActionConfFileType.java b/core/src/main/java/org/apache/oozie/service/ActionConfFileType.java new file mode 100644 index 0000000..1753b47 --- /dev/null +++ b/core/src/main/java/org/apache/oozie/service/ActionConfFileType.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.service; + +import com.google.common.base.Joiner; + +enum ActionConfFileType { + XML(".xml"), + PROPERTIES(".properties"); + + private final String extension; + + ActionConfFileType(String extension) { + this.extension = extension; + } + + public String getExtension() { + return this.extension; + } + + static boolean isSupportedFileType(String fileName) { + for (ActionConfFileType actionConfFileType : values()) { + if (fileName.endsWith(actionConfFileType.getExtension())) { + return true; + } + } + return false; + } + + static ActionConfFileType getFileType(String fileName) { + for (ActionConfFileType actionConfFileType : values()) { + if (fileName.endsWith(actionConfFileType.getExtension())) { + return actionConfFileType; + } + } + throw new UnsupportedOperationException(String.format( + "Unsupported action conf file extension for file %s, supported extension are %s", fileName, + Joiner.on(", ").join(values()))); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/58e510f3/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java index f171ea3..5377127 100644 --- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java @@ -31,6 +31,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.oozie.ErrorCode; import org.apache.oozie.action.hadoop.JavaActionExecutor; +import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.ParamChecker; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XLog; @@ -52,6 +53,7 @@ import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.HashSet; import java.util.concurrent.ConcurrentHashMap; @@ -367,64 +369,76 @@ public class HadoopAccessorService implements Service { File dir = actionConfigDirs.get(hostPort); XConfiguration actionConf = new XConfiguration(); if (dir != null) { - // See if a dir with the action name exists. If so, load all the xml files in the dir + // See if a dir with the action name exists. If so, load all the supported conf files in the dir File actionConfDir = new File(dir, action); if (actionConfDir.exists() && actionConfDir.isDirectory()) { LOG.info("Processing configuration files under [{0}]" + " for action [{1}] and hostPort [{2}]", actionConfDir.getAbsolutePath(), action, hostPort); - File[] xmlFiles = actionConfDir.listFiles( - new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".xml"); - }}); - Arrays.sort(xmlFiles, new Comparator<File>() { - @Override - public int compare(File o1, File o2) { - return o1.getName().compareTo(o2.getName()); - } - }); - for (File f : xmlFiles) { - if (f.isFile() && f.canRead()) { - LOG.info("Processing configuration file [{0}]", f.getName()); - FileInputStream fis = null; - try { - fis = new FileInputStream(f); - XConfiguration conf = new XConfiguration(fis); - XConfiguration.copy(conf, actionConf); - } - catch (IOException ex) { - LOG - .warn("Could not read file [{0}] for action [{1}] configuration and hostPort [{2}]", - f.getAbsolutePath(), action, hostPort); - } - finally { - if (fis != null) { - try { fis.close(); } catch(IOException ioe) { } - } - } - } - } + updateActionConfigWithDir(actionConf, actionConfDir); } } // Now check for <action.xml> This way <action.xml> has priority over <action-dir>/*.xml - File actionConfFile = new File(dir, action + ".xml"); + LOG.info("Processing configuration file [{0}] for action [{1}] and hostPort [{2}]", + actionConfFile.getAbsolutePath(), action, hostPort); if (actionConfFile.exists()) { - try { - XConfiguration conf = new XConfiguration(new FileInputStream(actionConfFile)); - XConfiguration.copy(conf, actionConf); + updateActionConfigWithFile(actionConf, actionConfFile); + } + + return actionConf; + } + + private void updateActionConfigWithFile(Configuration actionConf, File actionConfFile) { + try { + Configuration conf = readActionConfFile(actionConfFile); + XConfiguration.copy(conf, actionConf); + } catch (IOException e) { + LOG.warn("Could not read file [{0}].", actionConfFile.getAbsolutePath()); + } + } + + private void updateActionConfigWithDir(Configuration actionConf, File actionConfDir) { + File[] actionConfFiles = actionConfDir.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return ActionConfFileType.isSupportedFileType(name); + }}); + Arrays.sort(actionConfFiles, new Comparator<File>() { + @Override + public int compare(File o1, File o2) { + return o1.getName().compareTo(o2.getName()); } - catch (IOException ex) { - LOG.warn("Could not read file [{0}] for action [{1}] configuration for hostPort [{2}]", - actionConfFile.getAbsolutePath(), action, hostPort); + }); + for (File f : actionConfFiles) { + if (f.isFile() && f.canRead()) { + updateActionConfigWithFile(actionConf, f); } } - return actionConf; + } + + private Configuration readActionConfFile(File file) throws IOException { + InputStream fis = null; + try { + fis = new FileInputStream(file); + ActionConfFileType fileTyple = ActionConfFileType.getFileType(file.getName()); + switch (fileTyple) { + case XML: + return new XConfiguration(fis); + case PROPERTIES: + Properties properties = new Properties(); + properties.load(fis); + return new XConfiguration(properties); + default: + throw new UnsupportedOperationException( + String.format("Unable to parse action conf file of type %s", fileTyple)); + } + } finally { + IOUtils.closeSafely(fis); + } } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/58e510f3/core/src/test/java/org/apache/oozie/service/TestActionConfFileType.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestActionConfFileType.java b/core/src/test/java/org/apache/oozie/service/TestActionConfFileType.java new file mode 100644 index 0000000..c1dbe6f --- /dev/null +++ b/core/src/test/java/org/apache/oozie/service/TestActionConfFileType.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.service; + +import org.apache.oozie.test.XTestCase; + +public class TestActionConfFileType extends XTestCase { + + private static final String XML_FILE = "a.xml"; + private static final String PROPERTIES_FILE = "b.properties"; + private static final String JSON_FILE = "c.json"; + + public void testIsSupportedFileType() { + assertTrue(ActionConfFileType.isSupportedFileType(XML_FILE)); + assertTrue(ActionConfFileType.isSupportedFileType(PROPERTIES_FILE)); + assertFalse(ActionConfFileType.isSupportedFileType(JSON_FILE)); + } + + public void testGetFileType() { + assertEquals(ActionConfFileType.XML, ActionConfFileType.getFileType(XML_FILE)); + assertEquals(ActionConfFileType.PROPERTIES, ActionConfFileType.getFileType(PROPERTIES_FILE)); + + try { + ActionConfFileType.getFileType(JSON_FILE); + fail("Should throw exception"); + } + catch (Throwable e) { + assertTrue(e.getMessage().contains("Unsupported action conf file extension for file")); + } + } + + public void testGetExtension() { + assertEquals(".xml", ActionConfFileType.XML.getExtension()); + assertEquals(".properties", ActionConfFileType.PROPERTIES.getExtension()); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/58e510f3/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java index bbe2594..bbca479 100644 --- a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java +++ b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java @@ -71,6 +71,10 @@ public class TestHadoopAccessorService extends XTestCase { os = new FileOutputStream(new File(actConfXDir + "/default", "z-conf.xml")); IOUtils.copyStream(is, os); + is = Thread.currentThread().getContextClassLoader().getResourceAsStream("test-custom-log4j.properties"); + os = new FileOutputStream(new File(actConfXDir + "/action", "test-custom-log4j.properties")); + IOUtils.copyStream(is, os); + setSystemProperty("oozie.service.HadoopAccessorService.hadoop.configurations", "*=hadoop-conf,jt=hadoop-confx"); setSystemProperty("oozie.service.HadoopAccessorService.action.configurations", @@ -137,6 +141,10 @@ public class TestHadoopAccessorService extends XTestCase { assertEquals("100", conf.get("action.testprop")); assertEquals("1", conf.get("default.testprop")); + // Check that properties load correctly + assertEquals("org.apache.log4j.ConsoleAppender", conf.get("log4j.appender.oozie")); + assertEquals("NONE, null", conf.get("log4j.logger.a")); + } public void testAccessor() throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/58e510f3/docs/src/site/twiki/AG_ActionConfiguration.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/AG_ActionConfiguration.twiki b/docs/src/site/twiki/AG_ActionConfiguration.twiki index f8e6edc..f6ffdb2 100644 --- a/docs/src/site/twiki/AG_ActionConfiguration.twiki +++ b/docs/src/site/twiki/AG_ActionConfiguration.twiki @@ -9,7 +9,7 @@ action type and default configuration for all actions ---++ Hadoop Default Configuration Values -Oozie supports action configuration equivalent to the component's =*-site.xml= files. +Oozie supports action configuration equivalent to the component's =*-site.xml= and =*.properties= files. The configuration property in the =oozie-site.xml= is =oozie.service.HadoopAccessorService.action.configurations= and its value must follow the pattern _[AUTHORITY=ACTION-CONF-DIR_,]*. Where _AUTHORITY_ is the _HOST:PORT_ of @@ -18,7 +18,7 @@ also be specified. Oozie will load and process the action configuration files in 1. All files in _default_/*.xml (sorted by lexical name, files with names lexically lower have lesser precedence than the following ones), if present. 1. _default_.xml, if present. - 1. All files in _actionname_/*.xml (sorted by lexical name, files with names lexically lower have lesser precedence than the following ones), if present. + 1. All supported files in _actionname_/*, e.g. _actionname_/*.xml and _actionname_/*.properties (based on filename extension, sorted by lexical name, files with names lexically lower have lesser precedence than the following ones), if present. 1. _actionname_.xml, if present. @@ -26,7 +26,7 @@ For example, for _Hive_ action (which has the _actionname_ defined as _hive_ ), 1. All files in _default_/*.xml, if present 1. _default_.xml, if present. - 1. All files in _hive_/*.xml, if present + 1. All files in _hive_/*.xml and _hive_/*.properties, if present 1. _hive_.xml, if present. http://git-wip-us.apache.org/repos/asf/oozie/blob/58e510f3/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 2b90cf9..7355929 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2617 Read properties files in action configurations (wutaklon via jaydeepvishwakarma) OOZIE-2615 Flaky tests TestCoordActionsKillXCommand.testActionKillCommandActionNumbers and testActionKillCommandDate (pbacsko via rkanter) OOZIE-2632 Provide database dump/load functionality to make database migration easier (gezapeti, rjustice via rkanter) OOZIE-2243 Kill Command does not kill the child job for java action (jaydeepvishwakarma)
