Repository: oozie Updated Branches: refs/heads/master 805c5978e -> 0695c2033
OOZIE-1915 Move system properties to conf properties (puru via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0695c203 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0695c203 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0695c203 Branch: refs/heads/master Commit: 0695c20337283327fe700b136d27b7a2bc0e6846 Parents: 805c597 Author: Rohini Palaniswamy <[email protected]> Authored: Wed Jul 23 23:07:07 2014 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Wed Jul 23 23:07:07 2014 -0700 ---------------------------------------------------------------------- .../oozie/service/ConfigurationService.java | 22 ++++++++++++++------ .../oozie/service/JobsConcurrencyService.java | 8 +++---- .../apache/oozie/servlet/BaseAdminServlet.java | 5 ++++- .../java/org/apache/oozie/util/ConfigUtils.java | 5 +++-- .../java/org/apache/oozie/util/ZKUtils.java | 2 +- core/src/main/resources/oozie-default.xml | 11 +++++++++- .../service/TestJobsConcurrencyService.java | 17 ++++++++------- .../java/org/apache/oozie/test/ZKXTestCase.java | 2 +- docs/src/site/twiki/AG_Install.twiki | 16 +++++++++----- release-log.txt | 1 + 10 files changed, 59 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/java/org/apache/oozie/service/ConfigurationService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ConfigurationService.java b/core/src/main/java/org/apache/oozie/service/ConfigurationService.java index 798782e..8da50c3 100644 --- a/core/src/main/java/org/apache/oozie/service/ConfigurationService.java +++ b/core/src/main/java/org/apache/oozie/service/ConfigurationService.java @@ -81,26 +81,27 @@ public class ConfigurationService implements Service, Instrumentable { public static final String OOZIE_CONFIG_FILE = "oozie.config.file"; private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>(); + private static final Set<String> CONF_SYS_PROPS = new HashSet<String>(); + private static final String IGNORE_TEST_SYS_PROPS = "oozie.test."; private static final Set<String> MASK_PROPS = new HashSet<String>(); static { - IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS); //all this properties are seeded as system properties, no need to log changes - IGNORE_SYS_PROPS.add("oozie.http.hostname"); - IGNORE_SYS_PROPS.add("oozie.http.port"); - IGNORE_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID); - + IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS); IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR); IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR); IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE); IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR); - IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR); IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE); IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD); + CONF_SYS_PROPS.add("oozie.http.hostname"); + CONF_SYS_PROPS.add("oozie.http.port"); + CONF_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID); + // These properties should be masked when displayed because they contain sensitive info (e.g. password) MASK_PROPS.add(JPAService.CONF_PASSWORD); MASK_PROPS.add("oozie.authentication.signature.secret"); @@ -255,6 +256,15 @@ public class ConfigurationService implements Service, Instrumentable { } } + //Backward compatible, we should still support -Dparam. + for (String key : CONF_SYS_PROPS) { + String sysValue = System.getProperty(key); + if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) { + log.info("Overriding configuration with system property. Key [{0}], Value [{1}] ", key, sysValue); + configuration.set(key, sysValue); + } + } + return new LogChangesConfiguration(configuration); } http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/java/org/apache/oozie/service/JobsConcurrencyService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/JobsConcurrencyService.java b/core/src/main/java/org/apache/oozie/service/JobsConcurrencyService.java index b44d9d7..7508b21 100644 --- a/core/src/main/java/org/apache/oozie/service/JobsConcurrencyService.java +++ b/core/src/main/java/org/apache/oozie/service/JobsConcurrencyService.java @@ -33,11 +33,7 @@ import org.apache.oozie.util.ZKUtils; */ public class JobsConcurrencyService implements Service, Instrumentable { - private static final Map<String, String> urls; - static { - urls = new HashMap<String, String>(); - urls.put(System.getProperty(ZKUtils.OOZIE_INSTANCE_ID), ConfigUtils.getOozieEffectiveUrl()); - } + private static Map<String, String> urls; /** * Initialize the jobs concurrency service @@ -46,6 +42,8 @@ public class JobsConcurrencyService implements Service, Instrumentable { */ @Override public void init(Services services) throws ServiceException { + urls = new HashMap<String, String>(); + urls.put(services.getConf().get(ZKUtils.OOZIE_INSTANCE_ID), ConfigUtils.getOozieEffectiveUrl()); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java b/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java index 128347e..0e44bdf 100644 --- a/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java @@ -23,9 +23,11 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; import java.util.regex.Pattern; + import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; + import org.apache.hadoop.fs.Path; import org.apache.oozie.BuildInfo; import org.apache.oozie.client.rest.JsonBean; @@ -38,6 +40,7 @@ import org.apache.oozie.service.JobsConcurrencyService; import org.apache.oozie.service.Services; import org.apache.oozie.service.ShareLibService; import org.apache.oozie.util.AuthUrlClient; +import org.apache.oozie.util.ConfigUtils; import org.apache.oozie.util.Instrumentation; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -261,7 +264,7 @@ public abstract class BaseAdminServlet extends JsonRestServlet { private JSONObject updateLocalShareLib(HttpServletRequest request) { ShareLibService shareLibService = Services.get().get(ShareLibService.class); JSONObject json = new JSONObject(); - json.put(JsonTags.SHARELIB_UPDATE_HOST, Services.get().getConf().get("oozie.base.url")); + json.put(JsonTags.SHARELIB_UPDATE_HOST, ConfigUtils.getOozieEffectiveUrl()); try { json.putAll(shareLibService.updateShareLib()); json.put(JsonTags.SHARELIB_UPDATE_STATUS, "Successful"); http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/java/org/apache/oozie/util/ConfigUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/ConfigUtils.java b/core/src/main/java/org/apache/oozie/util/ConfigUtils.java index cdf6b48..c3ac01a 100644 --- a/core/src/main/java/org/apache/oozie/util/ConfigUtils.java +++ b/core/src/main/java/org/apache/oozie/util/ConfigUtils.java @@ -18,6 +18,7 @@ package org.apache.oozie.util; import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.service.Services; import org.apache.oozie.servlet.ServicesLoader; /** @@ -83,9 +84,9 @@ public class ConfigUtils { else { sb.append("http://";); } - sb.append(System.getProperty("oozie.http.hostname")); + sb.append(Services.get().getConf().get("oozie.http.hostname")); sb.append(":"); - sb.append(System.getProperty("oozie.http.port")); + sb.append(Services.get().getConf().get("oozie.http.port")); sb.append("/oozie"); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/java/org/apache/oozie/util/ZKUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/ZKUtils.java b/core/src/main/java/org/apache/oozie/util/ZKUtils.java index 74f6266..25ed2ee 100644 --- a/core/src/main/java/org/apache/oozie/util/ZKUtils.java +++ b/core/src/main/java/org/apache/oozie/util/ZKUtils.java @@ -118,7 +118,7 @@ public class ZKUtils { */ private ZKUtils() throws Exception { log = XLog.getLog(getClass()); - zkId = System.getProperty(OOZIE_INSTANCE_ID); + zkId = Services.get().getConf().get(OOZIE_INSTANCE_ID, Services.get().getConf().get("oozie.http.hostname")); createClient(); advertiseService(); checkAndSetACLs(); http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 3cb66e7..029ad1e 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -2065,7 +2065,16 @@ </description> </property> - +<!-- + <property> + <name>oozie.instance.id</name> + <value>${OOZIE_HTTP_HOSTNAME}</value> + <description> + Each Oozie server should have its own unique instance id. The default is system property + =${OOZIE_HTTP_HOSTNAME}= (i.e. the hostname). + </description> + </property> +--> <!-- Sharelib Configuration --> http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/test/java/org/apache/oozie/service/TestJobsConcurrencyService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestJobsConcurrencyService.java b/core/src/test/java/org/apache/oozie/service/TestJobsConcurrencyService.java index 53df650..473e120 100644 --- a/core/src/test/java/org/apache/oozie/service/TestJobsConcurrencyService.java +++ b/core/src/test/java/org/apache/oozie/service/TestJobsConcurrencyService.java @@ -31,6 +31,7 @@ public class TestJobsConcurrencyService extends XTestCase { @Override protected void setUp() throws Exception { super.setUp(); + new Services().init(); } @Override @@ -41,7 +42,7 @@ public class TestJobsConcurrencyService extends XTestCase { public void testIsLeader() throws Exception { JobsConcurrencyService jcs = new JobsConcurrencyService(); try { - jcs.init(null); + jcs.init(Services.get()); assertTrue(jcs.isLeader()); } finally { @@ -52,7 +53,7 @@ public class TestJobsConcurrencyService extends XTestCase { public void testIsJobIdForThisServer() throws Exception { JobsConcurrencyService jcs = new JobsConcurrencyService(); try { - jcs.init(null); + jcs.init(Services.get()); assertTrue(jcs.isJobIdForThisServer("blah")); } finally { @@ -63,7 +64,7 @@ public class TestJobsConcurrencyService extends XTestCase { public void testGetJobIdsForThisServer() throws Exception { JobsConcurrencyService jcs = new JobsConcurrencyService(); try { - jcs.init(null); + jcs.init(Services.get()); List<String> ids = new ArrayList<String>(); ids.add("blah"); ids.add("0000002-130521183438837-oozie-rkan-W"); @@ -80,11 +81,11 @@ public class TestJobsConcurrencyService extends XTestCase { public void testGetServerUrls() throws Exception { JobsConcurrencyService jcs = new JobsConcurrencyService(); try { - jcs.init(null); + jcs.init(Services.get()); Map<String, String> map = jcs.getServerUrls(); assertEquals(1, map.size()); - assertEquals(System.getProperty("oozie.instance.id"), map.keySet().iterator().next()); - assertEquals(ConfigUtils.getOozieURL(false), map.get(System.getProperty("oozie.instance.id"))); + assertEquals(Services.get().getConf().get("oozie.instance.id"), map.keySet().iterator().next()); + assertEquals(ConfigUtils.getOozieURL(false), map.get(Services.get().getConf().get("oozie.instance.id"))); } finally { jcs.destroy(); @@ -94,7 +95,7 @@ public class TestJobsConcurrencyService extends XTestCase { public void testsAllServerRequest() throws Exception { JobsConcurrencyService jcs = new JobsConcurrencyService(); try { - jcs.init(null); + jcs.init(Services.get()); assertFalse(jcs.isAllServerRequest(null)); Map<String, String[]> param = new HashMap<String, String[]>(); assertFalse(jcs.isAllServerRequest(param)); @@ -110,4 +111,4 @@ public class TestJobsConcurrencyService extends XTestCase { } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/test/java/org/apache/oozie/test/ZKXTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/ZKXTestCase.java b/core/src/test/java/org/apache/oozie/test/ZKXTestCase.java index 6a2dd32..4c7d0d7 100644 --- a/core/src/test/java/org/apache/oozie/test/ZKXTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/ZKXTestCase.java @@ -87,7 +87,7 @@ public abstract class ZKXTestCase extends XDataTestCase { private void setUpZK() throws Exception { zkServer = setupZKServer(); Services.get().getConf().set("oozie.zookeeper.connection.string", zkServer.getConnectString()); - setSystemProperty("oozie.instance.id", ZK_ID); + Services.get().getConf().set("oozie.instance.id", ZK_ID); createClient(); createServiceDiscovery(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/docs/src/site/twiki/AG_Install.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki index 8b0d110..06659f8 100644 --- a/docs/src/site/twiki/AG_Install.twiki +++ b/docs/src/site/twiki/AG_Install.twiki @@ -798,17 +798,23 @@ their own HA, they should have their own namespace. The default value is shown </property> </verbatim> -5. Change the value of =OOZIE_BASE_URL= in oozie-env.sh to point to the loadbalancer or virtual IP, for example: +5. Change the value of =OOZIE_BASE_URL= in oozie-site.xml to point to the loadbalancer or virtual IP, for example: <verbatim> -export OOZIE_BASE_URL="http://my.loadbalancer.hostname:11000/oozie"; +<property> + <name>oozie.base.url</name> + <value>http://my.loadbalancer.hostname:11000/oozie</value> +</property> </verbatim> -6. (Optional) Change the value of OOZIE_INSTANCE_ID in oozie-env.sh. Each Oozie server should have its own unique instance id. The -default is =${OOZIE_HTTP_HOSTNAME}= (i.e. the hostname). +6. (Optional) Add the following property to oozie-site.xml in all Oozie servers to specify the each host instance id. +Each Oozie server in HA should have its own unique instance id. The default is =${OOZIE_HTTP_HOSTNAME}= (i.e. the hostname). <verbatim> -export OOZIE_INSTANCE_ID="${OOZIE_HTTP_HOSTNAME}" +<property> + <name>oozie.instance.id</name> + <value>hostname</value> +</property> </verbatim> 7. (Optional) If using a secure cluster, see [[AG_Install#Security][Security]] below on configuring Kerberos with Oozie HA. http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index d048bca..70a7dfa 100644 --- a/release-log.txt +++ b/release-log.txt @@ -4,6 +4,7 @@ OOZIE-1943 Bump up trunk to 4.2.0-SNAPSHOT (bzhang) -- Oozie 4.1.0 release (4.1 - unreleased) +OOZIE-1915 Move system properties to conf properties (puru via rohini) OOZIE-1934 coordinator action repeatedly picked up by cachePurgeWorker of PartitionDependencyManagerService (ryota) OOZIE-1933 SLACalculatorMemory HA changes assume SLARegistrationBean exists for all jobs (mona) OOZIE-1935 Log level (ActionStartXCommand) and Doc fix (CoordinatorFunctionalSpec) (mona)
