This is an automated email from the ASF dual-hosted git repository. gezapeti pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push: new f0af3a0 OOZIE-3578 MapReduce counters cannot be used over 120 (dionusos via pbacsko, gezapeti) f0af3a0 is described below commit f0af3a01c68e373cba3122ac7a005b987d84c729 Author: Gezapeti Cseh <gezap...@apache.org> AuthorDate: Wed Jan 8 13:36:40 2020 +0100 OOZIE-3578 MapReduce counters cannot be used over 120 (dionusos via pbacsko, gezapeti) --- .../oozie/service/HadoopAccessorService.java | 12 ++++++++++ .../oozie/service/TestHadoopAccessorService.java | 13 +++++++++++ core/src/test/resources/test-mapred-site.xml | 27 ++++++++++++++++++++++ release-log.txt | 1 + 4 files changed, 53 insertions(+) diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java index 396ad29..ceb05ef 100644 --- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java @@ -30,6 +30,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.counters.Limits; import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest; @@ -185,6 +186,17 @@ public class HadoopAccessorService implements Service { } setConfigForHadoopSecurityUtil(conf); + initializeMRLimits(conf); + } + + /** + * This method initializes the MapReduce's Limits class so the user can define more than 120 counters + * in their map reduce action. For more details please see OOZIE-3578. + * + * @param conf the loaded hadoop configurations including mapred-site.xml + */ + private void initializeMRLimits(Configuration conf) { + Limits.init(conf); } private void setConfigForHadoopSecurityUtil(Configuration conf) { diff --git a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java index d3cb97a..582ad08 100644 --- a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java +++ b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java @@ -20,6 +20,7 @@ package org.apache.oozie.service; import org.apache.hadoop.fs.Path; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -35,6 +36,7 @@ import org.apache.oozie.util.IOUtils; import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; @@ -389,6 +391,17 @@ public class TestHadoopAccessorService extends XFsTestCase { assertSame("The two configuration object shall be the same", base, result); } + public void testIfMRLimitsIsInitialized() throws IOException, ServiceException { + InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("test-mapred-site.xml"); + OutputStream os = new FileOutputStream(new File(getTestCaseConfDir() + "/hadoop-confx", "mapred-site.xml")); + IOUtils.copyStream(is, os); + HadoopAccessorService has = new HadoopAccessorService(); + has.init(Services.get()); + Assert.assertEquals("Limits class shall not use default value for number of counters.", + 500, + has.createConfiguration("jt").getInt(MRJobConfig.COUNTERS_MAX_KEY, 120)); + } + public FileSystem createFileSystemWithCustomProperties( final String uri, final String fsKey, final String commaSeparatedKeyValues) throws Exception { final HadoopAccessorService has = new HadoopAccessorService(); diff --git a/core/src/test/resources/test-mapred-site.xml b/core/src/test/resources/test-mapred-site.xml new file mode 100644 index 0000000..53472e0 --- /dev/null +++ b/core/src/test/resources/test-mapred-site.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + + <property> + <name>mapreduce.job.counters.max</name> + <value>500</value> + </property> + +</configuration> diff --git a/release-log.txt b/release-log.txt index 46c1da8..5877766 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.3.0 release (trunk - unreleased) +OOZIE-3578 MapReduce counters cannot be used over 120 (dionusos via pbacsko, gezapeti) OOZIE-3573 Fix oozie-default.xml descriptions (arvaiadam via asalamon74) OOZIE-3571 Wrong log string substitution in CredentialsProviderFactory class (arvaiadam via asalamon74) OOZIE-3566 Wrong numbers provided by Oozie monitoring webservices.requests sampler (cas via asalamon74)