Author: rohini
Date: Fri May 26 22:21:06 2017
New Revision: 1796355
URL: http://svn.apache.org/viewvc?rev=1796355&view=rev
Log:
PIG-4924: Translate failures.maxpercent MR setting to Tez Tez (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/MRToTezHelper.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1796355&r1=1796354&r2=1796355&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 26 22:21:06 2017
@@ -36,6 +36,8 @@ PIG-5067: Revisit union on numeric type
Â
IMPROVEMENTS
+PIG-4924: Translate failures.maxpercent MR setting to Tez Tez (rohini)
+
PIG-5236: json simple jar not included automatically with piggybank
AvroStorage (satishsaley via rohini)
PIG-3103: make mockito a test dependency (instead of compile) (nkollar via
szita)
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/MRToTezHelper.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/MRToTezHelper.java?rev=1796355&r1=1796354&r2=1796355&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/MRToTezHelper.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/MRToTezHelper.java
Fri May 26 22:21:06 2017
@@ -89,21 +89,24 @@ public class MRToTezHelper {
mrAMParamToTezAMParamMap.put(MRConfiguration.JOB_CREDENTIALS_BINARY,
TezConfiguration.TEZ_CREDENTIALS_PATH);
mrAMParamToTezAMParamMap.put(MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN,
TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION);
- //Map settings
+ // Map settings for Scope.VERTEX
mrMapParamToTezVertexParamMap.put(MRJobConfig.MAP_MAX_ATTEMPTS,
TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS);
mrMapParamToTezVertexParamMap.put(MRJobConfig.MAP_SPECULATIVE,
TezConfiguration.TEZ_AM_SPECULATION_ENABLED);
mrMapParamToTezVertexParamMap.put(MRJobConfig.MAP_LOG_LEVEL,
TezConfiguration.TEZ_TASK_LOG_LEVEL);
- //TezConfiguration.TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY TEZ-2914 in Tez
0.8
+ // TezConfiguration.TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY TEZ-2914 in Tez
0.8
mrMapParamToTezVertexParamMap.put("mapreduce.job.running.map.limit",
"tez.am.vertex.max-task-concurrency");
- //TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS TEZ-808 in Tez
0.8
+ // TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS TEZ-808 in Tez
0.8
mrMapParamToTezVertexParamMap.put(MRJobConfig.TASK_TIMEOUT,
"tez.am.progress.stuck.interval-ms");
+ // TezConfiguration.TEZ_VERTEX_FAILURES_MAXPERCENT TEZ-3271 in Tez
0.8.4
+
mrMapParamToTezVertexParamMap.put(MRJobConfig.MAP_FAILURES_MAX_PERCENT,
"tez.vertex.failures.maxpercent");
- //Reduce settings
+ // Reduce settings for Scope.VERTEX
mrReduceParamToTezVertexParamMap.put(MRJobConfig.REDUCE_MAX_ATTEMPTS,
TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS);
mrReduceParamToTezVertexParamMap.put(MRJobConfig.REDUCE_SPECULATIVE,
TezConfiguration.TEZ_AM_SPECULATION_ENABLED);
mrReduceParamToTezVertexParamMap.put(MRJobConfig.REDUCE_LOG_LEVEL,
TezConfiguration.TEZ_TASK_LOG_LEVEL);
mrReduceParamToTezVertexParamMap.put("mapreduce.job.running.reduce.limit",
"tez.am.vertex.max-task-concurrency");
mrReduceParamToTezVertexParamMap.put(MRJobConfig.TASK_TIMEOUT,
"tez.am.progress.stuck.interval-ms");
+
mrReduceParamToTezVertexParamMap.put(MRJobConfig.REDUCE_FAILURES_MAXPERCENT,
"tez.vertex.failures.maxpercent");
}
private static void populateMRSettingsToRetain() {
