[16/17] hadoop git commit: MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml (rchiang via rkanter)
MAPREDUCE-6192. Create unit test to automatically compare MR related classes
and mapred-default.xml (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9809a16d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9809a16d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9809a16d
Branch: refs/heads/HDFS-7240
Commit: 9809a16d3c8068beccbf0106e99c7ede6ba11e0f
Parents: 0100b15
Author: Robert Kanter rkan...@apache.org
Authored: Mon May 4 17:48:10 2015 -0700
Committer: Robert Kanter rkan...@apache.org
Committed: Tue May 5 14:43:28 2015 -0700
--
.../conf/TestConfigurationFieldsBase.java | 58 ++-
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../mapred/TestMapreduceConfigFields.java | 76
3 files changed, 135 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index c3fe3a3..2e4d8b1 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -147,6 +147,12 @@ public abstract class TestConfigurationFieldsBase {
private SetString xmlFieldsMissingInConfiguration = null;
/**
+ * Member variable for debugging base class operation
+ */
+ protected boolean configDebug = false;
+ protected boolean xmlDebug = false;
+
+ /**
* Abstract method to be used by subclasses for initializing base
* members.
*/
@@ -168,13 +174,16 @@ public abstract class TestConfigurationFieldsBase {
HashMapString,String retVal = new HashMapString,String();
// Setup regexp for valid properties
-String propRegex = ^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$;
+String propRegex = ^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$;
Pattern p = Pattern.compile(propRegex);
// Iterate through class member variables
int totalFields = 0;
String value;
for (Field f : fields) {
+ if (configDebug) {
+System.out.println(Field: + f);
+ }
// Filter out anything that isn't public static final
if (!Modifier.isStatic(f.getModifiers()) ||
!Modifier.isPublic(f.getModifiers()) ||
@@ -192,6 +201,9 @@ public abstract class TestConfigurationFieldsBase {
} catch (IllegalAccessException iaException) {
continue;
}
+ if (configDebug) {
+System.out.println( Value: + value);
+ }
// Special Case: Detect and ignore partial properties (ending in x)
// or file properties (ending in .xml)
if (value.endsWith(.xml) ||
@@ -221,11 +233,23 @@ public abstract class TestConfigurationFieldsBase {
// something like: blah.blah2(.blah3.blah4...)
Matcher m = p.matcher(value);
if (!m.find()) {
+if (configDebug) {
+ System.out.println( Passes Regex: false);
+}
continue;
}
+ if (configDebug) {
+System.out.println( Passes Regex: true);
+ }
// Save member variable/value as hash
- retVal.put(value,f.getName());
+ if (!retVal.containsKey(value)) {
+retVal.put(value,f.getName());
+ } else {
+if (configDebug) {
+ System.out.println(ERROR: Already found key for property + value);
+}
+ }
}
return retVal;
@@ -256,6 +280,9 @@ public abstract class TestConfigurationFieldsBase {
// Ignore known xml props
if (xmlPropsToSkipCompare != null) {
if (xmlPropsToSkipCompare.contains(key)) {
+ if (xmlDebug) {
+System.out.println( Skipping Full Key: + key);
+ }
continue;
}
}
@@ -270,14 +297,23 @@ public abstract class TestConfigurationFieldsBase {
}
}
if (skipPrefix) {
+if (xmlDebug) {
+ System.out.println( Skipping Prefix Key: + key);
+}
continue;
}
if (conf.onlyKeyExists(key)) {
retVal.put(key,null);
+if (xmlDebug) {
+ System.out.println( XML Key,Null Value: + key);
+}
} else {
String value = conf.get(key);
if (value!=null) {
retVal.put(key,entry.getValue());
+ if (xmlDebug) {
+System.out.println( XML Key,Valid Value: +
hadoop git commit: MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml (rchiang via rkanter)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0100b1550 - 9809a16d3
MAPREDUCE-6192. Create unit test to automatically compare MR related classes
and mapred-default.xml (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9809a16d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9809a16d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9809a16d
Branch: refs/heads/trunk
Commit: 9809a16d3c8068beccbf0106e99c7ede6ba11e0f
Parents: 0100b15
Author: Robert Kanter rkan...@apache.org
Authored: Mon May 4 17:48:10 2015 -0700
Committer: Robert Kanter rkan...@apache.org
Committed: Tue May 5 14:43:28 2015 -0700
--
.../conf/TestConfigurationFieldsBase.java | 58 ++-
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../mapred/TestMapreduceConfigFields.java | 76
3 files changed, 135 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index c3fe3a3..2e4d8b1 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -147,6 +147,12 @@ public abstract class TestConfigurationFieldsBase {
private SetString xmlFieldsMissingInConfiguration = null;
/**
+ * Member variable for debugging base class operation
+ */
+ protected boolean configDebug = false;
+ protected boolean xmlDebug = false;
+
+ /**
* Abstract method to be used by subclasses for initializing base
* members.
*/
@@ -168,13 +174,16 @@ public abstract class TestConfigurationFieldsBase {
HashMapString,String retVal = new HashMapString,String();
// Setup regexp for valid properties
-String propRegex = ^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$;
+String propRegex = ^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$;
Pattern p = Pattern.compile(propRegex);
// Iterate through class member variables
int totalFields = 0;
String value;
for (Field f : fields) {
+ if (configDebug) {
+System.out.println(Field: + f);
+ }
// Filter out anything that isn't public static final
if (!Modifier.isStatic(f.getModifiers()) ||
!Modifier.isPublic(f.getModifiers()) ||
@@ -192,6 +201,9 @@ public abstract class TestConfigurationFieldsBase {
} catch (IllegalAccessException iaException) {
continue;
}
+ if (configDebug) {
+System.out.println( Value: + value);
+ }
// Special Case: Detect and ignore partial properties (ending in x)
// or file properties (ending in .xml)
if (value.endsWith(.xml) ||
@@ -221,11 +233,23 @@ public abstract class TestConfigurationFieldsBase {
// something like: blah.blah2(.blah3.blah4...)
Matcher m = p.matcher(value);
if (!m.find()) {
+if (configDebug) {
+ System.out.println( Passes Regex: false);
+}
continue;
}
+ if (configDebug) {
+System.out.println( Passes Regex: true);
+ }
// Save member variable/value as hash
- retVal.put(value,f.getName());
+ if (!retVal.containsKey(value)) {
+retVal.put(value,f.getName());
+ } else {
+if (configDebug) {
+ System.out.println(ERROR: Already found key for property + value);
+}
+ }
}
return retVal;
@@ -256,6 +280,9 @@ public abstract class TestConfigurationFieldsBase {
// Ignore known xml props
if (xmlPropsToSkipCompare != null) {
if (xmlPropsToSkipCompare.contains(key)) {
+ if (xmlDebug) {
+System.out.println( Skipping Full Key: + key);
+ }
continue;
}
}
@@ -270,14 +297,23 @@ public abstract class TestConfigurationFieldsBase {
}
}
if (skipPrefix) {
+if (xmlDebug) {
+ System.out.println( Skipping Prefix Key: + key);
+}
continue;
}
if (conf.onlyKeyExists(key)) {
retVal.put(key,null);
+if (xmlDebug) {
+ System.out.println( XML Key,Null Value: + key);
+}
} else {
String value = conf.get(key);
if (value!=null) {
retVal.put(key,entry.getValue());
+
hadoop git commit: MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml (rchiang via rkanter)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a9d12128f - bd207b6cc
MAPREDUCE-6192. Create unit test to automatically compare MR related classes
and mapred-default.xml (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd207b6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd207b6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd207b6c
Branch: refs/heads/branch-2
Commit: bd207b6cc0c223df6b3288c8a0249ee065ac9371
Parents: a9d1212
Author: Robert Kanter rkan...@apache.org
Authored: Tue May 5 14:47:52 2015 -0700
Committer: Robert Kanter rkan...@apache.org
Committed: Tue May 5 14:47:52 2015 -0700
--
.../conf/TestConfigurationFieldsBase.java | 58 +-
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../mapred/TestMapreduceConfigFields.java | 83
3 files changed, 142 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd207b6c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index c3fe3a3..2e4d8b1 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -147,6 +147,12 @@ public abstract class TestConfigurationFieldsBase {
private SetString xmlFieldsMissingInConfiguration = null;
/**
+ * Member variable for debugging base class operation
+ */
+ protected boolean configDebug = false;
+ protected boolean xmlDebug = false;
+
+ /**
* Abstract method to be used by subclasses for initializing base
* members.
*/
@@ -168,13 +174,16 @@ public abstract class TestConfigurationFieldsBase {
HashMapString,String retVal = new HashMapString,String();
// Setup regexp for valid properties
-String propRegex = ^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$;
+String propRegex = ^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$;
Pattern p = Pattern.compile(propRegex);
// Iterate through class member variables
int totalFields = 0;
String value;
for (Field f : fields) {
+ if (configDebug) {
+System.out.println(Field: + f);
+ }
// Filter out anything that isn't public static final
if (!Modifier.isStatic(f.getModifiers()) ||
!Modifier.isPublic(f.getModifiers()) ||
@@ -192,6 +201,9 @@ public abstract class TestConfigurationFieldsBase {
} catch (IllegalAccessException iaException) {
continue;
}
+ if (configDebug) {
+System.out.println( Value: + value);
+ }
// Special Case: Detect and ignore partial properties (ending in x)
// or file properties (ending in .xml)
if (value.endsWith(.xml) ||
@@ -221,11 +233,23 @@ public abstract class TestConfigurationFieldsBase {
// something like: blah.blah2(.blah3.blah4...)
Matcher m = p.matcher(value);
if (!m.find()) {
+if (configDebug) {
+ System.out.println( Passes Regex: false);
+}
continue;
}
+ if (configDebug) {
+System.out.println( Passes Regex: true);
+ }
// Save member variable/value as hash
- retVal.put(value,f.getName());
+ if (!retVal.containsKey(value)) {
+retVal.put(value,f.getName());
+ } else {
+if (configDebug) {
+ System.out.println(ERROR: Already found key for property + value);
+}
+ }
}
return retVal;
@@ -256,6 +280,9 @@ public abstract class TestConfigurationFieldsBase {
// Ignore known xml props
if (xmlPropsToSkipCompare != null) {
if (xmlPropsToSkipCompare.contains(key)) {
+ if (xmlDebug) {
+System.out.println( Skipping Full Key: + key);
+ }
continue;
}
}
@@ -270,14 +297,23 @@ public abstract class TestConfigurationFieldsBase {
}
}
if (skipPrefix) {
+if (xmlDebug) {
+ System.out.println( Skipping Prefix Key: + key);
+}
continue;
}
if (conf.onlyKeyExists(key)) {
retVal.put(key,null);
+if (xmlDebug) {
+ System.out.println( XML Key,Null Value: + key);
+}
} else {
String value = conf.get(key);
if (value!=null) {
retVal.put(key,entry.getValue());
+
