HADOOP-12101. Add automatic search of default Configuration variables to TestConfigurationFieldsBase. Contributed by Ray Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/355325bc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/355325bc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/355325bc Branch: refs/heads/HDFS-7240 Commit: 355325bcc7111fa4aac801fd23a26422ffabaf7c Parents: 75e0450 Author: Masatake Iwasaki <iwasak...@apache.org> Authored: Wed May 4 15:12:51 2016 +0900 Committer: Masatake Iwasaki <iwasak...@apache.org> Committed: Wed May 4 15:12:51 2016 +0900 ---------------------------------------------------------------------- dev-support/verify-xml.sh | 150 +++++++++++ .../conf/TestConfigurationFieldsBase.java | 255 +++++++++++++++++++ 2 files changed, 405 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/355325bc/dev-support/verify-xml.sh ---------------------------------------------------------------------- diff --git a/dev-support/verify-xml.sh b/dev-support/verify-xml.sh new file mode 100755 index 0000000..abab4e6 --- /dev/null +++ b/dev-support/verify-xml.sh @@ -0,0 +1,150 @@ +#!/bin/bash +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## +# Script to run unit tests for xml <-> 1 or more Configuration file verification +# usage: ./verify-xml.sh <mode> +# + +# Utility functions +function find_test_output_file() { + echo "Found test output file(s) at" + echo "" + if [ -n "$1" ] && [ -e "$1" ] ; then + echo " $1" + fi + if [ -n "$2" ] && [ -e "$2" ] ; then + echo " $2" + fi + if [ -n "$3" ] && [ -e "$3" ] ; then + echo " $3" + fi + if [ -n "$4" ] && [ -e "$4" ] ; then + echo " $4" + fi + echo "" + echo "Examine the file for specific information xml/Configuration mismatches." + echo "" +} + +function print_test_banner() { + local banner_text=$1 + local banner_length=${#banner_text} + local banner + banner=$( printf "%${banner_length}s" ' ' ) + echo "" + echo "${banner// /=}" + echo "${banner_text}" + echo "${banner// /=}" + echo "" +} + +# Wrapper functions for running unit tests +function run_all_xml_test() { + mvn test -Dtest=TestCommonConfigurationFields,TestHdfsConfigFields,TestMapreduceConfigFields,TestYarnConfigurationFields + if [ $? -ne 0 ] ; then + print_test_banner "All Test*ConfigFields FAIL" + else + print_test_banner "All Test*ConfigFields SUCCESS" + fi +} + +function run_common_xml_test() { + mvn test -Dtest=TestCommonConfigFields + if [ $? -ne 0 ] ; then + print_test_banner "TestCommonConfigurationFields FAIL" + else + print_test_banner "TestCommonConfigurationFields SUCCESS" + fi +} + +function run_hdfs_xml_test() { + mvn test -Dtest=TestHdfsConfigFields + if [ $? -ne 0 ] ; then + print_test_banner "TestHdfsConfigFields FAIL" + else + print_test_banner "TestHdfsConfigFields SUCCESS" + fi +} + +function run_mapreduce_xml_test() { + mvn test -Dtest=TestMapreduceConfigFields + if [ $? -ne 0 ] ; then + print_test_banner "TestMapreduceConfigFields FAIL" + else + print_test_banner "TestMapreduceConfigFields SUCCESS" + fi +} + +function run_yarn_xml_test() { + mvn test -Dtest=TestYarnConfigurationFields + if [ $? -ne 0 ] ; then + print_test_banner "TestYarnConfigurationFields FAIL" + else + print_test_banner "TestYarnConfigurationFields SUCCESS" + fi +} + +# Main body +cd -P -- "$(dirname -- "${BASH_SOURCE-$0}")/.." || exit +dir="$(pwd -P)" + +# - Create unit test file names +export commonOutputFile +commonOutputFile="$(find "${dir}" -name org.apache.hadoop.conf.TestCommonConfigurationFields-output.txt)" +export hdfsOutputFile +hdfsOutputFile="$(find "${dir}" -name org.apache.hadoop.tools.TestHdfsConfigFields-output.txt)" +export mrOutputFile +mrOutputFile="$(find "${dir}" -name org.apache.hadoop.mapreduce.TestMapreduceConfigFields-output.txt)" +export yarnOutputFile +yarnOutputFile="$(find "${dir}" -name org.apache.hadoop.yarn.conf.TestYarnConfigurationFields-output.txt)" + +# - Determine which tests to run +case "$1" in + + all) + run_all_xml_test + find_test_output_file "${commonOutputFile}" "${hdfsOutputFile}" "${mrOutputFile}" "${yarnOutputFile}" + ;; + + common) + run_common_xml_test + find_test_output_file "${commonOutputFile}" + ;; + + hdfs) + run_hdfs_xml_test + find_test_output_file "${hdfsOutputFile}" + ;; + + mr) + run_mapreduce_xml_test + find_test_output_file "${mrOutputFile}" + ;; + + yarn) + run_yarn_xml_test + find_test_output_file "${yarnOutputFile}" + ;; + + *) + echo "Usage: $0 <mode>" + echo " where <mode> is one of all, common, hdfs, mr, yarn" + exit 1 + ;; + +esac http://git-wip-us.apache.org/repos/asf/hadoop/blob/355325bc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java index e528602..eab0161 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java @@ -125,6 +125,11 @@ public abstract class TestConfigurationFieldsBase { private Map<String,String> configurationMemberVariables = null; /** + * Member variable to store Configuration variables for later reference. + */ + private Map<String,String> configurationDefaultVariables = null; + + /** * Member variable to store XML properties for later comparison. */ private Map<String,String> xmlKeyValueMap = null; @@ -146,6 +151,7 @@ public abstract class TestConfigurationFieldsBase { */ protected boolean configDebug = false; protected boolean xmlDebug = false; + protected boolean defaultDebug = false; /** * Abstract method to be used by subclasses for initializing base @@ -317,6 +323,79 @@ public abstract class TestConfigurationFieldsBase { } /** + * Utility function to extract "public static final" default + * member variables from a Configuration type class. + * + * @param fields The class member variables + * @return HashMap containing <DefaultVariableName,DefaultValue> entries + */ + private HashMap<String,String> + extractDefaultVariablesFromConfigurationFields(Field[] fields) { + // Sanity Check + if (fields==null) { + return null; + } + + HashMap<String,String> retVal = new HashMap<String,String>(); + + // Setup regexp for valid properties + String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$"; + Pattern p = Pattern.compile(propRegex); + + // Iterate through class member variables + int totalFields = 0; + String value; + for (Field f : fields) { + // Filter out anything that isn't "public static final" + if (!Modifier.isStatic(f.getModifiers()) || + !Modifier.isPublic(f.getModifiers()) || + !Modifier.isFinal(f.getModifiers())) { + continue; + } + // Special: Stuff any property beginning with "DEFAULT_" into a + // different hash for later processing + if (f.getName().startsWith("DEFAULT_") || + f.getName().endsWith("_DEFAULT")) { + if (retVal.containsKey(f.getName())) { + continue; + } + try { + if (f.getType().getName().equals("java.lang.String")) { + String sValue = (String) f.get(null); + retVal.put(f.getName(),sValue); + } else if (f.getType().getName().equals("short")) { + short shValue = (short) f.get(null); + retVal.put(f.getName(),Integer.toString(shValue)); + } else if (f.getType().getName().equals("int")) { + int iValue = (int) f.get(null); + retVal.put(f.getName(),Integer.toString(iValue)); + } else if (f.getType().getName().equals("long")) { + long lValue = (long) f.get(null); + retVal.put(f.getName(),Long.toString(lValue)); + } else if (f.getType().getName().equals("float")) { + float fValue = (float) f.get(null); + retVal.put(f.getName(),Float.toString(fValue)); + } else if (f.getType().getName().equals("double")) { + double dValue = (double) f.get(null); + retVal.put(f.getName(),Double.toString(dValue)); + } else if (f.getType().getName().equals("boolean")) { + boolean bValue = (boolean) f.get(null); + retVal.put(f.getName(),Boolean.toString(bValue)); + } else { + if (defaultDebug) { + System.out.println("Config variable " + f.getName() + " has unknown type " + f.getType().getName()); + } + } + } catch (IllegalAccessException iaException) { + iaException.printStackTrace(); + } + } + } + + return retVal; + } + + /** * Perform set difference operation on keyMap2 from keyMap1. * * @param keyMap1 The initial set @@ -374,6 +453,26 @@ public abstract class TestConfigurationFieldsBase { System.out.println(""); } + // Create default configuration variable key/value map + if (defaultDebug) { + System.out.println("Reading Config property files for defaults"); + System.out.println(""); + } + configurationDefaultVariables = new HashMap<String,String>(); + for (Class c : configurationClasses) { + Field[] fields = c.getDeclaredFields(); + Map<String,String> defaultMap = + extractDefaultVariablesFromConfigurationFields(fields); + if (defaultMap!=null) { + configurationDefaultVariables.putAll(defaultMap); + } + } + if (defaultDebug) { + System.out.println(""); + System.out.println("====="); + System.out.println(""); + } + // Find class members not in the XML file configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields (configurationMemberVariables, xmlKeyValueMap); @@ -464,4 +563,160 @@ public abstract class TestConfigurationFieldsBase { assertTrue(configErrorMsg.toString(), missingConfigSize==0); } } + + /** + * For each property in the XML file, verify that the value matches + * up to the default if one exists. + */ + @Test + public void testXmlAgainstDefaultValuesInConfigurationClass() { + // Error if subclass hasn't set class members + assertTrue(xmlFilename!=null); + assertTrue(configurationMemberVariables!=null); + assertTrue(configurationDefaultVariables!=null); + + HashSet<String> xmlPropertiesWithEmptyValue = new HashSet<String>(); + HashSet<String> configPropertiesWithNoDefaultConfig = new HashSet<String>(); + HashMap<String,String> xmlPropertiesMatchingConfigDefault = + new HashMap<String,String>(); + // Ugly solution. Should have tuple-based solution. + HashMap<HashMap<String,String>,HashMap<String,String>> mismatchingXmlConfig = + new HashMap<HashMap<String,String>,HashMap<String,String>>(); + + for (Map.Entry<String,String> xEntry : xmlKeyValueMap.entrySet()) { + String xmlProperty = xEntry.getKey(); + String xmlDefaultValue = xEntry.getValue(); + String configProperty = configurationMemberVariables.get(xmlProperty); + if (configProperty!=null) { + String defaultConfigName = null; + String defaultConfigValue = null; + + // Type 1: Prepend DEFAULT_ + String defaultNameCheck1 = "DEFAULT_" + configProperty; + String defaultValueCheck1 = configurationDefaultVariables + .get(defaultNameCheck1); + // Type 2: Swap _KEY suffix with _DEFAULT suffix + String defaultNameCheck2 = null; + if (configProperty.endsWith("_KEY")) { + defaultNameCheck2 = configProperty + .substring(0,configProperty.length()-4) + "_DEFAULT"; + } + String defaultValueCheck2 = configurationDefaultVariables + .get(defaultNameCheck2); + // Type Last: Append _DEFAULT suffix + String defaultNameCheck3 = configProperty + "_DEFAULT"; + String defaultValueCheck3 = configurationDefaultVariables + .get(defaultNameCheck3); + + // Pick the default value that exists + if (defaultValueCheck1!=null) { + defaultConfigName = defaultNameCheck1; + defaultConfigValue = defaultValueCheck1; + } else if (defaultValueCheck2!=null) { + defaultConfigName = defaultNameCheck2; + defaultConfigValue = defaultValueCheck2; + } else if (defaultValueCheck3!=null) { + defaultConfigName = defaultNameCheck3; + defaultConfigValue = defaultValueCheck3; + } + + if (defaultConfigValue!=null) { + if (xmlDefaultValue==null) { + xmlPropertiesWithEmptyValue.add(xmlProperty); + } else if (!xmlDefaultValue.equals(defaultConfigValue)) { + HashMap<String,String> xmlEntry = + new HashMap<String,String>(); + xmlEntry.put(xmlProperty,xmlDefaultValue); + HashMap<String,String> configEntry = + new HashMap<String,String>(); + configEntry.put(defaultConfigName,defaultConfigValue); + mismatchingXmlConfig.put(xmlEntry,configEntry); + } else { + xmlPropertiesMatchingConfigDefault + .put(xmlProperty, defaultConfigName); + } + } else { + configPropertiesWithNoDefaultConfig.add(configProperty); + } + } else { + } + } + + // Print out any unknown mismatching XML value/Config default value + System.out.println(this.xmlFilename + " has " + + mismatchingXmlConfig.size() + + " properties that do not match the default Config value"); + if (mismatchingXmlConfig.size()==0) { + System.out.println(" (None)"); + } else { + for (Map.Entry<HashMap<String,String>,HashMap<String,String>> xcEntry : + mismatchingXmlConfig.entrySet()) { + HashMap<String,String> xmlMap = xcEntry.getKey(); + HashMap<String,String> configMap = xcEntry.getValue(); + for (Map.Entry<String,String> xmlEntry : xmlMap.entrySet()) { + System.out.println(" XML Property: " + xmlEntry.getKey()); + System.out.println(" XML Value: " + xmlEntry.getValue()); + } + for (Map.Entry<String,String> configEntry : configMap.entrySet()) { + System.out.println(" Config Name: " + configEntry.getKey()); + System.out.println(" Config Value: " + configEntry.getValue()); + } + System.out.println(""); + } + } + System.out.println(); + + // Print out Config properties that have no corresponding DEFAULT_* + // variable and cannot do any XML comparison (i.e. probably needs to + // be checked by hand) + System.out.println("Configuration(s) have " + + configPropertiesWithNoDefaultConfig.size() + + " properties with no corresponding default member variable. These" + + " will need to be verified manually."); + if (configPropertiesWithNoDefaultConfig.size()==0) { + System.out.println(" (None)"); + } else { + Iterator<String> cItr = configPropertiesWithNoDefaultConfig.iterator(); + while (cItr.hasNext()) { + System.out.println(" " + cItr.next()); + } + } + System.out.println(); + + // MAYBE TODO Print out any known mismatching XML value/Config default + + // Print out XML properties that have empty values (i.e. should result + // in code-based default) + System.out.println(this.xmlFilename + " has " + + xmlPropertiesWithEmptyValue.size() + " properties with empty values"); + if (xmlPropertiesWithEmptyValue.size()==0) { + System.out.println(" (None)"); + } else { + Iterator<String> xItr = xmlPropertiesWithEmptyValue.iterator(); + while (xItr.hasNext()) { + System.out.println(" " + xItr.next()); + } + } + System.out.println(); + + // Print out any matching XML value/Config default value + System.out.println(this.xmlFilename + " has " + + xmlPropertiesMatchingConfigDefault.size() + + " properties which match a corresponding Config variable"); + if (xmlPropertiesMatchingConfigDefault.size()==0) { + System.out.println(" (None)"); + } else { + for (Map.Entry<String,String> xcEntry : + xmlPropertiesMatchingConfigDefault.entrySet()) { + System.out.println(" " + xcEntry.getKey() + " / " + + xcEntry.getValue()); + } + } + System.out.println(); + + // Test separator + System.out.println(); + System.out.println("====="); + System.out.println(); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
