Author: omalley
Date: Fri Mar 4 03:28:08 2011
New Revision: 1076985
URL: http://svn.apache.org/viewvc?rev=1076985&view=rev
Log:
commit 40aaaa675ec4fae57eff339a849d8a593eb65f32
Author: Yahoo\! <[email protected]>
Date: Mon Aug 24 09:07:20 2009 -0700
HADOOP-6184 from patch
http://issues.apache.org/jira/secure/attachment/12417472/HADOOP-6184-ydist-1.patch
Modified:
hadoop/common/branches/branch-0.20-security-patches/ivy.xml
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/conf/TestConfiguration.java
Modified: hadoop/common/branches/branch-0.20-security-patches/ivy.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/ivy.xml?rev=1076985&r1=1076984&r2=1076985&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/ivy.xml (original)
+++ hadoop/common/branches/branch-0.20-security-patches/ivy.xml Fri Mar 4
03:28:08 2011
@@ -256,6 +256,10 @@
rev="${slf4j-log4j12.version}"
conf="common->master">
</dependency>
+ <dependency org="org.codehaus.jackson"
+ name="jackson-mapper-asl"
+ rev="1.0.1"
+ conf="common->default"/>
</dependencies>
</ivy-module>
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/conf/Configuration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/conf/Configuration.java?rev=1076985&r1=1076984&r2=1076985&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/conf/Configuration.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/conf/Configuration.java
Fri Mar 4 03:28:08 2011
@@ -28,6 +28,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
+import java.io.Writer;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
@@ -60,6 +61,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.util.StringUtils;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@@ -153,7 +156,7 @@ public class Configuration implements It
private boolean loadDefaults = true;
/**
- * Configurtion objects
+ * Configuration objects
*/
private static final WeakHashMap<Configuration,Object> REGISTRY =
new WeakHashMap<Configuration,Object>();
@@ -165,6 +168,18 @@ public class Configuration implements It
private static final ArrayList<String> defaultResources =
new ArrayList<String>();
+ /**
+ * Flag to indicate if the storage of resource which updates a key needs
+ * to be stored for each key
+ */
+ private boolean storeResource;
+
+ /**
+ * Stores the mapping of key to the resource which modifies or loads
+ * the key most recently
+ */
+ private HashMap<String, String> updatingResource;
+
static{
//print deprecation warning if hadoop-site.xml is found in classpath
ClassLoader cL = Thread.currentThread().getContextClassLoader();
@@ -212,6 +227,24 @@ public class Configuration implements It
synchronized(Configuration.class) {
REGISTRY.put(this, null);
}
+ this.storeResource = false;
+ }
+
+ /**
+ * A new configuration with the same settings and additional facility for
+ * storage of resource to each key which loads or updates
+ * the key most recently
+ * @param other the configuration from which to clone settings
+ * @param storeResource flag to indicate if the storage of resource to
+ * each key is to be stored
+ */
+ private Configuration(Configuration other, boolean storeResource) {
+ this(other);
+ this.loadDefaults = other.loadDefaults;
+ this.storeResource = storeResource;
+ if (storeResource) {
+ updatingResource = new HashMap<String, String>();
+ }
}
/**
@@ -977,8 +1010,14 @@ public class Configuration implements It
if (properties == null) {
properties = new Properties();
loadResources(properties, resources, quietmode);
- if (overlay!= null)
+ if (overlay!= null) {
properties.putAll(overlay);
+ if (storeResource) {
+ for (Map.Entry<Object,Object> item: overlay.entrySet()) {
+ updatingResource.put((String) item.getKey(), "Unknown");
+ }
+ }
+ }
}
return properties;
}
@@ -1147,6 +1186,9 @@ public class Configuration implements It
if (attr != null && value != null) {
if (!finalParameters.contains(attr)) {
properties.setProperty(attr, value);
+ if (storeResource) {
+ updatingResource.put(attr, name.toString());
+ }
if (finalParameter)
finalParameters.add(attr);
} else {
@@ -1219,6 +1261,43 @@ public class Configuration implements It
}
/**
+ * Writes out all the parameters and their properties (final and resource)
to
+ * the given {@link Writer}
+ * The format of the output would be
+ * { "properties" : [ {key1,value1,key1.isFinal,key1.resource},
{key2,value2,
+ * key2.isFinal,key2.resource}... ] }
+ * It does not output the parameters of the configuration object which is
+ * loaded from an input stream.
+ * @param out the Writer to write to
+ * @throws IOException
+ */
+ public static void dumpConfiguration(Configuration conf,
+ Writer out) throws IOException {
+ Configuration config = new Configuration(conf,true);
+ config.reloadConfiguration();
+ JsonFactory dumpFactory = new JsonFactory();
+ JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
+ dumpGenerator.writeStartObject();
+ dumpGenerator.writeFieldName("properties");
+ dumpGenerator.writeStartArray();
+ dumpGenerator.flush();
+ for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
+ dumpGenerator.writeStartObject();
+ dumpGenerator.writeStringField("key", (String) item.getKey());
+ dumpGenerator.writeStringField("value",
+ config.get((String) item.getKey()));
+ dumpGenerator.writeBooleanField("isFinal",
+ config.finalParameters.contains(item.getKey()));
+ dumpGenerator.writeStringField("resource",
+ config.updatingResource.get(item.getKey()));
+ dumpGenerator.writeEndObject();
+ }
+ dumpGenerator.writeEndArray();
+ dumpGenerator.writeEndObject();
+ dumpGenerator.flush();
+ }
+
+ /**
* Get the {@link ClassLoader} for this job.
*
* @return the correct class loader.
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/conf/TestConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/conf/TestConfiguration.java?rev=1076985&r1=1076984&r2=1076985&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/conf/TestConfiguration.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/conf/TestConfiguration.java
Fri Mar 4 03:28:08 2011
@@ -25,12 +25,15 @@ import java.io.DataInputStream;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.io.DataOutputStream;
+import java.io.StringWriter;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Random;
import junit.framework.TestCase;
import org.apache.hadoop.fs.Path;
+import org.codehaus.jackson.map.ObjectMapper;
public class TestConfiguration extends TestCase {
@@ -389,4 +392,181 @@ public class TestConfiguration extends T
TestConfiguration.class.getName()
});
}
+
+ static class JsonConfiguration {
+ JsonProperty[] properties;
+
+ public JsonProperty[] getProperties() {
+ return properties;
+ }
+
+ public void setProperties(JsonProperty[] properties) {
+ this.properties = properties;
+ }
+ }
+
+ static class JsonProperty {
+ String key;
+ public String getKey() {
+ return key;
+ }
+ public void setKey(String key) {
+ this.key = key;
+ }
+ public String getValue() {
+ return value;
+ }
+ public void setValue(String value) {
+ this.value = value;
+ }
+ public boolean getIsFinal() {
+ return isFinal;
+ }
+ public void setIsFinal(boolean isFinal) {
+ this.isFinal = isFinal;
+ }
+ public String getResource() {
+ return resource;
+ }
+ public void setResource(String resource) {
+ this.resource = resource;
+ }
+ String value;
+ boolean isFinal;
+ String resource;
+ }
+
+ public void testDumpConfiguration () throws IOException {
+ StringWriter outWriter = new StringWriter();
+ Configuration.dumpConfiguration(conf, outWriter);
+ String jsonStr = outWriter.toString();
+ ObjectMapper mapper = new ObjectMapper();
+ JsonConfiguration jconf =
+ mapper.readValue(jsonStr, JsonConfiguration.class);
+ int defaultLength = jconf.getProperties().length;
+
+ // add 3 keys to the existing configuration properties
+ out=new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.key1", "value1");
+ appendProperty("test.key2", "value2",true);
+ appendProperty("test.key3", "value3");
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ conf.addResource(fileResource);
+ out.close();
+
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(conf, outWriter);
+ jsonStr = outWriter.toString();
+ mapper = new ObjectMapper();
+ jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+ int length = jconf.getProperties().length;
+ // check for consistency in the number of properties parsed in Json format.
+ assertEquals(length, defaultLength+3);
+
+ //change few keys in another resource file
+ out=new BufferedWriter(new FileWriter(CONFIG2));
+ startConfig();
+ appendProperty("test.key1", "newValue1");
+ appendProperty("test.key2", "newValue2");
+ endConfig();
+ Path fileResource1 = new Path(CONFIG2);
+ conf.addResource(fileResource1);
+ out.close();
+
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(conf, outWriter);
+ jsonStr = outWriter.toString();
+ mapper = new ObjectMapper();
+ jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+
+ // put the keys and their corresponding attributes into a hashmap for
their
+ // efficient retrieval
+ HashMap<String,JsonProperty> confDump = new HashMap<String,JsonProperty>();
+ for(JsonProperty prop : jconf.getProperties()) {
+ confDump.put(prop.getKey(), prop);
+ }
+ // check if the value and resource of test.key1 is changed
+ assertEquals("newValue1", confDump.get("test.key1").getValue());
+ assertEquals(false, confDump.get("test.key1").getIsFinal());
+ assertEquals(fileResource1.toString(),
+ confDump.get("test.key1").getResource());
+ // check if final parameter test.key2 is not changed, since it is first
+ // loaded as final parameter
+ assertEquals("value2", confDump.get("test.key2").getValue());
+ assertEquals(true, confDump.get("test.key2").getIsFinal());
+ assertEquals(fileResource.toString(),
+ confDump.get("test.key2").getResource());
+ // check for other keys which are not modified later
+ assertEquals("value3", confDump.get("test.key3").getValue());
+ assertEquals(false, confDump.get("test.key3").getIsFinal());
+ assertEquals(fileResource.toString(),
+ confDump.get("test.key3").getResource());
+ // check for resource to be "Unknown" for keys which are loaded using
'set'
+ // and expansion of properties
+ conf.set("test.key4", "value4");
+ conf.set("test.key5", "value5");
+ conf.set("test.key6", "${test.key5}");
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(conf, outWriter);
+ jsonStr = outWriter.toString();
+ mapper = new ObjectMapper();
+ jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+ confDump = new HashMap<String, JsonProperty>();
+ for(JsonProperty prop : jconf.getProperties()) {
+ confDump.put(prop.getKey(), prop);
+ }
+ assertEquals("value5",confDump.get("test.key6").getValue());
+ assertEquals("Unknown", confDump.get("test.key4").getResource());
+ outWriter.close();
+ }
+
+ public void testDumpConfiguratioWithoutDefaults() throws IOException {
+ // check for case when default resources are not loaded
+ Configuration config = new Configuration(false);
+ StringWriter outWriter = new StringWriter();
+ Configuration.dumpConfiguration(config, outWriter);
+ String jsonStr = outWriter.toString();
+ ObjectMapper mapper = new ObjectMapper();
+ JsonConfiguration jconf =
+ mapper.readValue(jsonStr, JsonConfiguration.class);
+
+ //ensure that no properties are loaded.
+ assertEquals(0, jconf.getProperties().length);
+
+ // add 2 keys
+ out=new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.key1", "value1");
+ appendProperty("test.key2", "value2",true);
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ config.addResource(fileResource);
+ out.close();
+
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(config, outWriter);
+ jsonStr = outWriter.toString();
+ mapper = new ObjectMapper();
+ jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+
+ HashMap<String, JsonProperty>confDump = new HashMap<String,
JsonProperty>();
+ for (JsonProperty prop : jconf.getProperties()) {
+ confDump.put(prop.getKey(), prop);
+ }
+ //ensure only 2 keys are loaded
+ assertEquals(2,jconf.getProperties().length);
+ //ensure the values are consistent
+ assertEquals(confDump.get("test.key1").getValue(),"value1");
+ assertEquals(confDump.get("test.key2").getValue(),"value2");
+ //check the final tag
+ assertEquals(false, confDump.get("test.key1").getIsFinal());
+ assertEquals(true, confDump.get("test.key2").getIsFinal());
+ //check the resource for each property
+ for (JsonProperty prop : jconf.getProperties()) {
+ assertEquals(fileResource.toString(),prop.getResource());
+ }
+ }
}
+
