Author: szetszwo
Date: Tue Sep 15 22:18:21 2009
New Revision: 815524
URL: http://svn.apache.org/viewvc?rev=815524&view=rev
Log:
HADOOP-6216. Support comments in host files. Contributed by Ravi Phulari and
Dmytro Molkov
Added:
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestHostsFileReader.java
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/util/HostsFileReader.java
Modified: hadoop/common/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=815524&r1=815523&r2=815524&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Tue Sep 15 22:18:21 2009
@@ -556,6 +556,9 @@
HADOOP-5879. Read compression level and strategy from Configuration for
gzip compression. (He Yongqiang via cdouglas)
+ HADOOP-6216. Support comments in host files. (Ravi Phulari and Dmytro
+ Molkov via szetszwo)
+
OPTIMIZATIONS
HADOOP-5595. NameNode does not need to run a replicator to choose a
Modified:
hadoop/common/trunk/src/java/org/apache/hadoop/util/HostsFileReader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/util/HostsFileReader.java?rev=815524&r1=815523&r2=815524&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/util/HostsFileReader.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/util/HostsFileReader.java
Tue Sep 15 22:18:21 2009
@@ -58,7 +58,12 @@
String[] nodes = line.split("[ \t\n\f\r]+");
if (nodes != null) {
for (int i = 0; i < nodes.length; i++) {
+ if (nodes[i].trim().startsWith("#")) {
+ // Everything from now on is a comment
+ break;
+ }
if (!nodes[i].equals("")) {
+ LOG.info("Adding " + nodes[i] + " to the list of hosts from " +
filename);
set.add(nodes[i]); // might need to add canonical name
}
}
Added:
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestHostsFileReader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestHostsFileReader.java?rev=815524&view=auto
==============================================================================
---
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestHostsFileReader.java
(added)
+++
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestHostsFileReader.java
Tue Sep 15 22:18:21 2009
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.util.Set;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+/*
+ * Test for HostsFileReader.java
+ *
+ */
+public class TestHostsFileReader {
+
+ // Using /test/build/data/tmp directory to store temprory files
+ final String HOSTS_TEST_DIR = new File(System.getProperty(
+ "test.build.data", "/tmp")).getAbsolutePath();
+ File EXCLUDES_FILE = new File(HOSTS_TEST_DIR, "dfs.exclude");
+ File INCLUDES_FILE = new File(HOSTS_TEST_DIR, "dfs.include");
+ String excludesFile = HOSTS_TEST_DIR + "/dfs.exclude";
+ String includesFile = HOSTS_TEST_DIR + "/dfs.include";
+ private Set<String> includes;
+ private Set<String> excludes;
+
+ @Before
+ public void setUp() throws Exception {
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ // Delete test files after running tests
+ EXCLUDES_FILE.delete();
+ INCLUDES_FILE.delete();
+
+ }
+
+ /*
+ * 1.Create dfs.exclude,dfs.include file
+ * 2.Write host names per line
+ * 3.Write comments starting with #
+ * 4.Close file
+ * 5.Compare if number of hosts reported by HostsFileReader
+ * are equal to the number of hosts written
+ */
+ @Test
+ public void testHostsFileReader() throws Exception {
+
+ FileWriter efw = new FileWriter(excludesFile);
+ FileWriter ifw = new FileWriter(includesFile);
+
+ efw.write("#DFS-Hosts-excluded\n");
+ efw.write("somehost1\n");
+ efw.write("#This-is-comment\n");
+ efw.write("somehost2\n");
+ efw.write("somehost3 # host3\n");
+ efw.write("somehost4\n");
+ efw.write("somehost4 somehost5\n");
+ efw.close();
+
+ ifw.write("#Hosts-in-DFS\n");
+ ifw.write("somehost1\n");
+ ifw.write("somehost2\n");
+ ifw.write("somehost3\n");
+ ifw.write("#This-is-comment\n");
+ ifw.write("somehost4 # host4\n");
+ ifw.write("somehost4 somehost5\n");
+ ifw.close();
+
+ HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile);
+
+ int includesLen = hfp.getHosts().size();
+ int excludesLen = hfp.getExcludedHosts().size();
+
+ assertEquals(5, includesLen);
+ assertEquals(5, excludesLen);
+
+ assertTrue(hfp.getHosts().contains("somehost5"));
+ assertFalse(hfp.getHosts().contains("host3"));
+
+ assertTrue(hfp.getExcludedHosts().contains("somehost5"));
+ assertFalse(hfp.getExcludedHosts().contains("host4"));
+
+ }
+
+ /*
+ * Test for null file
+ */
+ @Test
+ public void testHostFileReaderWithNull() throws Exception {
+ FileWriter efw = new FileWriter(excludesFile);
+ FileWriter ifw = new FileWriter(includesFile);
+
+ efw.close();
+
+ ifw.close();
+
+ HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile);
+
+ int includesLen = hfp.getHosts().size();
+ int excludesLen = hfp.getExcludedHosts().size();
+
+ // TestCase1: Check if lines beginning with # are ignored
+ assertEquals(0, includesLen);
+ assertEquals(0, excludesLen);
+
+ // TestCase2: Check if given host names are reported by getHosts and
+ // getExcludedHosts
+ assertFalse(hfp.getHosts().contains("somehost5"));
+
+ assertFalse(hfp.getExcludedHosts().contains("somehost5"));
+ }
+
+ /*
+ * Check if only comments can be written to hosts file
+ */
+ @Test
+ public void testHostFileReaderWithCommentsOnly() throws Exception {
+ FileWriter efw = new FileWriter(excludesFile);
+ FileWriter ifw = new FileWriter(includesFile);
+
+ efw.write("#DFS-Hosts-excluded\n");
+ efw.close();
+
+ ifw.write("#Hosts-in-DFS\n");
+ ifw.close();
+
+ HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile);
+
+ int includesLen = hfp.getHosts().size();
+ int excludesLen = hfp.getExcludedHosts().size();
+
+ assertEquals(0, includesLen);
+ assertEquals(0, excludesLen);
+
+ assertFalse(hfp.getHosts().contains("somehost5"));
+
+ assertFalse(hfp.getExcludedHosts().contains("somehost5"));
+
+ }
+
+ /*
+ * Test if spaces are allowed in host names
+ */
+ @Test
+ public void testHostFileReaderWithSpaces() throws Exception {
+ FileWriter efw = new FileWriter(excludesFile);
+ FileWriter ifw = new FileWriter(includesFile);
+
+ efw.write("#DFS-Hosts-excluded\n");
+ efw.write(" somehost somehost2");
+ efw.write(" somehost3 # somehost4");
+ efw.close();
+
+ ifw.write("#Hosts-in-DFS\n");
+ ifw.write(" somehost somehost2");
+ ifw.write(" somehost3 # somehost4");
+ ifw.close();
+
+ HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile);
+
+ int includesLen = hfp.getHosts().size();
+ int excludesLen = hfp.getExcludedHosts().size();
+
+ assertEquals(3, includesLen);
+ assertEquals(3, excludesLen);
+
+ assertTrue(hfp.getHosts().contains("somehost3"));
+ assertFalse(hfp.getHosts().contains("somehost5"));
+ assertFalse(hfp.getHosts().contains("somehost4"));
+
+ assertTrue(hfp.getExcludedHosts().contains("somehost3"));
+ assertFalse(hfp.getExcludedHosts().contains("somehost5"));
+ assertFalse(hfp.getExcludedHosts().contains("somehost4"));
+
+ }
+
+ /*
+ * Test if spaces , tabs and new lines are allowed
+ */
+ @Test
+ public void testHostFileReaderWithTabs() throws Exception {
+ FileWriter efw = new FileWriter(excludesFile);
+ FileWriter ifw = new FileWriter(includesFile);
+
+ efw.write("#DFS-Hosts-excluded\n");
+ efw.write(" \n");
+ efw.write(" somehost \t somehost2 \n somehost4");
+ efw.write(" somehost3 \t # somehost5");
+ efw.close();
+
+ ifw.write("#Hosts-in-DFS\n");
+ ifw.write(" \n");
+ ifw.write(" somehost \t somehost2 \n somehost4");
+ ifw.write(" somehost3 \t # somehost5");
+ ifw.close();
+
+ HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile);
+
+ int includesLen = hfp.getHosts().size();
+ int excludesLen = hfp.getExcludedHosts().size();
+
+ assertEquals(4, includesLen);
+ assertEquals(4, excludesLen);
+
+ assertTrue(hfp.getHosts().contains("somehost2"));
+ assertFalse(hfp.getHosts().contains("somehost5"));
+
+ assertTrue(hfp.getExcludedHosts().contains("somehost2"));
+ assertFalse(hfp.getExcludedHosts().contains("somehost5"));
+
+ }
+}
