Author: hairong
Date: Tue Nov 23 07:15:49 2010
New Revision: 1038003
URL: http://svn.apache.org/viewvc?rev=1038003&view=rev
Log:
HADOOP-7023. Add listCorruptFileBlocks to FileSystem. Contributed by Patrick
Kling.
Added:
hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java
Modified: hadoop/common/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Tue Nov 23 07:15:49 2010
@@ -6,6 +6,9 @@ Trunk (unreleased changes)
NEW FEATURES
+ HADOOP-7023. Add listCorruptFileBlocks to Filesysem. (Patrick Kling
+ via hairong)
+
IMPROVEMENTS
HADOOP-7042. Updates to test-patch.sh to include failed test names and
Modified:
hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
Tue Nov 23 07:15:49 2010
@@ -835,6 +835,18 @@ public abstract class AbstractFileSystem
UnresolvedLinkException, IOException;
/**
+ * @return a list in which each entry describes a corrupt file/block
+ * @throws IOException
+ */
+ public CorruptFileBlocks listCorruptFileBlocks(String path,
+ String cookie)
+ throws IOException {
+ throw new UnsupportedOperationException(getClass().getCanonicalName() +
+ " does not support" +
+ " listCorruptFileBlocks");
+ }
+
+ /**
* The specification of this method matches that of
* {...@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
* must be for this file system.
Added: hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java?rev=1038003&view=auto
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
(added)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
Tue Nov 23 07:15:49 2010
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.Text;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Contains a list of paths corresponding to corrupt files and a cookie
+ * used for iterative calls to NameNode.listCorruptFileBlocks.
+ *
+ */
+public class CorruptFileBlocks implements Writable {
+ // used for hashCode
+ private static final int PRIME = 16777619;
+
+ private String[] files;
+ private String cookie;
+
+ public CorruptFileBlocks() {
+ this(new String[0], "");
+ }
+
+ public CorruptFileBlocks(String[] files, String cookie) {
+ this.files = files;
+ this.cookie = cookie;
+ }
+
+ public String[] getFiles() {
+ return files;
+ }
+
+ public String getCookie() {
+ return cookie;
+ }
+
+ /**
+ * {...@inheritdoc}
+ */
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ int fileCount = in.readInt();
+ files = new String[fileCount];
+ for (int i = 0; i < fileCount; i++) {
+ files[i] = Text.readString(in);
+ }
+ cookie = Text.readString(in);
+ }
+
+ /**
+ * {...@inheritdoc}
+ */
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(files.length);
+ for (int i = 0; i < files.length; i++) {
+ Text.writeString(out, files[i]);
+ }
+ Text.writeString(out, cookie);
+ }
+
+ /**
+ * {...@inheritdoc}
+ */
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof CorruptFileBlocks)) {
+ return false;
+ }
+ CorruptFileBlocks other = (CorruptFileBlocks) obj;
+ return cookie.equals(other.cookie) &&
+ Arrays.equals(files, other.files);
+ }
+
+ /**
+ * {...@inheritdoc}
+ */
+ public int hashCode() {
+ int result = cookie.hashCode();
+
+ for (String file : files) {
+ result = PRIME * result + file.hashCode();
+ }
+
+ return result;
+ }
+}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java Tue Nov
23 07:15:49 2010
@@ -1296,6 +1296,23 @@ public final class FileContext {
}
}.resolve(this, absF);
}
+
+ /**
+ * @return a list in which each entry describes a corrupt file/block
+ * @throws IOException
+ */
+ public CorruptFileBlocks listCorruptFileBlocks(final String path,
+ final String cookie)
+ throws IOException {
+ final Path absF = fixRelativePart(new Path(path));
+ return new FSLinkResolver<CorruptFileBlocks>() {
+ @Override
+ public CorruptFileBlocks next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.listCorruptFileBlocks(p.toUri().getPath(), cookie);
+ }
+ }.resolve(this, absF);
+ }
/**
* List the statuses of the files/directories in the given path if the path
is
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Tue Nov
23 07:15:49 2010
@@ -1091,6 +1091,18 @@ public abstract class FileSystem extends
}
/**
+ * @return a list in which each entry describes a corrupt file/block
+ * @throws IOException
+ */
+ public CorruptFileBlocks listCorruptFileBlocks(String path,
+ String cookie)
+ throws IOException {
+ throw new UnsupportedOperationException(getClass().getCanonicalName() +
+ " does not support" +
+ " listCorruptFileBlocks");
+ }
+
+ /**
* Filter files/directories in the given path using the user-supplied path
* filter.
*
Modified:
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java Tue
Nov 23 07:15:49 2010
@@ -165,7 +165,17 @@ public class FilterFileSystem extends Fi
public FileStatus[] listStatus(Path f) throws IOException {
return fs.listStatus(f);
}
-
+
+ /**
+ * {...@inheritdoc}
+ */
+ @Override
+ public CorruptFileBlocks listCorruptFileBlocks(String path,
+ String cookie)
+ throws IOException {
+ return fs.listCorruptFileBlocks(path, cookie);
+ }
+
/** List files and its block locations in a directory. */
public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
throws IOException {
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java Tue Nov 23
07:15:49 2010
@@ -164,6 +164,16 @@ public abstract class FilterFs extends A
return myFs.listStatus(f);
}
+ /**
+ * {...@inheritdoc}
+ */
+ @Override
+ public CorruptFileBlocks listCorruptFileBlocks(String path,
+ String cookie)
+ throws IOException {
+ return myFs.listCorruptFileBlocks(path, cookie);
+ }
+
@Override
public void mkdir(Path dir, FsPermission permission, boolean createParent)
throws IOException, UnresolvedLinkException {
Added:
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java?rev=1038003&view=auto
==============================================================================
---
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
(added)
+++
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
Tue Nov 23 07:15:49 2010
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.*;
+import org.junit.Test;
+
+import org.apache.hadoop.io.DataOutputBuffer;
+
+public class TestCorruptFileBlocks {
+
+ /**
+ * Serialize the cfb given, deserialize and return the result.
+ */
+ static CorruptFileBlocks serializeAndDeserialize(CorruptFileBlocks cfb)
+ throws IOException {
+ DataOutputBuffer buf = new DataOutputBuffer();
+ cfb.write(buf);
+
+ byte[] data = buf.getData();
+ DataInputStream input = new DataInputStream(new
ByteArrayInputStream(data));
+
+ CorruptFileBlocks result = new CorruptFileBlocks();
+ result.readFields(input);
+
+ return result;
+ }
+
+ /**
+ * Check whether cfb is unchanged after serialization and deserialization.
+ */
+ static boolean checkSerialize(CorruptFileBlocks cfb)
+ throws IOException {
+ return cfb.equals(serializeAndDeserialize(cfb));
+ }
+
+ /**
+ * Test serialization and deserializaton of CorruptFileBlocks.
+ */
+ @Test
+ public void testSerialization() throws IOException {
+ {
+ CorruptFileBlocks cfb = new CorruptFileBlocks();
+ assertTrue(checkSerialize(cfb));
+ }
+
+ {
+ String[] files = new String[0];
+ CorruptFileBlocks cfb = new CorruptFileBlocks(files, "");
+ assertTrue(checkSerialize(cfb));
+ }
+
+ {
+ String[] files = { "a", "bb", "ccc" };
+ CorruptFileBlocks cfb = new CorruptFileBlocks(files, "test");
+ assertTrue(checkSerialize(cfb));
+ }
+ }
+}
\ No newline at end of file
