Added: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/FSDatasetTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/FSDatasetTestUtil.java?rev=1164719&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/FSDatasetTestUtil.java (added) +++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/FSDatasetTestUtil.java Fri Sep 2 21:38:27 2011 @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode; + +import java.io.IOException; +import java.io.File; +import org.apache.hadoop.hdfs.protocol.Block; + +public abstract class FSDatasetTestUtil { + + /** + * Truncate the given block in place, such that the new truncated block + * is still valid (ie checksums are updated to stay in sync with block file) + */ + public static void truncateBlock(DataNode dn, + Block block, + long newLength) + throws IOException + { + FSDataset ds = (FSDataset)dn.data; + + File blockFile = ds.findBlockFile(block.getBlockId()); + if (blockFile == null) { + throw new IOException("Can't find block file for block " + + block + " on DN " + dn); + } + File metaFile = ds.findMetaFile(blockFile); + FSDataset.truncateBlock(blockFile, metaFile, + block.getNumBytes(), newLength); + } + +} \ No newline at end of file
Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1164719&r1=1164718&r2=1164719&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original) +++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Fri Sep 2 21:38:27 2011 @@ -265,7 +265,18 @@ public class SimulatedFSDataset impleme } } - public synchronized void finalizeBlock(Block b) throws IOException { + @Override + public void finalizeBlock(Block b) throws IOException { + finalizeBlockInternal(b, false); + } + + @Override + public void finalizeBlockIfNeeded(Block b) throws IOException { + finalizeBlockInternal(b, true); + } + + private synchronized void finalizeBlockInternal(Block b, boolean refinalizeOk) + throws IOException { BInfo binfo = blockMap.get(b); if (binfo == null) { throw new IOException("Finalizing a non existing block " + b); @@ -381,7 +392,8 @@ public class SimulatedFSDataset impleme } public synchronized BlockWriteStreams writeToBlock(Block b, - boolean isRecovery) + boolean isRecovery, + boolean isReplicationRequest) throws IOException { if (isValidBlock(b)) { throw new BlockAlreadyExistsException("Block " + b + Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java?rev=1164719&r1=1164718&r2=1164719&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java (original) +++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Fri Sep 2 21:38:27 2011 @@ -57,8 +57,8 @@ public class TestDiskError extends TestC FileSystem fs = cluster.getFileSystem(); final int dnIndex = 0; String dataDir = cluster.getDataDirectory(); - File dir1 = new File(new File(dataDir, "data"+(2*dnIndex+1)), "tmp"); - File dir2 = new File(new File(dataDir, "data"+(2*dnIndex+2)), "tmp"); + File dir1 = new File(new File(dataDir, "data"+(2*dnIndex+1)), "blocksBeingWritten"); + File dir2 = new File(new File(dataDir, "data"+(2*dnIndex+2)), "blocksBeingWritten"); try { // make the data directory of the first datanode to be readonly assertTrue(dir1.setReadOnly()); Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java?rev=1164719&r1=1164718&r2=1164719&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java (original) +++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java Fri Sep 2 21:38:27 2011 @@ -62,7 +62,7 @@ public class TestSimulatedFSDataset exte int bytesAdded = 0; for (int i = startingBlockId; i < startingBlockId+NUMBLOCKS; ++i) { Block b = new Block(i, 0, 0); // we pass expected len as zero, - fsdataset should use the sizeof actual data written - OutputStream dataOut = fsdataset.writeToBlock(b, false).dataOut; + OutputStream dataOut = fsdataset.writeToBlock(b, false, false).dataOut; assertEquals(0, fsdataset.getLength(b)); for (int j=1; j <= blockIdToLen(i); ++j) { dataOut.write(j); Added: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/FSImageAdapter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/FSImageAdapter.java?rev=1164719&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/FSImageAdapter.java (added) +++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/FSImageAdapter.java Fri Sep 2 21:38:27 2011 @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.mockito.Mockito.spy; + +public abstract class FSImageAdapter { + public static FSEditLog injectEditLogSpy(FSNamesystem ns) { + FSImage image = ns.getFSImage(); + image.editLog = spy(image.editLog); + return image.editLog; + } +} \ No newline at end of file Added: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestDFSConcurrentFileOperations.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestDFSConcurrentFileOperations.java?rev=1164719&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestDFSConcurrentFileOperations.java (added) +++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestDFSConcurrentFileOperations.java Fri Sep 2 21:38:27 2011 @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; + +import java.io.IOException; + +public class TestDFSConcurrentFileOperations extends TestCase { + + MiniDFSCluster cluster; + FileSystem fs; + private int writeSize; + private long blockSize; + + @Override + protected void setUp() throws Exception { + super.setUp(); + writeSize = 64 * 1024; + blockSize = 2 * writeSize; + } + + private void init() throws IOException { + init(new Configuration()); + } + + private void init(Configuration conf) throws IOException { + cluster = new MiniDFSCluster(conf, 3, true, new String[]{"/rack1", "/rack2", "/rack1"}); + cluster.waitClusterUp(); + fs = cluster.getFileSystem(); + } + + @Override + protected void tearDown() throws Exception { + fs.close(); + cluster.shutdown(); + super.tearDown(); + } + + /* + * test case: + * 1. file is opened + * 2. file is moved while being written to (including move to trash on delete) + * 3. blocks complete and are finalized + * 4. close fails + * 5. lease recovery tries to finalize blocks and should succeed + */ + public void testLeaseRecoveryOnTrashedFile() throws Exception { + Configuration conf = new Configuration(); + + conf.setLong("dfs.block.size", blockSize); + + init(conf); + + String src = "/file-1"; + String dst = "/file-2"; + Path srcPath = new Path(src); + Path dstPath = new Path(dst); + FSDataOutputStream fos = fs.create(srcPath); + + fos.write(DFSTestUtil.generateSequentialBytes(0, writeSize)); + fos.sync(); + + LocatedBlocks blocks; + int i = 0; + do { + blocks = cluster + .getNameNode() + .getNamesystem() + .getBlockLocations(src, 0, writeSize); + } while (blocks.getLocatedBlocks().isEmpty() && ++i < 1000); + + assertTrue("failed to get block for file", i < 1000); + + Block block = blocks.get(blocks.getLocatedBlocks().size()-1).getBlock(); + + // renaming a file out from under a client will cause close to fail + // and result in the lease remaining while the blocks are finalized on + // the DNs + fs.rename(srcPath, dstPath); + + try { + fos.close(); + fail("expected IOException"); + } catch (IOException e) { + //expected + } + + // simulate what lease recovery does--tries to update block and finalize + cluster.getDataNodes().get(0).updateBlock(block, block, true); + } +}
