Author: jing9 Date: Tue Jan 21 02:08:36 2014 New Revision: 1559904 URL: http://svn.apache.org/r1559904 Log: Merging r1558604 through r1559901 from trunk.
Added: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutFlags.java - copied unchanged from r1559901, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutFlags.java Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed) hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed) hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed) hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed) hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed) hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1558604-1559901 Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Jan 21 02:08:36 2014 @@ -484,6 +484,9 @@ Trunk (Unreleased) HDFS-5775. Consolidate the code for serialization in CacheManager (Haohui Mai via brandonli) + HDFS-5794. Fix the inconsistency of layout version number of + ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9) + Release 2.4.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -518,6 +521,9 @@ Release 2.4.0 - UNRELEASED as a collection of storages (see breakdown of tasks below for features and contributors). + HDFS-5784. reserve space in edit log header and fsimage header for feature + flag section (cmccabe) + IMPROVEMENTS HDFS-5267. Remove volatile from LightWeightHashSet. (Junping Du via llu) @@ -687,6 +693,8 @@ Release 2.4.0 - UNRELEASED InvalidEncryptionKeyException in fetchBlockByteRange (Liang Xie via Colin Patrick McCabe) + HDFS-5704. Change OP_UPDATE_BLOCKS with a new OP_ADD_BLOCK. (jing9) + OPTIMIZATIONS HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) @@ -765,6 +773,11 @@ Release 2.4.0 - UNRELEASED HDFS-5579. Under construction files make DataNode decommission take very long hours. (zhaoyunjiong via jing9) + HDFS-5777. Update LayoutVersion for the new editlog op OP_ADD_BLOCK. (jing9) + + HDFS-5800. Fix a typo in DFSClient.renewLease(). (Kousuke Saruta + via szetszwo) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report @@ -940,8 +953,6 @@ Release 2.3.0 - UNRELEASED HDFS-5677. Need error checking for HA cluster configuration. (Vincent Sheffer via cos) - HDFS-5704. Change OP_UPDATE_BLOCKS with a new OP_ADD_BLOCK. (jing9) - OPTIMIZATIONS BUG FIXES Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1558604-1559901 Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Tue Jan 21 02:08:36 2014 @@ -770,7 +770,7 @@ public class DFSClient implements java.i final long elapsed = Time.now() - getLastLeaseRenewal(); if (elapsed > HdfsConstants.LEASE_HARDLIMIT_PERIOD) { LOG.warn("Failed to renew lease for " + clientName + " for " - + (elapsed/1000) + " seconds (>= soft-limit =" + + (elapsed/1000) + " seconds (>= hard-limit =" + (HdfsConstants.LEASE_HARDLIMIT_PERIOD/1000) + " seconds.) " + "Closing all files being written ...", e); closeAllFilesBeingWritten(true); Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java Tue Jan 21 02:08:36 2014 @@ -107,12 +107,14 @@ public class LayoutVersion { "block IDs in the edits log and image files"), EDITLOG_SUPPORT_RETRYCACHE(-47, "Record ClientId and CallId in editlog to " + "enable rebuilding retry cache in case of HA failover"), - CACHING(-48, "Support for cache pools and path-based caching"), + EDITLOG_ADD_BLOCK(-48, "Add new editlog that only records allocation of " + + "the new block instead of the entire block list"), ADD_DATANODE_AND_STORAGE_UUIDS(-49, "Replace StorageID with DatanodeUuid." + " Use distinct StorageUuid per storage directory."), - PROTOBUF_FORMAT(-50, "Use protobuf to serialize FSImage"); + ADD_LAYOUT_FLAGS(-50, "Add support for layout flags."), + CACHING(-51, "Support for cache pools and path-based caching"), + PROTOBUF_FORMAT(-52, "Use protobuf to serialize FSImage"); - final int lv; final int ancestorLV; final String description; Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java Tue Jan 21 02:08:36 2014 @@ -34,6 +34,9 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.LayoutFlags; +import org.apache.hadoop.hdfs.protocol.LayoutVersion; +import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.namenode.TransferFsImage.HttpGetFailedException; import org.apache.hadoop.hdfs.web.URLConnectionFactory; @@ -146,6 +149,14 @@ public class EditLogFileInputStream exte } catch (EOFException eofe) { throw new LogHeaderCorruptException("No header found in log"); } + if (LayoutVersion.supports(Feature.ADD_LAYOUT_FLAGS, logVersion)) { + try { + LayoutFlags.read(dataIn); + } catch (EOFException eofe) { + throw new LogHeaderCorruptException("EOF while reading layout " + + "flags from log"); + } + } reader = new FSEditLogOp.Reader(dataIn, tracker, logVersion); reader.setMaxOpSize(maxOpSize); state = State.OPEN; Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java Tue Jan 21 02:08:36 2014 @@ -32,6 +32,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.io.IOUtils; import com.google.common.annotations.VisibleForTesting; @@ -132,6 +133,7 @@ public class EditLogFileOutputStream ext @VisibleForTesting public static void writeHeader(DataOutputStream out) throws IOException { out.writeInt(HdfsConstants.LAYOUT_VERSION); + LayoutFlags.write(out); } @Override Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java Tue Jan 21 02:08:36 2014 @@ -60,13 +60,13 @@ public enum FSEditLogOpCodes { OP_DISALLOW_SNAPSHOT ((byte) 30), OP_SET_GENSTAMP_V2 ((byte) 31), OP_ALLOCATE_BLOCK_ID ((byte) 32), - OP_ADD_CACHE_DIRECTIVE ((byte) 33), - OP_REMOVE_CACHE_DIRECTIVE ((byte) 34), - OP_ADD_CACHE_POOL ((byte) 35), - OP_MODIFY_CACHE_POOL ((byte) 36), - OP_REMOVE_CACHE_POOL ((byte) 37), - OP_MODIFY_CACHE_DIRECTIVE ((byte) 38), - OP_ADD_BLOCK ((byte) 39), + OP_ADD_BLOCK ((byte) 33), + OP_ADD_CACHE_DIRECTIVE ((byte) 34), + OP_REMOVE_CACHE_DIRECTIVE ((byte) 35), + OP_ADD_CACHE_POOL ((byte) 36), + OP_MODIFY_CACHE_POOL ((byte) 37), + OP_REMOVE_CACHE_POOL ((byte) 38), + OP_MODIFY_CACHE_DIRECTIVE ((byte) 39), // Note that fromByte(..) depends on OP_INVALID being at the last position. OP_INVALID ((byte) -1); Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Tue Jan 21 02:08:36 2014 @@ -48,6 +48,7 @@ import org.apache.hadoop.fs.permission.P import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; +import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; @@ -324,6 +325,9 @@ public class FSImageFormat { } boolean supportSnapshot = LayoutVersion.supports(Feature.SNAPSHOT, imgVersion); + if (LayoutVersion.supports(Feature.ADD_LAYOUT_FLAGS, imgVersion)) { + LayoutFlags.read(in); + } // read namespaceID: first appeared in version -2 in.readInt(); @@ -1053,6 +1057,7 @@ public class FSImageFormat { DataOutputStream out = new DataOutputStream(fos); try { out.writeInt(HdfsConstants.LAYOUT_VERSION); + LayoutFlags.write(out); // We use the non-locked version of getNamespaceInfo here since // the coordinating thread of saveNamespace already has read-locked // the namespace for us. If we attempt to take another readlock Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Tue Jan 21 02:08:36 2014 @@ -28,6 +28,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; +import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -126,7 +127,7 @@ class ImageLoaderCurrent implements Imag new SimpleDateFormat("yyyy-MM-dd HH:mm"); private static int[] versions = { -16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36, -37, -38, -39, - -40, -41, -42, -43, -44, -45, -46, -47, -48, -49, -50 }; + -40, -41, -42, -43, -44, -45, -46, -47, -48, -49, -50, -51, -52 }; private int imageVersion = 0; private final Map<Long, Boolean> subtreeMap = new HashMap<Long, Boolean>(); @@ -157,6 +158,9 @@ class ImageLoaderCurrent implements Imag imageVersion = in.readInt(); if( !canLoadVersion(imageVersion)) throw new IOException("Cannot process fslayout version " + imageVersion); + if (LayoutVersion.supports(Feature.ADD_LAYOUT_FLAGS, imageVersion)) { + LayoutFlags.read(in); + } v.visit(ImageElement.IMAGE_VERSION, imageVersion); v.visit(ImageElement.NAMESPACE_ID, in.readInt()); Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1558604-1559901 Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1558604-1559901 Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1558604-1559901 Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1558604-1559901 Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1558604-1559901 Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java Tue Jan 21 02:08:36 2014 @@ -191,6 +191,7 @@ public class TestJournalNode { "/getJournal?segmentTxId=1&jid=" + journalId)); byte[] expected = Bytes.concat( Ints.toByteArray(HdfsConstants.LAYOUT_VERSION), + (new byte[] { 0, 0, 0, 0 }), // layout flags section EDITS_DATA); assertArrayEquals(expected, retrievedViaHttp); Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java Tue Jan 21 02:08:36 2014 @@ -377,8 +377,9 @@ public class TestFSEditLogLoader { File testDir = new File(TEST_DIR, "testValidateEmptyEditLog"); SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap(); File logFile = prepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId); - // Truncate the file so that there is nothing except the header - truncateFile(logFile, 4); + // Truncate the file so that there is nothing except the header and + // layout flags section. + truncateFile(logFile, 8); EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile); assertTrue(!validation.hasCorruptHeader()); Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== Binary files - no diff available. Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml?rev=1559904&r1=1559903&r2=1559904&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml Tue Jan 21 02:08:36 2014 @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <EDITS> - <EDITS_VERSION>-49</EDITS_VERSION> + <EDITS_VERSION>-51</EDITS_VERSION> <RECORD> <OPCODE>OP_START_LOG_SEGMENT</OPCODE> <DATA> @@ -13,8 +13,8 @@ <TXID>2</TXID> <DELEGATION_KEY> <KEY_ID>1</KEY_ID> - <EXPIRY_DATE>1389736494300</EXPIRY_DATE> - <KEY>d1a0861e6b9e394e</KEY> + <EXPIRY_DATE>1390519460949</EXPIRY_DATE> + <KEY>dc8d30edc97df67d</KEY> </DELEGATION_KEY> </DATA> </RECORD> @@ -24,8 +24,8 @@ <TXID>3</TXID> <DELEGATION_KEY> <KEY_ID>2</KEY_ID> - <EXPIRY_DATE>1389736494302</EXPIRY_DATE> - <KEY>8239b8f0ed7e6ce6</KEY> + <EXPIRY_DATE>1390519460952</EXPIRY_DATE> + <KEY>096bc20b6debed03</KEY> </DELEGATION_KEY> </DATA> </RECORD> @@ -37,18 +37,18 @@ <INODEID>16386</INODEID> <PATH>/file_create</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295288</MTIME> - <ATIME>1389045295288</ATIME> + <MTIME>1389828264873</MTIME> + <ATIME>1389828264873</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_-216163116_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_16108824_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <PERMISSION_STATUS> <USERNAME>jing</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>7</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>9</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -59,8 +59,8 @@ <INODEID>0</INODEID> <PATH>/file_create</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295307</MTIME> - <ATIME>1389045295288</ATIME> + <MTIME>1389828265699</MTIME> + <ATIME>1389828264873</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -78,9 +78,9 @@ <LENGTH>0</LENGTH> <SRC>/file_create</SRC> <DST>/file_moved</DST> - <TIMESTAMP>1389045295311</TIMESTAMP> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>9</RPC_CALLID> + <TIMESTAMP>1389828265705</TIMESTAMP> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>11</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -89,9 +89,9 @@ <TXID>7</TXID> <LENGTH>0</LENGTH> <PATH>/file_moved</PATH> - <TIMESTAMP>1389045295318</TIMESTAMP> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>10</RPC_CALLID> + <TIMESTAMP>1389828265712</TIMESTAMP> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>12</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -101,7 +101,7 @@ <LENGTH>0</LENGTH> <INODEID>16387</INODEID> <PATH>/directory_mkdir</PATH> - <TIMESTAMP>1389045295326</TIMESTAMP> + <TIMESTAMP>1389828265722</TIMESTAMP> <PERMISSION_STATUS> <USERNAME>jing</USERNAME> <GROUPNAME>supergroup</GROUPNAME> @@ -136,8 +136,8 @@ <TXID>12</TXID> <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT> <SNAPSHOTNAME>snapshot1</SNAPSHOTNAME> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>15</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>17</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -147,8 +147,8 @@ <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT> <SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME> <SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>16</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>18</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -157,8 +157,8 @@ <TXID>14</TXID> <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT> <SNAPSHOTNAME>snapshot2</SNAPSHOTNAME> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>17</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>19</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -169,18 +169,18 @@ <INODEID>16388</INODEID> <PATH>/file_create</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295354</MTIME> - <ATIME>1389045295354</ATIME> + <MTIME>1389828265757</MTIME> + <ATIME>1389828265757</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_-216163116_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_16108824_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <PERMISSION_STATUS> <USERNAME>jing</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>18</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>20</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -191,8 +191,8 @@ <INODEID>0</INODEID> <PATH>/file_create</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295357</MTIME> - <ATIME>1389045295354</ATIME> + <MTIME>1389828265759</MTIME> + <ATIME>1389828265757</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -253,10 +253,10 @@ <LENGTH>0</LENGTH> <SRC>/file_create</SRC> <DST>/file_moved</DST> - <TIMESTAMP>1389045295378</TIMESTAMP> + <TIMESTAMP>1389828265782</TIMESTAMP> <OPTIONS>NONE</OPTIONS> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>25</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>27</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -267,18 +267,18 @@ <INODEID>16389</INODEID> <PATH>/file_concat_target</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295383</MTIME> - <ATIME>1389045295383</ATIME> + <MTIME>1389828265787</MTIME> + <ATIME>1389828265787</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_-216163116_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_16108824_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <PERMISSION_STATUS> <USERNAME>jing</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>27</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>29</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -383,8 +383,8 @@ <INODEID>0</INODEID> <PATH>/file_concat_target</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295484</MTIME> - <ATIME>1389045295383</ATIME> + <MTIME>1389828266540</MTIME> + <ATIME>1389828265787</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -418,18 +418,18 @@ <INODEID>16390</INODEID> <PATH>/file_concat_0</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295486</MTIME> - <ATIME>1389045295486</ATIME> + <MTIME>1389828266544</MTIME> + <ATIME>1389828266544</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_-216163116_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_16108824_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <PERMISSION_STATUS> <USERNAME>jing</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>40</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>41</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -534,8 +534,8 @@ <INODEID>0</INODEID> <PATH>/file_concat_0</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295507</MTIME> - <ATIME>1389045295486</ATIME> + <MTIME>1389828266569</MTIME> + <ATIME>1389828266544</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -569,18 +569,18 @@ <INODEID>16391</INODEID> <PATH>/file_concat_1</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295509</MTIME> - <ATIME>1389045295509</ATIME> + <MTIME>1389828266572</MTIME> + <ATIME>1389828266572</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_-216163116_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_16108824_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <PERMISSION_STATUS> <USERNAME>jing</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>52</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>53</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -685,8 +685,8 @@ <INODEID>0</INODEID> <PATH>/file_concat_1</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295532</MTIME> - <ATIME>1389045295509</ATIME> + <MTIME>1389828266599</MTIME> + <ATIME>1389828266572</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -718,13 +718,13 @@ <TXID>56</TXID> <LENGTH>0</LENGTH> <TRG>/file_concat_target</TRG> - <TIMESTAMP>1389045295535</TIMESTAMP> + <TIMESTAMP>1389828266603</TIMESTAMP> <SOURCES> <SOURCE1>/file_concat_0</SOURCE1> <SOURCE2>/file_concat_1</SOURCE2> </SOURCES> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>63</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>64</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -735,15 +735,15 @@ <INODEID>16392</INODEID> <PATH>/file_symlink</PATH> <VALUE>/file_concat_target</VALUE> - <MTIME>1389045295540</MTIME> - <ATIME>1389045295540</ATIME> + <MTIME>1389828266633</MTIME> + <ATIME>1389828266633</ATIME> <PERMISSION_STATUS> <USERNAME>jing</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>511</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>64</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>66</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -754,18 +754,18 @@ <INODEID>16393</INODEID> <PATH>/hard-lease-recovery-test</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045295543</MTIME> - <ATIME>1389045295543</ATIME> + <MTIME>1389828266637</MTIME> + <ATIME>1389828266637</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_-216163116_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_16108824_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <PERMISSION_STATUS> <USERNAME>jing</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>65</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>67</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -821,7 +821,7 @@ <OPCODE>OP_REASSIGN_LEASE</OPCODE> <DATA> <TXID>64</TXID> - <LEASEHOLDER>DFSClient_NONMAPREDUCE_-216163116_1</LEASEHOLDER> + <LEASEHOLDER>DFSClient_NONMAPREDUCE_16108824_1</LEASEHOLDER> <PATH>/hard-lease-recovery-test</PATH> <NEWHOLDER>HDFS_NameNode</NEWHOLDER> </DATA> @@ -834,8 +834,8 @@ <INODEID>0</INODEID> <PATH>/hard-lease-recovery-test</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1389045298180</MTIME> - <ATIME>1389045295543</ATIME> + <MTIME>1389828269751</MTIME> + <ATIME>1389828266637</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -861,8 +861,8 @@ <MODE>493</MODE> <LIMIT>9223372036854775807</LIMIT> <MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>72</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>74</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -871,8 +871,8 @@ <TXID>67</TXID> <POOLNAME>pool1</POOLNAME> <LIMIT>99</LIMIT> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>73</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>75</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -883,9 +883,9 @@ <PATH>/path</PATH> <REPLICATION>1</REPLICATION> <POOL>pool1</POOL> - <EXPIRATION>2305844398258992525</EXPIRATION> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>74</RPC_CALLID> + <EXPIRATION>2305844399041964876</EXPIRATION> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>76</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -894,8 +894,8 @@ <TXID>69</TXID> <ID>1</ID> <REPLICATION>2</REPLICATION> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>75</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>77</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -903,8 +903,8 @@ <DATA> <TXID>70</TXID> <ID>1</ID> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>76</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>78</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -912,8 +912,8 @@ <DATA> <TXID>71</TXID> <POOLNAME>pool1</POOLNAME> - <RPC_CLIENTID>48c96601-9238-4d1f-b78b-ef0f1e922ba2</RPC_CLIENTID> - <RPC_CALLID>77</RPC_CALLID> + <RPC_CLIENTID>b5928e80-e373-4807-a688-f94483d08ce5</RPC_CLIENTID> + <RPC_CALLID>79</RPC_CALLID> </DATA> </RECORD> <RECORD>