Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 62da8f6fa -> 8e784afb1
Revert "HDFS-8791. block ID-based DN storage layout can be very slow for
datanode on ext4. Contributed by Chris Trezzo."
This reverts commit 9bc9e13a979c775354b2394d926f466a0d5be514.
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e784afb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e784afb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e784afb
Branch: refs/heads/branch-2.7
Commit: 8e784afb1596b06c08493d4ceb1b743e5ba61edc
Parents: 62da8f6
Author: Kihwal Lee <kih...@apache.org>
Authored: Wed Apr 6 17:35:54 2016 -0500
Committer: Kihwal Lee <kih...@apache.org>
Committed: Wed Apr 6 17:35:54 2016 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 --
.../server/datanode/DataNodeLayoutVersion.java | 5 +---
.../hdfs/server/datanode/DataStorage.java | 11 +++----
.../hdfs/server/datanode/DatanodeUtil.java | 4 +--
.../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 29 ++-----------------
.../resources/hadoop-56-layout-datanode-dir.tgz | Bin 198996 -> 0 bytes
.../resources/hadoop-to-57-dn-layout-dir.txt | 24 ---------------
7 files changed, 9 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index bb37eb8..2441d09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -38,9 +38,6 @@ Release 2.7.3 - UNRELEASED
HDFS-9395. Make HDFS audit logging consistant (Kuhu Shukla via kihwal)
- HDFS-8791. block ID-based DN storage layout can be very slow for datanode
- on ext4 (Chris Trezzo via kihwal)
-
HDFS-9048. DistCp documentation is out-of-dated
(Daisuke Kobayashi via iwasakims)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
index 609a740..23e7cfe 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
@@ -65,10 +65,7 @@ public class DataNodeLayoutVersion {
FIRST_LAYOUT(-55, -53, "First datanode layout", false),
BLOCKID_BASED_LAYOUT(-56,
"The block ID of a finalized block uniquely determines its position " +
- "in the directory structure"),
- BLOCKID_BASED_LAYOUT_32_by_32(-57,
- "Identical to the block id based layout (-56) except it uses a smaller"
- + " directory structure (32x32)");
+ "in the directory structure");
private final FeatureInfo info;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 7681d3c..8656ae9 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -1117,13 +1117,10 @@ public class DataStorage extends Storage {
LOG.info("Start linking block files from " + from + " to " + to);
boolean upgradeToIdBasedLayout = false;
// If we are upgrading from a version older than the one where we
introduced
- // block ID-based layout (32x32) AND we're working with the finalized
- // directory, we'll need to upgrade from the old layout to the new one. The
- // upgrade path from pre-blockid based layouts (>-56) and blockid based
- // 256x256 layouts (-56) is fortunately the same.
- if (oldLV > DataNodeLayoutVersion.Feature.BLOCKID_BASED_LAYOUT_32_by_32
- .getInfo().getLayoutVersion()
- && to.getName().equals(STORAGE_DIR_FINALIZED)) {
+ // block ID-based layout AND we're working with the finalized directory,
+ // we'll need to upgrade from the old flat layout to the block ID-based one
+ if (oldLV > DataNodeLayoutVersion.Feature.BLOCKID_BASED_LAYOUT.getInfo().
+ getLayoutVersion() && to.getName().equals(STORAGE_DIR_FINALIZED)) {
upgradeToIdBasedLayout = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index ad054a8..746c3f6 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -113,8 +113,8 @@ public class DatanodeUtil {
* @return
*/
public static File idToBlockDir(File root, long blockId) {
- int d1 = (int) ((blockId >> 16) & 0x1F);
- int d2 = (int) ((blockId >> 8) & 0x1F);
+ int d1 = (int)((blockId >> 16) & 0xff);
+ int d2 = (int)((blockId >> 8) & 0xff);
String path = DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
DataStorage.BLOCK_SUBDIR_PREFIX + d2;
return new File(root, path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
index 97769ba..343320c 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
@@ -28,16 +28,10 @@ public class TestDatanodeLayoutUpgrade {
private static final String HADOOP_DATANODE_DIR_TXT =
"hadoop-datanode-dir.txt";
private static final String HADOOP24_DATANODE = "hadoop-24-datanode-dir.tgz";
- private static final String HADOOP_56_DN_LAYOUT_TXT =
- "hadoop-to-57-dn-layout-dir.txt";
- private static final String HADOOP_56_DN_LAYOUT =
- "hadoop-56-layout-datanode-dir.tgz";
- /**
- * Upgrade from LDir-based layout to 32x32 block ID-based layout (-57) --
- * change described in HDFS-6482 and HDFS-8791
- */
@Test
+ // Upgrade from LDir-based layout to block ID-based layout -- change
described
+ // in HDFS-6482
public void testUpgradeToIdBasedLayout() throws IOException {
TestDFSUpgradeFromImage upgrade = new TestDFSUpgradeFromImage();
upgrade.unpackStorage(HADOOP24_DATANODE, HADOOP_DATANODE_DIR_TXT);
@@ -51,23 +45,4 @@ public class TestDatanodeLayoutUpgrade {
upgrade.upgradeAndVerify(new MiniDFSCluster.Builder(conf).numDataNodes(1)
.manageDataDfsDirs(false).manageNameDfsDirs(false), null);
}
-
- /**
- * Test upgrade from block ID-based layout 256x256 (-56) to block ID-based
- * layout 32x32 (-57)
- */
- @Test
- public void testUpgradeFrom256To32Layout() throws IOException {
- TestDFSUpgradeFromImage upgrade = new TestDFSUpgradeFromImage();
- upgrade.unpackStorage(HADOOP_56_DN_LAYOUT, HADOOP_56_DN_LAYOUT_TXT);
- Configuration conf = new
Configuration(TestDFSUpgradeFromImage.upgradeConf);
- conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
- new File(System.getProperty("test.build.data"), "dfs" + File.separator
- + "data").toURI().toString());
- conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
- new File(System.getProperty("test.build.data"), "dfs" + File.separator
- + "name").toURI().toString());
- upgrade.upgradeAndVerify(new MiniDFSCluster.Builder(conf).numDataNodes(1)
- .manageDataDfsDirs(false).manageNameDfsDirs(false), null);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz
b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz
deleted file mode 100644
index b43ac17..0000000
Binary files
a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz
and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt
b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt
deleted file mode 100644
index 2dd1766..0000000
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Similar to hadoop-dfs-dir.txt, except this is used for a datanode layout
-# upgrade test to 32x32 (layout -57)
-# Uncomment the following line to produce checksum info for a new DFS image.
-#printChecksums
-/blocks/part1 286881285
-/blocks/part12922 1068680946
-/blocks/part972 2479788008
-/blocks/part973 1221039573
-overallCRC 1902127725
