Repository: hadoop Updated Branches: refs/heads/branch-2.7 62da8f6fa -> 8e784afb1
Revert "HDFS-8791. block ID-based DN storage layout can be very slow for datanode on ext4. Contributed by Chris Trezzo." This reverts commit 9bc9e13a979c775354b2394d926f466a0d5be514. Conflicts: hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e784afb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e784afb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e784afb Branch: refs/heads/branch-2.7 Commit: 8e784afb1596b06c08493d4ceb1b743e5ba61edc Parents: 62da8f6 Author: Kihwal Lee <kih...@apache.org> Authored: Wed Apr 6 17:35:54 2016 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Wed Apr 6 17:35:54 2016 -0500 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 -- .../server/datanode/DataNodeLayoutVersion.java | 5 +--- .../hdfs/server/datanode/DataStorage.java | 11 +++---- .../hdfs/server/datanode/DatanodeUtil.java | 4 +-- .../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 29 ++----------------- .../resources/hadoop-56-layout-datanode-dir.tgz | Bin 198996 -> 0 bytes .../resources/hadoop-to-57-dn-layout-dir.txt | 24 --------------- 7 files changed, 9 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bb37eb8..2441d09 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -38,9 +38,6 @@ Release 2.7.3 - UNRELEASED HDFS-9395. Make HDFS audit logging consistant (Kuhu Shukla via kihwal) - HDFS-8791. block ID-based DN storage layout can be very slow for datanode - on ext4 (Chris Trezzo via kihwal) - HDFS-9048. DistCp documentation is out-of-dated (Daisuke Kobayashi via iwasakims) http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java index 609a740..23e7cfe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java @@ -65,10 +65,7 @@ public class DataNodeLayoutVersion { FIRST_LAYOUT(-55, -53, "First datanode layout", false), BLOCKID_BASED_LAYOUT(-56, "The block ID of a finalized block uniquely determines its position " + - "in the directory structure"), - BLOCKID_BASED_LAYOUT_32_by_32(-57, - "Identical to the block id based layout (-56) except it uses a smaller" - + " directory structure (32x32)"); + "in the directory structure"); private final FeatureInfo info; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index 7681d3c..8656ae9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -1117,13 +1117,10 @@ public class DataStorage extends Storage { LOG.info("Start linking block files from " + from + " to " + to); boolean upgradeToIdBasedLayout = false; // If we are upgrading from a version older than the one where we introduced - // block ID-based layout (32x32) AND we're working with the finalized - // directory, we'll need to upgrade from the old layout to the new one. The - // upgrade path from pre-blockid based layouts (>-56) and blockid based - // 256x256 layouts (-56) is fortunately the same. - if (oldLV > DataNodeLayoutVersion.Feature.BLOCKID_BASED_LAYOUT_32_by_32 - .getInfo().getLayoutVersion() - && to.getName().equals(STORAGE_DIR_FINALIZED)) { + // block ID-based layout AND we're working with the finalized directory, + // we'll need to upgrade from the old flat layout to the block ID-based one + if (oldLV > DataNodeLayoutVersion.Feature.BLOCKID_BASED_LAYOUT.getInfo(). + getLayoutVersion() && to.getName().equals(STORAGE_DIR_FINALIZED)) { upgradeToIdBasedLayout = true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java index ad054a8..746c3f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java @@ -113,8 +113,8 @@ public class DatanodeUtil { * @return */ public static File idToBlockDir(File root, long blockId) { - int d1 = (int) ((blockId >> 16) & 0x1F); - int d2 = (int) ((blockId >> 8) & 0x1F); + int d1 = (int)((blockId >> 16) & 0xff); + int d2 = (int)((blockId >> 8) & 0xff); String path = DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP + DataStorage.BLOCK_SUBDIR_PREFIX + d2; return new File(root, path); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java index 97769ba..343320c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java @@ -28,16 +28,10 @@ public class TestDatanodeLayoutUpgrade { private static final String HADOOP_DATANODE_DIR_TXT = "hadoop-datanode-dir.txt"; private static final String HADOOP24_DATANODE = "hadoop-24-datanode-dir.tgz"; - private static final String HADOOP_56_DN_LAYOUT_TXT = - "hadoop-to-57-dn-layout-dir.txt"; - private static final String HADOOP_56_DN_LAYOUT = - "hadoop-56-layout-datanode-dir.tgz"; - /** - * Upgrade from LDir-based layout to 32x32 block ID-based layout (-57) -- - * change described in HDFS-6482 and HDFS-8791 - */ @Test + // Upgrade from LDir-based layout to block ID-based layout -- change described + // in HDFS-6482 public void testUpgradeToIdBasedLayout() throws IOException { TestDFSUpgradeFromImage upgrade = new TestDFSUpgradeFromImage(); upgrade.unpackStorage(HADOOP24_DATANODE, HADOOP_DATANODE_DIR_TXT); @@ -51,23 +45,4 @@ public class TestDatanodeLayoutUpgrade { upgrade.upgradeAndVerify(new MiniDFSCluster.Builder(conf).numDataNodes(1) .manageDataDfsDirs(false).manageNameDfsDirs(false), null); } - - /** - * Test upgrade from block ID-based layout 256x256 (-56) to block ID-based - * layout 32x32 (-57) - */ - @Test - public void testUpgradeFrom256To32Layout() throws IOException { - TestDFSUpgradeFromImage upgrade = new TestDFSUpgradeFromImage(); - upgrade.unpackStorage(HADOOP_56_DN_LAYOUT, HADOOP_56_DN_LAYOUT_TXT); - Configuration conf = new Configuration(TestDFSUpgradeFromImage.upgradeConf); - conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, - new File(System.getProperty("test.build.data"), "dfs" + File.separator - + "data").toURI().toString()); - conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, - new File(System.getProperty("test.build.data"), "dfs" + File.separator - + "name").toURI().toString()); - upgrade.upgradeAndVerify(new MiniDFSCluster.Builder(conf).numDataNodes(1) - .manageDataDfsDirs(false).manageNameDfsDirs(false), null); - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz deleted file mode 100644 index b43ac17..0000000 Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz and /dev/null differ http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e784afb/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt deleted file mode 100644 index 2dd1766..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Similar to hadoop-dfs-dir.txt, except this is used for a datanode layout -# upgrade test to 32x32 (layout -57) -# Uncomment the following line to produce checksum info for a new DFS image. -#printChecksums -/blocks/part1 286881285 -/blocks/part12922 1068680946 -/blocks/part972 2479788008 -/blocks/part973 1221039573 -overallCRC 1902127725