HDFS-9461. DiskBalancer: Add Report Command. Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/201f82b9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/201f82b9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/201f82b9 Branch: refs/heads/HDFS-1312 Commit: 201f82b9a6be0281fe67cd268e0edbde53989021 Parents: f56ab2e Author: Anu Engineer <aengin...@apache.org> Authored: Fri Jun 10 21:15:54 2016 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Fri Jun 10 21:15:54 2016 -0700 ---------------------------------------------------------------------- .../server/diskbalancer/command/Command.java | 67 + .../diskbalancer/command/ReportCommand.java | 197 + .../datamodel/DiskBalancerVolume.java | 30 + .../apache/hadoop/hdfs/tools/DiskBalancer.java | 61 +- .../command/TestDiskBalancerCommand.java | 299 + .../diskBalancer/data-cluster-64node-3disk.json | 9484 ++++++++++++++++++ 6 files changed, 10136 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/201f82b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java index 94a21d1..bbf91ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java @@ -21,6 +21,8 @@ package org.apache.hadoop.hdfs.server.diskbalancer.command; import com.google.common.base.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.text.StrBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -70,6 +72,7 @@ public abstract class Command extends Configured { private URI clusterURI; private FileSystem fs = null; private DiskBalancerCluster cluster = null; + private int topNodes; private static final Path DEFAULT_LOG_DIR = new Path("/system/diskbalancer"); @@ -83,6 +86,7 @@ public abstract class Command extends Configured { // These arguments are valid for all commands. addValidCommandParameters(DiskBalancer.HELP, "Help for this command"); addValidCommandParameters("arg", ""); + topNodes = 0; } /** @@ -391,4 +395,67 @@ public abstract class Command extends Configured { protected DiskBalancerCluster getCluster() { return cluster; } + + /** + * returns default top number of nodes. + * @return default top number of nodes. + */ + protected int getDefaultTop() { + return DiskBalancer.DEFAULT_TOP; + } + + /** + * Put output line to log and string buffer. + * */ + protected void recordOutput(final StrBuilder result, + final String outputLine) { + LOG.info(outputLine); + result.appendln(outputLine); + } + + /** + * Parse top number of nodes to be processed. + * @return top number of nodes to be processed. + */ + protected int parseTopNodes(final CommandLine cmd, final StrBuilder result) { + String outputLine = ""; + int nodes = 0; + final String topVal = cmd.getOptionValue(DiskBalancer.TOP); + if (StringUtils.isBlank(topVal)) { + outputLine = String.format( + "No top limit specified, using default top value %d.", + getDefaultTop()); + LOG.info(outputLine); + result.appendln(outputLine); + nodes = getDefaultTop(); + } else { + try { + nodes = Integer.parseInt(topVal); + } catch (NumberFormatException nfe) { + outputLine = String.format( + "Top limit input is not numeric, using default top value %d.", + getDefaultTop()); + LOG.info(outputLine); + result.appendln(outputLine); + nodes = getDefaultTop(); + } + } + + return Math.min(nodes, cluster.getNodes().size()); + } + + /** + * Set top number of nodes to be processed. + * */ + public void setTopNodes(int topNodes) { + this.topNodes = topNodes; + } + + /** + * Get top number of nodes to be processed. + * @return top number of nodes to be processed. + * */ + public int getTopNodes() { + return topNodes; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/201f82b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java new file mode 100644 index 0000000..acf9ff2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdfs.server.diskbalancer.command; + +import java.io.PrintStream; +import java.util.Collections; +import java.util.ListIterator; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.text.StrBuilder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerDataNode; +import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolume; +import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolumeSet; +import org.apache.hadoop.hdfs.tools.DiskBalancer; + +import com.google.common.base.Preconditions; + +/** + * Executes the report command. + * + * This command will report volume information for a specific DataNode or top X + * DataNode(s) benefiting from running DiskBalancer. + * + * This is done by reading the cluster info, sorting the DiskbalancerNodes by + * their NodeDataDensity and printing out the info. + */ +public class ReportCommand extends Command { + + private PrintStream out; + + public ReportCommand(Configuration conf, final PrintStream out) { + super(conf); + this.out = out; + + addValidCommandParameters(DiskBalancer.REPORT, + "Report volume information of nodes."); + + String desc = String.format( + "Top number of nodes to be processed. Default: %d", getDefaultTop()); + addValidCommandParameters(DiskBalancer.TOP, desc); + + desc = String.format("Print out volume information for a DataNode."); + addValidCommandParameters(DiskBalancer.NODE, desc); + } + + @Override + public void execute(CommandLine cmd) throws Exception { + StrBuilder result = new StrBuilder(); + String outputLine = "Processing report command"; + recordOutput(result, outputLine); + + Preconditions.checkState(cmd.hasOption(DiskBalancer.REPORT)); + verifyCommandOptions(DiskBalancer.REPORT, cmd); + readClusterInfo(cmd); + + final String nodeFormat = + "%d/%d %s[%s:%d] - <%s>: %d volumes with node data density %.2f."; + final String nodeFormatWithoutSequence = + "%s[%s:%d] - <%s>: %d volumes with node data density %.2f."; + final String volumeFormat = + "[%s: volume-%s] - %.2f used: %d/%d, %.2f free: %d/%d, " + + "isFailed: %s, isReadOnly: %s, isSkip: %s, isTransient: %s."; + + if (cmd.hasOption(DiskBalancer.NODE)) { + /* + * Reporting volume information for a specific DataNode + */ + handleNodeReport(cmd, result, nodeFormatWithoutSequence, volumeFormat); + + } else { // handle TOP + /* + * Reporting volume information for top X DataNode(s) + */ + handleTopReport(cmd, result, nodeFormat); + } + + out.println(result.toString()); + } + + private void handleTopReport(final CommandLine cmd, final StrBuilder result, + final String nodeFormat) { + Collections.sort(getCluster().getNodes(), Collections.reverseOrder()); + + /* extract value that identifies top X DataNode(s) */ + setTopNodes(parseTopNodes(cmd, result)); + + /* + * Reporting volume information of top X DataNode(s) in summary + */ + final String outputLine = String.format( + "Reporting top %d DataNode(s) benefiting from running DiskBalancer.", + getTopNodes()); + recordOutput(result, outputLine); + + ListIterator<DiskBalancerDataNode> li = getCluster().getNodes() + .listIterator(); + + for (int i = 0; i < getTopNodes() && li.hasNext(); i++) { + DiskBalancerDataNode dbdn = li.next(); + result.appendln(String.format(nodeFormat, + i+1, + getTopNodes(), + dbdn.getDataNodeName(), + dbdn.getDataNodeIP(), + dbdn.getDataNodePort(), + dbdn.getDataNodeUUID(), + dbdn.getVolumeCount(), + dbdn.getNodeDataDensity())); + } + } + + private void handleNodeReport(final CommandLine cmd, StrBuilder result, + final String nodeFormat, final String volumeFormat) { + String outputLine = ""; + /* + * get value that identifies a DataNode from command line, it could be UUID, + * IP address or host name. + */ + final String nodeVal = cmd.getOptionValue(DiskBalancer.NODE); + + if (StringUtils.isBlank(nodeVal)) { + outputLine = "The value for '-node' is neither specified or empty."; + recordOutput(result, outputLine); + } else { + /* + * Reporting volume information for a specific DataNode + */ + outputLine = String.format( + "Reporting volume information for DataNode '%s'.", nodeVal); + recordOutput(result, outputLine); + + final String trueStr = "True"; + final String falseStr = "False"; + DiskBalancerDataNode dbdn = getNode(nodeVal); + + if (dbdn == null) { + outputLine = String.format( + "Can't find a DataNode that matches '%s'.", nodeVal); + recordOutput(result, outputLine); + } else { + result.appendln(String.format(nodeFormat, + dbdn.getDataNodeName(), + dbdn.getDataNodeIP(), + dbdn.getDataNodePort(), + dbdn.getDataNodeUUID(), + dbdn.getVolumeCount(), + dbdn.getNodeDataDensity())); + + for (DiskBalancerVolumeSet vset : dbdn.getVolumeSets().values()) { + for (DiskBalancerVolume vol : vset.getVolumes()) { + result.appendln(String.format(volumeFormat, + vol.getStorageType(), + vol.getPath(), + vol.getUsedRatio(), + vol.getUsed(), + vol.getCapacity(), + vol.getFreeRatio(), + vol.getFreeSpace(), + vol.getCapacity(), + vol.isFailed() ? trueStr : falseStr, + vol.isReadOnly() ? trueStr : falseStr, + vol.isSkip() ? trueStr : falseStr, + vol.isTransient() ? trueStr : falseStr)); + } + } + } + } + } + + @Override + protected String getHelp() { + return "Report volume information for a specific DataNode or top X " + + "one(s) benefiting from running DiskBalancer, " + + "top defaults to " + getDefaultTop() + ". E.g.:\n" + + "hdfs diskbalancer -uri http://namenode.uri -report\n" + + "hdfs diskbalancer -uri http://namenode.uri -report -top 5\n" + + "hdfs diskbalancer -uri http://namenode.uri -report " + + "-node {DataNodeID | IP | Hostname}"; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/201f82b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java index 2a39609..a6a8bdc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java @@ -140,6 +140,36 @@ public class DiskBalancerVolume { } /** + * Get free space of the volume. + * + * @return long + */ + @JsonIgnore + public long getFreeSpace() { + return getCapacity() - getUsed(); + } + + /** + * Get ratio between used space and capacity. + * + * @return double + */ + @JsonIgnore + public double getUsedRatio() { + return (1.0 * getUsed()) / getCapacity(); + } + + /** + * Get ratio between free space and capacity. + * + * @return double + */ + @JsonIgnore + public double getFreeRatio() { + return (1.0 * getFreeSpace()) / getCapacity(); + } + + /** * Sets the capacity of this volume. * * @param totalCapacity long http://git-wip-us.apache.org/repos/asf/hadoop/blob/201f82b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java index dde2ce4..1251e96 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java @@ -29,12 +29,14 @@ import org.apache.hadoop.hdfs.server.diskbalancer.command.Command; import org.apache.hadoop.hdfs.server.diskbalancer.command.ExecuteCommand; import org.apache.hadoop.hdfs.server.diskbalancer.command.PlanCommand; import org.apache.hadoop.hdfs.server.diskbalancer.command.QueryCommand; +import org.apache.hadoop.hdfs.server.diskbalancer.command.ReportCommand; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.PrintStream; import java.net.URISyntaxException; /** @@ -94,6 +96,22 @@ public class DiskBalancer extends Configured implements Tool { */ public static final String EXECUTE = "execute"; /** + * The report command prints out a disk fragmentation report about the data + * cluster. By default it prints the DEFAULT_TOP machines names with high + * nodeDataDensity {DiskBalancerDataNode#getNodeDataDensity} values. This + * means that these are the nodes that deviates from the ideal data + * distribution. + */ + public static final String REPORT = "report"; + /** + * specify top number of nodes to be processed. + */ + public static final String TOP = "top"; + /** + * specify default top number of nodes to be processed. + */ + public static final int DEFAULT_TOP = 100; + /** * Name or address of the node to execute against. */ public static final String NODE = "node"; @@ -157,9 +175,21 @@ public class DiskBalancer extends Configured implements Tool { */ @Override public int run(String[] args) throws Exception { + return run(args, System.out); + } + + /** + * Execute the command with the given arguments. + * + * @param args command specific arguments. + * @param out the output stream used for printing + * @return exit code. + * @throws Exception + */ + public int run(String[] args, final PrintStream out) throws Exception { Options opts = getOpts(); CommandLine cmd = parseArgs(args, opts); - return dispatch(cmd, opts); + return dispatch(cmd, opts, out); } /** @@ -173,6 +203,7 @@ public class DiskBalancer extends Configured implements Tool { addExecuteCommands(opts); addQueryCommands(opts); addCancelCommands(opts); + addReportCommands(opts); return opts; } @@ -256,6 +287,26 @@ public class DiskBalancer extends Configured implements Tool { } /** + * Adds report command options. + * @param opt Options + */ + private void addReportCommands(Options opt) { + Option report = new Option(REPORT, false, + "Report volume information of DataNode(s)" + + " benefiting from running DiskBalancer. " + + "-report [top -X] | [-node {DataNodeID | IP | Hostname}]."); + opt.addOption(report); + + Option top = new Option(TOP, true, + "specify the top number of nodes to be processed."); + opt.addOption(top); + + Option node = new Option(NODE, true, + "Name of the datanode in the format of DataNodeID, IP or hostname."); + opt.addOption(node); + } + + /** * This function parses all command line arguments and returns the appropriate * values. * @@ -272,10 +323,12 @@ public class DiskBalancer extends Configured implements Tool { * Dispatches calls to the right command Handler classes. * * @param cmd - CommandLine + * @param opts options of command line + * @param out the output stream used for printing * @throws IOException * @throws URISyntaxException */ - private int dispatch(CommandLine cmd, Options opts) + private int dispatch(CommandLine cmd, Options opts, final PrintStream out) throws IOException, URISyntaxException { Command currentCommand = null; @@ -297,6 +350,10 @@ public class DiskBalancer extends Configured implements Tool { currentCommand = new CancelCommand(getConf()); } + if (cmd.hasOption(DiskBalancer.REPORT)) { + currentCommand = new ReportCommand(getConf(), out); + } + if(currentCommand == null) { HelpFormatter helpFormatter = new HelpFormatter(); helpFormatter.printHelp(80, "hdfs diskbalancer -uri [args]", http://git-wip-us.apache.org/repos/asf/hadoop/blob/201f82b9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java new file mode 100644 index 0000000..57e59f6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java @@ -0,0 +1,299 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdfs.server.diskbalancer.command; + +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.net.URI; +import java.util.List; +import java.util.Scanner; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ClusterConnector; +import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ConnectorFactory; +import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.Lists; + +/** + * Tests various CLI commands of DiskBalancer. + */ +public class TestDiskBalancerCommand { + private MiniDFSCluster cluster; + private URI clusterJson; + + @Before + public void setUp() throws Exception { + Configuration conf = new HdfsConfiguration(); + conf.setBoolean(DFSConfigKeys.DFS_DISK_BALANCER_ENABLED, true); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3) + .storagesPerDatanode(2).build(); + cluster.waitActive(); + + clusterJson = getClass().getResource( + "/diskBalancer/data-cluster-64node-3disk.json").toURI(); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + // Just make sure we can shutdown datanodes. + cluster.getDataNodes().get(0).shutdown(); + cluster.shutdown(); + } + } + + private void testReportSimple() throws Exception { + final String cmdLine = String.format("hdfs diskbalancer -uri %s -report", + clusterJson.toString()); + final List<String> outputs = runCommand(cmdLine); + + assertThat( + outputs.get(0), + containsString("Processing report command")); + assertThat( + outputs.get(1), + is(allOf(containsString("No top limit specified"), + containsString("using default top value"), containsString("100")))); + assertThat( + outputs.get(2), + is(allOf( + containsString("Reporting top"), + containsString("64"), + containsString( + "DataNode(s) benefiting from running DiskBalancer")))); + assertThat( + outputs.get(32), + is(allOf(containsString("30/64 null[null:0]"), + containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"), + containsString("9 volumes with node data density 1.97")))); + + } + + private void testReportLessThanTotal() throws Exception { + final String cmdLine = String.format( + "hdfs diskbalancer -uri %s -report -top 32", clusterJson.toString()); + final List<String> outputs = runCommand(cmdLine); + + assertThat( + outputs.get(0), + containsString("Processing report command")); + assertThat( + outputs.get(1), + is(allOf( + containsString("Reporting top"), + containsString("32"), + containsString( + "DataNode(s) benefiting from running DiskBalancer")))); + assertThat( + outputs.get(31), + is(allOf(containsString("30/32 null[null:0]"), + containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"), + containsString("9 volumes with node data density 1.97")))); + } + + private void testReportMoreThanTotal() throws Exception { + final String cmdLine = String.format( + "hdfs diskbalancer -uri %s -report -top 128", clusterJson.toString()); + final List<String> outputs = runCommand(cmdLine); + + assertThat( + outputs.get(0), + containsString("Processing report command")); + assertThat( + outputs.get(1), + is(allOf( + containsString("Reporting top"), + containsString("64"), + containsString( + "DataNode(s) benefiting from running DiskBalancer")))); + assertThat( + outputs.get(31), + is(allOf(containsString("30/64 null[null:0]"), + containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"), + containsString("9 volumes with node data density 1.97")))); + + } + + private void testReportInvalidTopLimit() throws Exception { + final String cmdLine = String.format( + "hdfs diskbalancer -uri %s -report -top xx", clusterJson.toString()); + final List<String> outputs = runCommand(cmdLine); + + assertThat( + outputs.get(0), + containsString("Processing report command")); + assertThat( + outputs.get(1), + is(allOf(containsString("Top limit input is not numeric"), + containsString("using default top value"), containsString("100")))); + assertThat( + outputs.get(2), + is(allOf( + containsString("Reporting top"), + containsString("64"), + containsString( + "DataNode(s) benefiting from running DiskBalancer")))); + assertThat( + outputs.get(32), + is(allOf(containsString("30/64 null[null:0]"), + containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"), + containsString("9 volumes with node data density 1.97")))); + } + + private void testReportNode() throws Exception { + final String cmdLine = String + .format( + "hdfs diskbalancer -uri %s -report -node " + + "a87654a9-54c7-4693-8dd9-c9c7021dc340", + clusterJson.toString()); + final List<String> outputs = runCommand(cmdLine); + + assertThat( + outputs.get(0), + containsString("Processing report command")); + assertThat( + outputs.get(1), + is(allOf(containsString("Reporting volume information for DataNode"), + containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340")))); + assertThat( + outputs.get(2), + is(allOf(containsString("null[null:0]"), + containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"), + containsString("9 volumes with node data density 1.97")))); + assertThat( + outputs.get(3), + is(allOf(containsString("DISK"), + containsString("/tmp/disk/xx3j3ph3zd"), + containsString("0.72 used: 289544224916/400000000000"), + containsString("0.28 free: 110455775084/400000000000")))); + assertThat( + outputs.get(4), + is(allOf(containsString("DISK"), + containsString("/tmp/disk/Mxfcfmb24Y"), + containsString("0.92 used: 733099315216/800000000000"), + containsString("0.08 free: 66900684784/800000000000")))); + assertThat( + outputs.get(5), + is(allOf(containsString("DISK"), + containsString("DISK"), + containsString("/tmp/disk/KmHefYNURo"), + containsString("0.20 used: 39160240782/200000000000"), + containsString("0.80 free: 160839759218/200000000000")))); + assertThat( + outputs.get(6), + is(allOf(containsString("RAM_DISK"), + containsString("/tmp/disk/MXRyYsCz3U"), + containsString("0.55 used: 438102096853/800000000000"), + containsString("0.45 free: 361897903147/800000000000")))); + assertThat( + outputs.get(7), + is(allOf(containsString("RAM_DISK"), + containsString("/tmp/disk/DtmAygEU6f"), + containsString("0.34 used: 134602910470/400000000000"), + containsString("0.66 free: 265397089530/400000000000")))); + assertThat( + outputs.get(8), + is(allOf(containsString("RAM_DISK"), + containsString("/tmp/disk/BoBlQFxhfw"), + containsString("0.60 used: 477590453390/800000000000"), + containsString("0.40 free: 322409546610/800000000000")))); + assertThat( + outputs.get(9), + is(allOf(containsString("SSD"), + containsString("/tmp/disk/BGe09Y77dI"), + containsString("0.89 used: 890446265501/1000000000000"), + containsString("0.11 free: 109553734499/1000000000000")))); + assertThat( + outputs.get(10), + is(allOf(containsString("SSD"), + containsString("/tmp/disk/JX3H8iHggM"), + containsString("0.31 used: 2782614512957/9000000000000"), + containsString("0.69 free: 6217385487043/9000000000000")))); + assertThat( + outputs.get(11), + is(allOf(containsString("SSD"), + containsString("/tmp/disk/uLOYmVZfWV"), + containsString("0.75 used: 1509592146007/2000000000000"), + containsString("0.25 free: 490407853993/2000000000000")))); + } + + @Test(timeout=60000) + public void testReportCommmand() throws Exception { + + /* test basic report */ + testReportSimple(); + + /* test less than 64 DataNode(s) as total, e.g., -report -top 32 */ + testReportLessThanTotal(); + + /* test more than 64 DataNode(s) as total, e.g., -report -top 128 */ + testReportMoreThanTotal(); + + /* test invalid top limit, e.g., -report -top xx */ + testReportInvalidTopLimit(); + + /* test -report -node DataNodeID */ + testReportNode(); + } + + @Test + public void testReadClusterFromJson() throws Exception { + Configuration conf = new HdfsConfiguration(); + conf.setBoolean(DFSConfigKeys.DFS_DISK_BALANCER_ENABLED, true); + + ClusterConnector jsonConnector = ConnectorFactory.getCluster(clusterJson, + conf); + DiskBalancerCluster diskBalancerCluster = new DiskBalancerCluster( + jsonConnector); + diskBalancerCluster.readClusterInfo(); + assertEquals(64, diskBalancerCluster.getNodes().size()); + } + + private List<String> runCommand(final String cmdLine) throws Exception { + + String[] cmds = StringUtils.split(cmdLine, ' '); + Configuration conf = new HdfsConfiguration(); + org.apache.hadoop.hdfs.tools.DiskBalancer db = + new org.apache.hadoop.hdfs.tools.DiskBalancer(conf); + + ByteArrayOutputStream bufOut = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bufOut); + db.run(cmds, out); + + Scanner scanner = new Scanner(bufOut.toString()); + List<String> outputs = Lists.newArrayList(); + while (scanner.hasNextLine()) { + outputs.add(scanner.nextLine()); + } + return outputs; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
