HDFS-10518. DiskBalancer: Pretty-print json in Query command. Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e2be5c4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e2be5c4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e2be5c4 Branch: refs/heads/trunk Commit: 7e2be5c4a0b68b556ec6afcb0e14e0ab5ef1a9b2 Parents: af11ab3 Author: Anu Engineer <[email protected]> Authored: Mon Jun 13 14:11:23 2016 -0700 Committer: Arpit Agarwal <[email protected]> Committed: Thu Jun 23 18:21:08 2016 -0700 ---------------------------------------------------------------------- .../server/datanode/DiskBalancerWorkItem.java | 42 ++++++++++++++++ .../server/datanode/DiskBalancerWorkStatus.java | 2 + .../hdfs/server/datanode/DiskBalancer.java | 53 ++++++++++++++++++-- .../diskbalancer/command/QueryCommand.java | 5 +- .../TestDiskBalancerWithMockMover.java | 20 ++++++++ 5 files changed, 116 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e2be5c4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java index fe908d8..f46a987 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java @@ -34,6 +34,8 @@ import java.io.IOException; @InterfaceStability.Unstable @JsonInclude(JsonInclude.Include.NON_DEFAULT) public class DiskBalancerWorkItem { + private long startTime; + private long secondsElapsed; private long bytesToCopy; private long bytesCopied; private long errorCount; @@ -242,4 +244,44 @@ public class DiskBalancerWorkItem { public void setBandwidth(long bandwidth) { this.bandwidth = bandwidth; } + + + /** + * Records the Start time of execution. + * @return startTime + */ + public long getStartTime() { + return startTime; + } + + /** + * Sets the Start time. + * @param startTime - Time stamp for start of execution. + */ + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + /** + * Gets the number of seconds elapsed from the start time. + * + * The reason why we have this is of time skews. The client's current time + * may not match with the server time stamp, hence the elapsed second + * cannot be computed from only startTime. + * + * @return seconds elapsed from start time. + */ + public long getSecondsElapsed() { + return secondsElapsed; + } + + /** + * Sets number of seconds elapsed. + * + * This is updated whenever we update the other counters. + * @param secondsElapsed - seconds elapsed. + */ + public void setSecondsElapsed(long secondsElapsed) { + this.secondsElapsed = secondsElapsed; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e2be5c4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java index ca5e5f0..1f62f47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.SerializationConfig; import static org.codehaus.jackson.map.type.TypeFactory.defaultInstance; @@ -128,6 +129,7 @@ public class DiskBalancerWorkStatus { **/ public String currentStateString() throws IOException { ObjectMapper mapper = new ObjectMapper(); + mapper.enable(SerializationConfig.Feature.INDENT_OUTPUT); return mapper.writeValueAsString(currentState); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e2be5c4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java index 7f768ea..5fde7c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java @@ -552,6 +552,21 @@ public class DiskBalancer { * @return FsDatasetSpi */ FsDatasetSpi getDataset(); + + /** + * Returns time when this plan started executing. + * + * @return Start time in milliseconds. + */ + long getStartTime(); + + /** + * Number of seconds elapsed. + * + * @return time in seconds + */ + long getElapsedSeconds(); + } /** @@ -622,6 +637,8 @@ public class DiskBalancer { private long maxDiskErrors; private int poolIndex; private AtomicBoolean shouldRun; + private long startTime; + private long secondsElapsed; /** * Constructs diskBalancerMover. @@ -897,6 +914,9 @@ public class DiskBalancer { FsVolumeSpi source = pair.getSource(); FsVolumeSpi dest = pair.getDest(); List<FsVolumeSpi.BlockIterator> poolIters = new LinkedList<>(); + startTime = Time.now(); + item.setStartTime(startTime); + secondsElapsed = 0; if (source.isTransientStorage() || dest.isTransientStorage()) { return; @@ -937,7 +957,7 @@ public class DiskBalancer { if (block == null) { this.setExitFlag(); LOG.error("No source blocks, exiting the copy. Source: {}, " + - "dest:{}", source.getBasePath(), dest.getBasePath()); + "dest:{}", source.getBasePath(), dest.getBasePath()); continue; } @@ -973,9 +993,6 @@ public class DiskBalancer { block.getNumBytes(), source.getBasePath(), dest.getBasePath()); - item.incCopiedSoFar(block.getNumBytes()); - item.incBlocksCopied(); - // Check for the max throughput constraint. // We sleep here to keep the promise that we will not // copy more than Max MB/sec. we sleep enough time @@ -984,6 +1001,14 @@ public class DiskBalancer { // we exit via Thread Interrupted exception. Thread.sleep(computeDelay(block.getNumBytes(), timeUsed, item)); + // We delay updating the info to avoid confusing the user. + // This way we report the copy only if it is under the + // throughput threshold. + item.incCopiedSoFar(block.getNumBytes()); + item.incBlocksCopied(); + secondsElapsed = TimeUnit.MILLISECONDS.toSeconds(Time.now() - + startTime); + item.setSecondsElapsed(secondsElapsed); } catch (IOException ex) { LOG.error("Exception while trying to copy blocks. error: {}", ex); item.incErrorCount(); @@ -1009,5 +1034,25 @@ public class DiskBalancer { public FsDatasetSpi getDataset() { return dataset; } + + /** + * Returns time when this plan started executing. + * + * @return Start time in milliseconds. + */ + @Override + public long getStartTime() { + return startTime; + } + + /** + * Number of seconds elapsed. + * + * @return time in seconds + */ + @Override + public long getElapsedSeconds() { + return secondsElapsed; + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e2be5c4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java index 3a3b97f..6c759e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java @@ -46,6 +46,7 @@ public class QueryCommand extends Command { " plan running on a given datanode."); addValidCommandParameters(DiskBalancer.VERBOSE, "Prints verbose results."); } + /** * Executes the Client Calls. * @@ -62,7 +63,7 @@ public class QueryCommand extends Command { String nodeAddress = nodeName; // if the string is not name:port format use the default port. - if(!nodeName.matches("^.*:\\d$")) { + if (!nodeName.matches("^.*:\\d$")) { int defaultIPC = NetUtils.createSocketAddr( getConf().getTrimmed(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_DEFAULT)).getPort(); @@ -76,7 +77,7 @@ public class QueryCommand extends Command { System.out.printf("Plan ID: %s %nResult: %s%n", workStatus.getPlanID(), workStatus.getResult().toString()); - if(cmd.hasOption(DiskBalancer.VERBOSE)) { + if (cmd.hasOption(DiskBalancer.VERBOSE)) { System.out.printf("%s", workStatus.currentStateString()); } } catch (DiskBalancerException ex) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e2be5c4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java index 491fccb..b73b290 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java @@ -436,6 +436,26 @@ public class TestDiskBalancerWithMockMover { return this.dataset; } + /** + * Returns time when this plan started executing. + * + * @return Start time in milliseconds. + */ + @Override + public long getStartTime() { + return 0; + } + + /** + * Number of seconds elapsed. + * + * @return time in seconds + */ + @Override + public long getElapsedSeconds() { + return 0; + } + public int getRunCount() { synchronized (runCount) { LOG.info("Run count : " + runCount.intValue()); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
