HDDS-754. VolumeInfo#getScmUsed throws NPE. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/773f0d15 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/773f0d15 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/773f0d15 Branch: refs/heads/HDDS-4 Commit: 773f0d1519715e3ddf77c139998cc12d7447da66 Parents: e33b61f Author: Anu Engineer <aengin...@apache.org> Authored: Tue Oct 30 19:17:57 2018 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Tue Oct 30 19:17:57 2018 -0700 ---------------------------------------------------------------------- .../container/common/volume/VolumeInfo.java | 19 +++++++++++++++++-- .../ozone/container/common/volume/VolumeSet.java | 11 +++++++---- .../container/common/volume/TestHddsVolume.java | 9 ++++++--- .../container/common/volume/TestVolumeSet.java | 4 +++- .../hdds/scm/pipeline/TestNodeFailure.java | 3 ++- .../apache/hadoop/ozone/MiniOzoneCluster.java | 8 ++++++++ .../hadoop/ozone/MiniOzoneClusterImpl.java | 16 +++++++++++++++- 7 files changed, 58 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java index 62fca63..0de9f18 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java @@ -95,15 +95,30 @@ public class VolumeInfo { this.usage = new VolumeUsage(root, b.conf); } - public long getCapacity() { - return configuredCapacity < 0 ? usage.getCapacity() : configuredCapacity; + public long getCapacity() throws IOException { + if (configuredCapacity < 0) { + if (usage == null) { + throw new IOException("Volume Usage thread is not running. This error" + + " is usually seen during DataNode shutdown."); + } + return usage.getCapacity(); + } + return configuredCapacity; } public long getAvailable() throws IOException { + if (usage == null) { + throw new IOException("Volume Usage thread is not running. This error " + + "is usually seen during DataNode shutdown."); + } return usage.getAvailable(); } public long getScmUsed() throws IOException { + if (usage == null) { + throw new IOException("Volume Usage thread is not running. This error " + + "is usually seen during DataNode shutdown."); + } return usage.getScmUsed(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java index 5b6b823..d30dd89 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java @@ -372,18 +372,21 @@ public class VolumeSet { for (Map.Entry<String, HddsVolume> entry : volumeMap.entrySet()) { hddsVolume = entry.getValue(); VolumeInfo volumeInfo = hddsVolume.getVolumeInfo(); - long scmUsed = 0; - long remaining = 0; + long scmUsed; + long remaining; + long capacity; failed = false; try { scmUsed = volumeInfo.getScmUsed(); remaining = volumeInfo.getAvailable(); + capacity = volumeInfo.getCapacity(); } catch (IOException ex) { LOG.warn("Failed to get scmUsed and remaining for container " + - "storage location {}", volumeInfo.getRootDir()); + "storage location {}", volumeInfo.getRootDir(), ex); // reset scmUsed and remaining if df/du failed. scmUsed = 0; remaining = 0; + capacity = 0; failed = true; } @@ -392,7 +395,7 @@ public class VolumeSet { builder.setStorageLocation(volumeInfo.getRootDir()) .setId(hddsVolume.getStorageID()) .setFailed(failed) - .setCapacity(hddsVolume.getCapacity()) + .setCapacity(capacity) .setRemaining(remaining) .setScmUsed(scmUsed) .setStorageType(hddsVolume.getStorageType()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java index 7755345..6b46762 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java @@ -31,6 +31,7 @@ import org.junit.rules.TemporaryFolder; import org.mockito.Mockito; import java.io.File; +import java.io.IOException; import java.util.Properties; import java.util.UUID; @@ -134,12 +135,14 @@ public class TestHddsVolume { scmUsedFile.exists()); try { - // Volume.getAvailable() should fail with NullPointerException as usage - // is shutdown. + // Volume.getAvailable() should fail with IOException + // as usage thread is shutdown. volume.getAvailable(); fail("HddsVolume#shutdown test failed"); } catch (Exception ex){ - assertTrue(ex instanceof NullPointerException); + assertTrue(ex instanceof IOException); + assertTrue(ex.getMessage().contains( + "Volume Usage thread is not running.")); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java index fca68b1..7bb8a43 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java @@ -222,8 +222,10 @@ public class TestVolumeSet { // getAvailable() should throw null pointer exception as usage is null. volume.getAvailable(); fail("Volume shutdown failed."); - } catch (NullPointerException ex) { + } catch (IOException ex) { // Do Nothing. Exception is expected. + assertTrue(ex.getMessage().contains( + "Volume Usage thread is not running.")); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java index 9a1c705..618cd8e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java @@ -99,7 +99,7 @@ public class TestNodeFailure { } } - @Test + @Test(timeout = 300_000L) public void testPipelineFail() throws InterruptedException, IOException, TimeoutException { Assert.assertEquals(ratisContainer1.getPipeline().getPipelineState(), @@ -118,6 +118,7 @@ public class TestNodeFailure { pipelineManager.getPipeline(ratisContainer2.getPipeline().getId()) .getPipelineState()); // Now restart the datanode and make sure that a new pipeline is created. + cluster.setWaitForClusterToBeReadyTimeout(300000); cluster.restartHddsDatanode(dnToFail, true); ContainerWithPipeline ratisContainer3 = containerManager.allocateContainer(RATIS, THREE, "testOwner"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index 3aad7f7..15bf8d0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -67,6 +67,14 @@ public interface MiniOzoneCluster { void waitForClusterToBeReady() throws TimeoutException, InterruptedException; /** + * Sets the timeout value after which + * {@link MiniOzoneCluster#waitForClusterToBeReady} times out. + * + * @param timeoutInMs timeout value in milliseconds + */ + void setWaitForClusterToBeReadyTimeout(int timeoutInMs); + + /** * Waits/blocks till the cluster is out of chill mode. * * @throws TimeoutException TimeoutException In case of timeout http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 11bc0e0..6c0f408 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -90,6 +90,9 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster { private final OzoneManager ozoneManager; private final List<HddsDatanodeService> hddsDatanodes; + // Timeout for the cluster to be ready + private int waitForClusterToBeReadyTimeout = 60000; // 1 min + /** * Creates a new MiniOzoneCluster. * @@ -122,7 +125,18 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster { isReady? "Cluster is ready" : "Waiting for cluster to be ready", healthy, hddsDatanodes.size()); return isReady; - }, 1000, 60 * 1000); //wait for 1 min. + }, 1000, waitForClusterToBeReadyTimeout); + } + + /** + * Sets the timeout value after which + * {@link MiniOzoneClusterImpl#waitForClusterToBeReady} times out. + * + * @param timeoutInMs timeout value in milliseconds + */ + @Override + public void setWaitForClusterToBeReadyTimeout(int timeoutInMs) { + waitForClusterToBeReadyTimeout = timeoutInMs; } /** --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
