http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java deleted file mode 100644 index 6044c0a..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.audit; - -/** - * Enum to define Dummy AuditAction Type for test. - */ -public enum DummyAction implements AuditAction { - - CREATE_VOLUME("CREATE_VOLUME"), - CREATE_BUCKET("CREATE_BUCKET"), - CREATE_KEY("CREATE_KEY"), - READ_VOLUME("READ_VOLUME"), - READ_BUCKET("READ_BUCKET"), - READ_KEY("READ_BUCKET"), - UPDATE_VOLUME("UPDATE_VOLUME"), - UPDATE_BUCKET("UPDATE_BUCKET"), - UPDATE_KEY("UPDATE_KEY"), - DELETE_VOLUME("DELETE_VOLUME"), - DELETE_BUCKET("DELETE_BUCKET"), - DELETE_KEY("DELETE_KEY"), - SET_OWNER("SET_OWNER"), - SET_QUOTA("SET_QUOTA"); - - private String action; - - DummyAction(String action) { - this.action = action; - } - - @Override - public String getAction() { - return this.action; - } - -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java deleted file mode 100644 index 0c2d98f..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.ozone.audit; - -import java.util.HashMap; -import java.util.Map; - -/** - * DummyEntity that implements Auditable for test purpose. - */ -public class DummyEntity implements Auditable { - - private String key1; - private String key2; - - public DummyEntity(){ - this.key1 = "value1"; - this.key2 = "value2"; - } - public String getKey1() { - return key1; - } - - public void setKey1(String key1) { - this.key1 = key1; - } - - public String getKey2() { - return key2; - } - - public void setKey2(String key2) { - this.key2 = key2; - } - - @Override - public Map<String, String> toAuditMap() { - Map<String, String> auditMap = new HashMap<>(); - auditMap.put("key1", this.key1); - auditMap.put("key2", this.key2); - return auditMap; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java deleted file mode 100644 index 77a6c0b..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java +++ /dev/null @@ -1,166 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.audit; - -import org.apache.commons.io.FileUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertTrue; - -/** - * Test Ozone Audit Logger. - */ -public class TestOzoneAuditLogger { - - private static final Logger LOG = - LoggerFactory.getLogger(TestOzoneAuditLogger.class.getName()); - - private static final AuditLogger AUDIT = - new AuditLogger(AuditLoggerType.OMLOGGER); - - private static final String SUCCESS = AuditEventStatus.SUCCESS.name(); - private static final String FAILURE = AuditEventStatus.FAILURE.name(); - - private static final Map<String, String> PARAMS = - new DummyEntity().toAuditMap(); - - private static final AuditMessage WRITE_FAIL_MSG = - new AuditMessage.Builder() - .setUser("john") - .atIp("192.168.0.1") - .forOperation(DummyAction.CREATE_VOLUME.name()) - .withParams(PARAMS) - .withResult(FAILURE) - .withException(null).build(); - - private static final AuditMessage WRITE_SUCCESS_MSG = - new AuditMessage.Builder() - .setUser("john") - .atIp("192.168.0.1") - .forOperation(DummyAction.CREATE_VOLUME.name()) - .withParams(PARAMS) - .withResult(SUCCESS) - .withException(null).build(); - - private static final AuditMessage READ_FAIL_MSG = - new AuditMessage.Builder() - .setUser("john") - .atIp("192.168.0.1") - .forOperation(DummyAction.READ_VOLUME.name()) - .withParams(PARAMS) - .withResult(FAILURE) - .withException(null).build(); - - private static final AuditMessage READ_SUCCESS_MSG = - new AuditMessage.Builder() - .setUser("john") - .atIp("192.168.0.1") - .forOperation(DummyAction.READ_VOLUME.name()) - .withParams(PARAMS) - .withResult(SUCCESS) - .withException(null).build(); - - @BeforeClass - public static void setUp(){ - System.setProperty("log4j.configurationFile", "log4j2.properties"); - } - - @AfterClass - public static void tearDown() { - File file = new File("audit.log"); - if (FileUtils.deleteQuietly(file)) { - LOG.info(file.getName() + - " has been deleted as all tests have completed."); - } else { - LOG.info("audit.log could not be deleted."); - } - } - - /** - * Test to verify default log level is INFO when logging success events. - */ - @Test - public void verifyDefaultLogLevelForSuccess() throws IOException { - AUDIT.logWriteSuccess(WRITE_SUCCESS_MSG); - String expected = - "INFO | OMAudit | " + WRITE_SUCCESS_MSG.getFormattedMessage(); - verifyLog(expected); - } - - /** - * Test to verify default log level is ERROR when logging failure events. - */ - @Test - public void verifyDefaultLogLevelForFailure() throws IOException { - AUDIT.logWriteFailure(WRITE_FAIL_MSG); - String expected = - "ERROR | OMAudit | " + WRITE_FAIL_MSG.getFormattedMessage(); - verifyLog(expected); - } - - /** - * Test to verify no READ event is logged. - */ - @Test - public void notLogReadEvents() throws IOException { - AUDIT.logReadSuccess(READ_SUCCESS_MSG); - AUDIT.logReadFailure(READ_FAIL_MSG); - verifyNoLog(); - } - - private void verifyLog(String expected) throws IOException { - File file = new File("audit.log"); - List<String> lines = FileUtils.readLines(file, (String)null); - final int retry = 5; - int i = 0; - while (lines.isEmpty() && i < retry) { - lines = FileUtils.readLines(file, (String)null); - try { - Thread.sleep(500 * (i + 1)); - } catch(InterruptedException ie) { - Thread.currentThread().interrupt(); - break; - } - i++; - } - - // When log entry is expected, the log file will contain one line and - // that must be equal to the expected string - assertTrue(lines.size() != 0); - assertTrue(expected.equalsIgnoreCase(lines.get(0))); - //empty the file - lines.remove(0); - FileUtils.writeLines(file, lines, false); - } - - private void verifyNoLog() throws IOException { - File file = new File("audit.log"); - List<String> lines = FileUtils.readLines(file, (String)null); - // When no log entry is expected, the log file must be empty - assertTrue(lines.size() == 0); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java deleted file mode 100644 index 1222ad0..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.audit; -/** - * Unit tests of Ozone Audit Logger. - * For test purpose, the log4j2 configuration is loaded from file at: - * src/test/resources/log4j2.properties - */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java deleted file mode 100644 index c1470bb..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.common; - -import org.apache.commons.collections.SetUtils; -import org.apache.hadoop.ozone.common.statemachine - .InvalidStateTransitionException; -import org.apache.hadoop.ozone.common.statemachine.StateMachine; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.util.HashSet; -import java.util.Set; - -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLEANUP; -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLOSED; -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CREATING; -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.FINAL; -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.INIT; -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES - .OPERATIONAL; - -/** - * This class is to test ozone common state machine. - */ -public class TestStateMachine { - - /** - * STATES used by the test state machine. - */ - public enum STATES {INIT, CREATING, OPERATIONAL, CLOSED, CLEANUP, FINAL}; - - /** - * EVENTS used by the test state machine. - */ - public enum EVENTS {ALLOCATE, CREATE, UPDATE, CLOSE, DELETE, TIMEOUT}; - - @Rule - public ExpectedException exception = ExpectedException.none(); - - @Test - public void testStateMachineStates() throws InvalidStateTransitionException { - Set<STATES> finals = new HashSet<>(); - finals.add(FINAL); - - StateMachine<STATES, EVENTS> stateMachine = - new StateMachine<>(INIT, finals); - - stateMachine.addTransition(INIT, CREATING, EVENTS.ALLOCATE); - stateMachine.addTransition(CREATING, OPERATIONAL, EVENTS.CREATE); - stateMachine.addTransition(OPERATIONAL, OPERATIONAL, EVENTS.UPDATE); - stateMachine.addTransition(OPERATIONAL, CLEANUP, EVENTS.DELETE); - stateMachine.addTransition(OPERATIONAL, CLOSED, EVENTS.CLOSE); - stateMachine.addTransition(CREATING, CLEANUP, EVENTS.TIMEOUT); - - // Initial and Final states - Assert.assertEquals("Initial State", INIT, stateMachine.getInitialState()); - Assert.assertTrue("Final States", SetUtils.isEqualSet(finals, - stateMachine.getFinalStates())); - - // Valid state transitions - Assert.assertEquals("STATE should be OPERATIONAL after being created", - OPERATIONAL, stateMachine.getNextState(CREATING, EVENTS.CREATE)); - Assert.assertEquals("STATE should be OPERATIONAL after being updated", - OPERATIONAL, stateMachine.getNextState(OPERATIONAL, EVENTS.UPDATE)); - Assert.assertEquals("STATE should be CLEANUP after being deleted", - CLEANUP, stateMachine.getNextState(OPERATIONAL, EVENTS.DELETE)); - Assert.assertEquals("STATE should be CLEANUP after being timeout", - CLEANUP, stateMachine.getNextState(CREATING, EVENTS.TIMEOUT)); - Assert.assertEquals("STATE should be CLOSED after being closed", - CLOSED, stateMachine.getNextState(OPERATIONAL, EVENTS.CLOSE)); - - // Negative cases: invalid transition - expectException(); - stateMachine.getNextState(OPERATIONAL, EVENTS.CREATE); - - expectException(); - stateMachine.getNextState(CREATING, EVENTS.CLOSE); - } - - /** - * We expect an InvalidStateTransitionException. - */ - private void expectException() { - exception.expect(InvalidStateTransitionException.class); - exception.expectMessage("Invalid event"); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java deleted file mode 100644 index bdc70fc..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java +++ /dev/null @@ -1,373 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -/** - * A generic lease management API which can be used if a service - * needs any kind of lease management. - */ - -package org.apache.hadoop.ozone.lease; - -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.util.HashMap; -import java.util.Map; - -/** - * Test class to check functionality and consistency of LeaseManager. - */ -public class TestLeaseManager { - - @Rule - public ExpectedException exception = ExpectedException.none(); - - /** - * Dummy resource on which leases can be acquired. - */ - private final class DummyResource { - - private final String name; - - private DummyResource(String name) { - this.name = name; - } - - @Override - public int hashCode() { - return name.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if(obj instanceof DummyResource) { - return name.equals(((DummyResource) obj).name); - } - return false; - } - } - - @Test - public void testLeaseAcquireAndRelease() throws LeaseException { - //It is assumed that the test case execution won't take more than 5 seconds, - //if it takes more time increase the defaultTimeout value of LeaseManager. - LeaseManager<DummyResource> manager = new LeaseManager<>("Test", 5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - DummyResource resourceTwo = new DummyResource("two"); - DummyResource resourceThree = new DummyResource("three"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo); - Lease<DummyResource> leaseThree = manager.acquire(resourceThree); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertEquals(leaseTwo, manager.get(resourceTwo)); - Assert.assertEquals(leaseThree, manager.get(resourceThree)); - Assert.assertFalse(leaseOne.hasExpired()); - Assert.assertFalse(leaseTwo.hasExpired()); - Assert.assertFalse(leaseThree.hasExpired()); - //The below releases should not throw LeaseNotFoundException. - manager.release(resourceOne); - manager.release(resourceTwo); - manager.release(resourceThree); - Assert.assertTrue(leaseOne.hasExpired()); - Assert.assertTrue(leaseTwo.hasExpired()); - Assert.assertTrue(leaseThree.hasExpired()); - manager.shutdown(); - } - - @Test - public void testLeaseAlreadyExist() throws LeaseException { - LeaseManager<DummyResource> manager = new LeaseManager<>("Test", 5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - DummyResource resourceTwo = new DummyResource("two"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertEquals(leaseTwo, manager.get(resourceTwo)); - - exception.expect(LeaseAlreadyExistException.class); - exception.expectMessage("Resource: " + resourceOne); - manager.acquire(resourceOne); - - manager.release(resourceOne); - manager.release(resourceTwo); - manager.shutdown(); - } - - @Test - public void testLeaseNotFound() throws LeaseException, InterruptedException { - LeaseManager<DummyResource> manager = new LeaseManager<>("Test", 5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - DummyResource resourceTwo = new DummyResource("two"); - DummyResource resourceThree = new DummyResource("three"); - - //Case 1: lease was never acquired. - exception.expect(LeaseNotFoundException.class); - exception.expectMessage("Resource: " + resourceOne); - manager.get(resourceOne); - - //Case 2: lease is acquired and released. - Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo); - Assert.assertEquals(leaseTwo, manager.get(resourceTwo)); - Assert.assertFalse(leaseTwo.hasExpired()); - manager.release(resourceTwo); - Assert.assertTrue(leaseTwo.hasExpired()); - exception.expect(LeaseNotFoundException.class); - exception.expectMessage("Resource: " + resourceTwo); - manager.get(resourceTwo); - - //Case 3: lease acquired and timed out. - Lease<DummyResource> leaseThree = manager.acquire(resourceThree); - Assert.assertEquals(leaseThree, manager.get(resourceThree)); - Assert.assertFalse(leaseThree.hasExpired()); - long sleepTime = leaseThree.getRemainingTime() + 1000; - try { - Thread.sleep(sleepTime); - } catch (InterruptedException ex) { - //even in case of interrupt we have to wait till lease times out. - Thread.sleep(sleepTime); - } - Assert.assertTrue(leaseThree.hasExpired()); - exception.expect(LeaseNotFoundException.class); - exception.expectMessage("Resource: " + resourceThree); - manager.get(resourceThree); - manager.shutdown(); - } - - @Test - public void testCustomLeaseTimeout() throws LeaseException { - LeaseManager<DummyResource> manager = new LeaseManager<>("Test", 5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - DummyResource resourceTwo = new DummyResource("two"); - DummyResource resourceThree = new DummyResource("three"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo, 10000); - Lease<DummyResource> leaseThree = manager.acquire(resourceThree, 50000); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertEquals(leaseTwo, manager.get(resourceTwo)); - Assert.assertEquals(leaseThree, manager.get(resourceThree)); - Assert.assertFalse(leaseOne.hasExpired()); - Assert.assertFalse(leaseTwo.hasExpired()); - Assert.assertFalse(leaseThree.hasExpired()); - Assert.assertEquals(5000, leaseOne.getLeaseLifeTime()); - Assert.assertEquals(10000, leaseTwo.getLeaseLifeTime()); - Assert.assertEquals(50000, leaseThree.getLeaseLifeTime()); - // Releasing of leases is done in shutdown, so don't have to worry about - // lease release - manager.shutdown(); - } - - @Test - public void testLeaseCallback() throws LeaseException, InterruptedException { - Map<DummyResource, String> leaseStatus = new HashMap<>(); - LeaseManager<DummyResource> manager = new LeaseManager<>("Test", 5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - leaseStatus.put(resourceOne, "lease in use"); - leaseOne.registerCallBack(() -> { - leaseStatus.put(resourceOne, "lease expired"); - return null; - }); - // wait for lease to expire - long sleepTime = leaseOne.getRemainingTime() + 1000; - try { - Thread.sleep(sleepTime); - } catch (InterruptedException ex) { - //even in case of interrupt we have to wait till lease times out. - Thread.sleep(sleepTime); - } - Assert.assertTrue(leaseOne.hasExpired()); - exception.expect(LeaseNotFoundException.class); - exception.expectMessage("Resource: " + resourceOne); - manager.get(resourceOne); - // check if callback has been executed - Assert.assertEquals("lease expired", leaseStatus.get(resourceOne)); - } - - @Test - public void testCallbackExecutionInCaseOfLeaseRelease() - throws LeaseException, InterruptedException { - // Callbacks should not be executed in case of lease release - Map<DummyResource, String> leaseStatus = new HashMap<>(); - LeaseManager<DummyResource> manager = new LeaseManager<>("Test", 5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - leaseStatus.put(resourceOne, "lease in use"); - leaseOne.registerCallBack(() -> { - leaseStatus.put(resourceOne, "lease expired"); - return null; - }); - leaseStatus.put(resourceOne, "lease released"); - manager.release(resourceOne); - Assert.assertTrue(leaseOne.hasExpired()); - exception.expect(LeaseNotFoundException.class); - exception.expectMessage("Resource: " + resourceOne); - manager.get(resourceOne); - Assert.assertEquals("lease released", leaseStatus.get(resourceOne)); - } - - @Test - public void testLeaseCallbackWithMultipleLeases() - throws LeaseException, InterruptedException { - Map<DummyResource, String> leaseStatus = new HashMap<>(); - LeaseManager<DummyResource> manager = new LeaseManager<>("Test", 5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - DummyResource resourceTwo = new DummyResource("two"); - DummyResource resourceThree = new DummyResource("three"); - DummyResource resourceFour = new DummyResource("four"); - DummyResource resourceFive = new DummyResource("five"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo); - Lease<DummyResource> leaseThree = manager.acquire(resourceThree); - Lease<DummyResource> leaseFour = manager.acquire(resourceFour); - Lease<DummyResource> leaseFive = manager.acquire(resourceFive); - leaseStatus.put(resourceOne, "lease in use"); - leaseStatus.put(resourceTwo, "lease in use"); - leaseStatus.put(resourceThree, "lease in use"); - leaseStatus.put(resourceFour, "lease in use"); - leaseStatus.put(resourceFive, "lease in use"); - leaseOne.registerCallBack(() -> { - leaseStatus.put(resourceOne, "lease expired"); - return null; - }); - leaseTwo.registerCallBack(() -> { - leaseStatus.put(resourceTwo, "lease expired"); - return null; - }); - leaseThree.registerCallBack(() -> { - leaseStatus.put(resourceThree, "lease expired"); - return null; - }); - leaseFour.registerCallBack(() -> { - leaseStatus.put(resourceFour, "lease expired"); - return null; - }); - leaseFive.registerCallBack(() -> { - leaseStatus.put(resourceFive, "lease expired"); - return null; - }); - - // release lease one, two and three - leaseStatus.put(resourceOne, "lease released"); - manager.release(resourceOne); - leaseStatus.put(resourceTwo, "lease released"); - manager.release(resourceTwo); - leaseStatus.put(resourceThree, "lease released"); - manager.release(resourceThree); - - // wait for other leases to expire - long sleepTime = leaseFive.getRemainingTime() + 1000; - - try { - Thread.sleep(sleepTime); - } catch (InterruptedException ex) { - //even in case of interrupt we have to wait till lease times out. - Thread.sleep(sleepTime); - } - Assert.assertTrue(leaseOne.hasExpired()); - Assert.assertTrue(leaseTwo.hasExpired()); - Assert.assertTrue(leaseThree.hasExpired()); - Assert.assertTrue(leaseFour.hasExpired()); - Assert.assertTrue(leaseFive.hasExpired()); - - Assert.assertEquals("lease released", leaseStatus.get(resourceOne)); - Assert.assertEquals("lease released", leaseStatus.get(resourceTwo)); - Assert.assertEquals("lease released", leaseStatus.get(resourceThree)); - Assert.assertEquals("lease expired", leaseStatus.get(resourceFour)); - Assert.assertEquals("lease expired", leaseStatus.get(resourceFive)); - manager.shutdown(); - } - - @Test - public void testReuseReleasedLease() throws LeaseException { - LeaseManager<DummyResource> manager = new LeaseManager<>("Test", 5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertFalse(leaseOne.hasExpired()); - - manager.release(resourceOne); - Assert.assertTrue(leaseOne.hasExpired()); - - Lease<DummyResource> sameResourceLease = manager.acquire(resourceOne); - Assert.assertEquals(sameResourceLease, manager.get(resourceOne)); - Assert.assertFalse(sameResourceLease.hasExpired()); - - manager.release(resourceOne); - Assert.assertTrue(sameResourceLease.hasExpired()); - manager.shutdown(); - } - - @Test - public void testReuseTimedOutLease() - throws LeaseException, InterruptedException { - LeaseManager<DummyResource> manager = new LeaseManager<>("Test", 5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertFalse(leaseOne.hasExpired()); - // wait for lease to expire - long sleepTime = leaseOne.getRemainingTime() + 1000; - try { - Thread.sleep(sleepTime); - } catch (InterruptedException ex) { - //even in case of interrupt we have to wait till lease times out. - Thread.sleep(sleepTime); - } - Assert.assertTrue(leaseOne.hasExpired()); - - Lease<DummyResource> sameResourceLease = manager.acquire(resourceOne); - Assert.assertEquals(sameResourceLease, manager.get(resourceOne)); - Assert.assertFalse(sameResourceLease.hasExpired()); - - manager.release(resourceOne); - Assert.assertTrue(sameResourceLease.hasExpired()); - manager.shutdown(); - } - - @Test - public void testRenewLease() throws LeaseException, InterruptedException { - LeaseManager<DummyResource> manager = new LeaseManager<>("Test", 5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertFalse(leaseOne.hasExpired()); - - // add 5 more seconds to the lease - leaseOne.renew(5000); - - Thread.sleep(5000); - - // lease should still be active - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertFalse(leaseOne.hasExpired()); - manager.release(resourceOne); - manager.shutdown(); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java deleted file mode 100644 index 1071309..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.lease; -/* - This package contains lease management unit test classes. - */ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lock/TestLockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lock/TestLockManager.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lock/TestLockManager.java deleted file mode 100644 index fa3030d..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lock/TestLockManager.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.lock; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.junit.Assert; -import org.junit.Test; - -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Test-cases to test LockManager. - */ -public class TestLockManager { - - @Test(timeout = 1000) - public void testWithDifferentResource() { - LockManager<String> manager = new LockManager<>(new OzoneConfiguration()); - manager.lock("/resourceOne"); - // This should work, as they are different resource. - manager.lock("/resourceTwo"); - manager.unlock("/resourceOne"); - manager.unlock("/resourceTwo"); - Assert.assertTrue(true); - } - - @Test - public void testWithSameResource() throws Exception { - LockManager<String> manager = new LockManager<>(new OzoneConfiguration()); - manager.lock("/resourceOne"); - AtomicBoolean gotLock = new AtomicBoolean(false); - new Thread(() -> { - manager.lock("/resourceOne"); - gotLock.set(true); - manager.unlock("/resourceOne"); - }).start(); - // Let's give some time for the new thread to run - Thread.sleep(100); - // Since the new thread is trying to get lock on same object, it will wait. - Assert.assertFalse(gotLock.get()); - manager.unlock("/resourceOne"); - // Since we have released the lock, the new thread should have the lock - // now - // Let's give some time for the new thread to run - Thread.sleep(100); - Assert.assertTrue(gotLock.get()); - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lock/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lock/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lock/package-info.java deleted file mode 100644 index a96bc16..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lock/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.lock; -/* - This package contains the lock related test classes. - */ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java deleted file mode 100644 index 0030d2e..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone; -/** - * Ozone related test helper classes and tests of common utils. - */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestHddsIdFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestHddsIdFactory.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestHddsIdFactory.java deleted file mode 100644 index a341ccc..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestHddsIdFactory.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.utils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import org.apache.hadoop.hdds.HddsIdFactory; -import org.junit.After; -import static org.junit.Assert.assertEquals; - -import org.junit.Assert; -import org.junit.Test; - -/** - * Test the JMX interface for the rocksdb metastore implementation. - */ -public class TestHddsIdFactory { - - private static final Set<Long> ID_SET = ConcurrentHashMap.newKeySet(); - private static final int IDS_PER_THREAD = 10000; - private static final int NUM_OF_THREADS = 5; - - @After - public void cleanup() { - ID_SET.clear(); - } - - @Test - public void testGetLongId() throws Exception { - - ExecutorService executor = Executors.newFixedThreadPool(5); - List<Callable<Integer>> tasks = new ArrayList<>(5); - addTasks(tasks); - List<Future<Integer>> result = executor.invokeAll(tasks); - assertEquals(IDS_PER_THREAD * NUM_OF_THREADS, ID_SET.size()); - for (Future<Integer> r : result) { - assertEquals(r.get().intValue(), IDS_PER_THREAD); - } - } - - private void addTasks(List<Callable<Integer>> tasks) { - for (int i = 0; i < NUM_OF_THREADS; i++) { - Callable<Integer> task = () -> { - for (int idNum = 0; idNum < IDS_PER_THREAD; idNum++) { - long var = HddsIdFactory.getLongId(); - if (ID_SET.contains(var)) { - Assert.fail("Duplicate id found"); - } - ID_SET.add(var); - } - return IDS_PER_THREAD; - }; - tasks.add(task); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java deleted file mode 100644 index 30fc7f3..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java +++ /dev/null @@ -1,590 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.utils; - -import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows; - -import com.google.common.collect.Lists; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.utils.MetadataStore.KeyValue; -import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter; -import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.event.Level; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.UUID; - -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import static org.junit.runners.Parameterized.Parameters; - -/** - * Test class for ozone metadata store. - */ -@RunWith(Parameterized.class) -public class TestMetadataStore { - - private final String storeImpl; - - public TestMetadataStore(String metadataImpl) { - this.storeImpl = metadataImpl; - } - - @Parameters - public static Collection<Object[]> data() { - return Arrays.asList(new Object[][] { - {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB}, - {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB} - }); - } - - private MetadataStore store; - private File testDir; - private final static int MAX_GETRANGE_LENGTH = 100; - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Before - public void init() throws IOException { - if (OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB.equals(storeImpl)) { - // The initialization of RocksDB fails on Windows - assumeNotWindows(); - } - - testDir = GenericTestUtils.getTestDir(getClass().getSimpleName() - + "-" + storeImpl.toLowerCase()); - - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl); - - store = MetadataStoreBuilder.newBuilder() - .setConf(conf) - .setCreateIfMissing(true) - .setDbFile(testDir) - .build(); - - // Add 20 entries. - // {a0 : a-value0} to {a9 : a-value9} - // {b0 : b-value0} to {b9 : b-value9} - for (int i=0; i<10; i++) { - store.put(getBytes("a" + i), getBytes("a-value" + i)); - store.put(getBytes("b" + i), getBytes("b-value" + i)); - } - } - - @Test - public void testIterator() throws Exception { - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl); - File dbDir = GenericTestUtils.getRandomizedTestDir(); - MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setConf(conf) - .setCreateIfMissing(true) - .setDbFile(dbDir) - .build(); - - //As database is empty, check whether iterator is working as expected or - // not. - MetaStoreIterator<KeyValue> metaStoreIterator = dbStore.iterator(); - assertFalse(metaStoreIterator.hasNext()); - try { - metaStoreIterator.next(); - fail("testIterator failed"); - } catch (NoSuchElementException ex) { - GenericTestUtils.assertExceptionContains("Store has no more elements", - ex); - } - - for (int i = 0; i < 10; i++) { - store.put(getBytes("a" + i), getBytes("a-value" + i)); - } - - metaStoreIterator = dbStore.iterator(); - - int i = 0; - while (metaStoreIterator.hasNext()) { - KeyValue val = metaStoreIterator.next(); - assertEquals("a" + i, getString(val.getKey())); - assertEquals("a-value" + i, getString(val.getValue())); - i++; - } - - // As we have iterated all the keys in database, hasNext should return - // false and next() should throw NoSuchElement exception. - - assertFalse(metaStoreIterator.hasNext()); - try { - metaStoreIterator.next(); - fail("testIterator failed"); - } catch (NoSuchElementException ex) { - GenericTestUtils.assertExceptionContains("Store has no more elements", - ex); - } - FileUtils.deleteDirectory(dbDir); - - } - - @Test - public void testMetaStoreConfigDifferentFromType() throws IOException { - - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl); - String dbType; - GenericTestUtils.setLogLevel(MetadataStoreBuilder.LOG, Level.DEBUG); - GenericTestUtils.LogCapturer logCapturer = - GenericTestUtils.LogCapturer.captureLogs(MetadataStoreBuilder.LOG); - if(storeImpl.equals(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB)) { - dbType = "RocksDB"; - } else { - dbType = "LevelDB"; - } - - File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName() - + "-" + dbType.toLowerCase() + "-test"); - MetadataStore dbStore = MetadataStoreBuilder.newBuilder().setConf(conf) - .setCreateIfMissing(true).setDbFile(dbDir).setDBType(dbType).build(); - assertTrue(logCapturer.getOutput().contains("Using dbType " + dbType + "" + - " for metastore")); - dbStore.close(); - dbStore.destroy(); - FileUtils.deleteDirectory(dbDir); - - } - - @Test - public void testdbTypeNotSet() throws IOException { - - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl); - GenericTestUtils.setLogLevel(MetadataStoreBuilder.LOG, Level.DEBUG); - GenericTestUtils.LogCapturer logCapturer = - GenericTestUtils.LogCapturer.captureLogs(MetadataStoreBuilder.LOG); - - - File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName() - + "-" + storeImpl.toLowerCase() + "-test"); - MetadataStore dbStore = MetadataStoreBuilder.newBuilder().setConf(conf) - .setCreateIfMissing(true).setDbFile(dbDir).build(); - assertTrue(logCapturer.getOutput().contains("dbType is null, using dbType" + - " " + storeImpl)); - dbStore.close(); - dbStore.destroy(); - FileUtils.deleteDirectory(dbDir); - - } - - @After - public void cleanup() throws IOException { - if (store != null) { - store.close(); - store.destroy(); - } - if (testDir != null) { - FileUtils.deleteDirectory(testDir); - } - } - - private byte[] getBytes(String str) { - return str == null ? null : - DFSUtilClient.string2Bytes(str); - } - - private String getString(byte[] bytes) { - return bytes == null ? null : - DFSUtilClient.bytes2String(bytes); - } - - @Test - public void testGetDelete() throws IOException { - for (int i=0; i<10; i++) { - byte[] va = store.get(getBytes("a" + i)); - assertEquals("a-value" + i, getString(va)); - - byte[] vb = store.get(getBytes("b" + i)); - assertEquals("b-value" + i, getString(vb)); - } - - String keyToDel = "del-" + UUID.randomUUID().toString(); - store.put(getBytes(keyToDel), getBytes(keyToDel)); - assertEquals(keyToDel, getString(store.get(getBytes(keyToDel)))); - store.delete(getBytes(keyToDel)); - assertEquals(null, store.get(getBytes(keyToDel))); - } - - @Test - public void testPeekFrom() throws IOException { - // Test peek from an element that has prev as well as next - testPeek("a3", "a2", "a4"); - - // Test peek from an element that only has prev - testPeek("b9", "b8", null); - - // Test peek from an element that only has next - testPeek("a0", null, "a1"); - } - - private String getExpectedValue(String key) { - if (key == null) { - return null; - } - char[] arr = key.toCharArray(); - return new StringBuffer().append(arr[0]).append("-value") - .append(arr[arr.length - 1]).toString(); - } - - private void testPeek(String peekKey, String prevKey, String nextKey) - throws IOException { - // Look for current - String k = null; - String v = null; - ImmutablePair<byte[], byte[]> current = - store.peekAround(0, getBytes(peekKey)); - if (current != null) { - k = getString(current.getKey()); - v = getString(current.getValue()); - } - assertEquals(peekKey, k); - assertEquals(v, getExpectedValue(peekKey)); - - // Look for prev - k = null; - v = null; - ImmutablePair<byte[], byte[]> prev = - store.peekAround(-1, getBytes(peekKey)); - if (prev != null) { - k = getString(prev.getKey()); - v = getString(prev.getValue()); - } - assertEquals(prevKey, k); - assertEquals(v, getExpectedValue(prevKey)); - - // Look for next - k = null; - v = null; - ImmutablePair<byte[], byte[]> next = - store.peekAround(1, getBytes(peekKey)); - if (next != null) { - k = getString(next.getKey()); - v = getString(next.getValue()); - } - assertEquals(nextKey, k); - assertEquals(v, getExpectedValue(nextKey)); - } - - @Test - public void testIterateKeys() throws IOException { - // iterate keys from b0 - ArrayList<String> result = Lists.newArrayList(); - store.iterate(getBytes("b0"), (k, v) -> { - // b-value{i} - String value = getString(v); - char num = value.charAt(value.length() - 1); - // each value adds 1 - int i = Character.getNumericValue(num) + 1; - value = value.substring(0, value.length() - 1) + i; - result.add(value); - return true; - }); - - assertFalse(result.isEmpty()); - for (int i=0; i<result.size(); i++) { - assertEquals("b-value" + (i+1), result.get(i)); - } - - // iterate from a non exist key - result.clear(); - store.iterate(getBytes("xyz"), (k, v) -> { - result.add(getString(v)); - return true; - }); - assertTrue(result.isEmpty()); - - // iterate from the beginning - result.clear(); - store.iterate(null, (k, v) -> { - result.add(getString(v)); - return true; - }); - assertEquals(20, result.size()); - } - - @Test - public void testGetRangeKVs() throws IOException { - List<Map.Entry<byte[], byte[]>> result = null; - - // Set empty startKey will return values from beginning. - result = store.getRangeKVs(null, 5); - assertEquals(5, result.size()); - assertEquals("a-value2", getString(result.get(2).getValue())); - - // Empty list if startKey doesn't exist. - result = store.getRangeKVs(getBytes("a12"), 5); - assertEquals(0, result.size()); - - // Returns max available entries after a valid startKey. - result = store.getRangeKVs(getBytes("b0"), MAX_GETRANGE_LENGTH); - assertEquals(10, result.size()); - assertEquals("b0", getString(result.get(0).getKey())); - assertEquals("b-value0", getString(result.get(0).getValue())); - result = store.getRangeKVs(getBytes("b0"), 5); - assertEquals(5, result.size()); - - // Both startKey and count are honored. - result = store.getRangeKVs(getBytes("a9"), 2); - assertEquals(2, result.size()); - assertEquals("a9", getString(result.get(0).getKey())); - assertEquals("a-value9", getString(result.get(0).getValue())); - assertEquals("b0", getString(result.get(1).getKey())); - assertEquals("b-value0", getString(result.get(1).getValue())); - - // Filter keys by prefix. - // It should returns all "b*" entries. - MetadataKeyFilter filter1 = new KeyPrefixFilter().addFilter("b"); - result = store.getRangeKVs(null, 100, filter1); - assertEquals(10, result.size()); - assertTrue(result.stream().allMatch(entry -> - new String(entry.getKey()).startsWith("b") - )); - assertEquals(20, filter1.getKeysScannedNum()); - assertEquals(10, filter1.getKeysHintedNum()); - result = store.getRangeKVs(null, 3, filter1); - assertEquals(3, result.size()); - result = store.getRangeKVs(getBytes("b3"), 1, filter1); - assertEquals("b-value3", getString(result.get(0).getValue())); - - // Define a customized filter that filters keys by suffix. - // Returns all "*2" entries. - MetadataKeyFilter filter2 = (preKey, currentKey, nextKey) - -> getString(currentKey).endsWith("2"); - result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter2); - assertEquals(2, result.size()); - assertEquals("a2", getString(result.get(0).getKey())); - assertEquals("b2", getString(result.get(1).getKey())); - result = store.getRangeKVs(null, 1, filter2); - assertEquals(1, result.size()); - assertEquals("a2", getString(result.get(0).getKey())); - - // Apply multiple filters. - result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter1, filter2); - assertEquals(1, result.size()); - assertEquals("b2", getString(result.get(0).getKey())); - assertEquals("b-value2", getString(result.get(0).getValue())); - - // If filter is null, no effect. - result = store.getRangeKVs(null, 1, null); - assertEquals(1, result.size()); - assertEquals("a0", getString(result.get(0).getKey())); - } - - @Test - public void testGetSequentialRangeKVs() throws IOException { - MetadataKeyFilter suffixFilter = (preKey, currentKey, nextKey) - -> DFSUtil.bytes2String(currentKey).endsWith("2"); - // Suppose to return a2 and b2 - List<Map.Entry<byte[], byte[]>> result = - store.getRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter); - assertEquals(2, result.size()); - assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey())); - assertEquals("b2", DFSUtil.bytes2String(result.get(1).getKey())); - - // Suppose to return just a2, because when it iterates to a3, - // the filter no long matches and it should stop from there. - result = store.getSequentialRangeKVs(null, - MAX_GETRANGE_LENGTH, suffixFilter); - assertEquals(1, result.size()); - assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey())); - } - - @Test - public void testGetRangeLength() throws IOException { - List<Map.Entry<byte[], byte[]>> result = null; - - result = store.getRangeKVs(null, 0); - assertEquals(0, result.size()); - - result = store.getRangeKVs(null, 1); - assertEquals(1, result.size()); - - // Count less than zero is invalid. - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Invalid count given"); - store.getRangeKVs(null, -1); - } - - @Test - public void testInvalidStartKey() throws IOException { - // If startKey is invalid, the returned list should be empty. - List<Map.Entry<byte[], byte[]>> kvs = - store.getRangeKVs(getBytes("unknownKey"), MAX_GETRANGE_LENGTH); - assertEquals(kvs.size(), 0); - } - - @Test - public void testDestroyDB() throws IOException { - // create a new DB to test db destroy - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl); - - File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName() - + "-" + storeImpl.toLowerCase() + "-toDestroy"); - MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setConf(conf) - .setCreateIfMissing(true) - .setDbFile(dbDir) - .build(); - - dbStore.put(getBytes("key1"), getBytes("value1")); - dbStore.put(getBytes("key2"), getBytes("value2")); - - assertFalse(dbStore.isEmpty()); - assertTrue(dbDir.exists()); - assertTrue(dbDir.listFiles().length > 0); - - dbStore.destroy(); - - assertFalse(dbDir.exists()); - } - - @Test - public void testBatchWrite() throws IOException { - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl); - - File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName() - + "-" + storeImpl.toLowerCase() + "-batchWrite"); - MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setConf(conf) - .setCreateIfMissing(true) - .setDbFile(dbDir) - .build(); - - List<String> expectedResult = Lists.newArrayList(); - for (int i = 0; i<10; i++) { - dbStore.put(getBytes("batch-" + i), getBytes("batch-value-" + i)); - expectedResult.add("batch-" + i); - } - - BatchOperation batch = new BatchOperation(); - batch.delete(getBytes("batch-2")); - batch.delete(getBytes("batch-3")); - batch.delete(getBytes("batch-4")); - batch.put(getBytes("batch-new-2"), getBytes("batch-new-value-2")); - - expectedResult.remove("batch-2"); - expectedResult.remove("batch-3"); - expectedResult.remove("batch-4"); - expectedResult.add("batch-new-2"); - - dbStore.writeBatch(batch); - - Iterator<String> it = expectedResult.iterator(); - AtomicInteger count = new AtomicInteger(0); - dbStore.iterate(null, (key, value) -> { - count.incrementAndGet(); - return it.hasNext() && it.next().equals(getString(key)); - }); - - assertEquals(8, count.get()); - } - - @Test - public void testKeyPrefixFilter() throws IOException { - List<Map.Entry<byte[], byte[]>> result = null; - RuntimeException exception = null; - - try { - new KeyPrefixFilter().addFilter("b0", true).addFilter("b"); - } catch (IllegalArgumentException e) { - exception = e; - } - assertTrue(exception.getMessage().contains("KeyPrefix: b already " + - "rejected")); - - try { - new KeyPrefixFilter().addFilter("b0").addFilter("b", true); - } catch (IllegalArgumentException e) { - exception = e; - } - assertTrue(exception.getMessage().contains("KeyPrefix: b already " + - "accepted")); - - try { - new KeyPrefixFilter().addFilter("b", true).addFilter("b0"); - } catch (IllegalArgumentException e) { - exception = e; - } - assertTrue(exception.getMessage().contains("KeyPrefix: b0 already " + - "rejected")); - - try { - new KeyPrefixFilter().addFilter("b").addFilter("b0", true); - } catch (IllegalArgumentException e) { - exception = e; - } - assertTrue(exception.getMessage().contains("KeyPrefix: b0 already " + - "accepted")); - - MetadataKeyFilter filter1 = new KeyPrefixFilter(true) - .addFilter("a0") - .addFilter("a1") - .addFilter("b", true); - result = store.getRangeKVs(null, 100, filter1); - assertEquals(2, result.size()); - assertTrue(result.stream().anyMatch(entry -> new String(entry.getKey()) - .startsWith("a0")) && result.stream().anyMatch(entry -> new String( - entry.getKey()).startsWith("a1"))); - - filter1 = new KeyPrefixFilter(true).addFilter("b", true); - result = store.getRangeKVs(null, 100, filter1); - assertEquals(0, result.size()); - - filter1 = new KeyPrefixFilter().addFilter("b", true); - result = store.getRangeKVs(null, 100, filter1); - assertEquals(10, result.size()); - assertTrue(result.stream().allMatch(entry -> new String(entry.getKey()) - .startsWith("a"))); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java deleted file mode 100644 index a7ce60b..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.utils; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.Test; - -import javax.management.MBeanServer; -import java.io.File; -import java.lang.management.ManagementFactory; - -/** - * Test the JMX interface for the rocksdb metastore implementation. - */ -public class TestRocksDBStoreMBean { - - @Test - public void testJmxBeans() throws Exception { - File testDir = - GenericTestUtils.getTestDir(getClass().getSimpleName() + "-withstat"); - - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, - OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB); - - RocksDBStore metadataStore = - (RocksDBStore) MetadataStoreBuilder.newBuilder().setConf(conf) - .setCreateIfMissing(true).setDbFile(testDir).build(); - - for (int i = 0; i < 10; i++) { - metadataStore.put("key".getBytes(), "value".getBytes()); - } - - MBeanServer platformMBeanServer = - ManagementFactory.getPlatformMBeanServer(); - Thread.sleep(2000); - - Object keysWritten = platformMBeanServer - .getAttribute(metadataStore.getStatMBeanName(), "NUMBER_KEYS_WRITTEN"); - - Assert.assertEquals(10L, keysWritten); - - Object dbWriteAverage = platformMBeanServer - .getAttribute(metadataStore.getStatMBeanName(), "DB_WRITE_AVERAGE"); - Assert.assertTrue((double) dbWriteAverage > 0); - - metadataStore.close(); - - } - - @Test() - public void testDisabledStat() throws Exception { - File testDir = GenericTestUtils - .getTestDir(getClass().getSimpleName() + "-withoutstat"); - - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, - OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS, - OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF); - - RocksDBStore metadataStore = - (RocksDBStore) MetadataStoreBuilder.newBuilder().setConf(conf) - .setCreateIfMissing(true).setDbFile(testDir).build(); - - Assert.assertNull(metadataStore.getStatMBeanName()); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBConfigFromFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBConfigFromFile.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBConfigFromFile.java deleted file mode 100644 index b20ca70..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBConfigFromFile.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.utils.db; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.hdfs.DFSUtil; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.rocksdb.ColumnFamilyDescriptor; -import org.rocksdb.ColumnFamilyOptions; -import org.rocksdb.DBOptions; -import org.rocksdb.RocksDB; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.apache.hadoop.utils.db.DBConfigFromFile.getOptionsFileNameFromDB; - -/** - * DBConf tests. - */ -public class TestDBConfigFromFile { - private final static String DB_FILE = "test.db"; - private final static String INI_FILE = getOptionsFileNameFromDB(DB_FILE); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Before - public void setUp() throws Exception { - System.setProperty(DBConfigFromFile.CONFIG_DIR, - folder.newFolder().toString()); - ClassLoader classLoader = getClass().getClassLoader(); - File testData = new File(classLoader.getResource(INI_FILE).getFile()); - File dest = Paths.get( - System.getProperty(DBConfigFromFile.CONFIG_DIR), INI_FILE).toFile(); - FileUtils.copyFile(testData, dest); - } - - @After - public void tearDown() throws Exception { - } - - @Test - public void readFromFile() throws IOException { - final List<String> families = - Arrays.asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY), - "First", "Second", "Third", - "Fourth", "Fifth", - "Sixth"); - final List<ColumnFamilyDescriptor> columnFamilyDescriptors = - new ArrayList<>(); - for (String family : families) { - columnFamilyDescriptors.add( - new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8), - new ColumnFamilyOptions())); - } - - final DBOptions options = DBConfigFromFile.readFromFile(DB_FILE, - columnFamilyDescriptors); - - // Some Random Values Defined in the test.db.ini, we verify that we are - // able to get values that are defined in the test.db.ini. - Assert.assertNotNull(options); - Assert.assertEquals(551615L, options.maxManifestFileSize()); - Assert.assertEquals(1000L, options.keepLogFileNum()); - Assert.assertEquals(1048576, options.writableFileMaxBufferSize()); - } - - @Test - public void readFromFileInvalidConfig() throws IOException { - final List<String> families = - Arrays.asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY), - "First", "Second", "Third", - "Fourth", "Fifth", - "Sixth"); - final List<ColumnFamilyDescriptor> columnFamilyDescriptors = - new ArrayList<>(); - for (String family : families) { - columnFamilyDescriptors.add( - new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8), - new ColumnFamilyOptions())); - } - - final DBOptions options = DBConfigFromFile.readFromFile("badfile.db.ini", - columnFamilyDescriptors); - - // This has to return a Null, since we have config defined for badfile.db - Assert.assertNull(options); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java deleted file mode 100644 index 3e1f364..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.utils.db; - -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; - -/** - * Tests RDBStore creation. - */ -public class TestDBStoreBuilder { - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Before - public void setUp() throws Exception { - System.setProperty(DBConfigFromFile.CONFIG_DIR, - folder.newFolder().toString()); - } - - @Test - public void builderWithoutAnyParams() throws IOException { - Configuration conf = new Configuration(); - thrown.expect(IOException.class); - DBStoreBuilder.newBuilder(conf).build(); - } - - @Test - public void builderWithOneParamV1() throws IOException { - Configuration conf = new Configuration(); - thrown.expect(IOException.class); - DBStoreBuilder.newBuilder(conf) - .setName("Test.db") - .build(); - } - - @Test - public void builderWithOneParamV2() throws IOException { - Configuration conf = new Configuration(); - File newFolder = folder.newFolder(); - if(!newFolder.exists()) { - Assert.assertTrue(newFolder.mkdirs()); - } - thrown.expect(IOException.class); - DBStoreBuilder.newBuilder(conf) - .setPath(newFolder.toPath()) - .build(); - } - - @Test - public void builderWithOpenClose() throws Exception { - Configuration conf = new Configuration(); - File newFolder = folder.newFolder(); - if(!newFolder.exists()) { - Assert.assertTrue(newFolder.mkdirs()); - } - DBStore dbStore = DBStoreBuilder.newBuilder(conf) - .setName("Test.db") - .setPath(newFolder.toPath()) - .build(); - // Nothing to do just open and Close. - dbStore.close(); - } - - @Test - public void builderWithDoubleTableName() throws Exception { - Configuration conf = new Configuration(); - File newFolder = folder.newFolder(); - if(!newFolder.exists()) { - Assert.assertTrue(newFolder.mkdirs()); - } - thrown.expect(IOException.class); - DBStoreBuilder.newBuilder(conf) - .setName("Test.db") - .setPath(newFolder.toPath()) - .addTable("FIRST") - .addTable("FIRST") - .build(); - // Nothing to do , This will throw so we do not have to close. - - } - - @Test - public void builderWithDataWrites() throws Exception { - Configuration conf = new Configuration(); - File newFolder = folder.newFolder(); - if(!newFolder.exists()) { - Assert.assertTrue(newFolder.mkdirs()); - } - try (DBStore dbStore = DBStoreBuilder.newBuilder(conf) - .setName("Test.db") - .setPath(newFolder.toPath()) - .addTable("First") - .addTable("Second") - .build()) { - try (Table firstTable = dbStore.getTable("First")) { - byte[] key = - RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8); - firstTable.put(key, value); - byte[] temp = firstTable.get(key); - Arrays.equals(value, temp); - } - - try (Table secondTable = dbStore.getTable("Second")) { - Assert.assertTrue(secondTable.isEmpty()); - } - } - } - - @Test - public void builderWithDiskProfileWrites() throws Exception { - Configuration conf = new Configuration(); - File newFolder = folder.newFolder(); - if(!newFolder.exists()) { - Assert.assertTrue(newFolder.mkdirs()); - } - try (DBStore dbStore = DBStoreBuilder.newBuilder(conf) - .setName("Test.db") - .setPath(newFolder.toPath()) - .addTable("First") - .addTable("Second") - .setProfile(DBProfile.DISK) - .build()) { - try (Table firstTable = dbStore.getTable("First")) { - byte[] key = - RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8); - firstTable.put(key, value); - byte[] temp = firstTable.get(key); - Arrays.equals(value, temp); - } - - try (Table secondTable = dbStore.getTable("Second")) { - Assert.assertTrue(secondTable.isEmpty()); - } - } - } - - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java deleted file mode 100644 index 0b673d3..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.utils.db; - -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.hadoop.hdfs.DFSUtil; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; -import org.rocksdb.ColumnFamilyOptions; -import org.rocksdb.DBOptions; -import org.rocksdb.RocksDB; -import org.rocksdb.Statistics; -import org.rocksdb.StatsLevel; - -import javax.management.MBeanServer; -import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * RDBStore Tests. - */ -public class TestRDBStore { - private final List<String> families = - Arrays.asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY), - "First", "Second", "Third", - "Fourth", "Fifth", - "Sixth"); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - @Rule - public ExpectedException thrown = ExpectedException.none(); - private RDBStore rdbStore = null; - private DBOptions options = null; - private Set<TableConfig> configSet; - - @Before - public void setUp() throws Exception { - options = new DBOptions(); - options.setCreateIfMissing(true); - options.setCreateMissingColumnFamilies(true); - - Statistics statistics = new Statistics(); - statistics.setStatsLevel(StatsLevel.ALL); - options = options.setStatistics(statistics); - configSet = new HashSet<>(); - for(String name : families) { - TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions()); - configSet.add(newConfig); - } - rdbStore = new RDBStore(folder.newFolder(), options, configSet); - } - - @After - public void tearDown() throws Exception { - if (rdbStore != null) { - rdbStore.close(); - } - } - - @Test - public void compactDB() throws Exception { - try (RDBStore newStore = - new RDBStore(folder.newFolder(), options, configSet)) { - Assert.assertNotNull("DB Store cannot be null", newStore); - try (Table firstTable = newStore.getTable(families.get(1))) { - Assert.assertNotNull("Table cannot be null", firstTable); - for (int x = 0; x < 100; x++) { - byte[] key = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - firstTable.put(key, value); - } - } - // This test does not assert anything if there is any error this test - // will throw and fail. - newStore.compactDB(); - } - } - - @Test - public void close() throws Exception { - RDBStore newStore = - new RDBStore(folder.newFolder(), options, configSet); - Assert.assertNotNull("DBStore cannot be null", newStore); - // This test does not assert anything if there is any error this test - // will throw and fail. - newStore.close(); - } - - @Test - public void moveKey() throws Exception { - byte[] key = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - - try (Table firstTable = rdbStore.getTable(families.get(1))) { - firstTable.put(key, value); - try (Table secondTable = rdbStore.getTable(families.get(2))) { - rdbStore.move(key, firstTable, secondTable); - byte[] newvalue = secondTable.get(key); - // Make sure we have value in the second table - Assert.assertNotNull(newvalue); - //and it is same as what we wrote to the FirstTable - Assert.assertArrayEquals(value, newvalue); - } - // After move this key must not exist in the first table. - Assert.assertNull(firstTable.get(key)); - } - } - - @Test - public void moveWithValue() throws Exception { - byte[] key = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - - byte[] nextValue = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - try (Table firstTable = rdbStore.getTable(families.get(1))) { - firstTable.put(key, value); - try (Table secondTable = rdbStore.getTable(families.get(2))) { - rdbStore.move(key, nextValue, firstTable, secondTable); - byte[] newvalue = secondTable.get(key); - // Make sure we have value in the second table - Assert.assertNotNull(newvalue); - //and it is not same as what we wrote to the FirstTable, and equals - // the new value. - Assert.assertArrayEquals(nextValue, nextValue); - } - } - - } - - @Test - public void getEstimatedKeyCount() throws Exception { - try (RDBStore newStore = - new RDBStore(folder.newFolder(), options, configSet)) { - Assert.assertNotNull("DB Store cannot be null", newStore); - // Write 100 keys to the first table. - try (Table firstTable = newStore.getTable(families.get(1))) { - Assert.assertNotNull("Table cannot be null", firstTable); - for (int x = 0; x < 100; x++) { - byte[] key = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - firstTable.put(key, value); - } - } - - // Write 100 keys to the secondTable table. - try (Table secondTable = newStore.getTable(families.get(2))) { - Assert.assertNotNull("Table cannot be null", secondTable); - for (int x = 0; x < 100; x++) { - byte[] key = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - secondTable.put(key, value); - } - } - // Let us make sure that our estimate is not off by 10% - Assert.assertTrue(newStore.getEstimatedKeyCount() > 180 - || newStore.getEstimatedKeyCount() < 220); - } - } - - @Test - public void getStatMBeanName() throws Exception { - - try (Table firstTable = rdbStore.getTable(families.get(1))) { - for (int y = 0; y < 100; y++) { - byte[] key = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - firstTable.put(key, value); - } - } - MBeanServer platformMBeanServer = - ManagementFactory.getPlatformMBeanServer(); - Thread.sleep(2000); - - Object keysWritten = platformMBeanServer - .getAttribute(rdbStore.getStatMBeanName(), "NUMBER_KEYS_WRITTEN"); - - Assert.assertTrue(((Long) keysWritten) >= 99L); - - Object dbWriteAverage = platformMBeanServer - .getAttribute(rdbStore.getStatMBeanName(), "DB_WRITE_AVERAGE"); - Assert.assertTrue((double) dbWriteAverage > 0); - } - - @Test - public void getTable() throws Exception { - for (String tableName : families) { - try (Table table = rdbStore.getTable(tableName)) { - Assert.assertNotNull(tableName + "is null", table); - } - } - thrown.expect(IOException.class); - rdbStore.getTable("ATableWithNoName"); - } - - @Test - public void listTables() throws Exception { - List<Table> tableList = rdbStore.listTables(); - Assert.assertNotNull("Table list cannot be null", tableList); - Map<String, Table> hashTable = new HashMap<>(); - - for (Table t : tableList) { - hashTable.put(t.getName(), t); - } - - int count = families.size(); - // Assert that we have all the tables in the list and no more. - for (String name : families) { - Assert.assertTrue(hashTable.containsKey(name)); - count--; - } - Assert.assertEquals(0, count); - } -} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
