http://git-wip-us.apache.org/repos/asf/hadoop/blob/36957f0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestProvidedStorageMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestProvidedStorageMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestProvidedStorageMap.java index 1ef2f2b..faf1f83 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestProvidedStorageMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestProvidedStorageMap.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.util.RwLock; import org.junit.Before; import org.junit.Test; - import java.io.IOException; import static org.junit.Assert.assertNotNull;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36957f0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java new file mode 100644 index 0000000..4a9661b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.common.blockaliasmap.impl; + +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer; +import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap; +import org.apache.hadoop.hdfs.server.common.FileRegion; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +/** + * Tests the {@link InMemoryLevelDBAliasMapClient}. + */ +public class TestInMemoryLevelDBAliasMapClient { + + private InMemoryLevelDBAliasMapServer levelDBAliasMapServer; + private InMemoryLevelDBAliasMapClient inMemoryLevelDBAliasMapClient; + private File tempDir; + private Configuration conf; + + @Before + public void setUp() throws IOException { + levelDBAliasMapServer = + new InMemoryLevelDBAliasMapServer(InMemoryAliasMap::init); + conf = new Configuration(); + int port = 9876; + + conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, + "localhost:" + port); + tempDir = Files.createTempDir(); + conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, + tempDir.getAbsolutePath()); + inMemoryLevelDBAliasMapClient = new InMemoryLevelDBAliasMapClient(); + } + + @After + public void tearDown() throws IOException { + levelDBAliasMapServer.close(); + inMemoryLevelDBAliasMapClient.close(); + FileUtils.deleteDirectory(tempDir); + } + + @Test + public void writeRead() throws Exception { + inMemoryLevelDBAliasMapClient.setConf(conf); + levelDBAliasMapServer.setConf(conf); + levelDBAliasMapServer.start(); + Block block = new Block(42, 43, 44); + byte[] nonce = "blackbird".getBytes(); + ProvidedStorageLocation providedStorageLocation + = new ProvidedStorageLocation(new Path("cuckoo"), + 45, 46, nonce); + BlockAliasMap.Writer<FileRegion> writer = + inMemoryLevelDBAliasMapClient.getWriter(null); + writer.store(new FileRegion(block, providedStorageLocation)); + + BlockAliasMap.Reader<FileRegion> reader = + inMemoryLevelDBAliasMapClient.getReader(null); + Optional<FileRegion> fileRegion = reader.resolve(block); + assertEquals(new FileRegion(block, providedStorageLocation), + fileRegion.get()); + } + + @Test + public void iterateSingleBatch() throws Exception { + inMemoryLevelDBAliasMapClient.setConf(conf); + levelDBAliasMapServer.setConf(conf); + levelDBAliasMapServer.start(); + Block block1 = new Block(42, 43, 44); + Block block2 = new Block(43, 44, 45); + byte[] nonce1 = "blackbird".getBytes(); + byte[] nonce2 = "cuckoo".getBytes(); + ProvidedStorageLocation providedStorageLocation1 = + new ProvidedStorageLocation(new Path("eagle"), + 46, 47, nonce1); + ProvidedStorageLocation providedStorageLocation2 = + new ProvidedStorageLocation(new Path("falcon"), + 46, 47, nonce2); + BlockAliasMap.Writer<FileRegion> writer1 = + inMemoryLevelDBAliasMapClient.getWriter(null); + writer1.store(new FileRegion(block1, providedStorageLocation1)); + BlockAliasMap.Writer<FileRegion> writer2 = + inMemoryLevelDBAliasMapClient.getWriter(null); + writer2.store(new FileRegion(block2, providedStorageLocation2)); + + BlockAliasMap.Reader<FileRegion> reader = + inMemoryLevelDBAliasMapClient.getReader(null); + List<FileRegion> actualFileRegions = + Lists.newArrayListWithCapacity(2); + for (FileRegion fileRegion : reader) { + actualFileRegions.add(fileRegion); + } + + assertArrayEquals( + new FileRegion[] {new FileRegion(block1, providedStorageLocation1), + new FileRegion(block2, providedStorageLocation2)}, + actualFileRegions.toArray()); + } + + @Test + public void iterateThreeBatches() throws Exception { + conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_BATCH_SIZE, "2"); + levelDBAliasMapServer.setConf(conf); + inMemoryLevelDBAliasMapClient.setConf(conf); + levelDBAliasMapServer.start(); + Block block1 = new Block(42, 43, 44); + Block block2 = new Block(43, 44, 45); + Block block3 = new Block(44, 45, 46); + Block block4 = new Block(47, 48, 49); + Block block5 = new Block(50, 51, 52); + Block block6 = new Block(53, 54, 55); + byte[] nonce1 = "blackbird".getBytes(); + byte[] nonce2 = "cuckoo".getBytes(); + byte[] nonce3 = "sparrow".getBytes(); + byte[] nonce4 = "magpie".getBytes(); + byte[] nonce5 = "seagull".getBytes(); + byte[] nonce6 = "finch".getBytes(); + ProvidedStorageLocation providedStorageLocation1 = + new ProvidedStorageLocation(new Path("eagle"), + 46, 47, nonce1); + ProvidedStorageLocation providedStorageLocation2 = + new ProvidedStorageLocation(new Path("falcon"), + 48, 49, nonce2); + ProvidedStorageLocation providedStorageLocation3 = + new ProvidedStorageLocation(new Path("robin"), + 50, 51, nonce3); + ProvidedStorageLocation providedStorageLocation4 = + new ProvidedStorageLocation(new Path("parakeet"), + 52, 53, nonce4); + ProvidedStorageLocation providedStorageLocation5 = + new ProvidedStorageLocation(new Path("heron"), + 54, 55, nonce5); + ProvidedStorageLocation providedStorageLocation6 = + new ProvidedStorageLocation(new Path("duck"), + 56, 57, nonce6); + inMemoryLevelDBAliasMapClient + .getWriter(null) + .store(new FileRegion(block1, providedStorageLocation1)); + inMemoryLevelDBAliasMapClient + .getWriter(null) + .store(new FileRegion(block2, providedStorageLocation2)); + inMemoryLevelDBAliasMapClient + .getWriter(null) + .store(new FileRegion(block3, providedStorageLocation3)); + inMemoryLevelDBAliasMapClient + .getWriter(null) + .store(new FileRegion(block4, providedStorageLocation4)); + inMemoryLevelDBAliasMapClient + .getWriter(null) + .store(new FileRegion(block5, providedStorageLocation5)); + inMemoryLevelDBAliasMapClient + .getWriter(null) + .store(new FileRegion(block6, providedStorageLocation6)); + + BlockAliasMap.Reader<FileRegion> reader = + inMemoryLevelDBAliasMapClient.getReader(null); + List<FileRegion> actualFileRegions = + Lists.newArrayListWithCapacity(6); + for (FileRegion fileRegion : reader) { + actualFileRegions.add(fileRegion); + } + + FileRegion[] expectedFileRegions = + new FileRegion[] {new FileRegion(block1, providedStorageLocation1), + new FileRegion(block2, providedStorageLocation2), + new FileRegion(block3, providedStorageLocation3), + new FileRegion(block4, providedStorageLocation4), + new FileRegion(block5, providedStorageLocation5), + new FileRegion(block6, providedStorageLocation6)}; + assertArrayEquals(expectedFileRegions, actualFileRegions.toArray()); + } + + + class ReadThread implements Runnable { + private final Block block; + private final BlockAliasMap.Reader<FileRegion> reader; + private int delay; + private Optional<FileRegion> fileRegionOpt; + + ReadThread(Block block, BlockAliasMap.Reader<FileRegion> reader, + int delay) { + this.block = block; + this.reader = reader; + this.delay = delay; + } + + public Optional<FileRegion> getFileRegion() { + return fileRegionOpt; + } + + @Override + public void run() { + try { + Thread.sleep(delay); + fileRegionOpt = reader.resolve(block); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + class WriteThread implements Runnable { + private final Block block; + private final BlockAliasMap.Writer<FileRegion> writer; + private final ProvidedStorageLocation providedStorageLocation; + private int delay; + + WriteThread(Block block, ProvidedStorageLocation providedStorageLocation, + BlockAliasMap.Writer<FileRegion> writer, int delay) { + this.block = block; + this.writer = writer; + this.providedStorageLocation = providedStorageLocation; + this.delay = delay; + } + + @Override + public void run() { + try { + Thread.sleep(delay); + writer.store(new FileRegion(block, providedStorageLocation)); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + public FileRegion generateRandomFileRegion(int seed) { + Block block = new Block(seed, seed + 1, seed + 2); + Path path = new Path("koekoek"); + byte[] nonce = new byte[0]; + ProvidedStorageLocation providedStorageLocation = + new ProvidedStorageLocation(path, seed + 3, seed + 4, nonce); + return new FileRegion(block, providedStorageLocation); + } + + @Test + public void multipleReads() throws IOException { + inMemoryLevelDBAliasMapClient.setConf(conf); + levelDBAliasMapServer.setConf(conf); + levelDBAliasMapServer.start(); + + Random r = new Random(); + List<FileRegion> expectedFileRegions = r.ints(0, 200) + .limit(50) + .boxed() + .map(i -> generateRandomFileRegion(i)) + .collect(Collectors.toList()); + + + BlockAliasMap.Reader<FileRegion> reader = + inMemoryLevelDBAliasMapClient.getReader(null); + BlockAliasMap.Writer<FileRegion> writer = + inMemoryLevelDBAliasMapClient.getWriter(null); + + ExecutorService executor = Executors.newCachedThreadPool(); + + List<ReadThread> readThreads = expectedFileRegions + .stream() + .map(fileRegion -> new ReadThread(fileRegion.getBlock(), + reader, + 4000)) + .collect(Collectors.toList()); + + + List<? extends Future<?>> readFutures = + readThreads.stream() + .map(readThread -> executor.submit(readThread)) + .collect(Collectors.toList()); + + List<? extends Future<?>> writeFutures = expectedFileRegions + .stream() + .map(fileRegion -> new WriteThread(fileRegion.getBlock(), + fileRegion.getProvidedStorageLocation(), + writer, + 1000)) + .map(writeThread -> executor.submit(writeThread)) + .collect(Collectors.toList()); + + readFutures.stream() + .map(readFuture -> { + try { + return readFuture.get(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + + List<FileRegion> actualFileRegions = readThreads.stream() + .map(readThread -> readThread.getFileRegion().get()) + .collect(Collectors.toList()); + + assertThat(actualFileRegions).containsExactlyInAnyOrder( + expectedFileRegions.toArray(new FileRegion[0])); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/36957f0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java new file mode 100644 index 0000000..43fc68c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.common.blockaliasmap.impl; + +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer; +import org.apache.hadoop.hdfs.server.common.FileRegion; +import org.iq80.leveldb.DBException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import java.io.File; +import java.io.IOException; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + +/** + * Tests the in-memory alias map with a mock level-db implementation. + */ +public class TestLevelDbMockAliasMapClient { + private InMemoryLevelDBAliasMapServer levelDBAliasMapServer; + private InMemoryLevelDBAliasMapClient inMemoryLevelDBAliasMapClient; + private File tempDir; + private Configuration conf; + private InMemoryAliasMap aliasMapMock; + + @Before + public void setUp() throws IOException { + aliasMapMock = mock(InMemoryAliasMap.class); + levelDBAliasMapServer = new InMemoryLevelDBAliasMapServer( + config -> aliasMapMock); + conf = new Configuration(); + int port = 9877; + + conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, + "localhost:" + port); + tempDir = Files.createTempDir(); + conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, + tempDir.getAbsolutePath()); + inMemoryLevelDBAliasMapClient = new InMemoryLevelDBAliasMapClient(); + inMemoryLevelDBAliasMapClient.setConf(conf); + levelDBAliasMapServer.setConf(conf); + levelDBAliasMapServer.start(); + } + + @After + public void tearDown() throws IOException { + levelDBAliasMapServer.close(); + inMemoryLevelDBAliasMapClient.close(); + FileUtils.deleteDirectory(tempDir); + } + + @Test + public void readFailure() throws Exception { + Block block = new Block(42, 43, 44); + doThrow(new IOException()) + .doThrow(new DBException()) + .when(aliasMapMock) + .read(block); + + assertThatExceptionOfType(IOException.class) + .isThrownBy(() -> + inMemoryLevelDBAliasMapClient.getReader(null).resolve(block)); + + assertThatExceptionOfType(IOException.class) + .isThrownBy(() -> + inMemoryLevelDBAliasMapClient.getReader(null).resolve(block)); + } + + @Test + public void writeFailure() throws IOException { + Block block = new Block(42, 43, 44); + byte[] nonce = new byte[0]; + Path path = new Path("koekoek"); + ProvidedStorageLocation providedStorageLocation = + new ProvidedStorageLocation(path, 45, 46, nonce); + + doThrow(new IOException()) + .when(aliasMapMock) + .write(block, providedStorageLocation); + + assertThatExceptionOfType(IOException.class) + .isThrownBy(() -> + inMemoryLevelDBAliasMapClient.getWriter(null) + .store(new FileRegion(block, providedStorageLocation))); + + assertThatExceptionOfType(IOException.class) + .isThrownBy(() -> + inMemoryLevelDBAliasMapClient.getWriter(null) + .store(new FileRegion(block, providedStorageLocation))); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/36957f0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java index 4190730..8bdbaa4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java @@ -43,6 +43,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.apache.commons.io.FileUtils; @@ -214,7 +215,8 @@ public class TestProvidedImpl { } @Override - public FileRegion resolve(Block ident) throws IOException { + public Optional<FileRegion> resolve(Block ident) + throws IOException { return null; } }; @@ -232,6 +234,11 @@ public class TestProvidedImpl { public void refresh() throws IOException { // do nothing! } + + @Override + public void close() throws IOException { + // do nothing + } } private static Storage.StorageDirectory createLocalStorageDirectory( http://git-wip-us.apache.org/repos/asf/hadoop/blob/36957f0d/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 04b93c4..b1a90c3 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1336,7 +1336,6 @@ <artifactId>mssql-jdbc</artifactId> <version>${mssql.version}</version> </dependency> - <dependency> <groupId>io.swagger</groupId> <artifactId>swagger-annotations</artifactId> @@ -1352,7 +1351,12 @@ <artifactId>snakeyaml</artifactId> <version>${snakeyaml.version}</version> </dependency> - + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <version>3.8.0</version> + <scope>test</scope> + </dependency> </dependencies> </dependencyManagement> http://git-wip-us.apache.org/repos/asf/hadoop/blob/36957f0d/hadoop-tools/hadoop-fs2img/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-fs2img/pom.xml b/hadoop-tools/hadoop-fs2img/pom.xml index e1411f8..8661c82 100644 --- a/hadoop-tools/hadoop-fs2img/pom.xml +++ b/hadoop-tools/hadoop-fs2img/pom.xml @@ -66,6 +66,12 @@ <artifactId>mockito-all</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <version>3.8.0</version> + <scope>test</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/hadoop/blob/36957f0d/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/NullBlockAliasMap.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/NullBlockAliasMap.java b/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/NullBlockAliasMap.java index 4cdf473..63d1f27 100644 --- a/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/NullBlockAliasMap.java +++ b/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/NullBlockAliasMap.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; +import java.util.Optional; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.FileRegion; @@ -57,14 +58,14 @@ public class NullBlockAliasMap extends BlockAliasMap<FileRegion> { } @Override - public FileRegion resolve(Block ident) throws IOException { + public Optional<FileRegion> resolve(Block ident) throws IOException { throw new UnsupportedOperationException(); } }; } @Override - public Writer<FileRegion> getWriter(Writer.Options opts) throws IOException { + public Writer getWriter(Writer.Options opts) throws IOException { return new Writer<FileRegion>() { @Override public void store(FileRegion token) throws IOException { @@ -83,4 +84,8 @@ public class NullBlockAliasMap extends BlockAliasMap<FileRegion> { // do nothing } + @Override + public void close() throws IOException { + + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/36957f0d/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java index 09e8f97..70e4c33 100644 --- a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java +++ b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java @@ -27,11 +27,13 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; import java.util.HashSet; import java.util.Iterator; import java.util.Random; import java.util.Set; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -39,6 +41,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.BlockMissingException; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; @@ -48,6 +51,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; @@ -56,6 +61,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.blockmanagement.ProvidedStorageMap; import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap; +import org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.InMemoryLevelDBAliasMapClient; import org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -172,16 +178,16 @@ public class TestNameNodeProvidedImplementation { void createImage(TreeWalk t, Path out, Class<? extends BlockResolver> blockIdsClass) throws Exception { - createImage(t, out, blockIdsClass, ""); + createImage(t, out, blockIdsClass, "", TextFileRegionAliasMap.class); } void createImage(TreeWalk t, Path out, - Class<? extends BlockResolver> blockIdsClass, String clusterID) - throws Exception { + Class<? extends BlockResolver> blockIdsClass, String clusterID, + Class<? extends BlockAliasMap> aliasMapClass) throws Exception { ImageWriter.Options opts = ImageWriter.defaults(); opts.setConf(conf); opts.output(out.toString()) - .blocks(TextFileRegionAliasMap.class) + .blocks(aliasMapClass) .blockIds(blockIdsClass) .clusterID(clusterID); try (ImageWriter w = new ImageWriter(opts)) { @@ -389,17 +395,8 @@ public class TestNameNodeProvidedImplementation { return ret; } - @Test(timeout=30000) - public void testBlockRead() throws Exception { - conf.setClass(ImageWriter.Options.UGI_CLASS, - FsUGIResolver.class, UGIResolver.class); - createImage(new FSTreeWalk(NAMEPATH, conf), NNDIRPATH, - FixedBlockResolver.class); - startCluster(NNDIRPATH, 3, - new StorageType[] {StorageType.PROVIDED, StorageType.DISK}, null, - false); + private void verifyFileSystemContents() throws Exception { FileSystem fs = cluster.getFileSystem(); - Thread.sleep(2000); int count = 0; // read NN metadata, verify contents match for (TreePath e : new FSTreeWalk(NAMEPATH, conf)) { @@ -683,7 +680,7 @@ public class TestNameNodeProvidedImplementation { public void testSetClusterID() throws Exception { String clusterID = "PROVIDED-CLUSTER"; createImage(new FSTreeWalk(NAMEPATH, conf), NNDIRPATH, - FixedBlockResolver.class, clusterID); + FixedBlockResolver.class, clusterID, TextFileRegionAliasMap.class); // 2 Datanodes, 1 PROVIDED and other DISK startCluster(NNDIRPATH, 2, null, new StorageType[][] { @@ -744,4 +741,42 @@ public class TestNameNodeProvidedImplementation { verifyFileLocation(i, expectedLocations); } } + + + // This test will fail until there is a refactoring of the FileRegion + // (HDFS-12713). + @Test(expected=BlockMissingException.class) + public void testInMemoryAliasMap() throws Exception { + conf.setClass(ImageWriter.Options.UGI_CLASS, + FsUGIResolver.class, UGIResolver.class); + conf.setClass(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_CLASS, + InMemoryLevelDBAliasMapClient.class, BlockAliasMap.class); + conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, + "localhost:32445"); + File tempDirectory = + Files.createTempDirectory("in-memory-alias-map").toFile(); + conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, + tempDirectory.getAbsolutePath()); + conf.setBoolean(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_ENABLED, true); + + InMemoryLevelDBAliasMapServer levelDBAliasMapServer = + new InMemoryLevelDBAliasMapServer(InMemoryAliasMap::init); + levelDBAliasMapServer.setConf(conf); + levelDBAliasMapServer.start(); + + createImage(new FSTreeWalk(NAMEPATH, conf), + NNDIRPATH, + FixedBlockResolver.class, "", + InMemoryLevelDBAliasMapClient.class); + levelDBAliasMapServer.close(); + + // start cluster with two datanodes, + // each with 1 PROVIDED volume and other DISK volume + startCluster(NNDIRPATH, 2, + new StorageType[] {StorageType.PROVIDED, StorageType.DISK}, + null, false); + verifyFileSystemContents(); + FileUtils.deleteDirectory(tempDirectory); + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org