Repository: tajo Updated Branches: refs/heads/branch-0.11.1 04d6d8e80 -> 0a17877f9
TAJO-2016: Fix storage tests hangs. Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/0a17877f Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/0a17877f Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/0a17877f Branch: refs/heads/branch-0.11.1 Commit: 0a17877f988e6fadd7cd052b7cb8f4072e0900e7 Parents: 04d6d8e Author: Jinho Kim <[email protected]> Authored: Thu Dec 10 15:50:53 2015 +0900 Committer: Jinho Kim <[email protected]> Committed: Thu Dec 10 15:50:53 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 ++ .../tajo/storage/TestByteBufLineReader.java | 7 +++--- .../tajo/storage/TestCompressionStorages.java | 15 ++++++------ .../apache/tajo/storage/TestFileTablespace.java | 25 ++++++++++---------- .../tajo/storage/raw/TestDirectRawFile.java | 2 +- 5 files changed, 27 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/0a17877f/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index d55e4b8..feb723e 100644 --- a/CHANGES +++ b/CHANGES @@ -24,6 +24,8 @@ Release 0.11.1 - unreleased BUG FIXES + TAJO-2016: Fix storage tests hangs. (jinho) + TAJO-2013: FilterPushDownRule fails due to the unsupported index. (jinho) TAJO-2014: TestRpcClientManager fails occasionally. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/0a17877f/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java index b6f65df..8472095 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java @@ -85,7 +85,7 @@ public class TestByteBufLineReader { fs.close(); } - @Test + @Test(timeout = 60000) public void testReaderWithDFS() throws Exception { final Configuration conf = new HdfsConfiguration(); String testDataPath = TEST_PATH + "/" + UUID.randomUUID().toString(); @@ -94,8 +94,7 @@ public class TestByteBufLineReader { conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, true); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(2).build(); - cluster.waitClusterUp(); + .numDataNodes(2).waitSafeMode(true).build(); TajoConf tajoConf = new TajoConf(conf); tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, cluster.getFileSystem().getUri() + "/tajo"); @@ -122,7 +121,7 @@ public class TestByteBufLineReader { lineReader.close(); fs.close(); } finally { - cluster.shutdown(true); + cluster.shutdown(); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/0a17877f/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java index 9c30202..96e58eb 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java @@ -27,6 +27,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.compress.*; import org.apache.hadoop.io.compress.zlib.ZlibFactory; import org.apache.hadoop.util.NativeCodeLoader; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; @@ -68,9 +69,9 @@ public class TestCompressionStorages { @Parameterized.Parameters public static Collection<Object[]> generateParameters() { return Arrays.asList(new Object[][]{ - {"TEXT"}, - {"RCFILE"}, - {"SEQUENCEFILE"} + {BuiltinStorages.TEXT}, + {BuiltinStorages.RCFILE}, + {BuiltinStorages.SEQUENCE_FILE} }); } @@ -81,11 +82,11 @@ public class TestCompressionStorages { @Test public void testGzipCodecCompressionData() throws IOException { - if (dataFormat.equalsIgnoreCase("RCFILE")) { + if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) { if( ZlibFactory.isNativeZlibLoaded(conf)) { storageCompressionTest(dataFormat, GzipCodec.class); } - } else if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) { + } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { if( ZlibFactory.isNativeZlibLoaded(conf)) { storageCompressionTest(dataFormat, GzipCodec.class); } @@ -131,7 +132,7 @@ public class TestCompressionStorages { extension = ((DelimitedTextFile.DelimitedTextFileAppender) appender).getExtension(); } - int tupleNum = 100000; + int tupleNum = 1000; VTuple vTuple; for (int i = 0; i < tupleNum; i++) { @@ -154,7 +155,7 @@ public class TestCompressionStorages { Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, tablets[0], schema); scanner.init(); - if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) { + if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { assertTrue(scanner instanceof SequenceFileScanner); Writable key = ((SequenceFileScanner) scanner).getKey(); assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName()); http://git-wip-us.apache.org/repos/asf/tajo/blob/0a17877f/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java index b11aa5a..c99e0dc 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java @@ -18,13 +18,13 @@ package org.apache.tajo.storage; -import com.google.common.base.Optional; import com.google.common.collect.Lists; import net.minidev.json.JSONObject; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.*; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; @@ -70,7 +70,7 @@ public class TestFileTablespace { schema.addColumn("age",Type.INT4); schema.addColumn("name",Type.TEXT); - TableMeta meta = CatalogUtil.newTableMeta("TEXT"); + TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); VTuple[] tuples = new VTuple[4]; for(int i=0; i < tuples.length; i++) { @@ -82,7 +82,7 @@ public class TestFileTablespace { Path path = StorageUtil.concatPath(testDir, "testGetScannerAndAppender", "table.csv"); localFs.mkdirs(path.getParent()); - FileTablespace fileStorageManager = (FileTablespace) TablespaceManager.getLocalFs(); + FileTablespace fileStorageManager = TablespaceManager.getLocalFs(); assertEquals(localFs.getUri(), fileStorageManager.getFileSystem().getUri()); Appender appender = fileStorageManager.getAppender(meta, schema, path); @@ -99,6 +99,7 @@ public class TestFileTablespace { i++; } assertEquals(4,i); + localFs.delete(path, true); } @Test(timeout = 60000) @@ -137,7 +138,7 @@ public class TestFileTablespace { schema.addColumn("id", Type.INT4); schema.addColumn("age",Type.INT4); schema.addColumn("name",Type.TEXT); - TableMeta meta = CatalogUtil.newTableMeta("TEXT"); + TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); List<Fragment> splits = Lists.newArrayList(); // Get FileFragments in partition batch @@ -154,7 +155,7 @@ public class TestFileTablespace { assertEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]); fs.close(); } finally { - cluster.shutdown(true); + cluster.shutdown(); } } @@ -203,7 +204,7 @@ public class TestFileTablespace { assertEquals(0, splits.size()); fs.close(); } finally { - cluster.shutdown(true); + cluster.shutdown(); } } @@ -254,7 +255,7 @@ public class TestFileTablespace { assertNotEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]); fs.close(); } finally { - cluster.shutdown(true); + cluster.shutdown(); } } @@ -271,7 +272,6 @@ public class TestFileTablespace { new MiniDFSCluster.Builder(hdfsConf).numDataNodes(1).build(); URI uri = URI.create(cluster.getFileSystem().getUri() + "/tajo"); - Optional<Tablespace> existingTs = Optional.absent(); try { /* Local FileSystem */ FileTablespace space = TablespaceManager.getLocalFs(); @@ -279,16 +279,17 @@ public class TestFileTablespace { FileTablespace distTablespace = new FileTablespace("testGetFileTablespace", uri, null); distTablespace.init(conf); - existingTs = TablespaceManager.addTableSpaceForTest(distTablespace); + + TablespaceManager.addTableSpaceForTest(distTablespace); /* Distributed FileSystem */ - space = (FileTablespace) TablespaceManager.get(uri); + space = TablespaceManager.get(uri); assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri()); - space = (FileTablespace) TablespaceManager.getByName("testGetFileTablespace"); + space = TablespaceManager.getByName("testGetFileTablespace"); assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri()); } finally { - cluster.shutdown(true); + cluster.shutdown(); } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/0a17877f/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java index 179c9eb..78e3390 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java @@ -117,7 +117,7 @@ public class TestDirectRawFile { @AfterClass public static void tearDownClass() throws InterruptedException { - cluster.shutdown(true); + cluster.shutdown(); } public Path getTestDir(FileSystem fs, String dir) throws IOException {
