Repository: tajo Updated Branches: refs/heads/branch-0.11.0 ade066578 -> 34062ea85
http://git-wip-us.apache.org/repos/asf/tajo/blob/34062ea8/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java index afe0f13..1051b3b 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.QueryId; import org.apache.tajo.TajoIdProtos; import org.apache.tajo.catalog.CatalogUtil; @@ -58,41 +59,41 @@ import static org.junit.Assert.assertTrue; @RunWith(Parameterized.class) public class TestStorages { - private TajoConf conf; - private static String TEST_PATH = "target/test-data/TestStorages"; + private TajoConf conf; + private static String TEST_PATH = "target/test-data/TestStorages"; private static String TEST_PROJECTION_AVRO_SCHEMA = "{\n" + - " \"type\": \"record\",\n" + - " \"namespace\": \"org.apache.tajo\",\n" + - " \"name\": \"testProjection\",\n" + - " \"fields\": [\n" + - " { \"name\": \"id\", \"type\": \"int\" },\n" + - " { \"name\": \"age\", \"type\": \"long\" },\n" + - " { \"name\": \"score\", \"type\": \"float\" }\n" + - " ]\n" + - "}\n"; + " \"type\": \"record\",\n" + + " \"namespace\": \"org.apache.tajo\",\n" + + " \"name\": \"testProjection\",\n" + + " \"fields\": [\n" + + " { \"name\": \"id\", \"type\": \"int\" },\n" + + " { \"name\": \"age\", \"type\": \"long\" },\n" + + " { \"name\": \"score\", \"type\": \"float\" }\n" + + " ]\n" + + "}\n"; private static String TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA = "{\n" + - " \"type\": \"record\",\n" + - " \"namespace\": \"org.apache.tajo\",\n" + - " \"name\": \"testNullHandlingTypes\",\n" + - " \"fields\": [\n" + - " { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" + - " { \"name\": \"col2\", \"type\": [\"null\", \"string\"] },\n" + - " { \"name\": \"col3\", \"type\": [\"null\", \"int\"] },\n" + - " { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" + - " { \"name\": \"col5\", \"type\": [\"null\", \"long\"] },\n" + - " { \"name\": \"col6\", \"type\": [\"null\", \"float\"] },\n" + - " { \"name\": \"col7\", \"type\": [\"null\", \"double\"] },\n" + - " { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" + - " { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" + - " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" + - " { \"name\": \"col11\", \"type\": \"null\" },\n" + - " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" + - " ]\n" + - "}\n"; + " \"type\": \"record\",\n" + + " \"namespace\": \"org.apache.tajo\",\n" + + " \"name\": \"testNullHandlingTypes\",\n" + + " \"fields\": [\n" + + " { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" + + " { \"name\": \"col2\", \"type\": [\"null\", \"string\"] },\n" + + " { \"name\": \"col3\", \"type\": [\"null\", \"int\"] },\n" + + " { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" + + " { \"name\": \"col5\", \"type\": [\"null\", \"long\"] },\n" + + " { \"name\": \"col6\", \"type\": [\"null\", \"float\"] },\n" + + " { \"name\": \"col7\", \"type\": [\"null\", \"double\"] },\n" + + " { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" + + " { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" + + " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" + + " { \"name\": \"col11\", \"type\": \"null\" },\n" + + " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" + + " ]\n" + + "}\n"; private static String TEST_MAX_VALUE_AVRO_SCHEMA = "{\n" + @@ -112,18 +113,20 @@ public class TestStorages { private boolean splitable; private boolean statsable; private boolean seekable; + private boolean internalType; private Path testDir; private FileSystem fs; - public TestStorages(String type, boolean splitable, boolean statsable, boolean seekable) throws IOException { + public TestStorages(String type, boolean splitable, boolean statsable, boolean seekable, boolean internalType) + throws IOException { this.storeType = type; this.splitable = splitable; this.statsable = statsable; this.seekable = seekable; - + this.internalType = internalType; conf = new TajoConf(); - if (storeType.equalsIgnoreCase("RCFILE")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) { conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100); } @@ -134,18 +137,19 @@ public class TestStorages { @Parameterized.Parameters public static Collection<Object[]> generateParameters() { return Arrays.asList(new Object[][] { - //type, splitable, statsable, seekable - {"RAW", false, true, true}, - {"RCFILE", true, true, false}, - {"PARQUET", false, false, false}, - {"SEQUENCEFILE", true, true, false}, - {"AVRO", false, false, false}, - {"TEXT", true, true, true}, - {"JSON", true, true, false}, + //type, splitable, statsable, seekable, internalType + {BuiltinStorages.RAW, false, true, true, true}, + {BuiltinStorages.DRAW, false, true, true, true}, + {BuiltinStorages.RCFILE, true, true, false, false}, + {BuiltinStorages.PARQUET, false, false, false, false}, + {BuiltinStorages.SEQUENCE_FILE, true, true, false, false}, + {BuiltinStorages.AVRO, false, false, false, false}, + {BuiltinStorages.TEXT, true, true, true, false}, + {BuiltinStorages.JSON, true, true, false, false}, }); } - @Test + @Test public void testSplitable() throws IOException { if (splitable) { Schema schema = new Schema(); @@ -198,11 +202,11 @@ public class TestStorages { assertEquals(tupleNum, tupleCnt); } - } + } @Test public void testRCFileSplitable() throws IOException { - if (storeType.equalsIgnoreCase("StoreType.RCFILE")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) { Schema schema = new Schema(); schema.addColumn("id", Type.INT4); schema.addColumn("age", Type.INT8); @@ -264,9 +268,9 @@ public class TestStorages { TableMeta meta = CatalogUtil.newTableMeta(storeType); meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); - if (storeType.equalsIgnoreCase("AVRO")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, - TEST_PROJECTION_AVRO_SCHEMA); + TEST_PROJECTION_AVRO_SCHEMA); } Path tablePath = new Path(testDir, "testProjection.data"); @@ -309,8 +313,8 @@ public class TestStorages { assertTrue(tupleCnt + 2 == tuple.getInt8(0)); assertTrue(tupleCnt + 3 == tuple.getFloat4(1)); } else { - // RAW and ROW always project all fields. - if (!storeType.equalsIgnoreCase("RAW") && !storeType.equalsIgnoreCase("ROWFILE")) { + // Internal storage always project all fields. + if (!internalType) { assertTrue(tuple.isBlankOrNull(0)); } assertTrue(tupleCnt + 2 == tuple.getInt8(1)); @@ -320,7 +324,7 @@ public class TestStorages { @Test public void testVariousTypes() throws IOException { - boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON"); + boolean handleProtobuf = !storeType.equalsIgnoreCase(BuiltinStorages.JSON); Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -341,7 +345,7 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(storeType, options); meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); - if (storeType.equalsIgnoreCase("AVRO")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) { String path = FileUtil.getResourcePath("dataset/testVariousTypes.avsc").toString(); meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path); } @@ -393,7 +397,7 @@ public class TestStorages { @Test public void testNullHandlingTypes() throws IOException { - boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON"); + boolean handleProtobuf = !storeType.equalsIgnoreCase(BuiltinStorages.JSON); Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -490,7 +494,7 @@ public class TestStorages { @Test public void testRCFileTextSerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase("RCFILE")) return; + if(!storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -559,7 +563,7 @@ public class TestStorages { @Test public void testRCFileBinarySerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase("RCFILE")) return; + if(!storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -628,7 +632,7 @@ public class TestStorages { @Test public void testSequenceFileTextSerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return; + if(!storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -701,7 +705,7 @@ public class TestStorages { @Test public void testSequenceFileBinarySerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return; + if(!storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -775,7 +779,7 @@ public class TestStorages { @Test public void testTime() throws IOException { - if (storeType.equalsIgnoreCase("TEXT") || storeType.equalsIgnoreCase("RAW")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.TEXT) || internalType) { Schema schema = new Schema(); schema.addColumn("col1", Type.DATE); schema.addColumn("col2", Type.TIME); @@ -869,7 +873,7 @@ public class TestStorages { long readRows = 0; for (long offset : offsets) { scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, - new FileFragment("table", tablePath, prevOffset, offset - prevOffset), schema); + new FileFragment("table", tablePath, prevOffset, offset - prevOffset), schema); scanner.init(); while (scanner.next() != null) { @@ -903,11 +907,11 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - if (storeType.equalsIgnoreCase("AVRO")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA); } - if (storeType.equalsIgnoreCase("RAW")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.RAW)) { OldStorageManager.clearCache(); /* TAJO-1250 reproduce BufferOverflow of RAWFile */ int headerSize = 4 + 2 + 1; //Integer record length + Short null-flag length + 1 byte null flags @@ -948,16 +952,16 @@ public class TestStorages { scanner.close(); - if (storeType.equalsIgnoreCase("RAW")){ + if (internalType){ OldStorageManager.clearCache(); } } @Test public void testLessThanSchemaSize() throws IOException { - /* RAW is internal storage. It must be same with schema size */ - if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO") - || storeType.equalsIgnoreCase("PARQUET")){ + /* Internal storage must be same with schema size */ + if (internalType || storeType.equalsIgnoreCase(BuiltinStorages.AVRO) + || storeType.equalsIgnoreCase(BuiltinStorages.PARQUET)) { return; } @@ -1020,10 +1024,10 @@ public class TestStorages { @Test public final void testInsertFixedCharTypeWithOverSize() throws Exception { - if (storeType.equalsIgnoreCase("TEXT") == false && - storeType.equalsIgnoreCase("SEQUENCEFILE") == false && - storeType.equalsIgnoreCase("RCFILE") == false && - storeType.equalsIgnoreCase("PARQUET") == false) { + if (!storeType.equalsIgnoreCase(BuiltinStorages.TEXT) && + !storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE) && + !storeType.equalsIgnoreCase(BuiltinStorages.RCFILE) && + !storeType.equalsIgnoreCase(BuiltinStorages.PARQUET)) { return; } @@ -1063,4 +1067,4 @@ public class TestStorages { assertTrue(ok); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/34062ea8/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java index 46c0d6e..bae81a9 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; @@ -40,8 +41,8 @@ import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.rawfile.DirectRawFileScanner; import org.apache.tajo.storage.rawfile.DirectRawFileWriter; -import org.apache.tajo.tuple.offheap.OffHeapRowBlock; -import org.apache.tajo.tuple.offheap.RowWriter; +import org.apache.tajo.tuple.memory.MemoryRowBlock; +import org.apache.tajo.tuple.memory.RowWriter; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.ProtoUtil; @@ -147,7 +148,7 @@ public class TestDirectRawFile { CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, PrimitiveProtos.StringProto.class.getName())); } - public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, OffHeapRowBlock rowBlock, Path outputFile) + public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, MemoryRowBlock rowBlock, Path outputFile) throws IOException { DirectRawFileWriter writer = new DirectRawFileWriter(conf, null, schema, meta, outputFile); writer.init(); @@ -160,7 +161,7 @@ public class TestDirectRawFile { return status; } - public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, OffHeapRowBlock rowBlock) throws IOException { + public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, MemoryRowBlock rowBlock) throws IOException { Path outputDir = new Path(testDir, UUID.randomUUID() + ""); outputDir.getFileSystem(conf).mkdirs(outputDir); Path outputFile = new Path(outputDir, "output.draw"); @@ -171,7 +172,7 @@ public class TestDirectRawFile { public void testRWForAllTypesWithNextTuple() throws IOException { int rowNum = 10000; - OffHeapRowBlock rowBlock = createRowBlock(rowNum); + MemoryRowBlock rowBlock = createRowBlock(rowNum); TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW); FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock); @@ -201,7 +202,7 @@ public class TestDirectRawFile { public void testRepeatedScan() throws IOException { int rowNum = 2; - OffHeapRowBlock rowBlock = createRowBlock(rowNum); + MemoryRowBlock rowBlock = createRowBlock(rowNum); TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW); FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock); @@ -229,7 +230,7 @@ public class TestDirectRawFile { public void testReset() throws IOException { int rowNum = 2; - OffHeapRowBlock rowBlock = createRowBlock(rowNum); + MemoryRowBlock rowBlock = createRowBlock(rowNum); TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW); FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock); @@ -264,11 +265,11 @@ public class TestDirectRawFile { reader.close(); } - public static OffHeapRowBlock createRowBlock(int rowNum) { + public static MemoryRowBlock createRowBlock(int rowNum) { long allocateStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 8); + MemoryRowBlock rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), StorageUnit.KB * 128); long allocatedEnd = System.currentTimeMillis(); - LOG.info(FileUtil.humanReadableByteCount(rowBlock.size(), true) + " bytes allocated " + LOG.info(FileUtil.humanReadableByteCount(rowBlock.capacity(), true) + " bytes allocated " + (allocatedEnd - allocateStart) + " msec"); long writeStart = System.currentTimeMillis(); http://git-wip-us.apache.org/repos/asf/tajo/blob/34062ea8/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml index 2de1617..b5a60fe 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml @@ -30,15 +30,11 @@ <name>tajo.storage.manager.hdfs.class</name> <value>org.apache.tajo.storage.FileTablespace</value> </property> - <property> - <name>tajo.storage.manager.hbase.class</name> - <value>org.apache.tajo.storage.hbase.HBaseTablespace</value> - </property> <!--- Registered Scanner Handler --> <property> <name>tajo.storage.scanner-handler</name> - <value>text,json,raw,rcfile,row,parquet,sequencefile,avro</value> + <value>text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro</value> </property> <!--- Fragment Class Configurations --> @@ -55,6 +51,10 @@ <value>org.apache.tajo.storage.fragment.FileFragment</value> </property> <property> + <name>tajo.storage.fragment.draw.class</name> + <value>org.apache.tajo.storage.fragment.FileFragment</value> + </property> + <property> <name>tajo.storage.fragment.rcfile.class</name> <value>org.apache.tajo.storage.fragment.FileFragment</value> </property> @@ -67,6 +67,10 @@ <value>org.apache.tajo.storage.fragment.FileFragment</value> </property> <property> + <name>tajo.storage.fragment.orc.class</name> + <value>org.apache.tajo.storage.fragment.FileFragment</value> + </property> + <property> <name>tajo.storage.fragment.sequencefile.class</name> <value>org.apache.tajo.storage.fragment.FileFragment</value> </property> @@ -92,6 +96,11 @@ </property> <property> + <name>tajo.storage.scanner-handler.draw.class</name> + <value>org.apache.tajo.storage.rawfile.DirectRawFileScanner</value> + </property> + + <property> <name>tajo.storage.scanner-handler.rcfile.class</name> <value>org.apache.tajo.storage.rcfile.RCFile$RCFileScanner</value> </property> @@ -107,6 +116,11 @@ </property> <property> + <name>tajo.storage.scanner-handler.orc.class</name> + <value>org.apache.tajo.storage.orc.ORCScanner</value> + </property> + + <property> <name>tajo.storage.scanner-handler.sequencefile.class</name> <value>org.apache.tajo.storage.sequencefile.SequenceFileScanner</value> </property> @@ -119,7 +133,7 @@ <!--- Appender Handler --> <property> <name>tajo.storage.appender-handler</name> - <value>text,raw,rcfile,row,parquet,sequencefile,avro</value> + <value>text,raw,draw,rcfile,row,parquet,sequencefile,avro</value> </property> <property> @@ -138,6 +152,11 @@ </property> <property> + <name>tajo.storage.appender-handler.draw.class</name> + <value>org.apache.tajo.storage.rawfile.DirectRawFileWriter</value> + </property> + + <property> <name>tajo.storage.appender-handler.rcfile.class</name> <value>org.apache.tajo.storage.rcfile.RCFile$RCFileAppender</value> </property> @@ -183,4 +202,4 @@ <value>131072</value> <description>128KB write buffer</description> </property> -</configuration> +</configuration> \ No newline at end of file
