http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java deleted file mode 100644 index ad5f408..0000000 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java +++ /dev/null @@ -1,857 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import com.google.common.collect.Lists; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Writable; -import org.apache.tajo.QueryId; -import org.apache.tajo.TajoIdProtos; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.datum.ProtobufDatumFactory; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.storage.rcfile.RCFile; -import org.apache.tajo.storage.sequencefile.SequenceFileScanner; -import org.apache.tajo.util.CommonTestingUtil; -import org.apache.tajo.util.FileUtil; -import org.apache.tajo.util.KeyValueSet; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@RunWith(Parameterized.class) -public class TestStorages { - private TajoConf conf; - private static String TEST_PATH = "target/test-data/TestStorages"; - - private static String TEST_PROJECTION_AVRO_SCHEMA = - "{\n" + - " \"type\": \"record\",\n" + - " \"namespace\": \"org.apache.tajo\",\n" + - " \"name\": \"testProjection\",\n" + - " \"fields\": [\n" + - " { \"name\": \"id\", \"type\": \"int\" },\n" + - " { \"name\": \"age\", \"type\": \"long\" },\n" + - " { \"name\": \"score\", \"type\": \"float\" }\n" + - " ]\n" + - "}\n"; - - private static String TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA = - "{\n" + - " \"type\": \"record\",\n" + - " \"namespace\": \"org.apache.tajo\",\n" + - " \"name\": \"testNullHandlingTypes\",\n" + - " \"fields\": [\n" + - " { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" + - " { \"name\": \"col2\", \"type\": [\"null\", \"int\"] },\n" + - " { \"name\": \"col3\", \"type\": [\"null\", \"string\"] },\n" + - " { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" + - " { \"name\": \"col5\", \"type\": [\"null\", \"int\"] },\n" + - " { \"name\": \"col6\", \"type\": [\"null\", \"long\"] },\n" + - " { \"name\": \"col7\", \"type\": [\"null\", \"float\"] },\n" + - " { \"name\": \"col8\", \"type\": [\"null\", \"double\"] },\n" + - " { \"name\": \"col9\", \"type\": [\"null\", \"string\"] },\n" + - " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" + - " { \"name\": \"col11\", \"type\": [\"null\", \"bytes\"] },\n" + - " { \"name\": \"col12\", \"type\": \"null\" },\n" + - " { \"name\": \"col13\", \"type\": [\"null\", \"bytes\"] }\n" + - " ]\n" + - "}\n"; - - private StoreType storeType; - private boolean splitable; - private boolean statsable; - private boolean seekable; - private Path testDir; - private FileSystem fs; - - public TestStorages(StoreType type, boolean splitable, boolean statsable, boolean seekable) throws IOException { - this.storeType = type; - this.splitable = splitable; - this.statsable = statsable; - this.seekable = seekable; - - conf = new TajoConf(); - - if (storeType == StoreType.RCFILE) { - conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100); - } - - testDir = CommonTestingUtil.getTestDir(TEST_PATH); - fs = testDir.getFileSystem(conf); - } - - @Parameterized.Parameters - public static Collection<Object[]> generateParameters() { - return Arrays.asList(new Object[][] { - //type, splitable, statsable, seekable - {StoreType.CSV, true, true, true}, - {StoreType.RAW, false, true, true}, - {StoreType.RCFILE, true, true, false}, - {StoreType.PARQUET, false, false, false}, - {StoreType.SEQUENCEFILE, true, true, false}, - {StoreType.AVRO, false, false, false}, - {StoreType.TEXTFILE, true, true, false}, - }); - } - - @Test - public void testSplitable() throws IOException { - if (splitable) { - Schema schema = new Schema(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - - TableMeta meta = CatalogUtil.newTableMeta(storeType); - Path tablePath = new Path(testDir, "Splitable.data"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); - appender.enableStats(); - appender.init(); - int tupleNum = 10000; - VTuple vTuple; - - for (int i = 0; i < tupleNum; i++) { - vTuple = new VTuple(2); - vTuple.put(0, DatumFactory.createInt4(i + 1)); - vTuple.put(1, DatumFactory.createInt8(25l)); - appender.addTuple(vTuple); - } - appender.close(); - TableStats stat = appender.getStats(); - assertEquals(tupleNum, stat.getNumRows().longValue()); - - FileStatus status = fs.getFileStatus(tablePath); - long fileLen = status.getLen(); - long randomNum = (long) (Math.random() * fileLen) + 1; - - FileFragment[] tablets = new FileFragment[2]; - tablets[0] = new FileFragment("Splitable", tablePath, 0, randomNum); - tablets[1] = new FileFragment("Splitable", tablePath, randomNum, (fileLen - randomNum)); - - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[0], schema); - assertTrue(scanner.isSplittable()); - scanner.init(); - int tupleCnt = 0; - while (scanner.next() != null) { - tupleCnt++; - } - scanner.close(); - - scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[1], schema); - assertTrue(scanner.isSplittable()); - scanner.init(); - while (scanner.next() != null) { - tupleCnt++; - } - scanner.close(); - - assertEquals(tupleNum, tupleCnt); - } - } - - @Test - public void testRCFileSplitable() throws IOException { - if (storeType == StoreType.RCFILE) { - Schema schema = new Schema(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - - TableMeta meta = CatalogUtil.newTableMeta(storeType); - Path tablePath = new Path(testDir, "Splitable.data"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); - appender.enableStats(); - appender.init(); - int tupleNum = 10000; - VTuple vTuple; - - for (int i = 0; i < tupleNum; i++) { - vTuple = new VTuple(2); - vTuple.put(0, DatumFactory.createInt4(i + 1)); - vTuple.put(1, DatumFactory.createInt8(25l)); - appender.addTuple(vTuple); - } - appender.close(); - TableStats stat = appender.getStats(); - assertEquals(tupleNum, stat.getNumRows().longValue()); - - FileStatus status = fs.getFileStatus(tablePath); - long fileLen = status.getLen(); - long randomNum = 122; // header size - - FileFragment[] tablets = new FileFragment[2]; - tablets[0] = new FileFragment("Splitable", tablePath, 0, randomNum); - tablets[1] = new FileFragment("Splitable", tablePath, randomNum, (fileLen - randomNum)); - - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[0], schema); - assertTrue(scanner.isSplittable()); - scanner.init(); - int tupleCnt = 0; - while (scanner.next() != null) { - tupleCnt++; - } - scanner.close(); - - scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[1], schema); - assertTrue(scanner.isSplittable()); - scanner.init(); - while (scanner.next() != null) { - tupleCnt++; - } - scanner.close(); - - assertEquals(tupleNum, tupleCnt); - } - } - - @Test - public void testProjection() throws IOException { - Schema schema = new Schema(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("score", Type.FLOAT4); - - TableMeta meta = CatalogUtil.newTableMeta(storeType); - meta.setOptions(CatalogUtil.newPhysicalProperties(storeType)); - if (storeType == StoreType.AVRO) { - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, - TEST_PROJECTION_AVRO_SCHEMA); - } - - Path tablePath = new Path(testDir, "testProjection.data"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); - appender.init(); - int tupleNum = 10000; - VTuple vTuple; - - for (int i = 0; i < tupleNum; i++) { - vTuple = new VTuple(3); - vTuple.put(0, DatumFactory.createInt4(i + 1)); - vTuple.put(1, DatumFactory.createInt8(i + 2)); - vTuple.put(2, DatumFactory.createFloat4(i + 3)); - appender.addTuple(vTuple); - } - appender.close(); - - FileStatus status = fs.getFileStatus(tablePath); - FileFragment fragment = new FileFragment("testReadAndWrite", tablePath, 0, status.getLen()); - - Schema target = new Schema(); - target.addColumn("age", Type.INT8); - target.addColumn("score", Type.FLOAT4); - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment, target); - scanner.init(); - int tupleCnt = 0; - Tuple tuple; - while ((tuple = scanner.next()) != null) { - if (storeType == StoreType.RCFILE - || storeType == StoreType.CSV - || storeType == StoreType.PARQUET - || storeType == StoreType.SEQUENCEFILE - || storeType == StoreType.AVRO) { - assertTrue(tuple.get(0) == null); - } - assertTrue(tupleCnt + 2 == tuple.get(1).asInt8()); - assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4()); - tupleCnt++; - } - scanner.close(); - - assertEquals(tupleNum, tupleCnt); - } - - @Test - public void testVariousTypes() throws IOException { - Schema schema = new Schema(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); - - KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(CatalogUtil.newPhysicalProperties(storeType)); - if (storeType == StoreType.AVRO) { - String path = FileUtil.getResourcePath("testVariousTypes.avsc").toString(); - meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path); - } - - Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); - appender.init(); - - QueryId queryid = new QueryId("12345", 5); - ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName()); - - Tuple tuple = new VTuple(13); - tuple.put(new Datum[] { - DatumFactory.createBool(true), - DatumFactory.createBit((byte) 0x99), - DatumFactory.createChar("hyunsik"), - DatumFactory.createInt2((short) 17), - DatumFactory.createInt4(59), - DatumFactory.createInt8(23l), - DatumFactory.createFloat4(77.9f), - DatumFactory.createFloat8(271.9f), - DatumFactory.createText("hyunsik"), - DatumFactory.createBlob("hyunsik".getBytes()), - DatumFactory.createInet4("192.168.0.1"), - NullDatum.get(), - factory.createDatum(queryid.getProto()) - }); - appender.addTuple(tuple); - appender.flush(); - appender.close(); - - FileStatus status = fs.getFileStatus(tablePath); - FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); - scanner.init(); - - Tuple retrieved; - while ((retrieved = scanner.next()) != null) { - for (int i = 0; i < tuple.size(); i++) { - assertEquals(tuple.get(i), retrieved.get(i)); - } - } - scanner.close(); - } - - @Test - public void testNullHandlingTypes() throws IOException { - Schema schema = new Schema(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); - - KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(CatalogUtil.newPhysicalProperties(storeType)); - meta.putOption(StorageConstants.TEXT_NULL, "\\\\N"); - meta.putOption(StorageConstants.RCFILE_NULL, "\\\\N"); - meta.putOption(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName()); - meta.putOption(StorageConstants.SEQUENCEFILE_NULL, "\\"); - if (storeType == StoreType.AVRO) { - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, - TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA); - } - - Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); - appender.init(); - - QueryId queryid = new QueryId("12345", 5); - ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName()); - - Tuple seedTuple = new VTuple(13); - seedTuple.put(new Datum[]{ - DatumFactory.createBool(true), // 0 - DatumFactory.createBit((byte) 0x99), // 1 - DatumFactory.createChar("hyunsik"), // 2 - DatumFactory.createInt2((short) 17), // 3 - DatumFactory.createInt4(59), // 4 - DatumFactory.createInt8(23l), // 5 - DatumFactory.createFloat4(77.9f), // 6 - DatumFactory.createFloat8(271.9f), // 7 - DatumFactory.createText("hyunsik"), // 8 - DatumFactory.createBlob("hyunsik".getBytes()),// 9 - DatumFactory.createInet4("192.168.0.1"), // 10 - NullDatum.get(), // 11 - factory.createDatum(queryid.getProto()) // 12 - }); - - // Making tuples with different null column positions - Tuple tuple; - for (int i = 0; i < 13; i++) { - tuple = new VTuple(13); - for (int j = 0; j < 13; j++) { - if (i == j) { // i'th column will have NULL value - tuple.put(j, NullDatum.get()); - } else { - tuple.put(j, seedTuple.get(j)); - } - } - appender.addTuple(tuple); - } - appender.flush(); - appender.close(); - - FileStatus status = fs.getFileStatus(tablePath); - FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); - scanner.init(); - - Tuple retrieved; - int i = 0; - while ((retrieved = scanner.next()) != null) { - assertEquals(13, retrieved.size()); - for (int j = 0; j < 13; j++) { - if (i == j) { - assertEquals(NullDatum.get(), retrieved.get(j)); - } else { - assertEquals(seedTuple.get(j), retrieved.get(j)); - } - } - - i++; - } - scanner.close(); - } - - @Test - public void testRCFileTextSerializeDeserialize() throws IOException { - if(storeType != StoreType.RCFILE) return; - - Schema schema = new Schema(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); - - KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.putOption(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName()); - - Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); - appender.enableStats(); - appender.init(); - - QueryId queryid = new QueryId("12345", 5); - ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName()); - - Tuple tuple = new VTuple(13); - tuple.put(new Datum[] { - DatumFactory.createBool(true), - DatumFactory.createBit((byte) 0x99), - DatumFactory.createChar("jinho"), - DatumFactory.createInt2((short) 17), - DatumFactory.createInt4(59), - DatumFactory.createInt8(23l), - DatumFactory.createFloat4(77.9f), - DatumFactory.createFloat8(271.9f), - DatumFactory.createText("jinho"), - DatumFactory.createBlob("hyunsik babo".getBytes()), - DatumFactory.createInet4("192.168.0.1"), - NullDatum.get(), - factory.createDatum(queryid.getProto()) - }); - appender.addTuple(tuple); - appender.flush(); - appender.close(); - - FileStatus status = fs.getFileStatus(tablePath); - assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen()); - - FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); - scanner.init(); - - Tuple retrieved; - while ((retrieved=scanner.next()) != null) { - for (int i = 0; i < tuple.size(); i++) { - assertEquals(tuple.get(i), retrieved.get(i)); - } - } - scanner.close(); - assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue()); - assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue()); - } - - @Test - public void testRCFileBinarySerializeDeserialize() throws IOException { - if(storeType != StoreType.RCFILE) return; - - Schema schema = new Schema(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); - - KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.putOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName()); - - Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); - appender.enableStats(); - appender.init(); - - QueryId queryid = new QueryId("12345", 5); - ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName()); - - Tuple tuple = new VTuple(13); - tuple.put(new Datum[] { - DatumFactory.createBool(true), - DatumFactory.createBit((byte) 0x99), - DatumFactory.createChar("jinho"), - DatumFactory.createInt2((short) 17), - DatumFactory.createInt4(59), - DatumFactory.createInt8(23l), - DatumFactory.createFloat4(77.9f), - DatumFactory.createFloat8(271.9f), - DatumFactory.createText("jinho"), - DatumFactory.createBlob("hyunsik babo".getBytes()), - DatumFactory.createInet4("192.168.0.1"), - NullDatum.get(), - factory.createDatum(queryid.getProto()) - }); - appender.addTuple(tuple); - appender.flush(); - appender.close(); - - FileStatus status = fs.getFileStatus(tablePath); - assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen()); - - FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); - scanner.init(); - - Tuple retrieved; - while ((retrieved=scanner.next()) != null) { - for (int i = 0; i < tuple.size(); i++) { - assertEquals(tuple.get(i), retrieved.get(i)); - } - } - scanner.close(); - assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue()); - assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue()); - } - - @Test - public void testSequenceFileTextSerializeDeserialize() throws IOException { - if(storeType != StoreType.SEQUENCEFILE) return; - - Schema schema = new Schema(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); - - KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); - - Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); - appender.enableStats(); - appender.init(); - - QueryId queryid = new QueryId("12345", 5); - ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName()); - - Tuple tuple = new VTuple(13); - tuple.put(new Datum[] { - DatumFactory.createBool(true), - DatumFactory.createBit((byte) 0x99), - DatumFactory.createChar("jinho"), - DatumFactory.createInt2((short) 17), - DatumFactory.createInt4(59), - DatumFactory.createInt8(23l), - DatumFactory.createFloat4(77.9f), - DatumFactory.createFloat8(271.9f), - DatumFactory.createText("jinho"), - DatumFactory.createBlob("hyunsik babo".getBytes()), - DatumFactory.createInet4("192.168.0.1"), - NullDatum.get(), - factory.createDatum(queryid.getProto()) - }); - appender.addTuple(tuple); - appender.flush(); - appender.close(); - - FileStatus status = fs.getFileStatus(tablePath); - assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen()); - - FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); - scanner.init(); - - assertTrue(scanner instanceof SequenceFileScanner); - Writable key = ((SequenceFileScanner) scanner).getKey(); - assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName()); - - Tuple retrieved; - while ((retrieved=scanner.next()) != null) { - for (int i = 0; i < tuple.size(); i++) { - assertEquals(tuple.get(i), retrieved.get(i)); - } - } - scanner.close(); - assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue()); - assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue()); - } - - @Test - public void testSequenceFileBinarySerializeDeserialize() throws IOException { - if(storeType != StoreType.SEQUENCEFILE) return; - - Schema schema = new Schema(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); - - KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName()); - - Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); - appender.enableStats(); - appender.init(); - - QueryId queryid = new QueryId("12345", 5); - ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName()); - - Tuple tuple = new VTuple(13); - tuple.put(new Datum[] { - DatumFactory.createBool(true), - DatumFactory.createBit((byte) 0x99), - DatumFactory.createChar("jinho"), - DatumFactory.createInt2((short) 17), - DatumFactory.createInt4(59), - DatumFactory.createInt8(23l), - DatumFactory.createFloat4(77.9f), - DatumFactory.createFloat8(271.9f), - DatumFactory.createText("jinho"), - DatumFactory.createBlob("hyunsik babo".getBytes()), - DatumFactory.createInet4("192.168.0.1"), - NullDatum.get(), - factory.createDatum(queryid.getProto()) - }); - appender.addTuple(tuple); - appender.flush(); - appender.close(); - - FileStatus status = fs.getFileStatus(tablePath); - assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen()); - - FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); - scanner.init(); - - assertTrue(scanner instanceof SequenceFileScanner); - Writable key = ((SequenceFileScanner) scanner).getKey(); - assertEquals(key.getClass().getCanonicalName(), BytesWritable.class.getCanonicalName()); - - Tuple retrieved; - while ((retrieved=scanner.next()) != null) { - for (int i = 0; i < tuple.size(); i++) { - assertEquals(tuple.get(i), retrieved.get(i)); - } - } - scanner.close(); - assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue()); - assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue()); - } - - @Test - public void testTime() throws IOException { - if (storeType == StoreType.CSV || storeType == StoreType.RAW) { - Schema schema = new Schema(); - schema.addColumn("col1", Type.DATE); - schema.addColumn("col2", Type.TIME); - schema.addColumn("col3", Type.TIMESTAMP); - - KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - - Path tablePath = new Path(testDir, "testTime.data"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); - appender.init(); - - Tuple tuple = new VTuple(3); - tuple.put(new Datum[]{ - DatumFactory.createDate("1980-04-01"), - DatumFactory.createTime("12:34:56"), - DatumFactory.createTimestmpDatumWithUnixTime((int)(System.currentTimeMillis() / 1000)) - }); - appender.addTuple(tuple); - appender.flush(); - appender.close(); - - FileStatus status = fs.getFileStatus(tablePath); - FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); - scanner.init(); - - Tuple retrieved; - while ((retrieved = scanner.next()) != null) { - for (int i = 0; i < tuple.size(); i++) { - assertEquals(tuple.get(i), retrieved.get(i)); - } - } - scanner.close(); - } - } - - @Test - public void testSeekableScanner() throws IOException { - if (!seekable) { - return; - } - - Schema schema = new Schema(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("comment", Type.TEXT); - - TableMeta meta = CatalogUtil.newTableMeta(storeType); - Path tablePath = new Path(testDir, "Seekable.data"); - FileAppender appender = (FileAppender) StorageManager.getFileStorageManager(conf).getAppender(meta, schema, - tablePath); - appender.enableStats(); - appender.init(); - int tupleNum = 100000; - VTuple vTuple; - - List<Long> offsets = Lists.newArrayList(); - offsets.add(0L); - for (int i = 0; i < tupleNum; i++) { - vTuple = new VTuple(3); - vTuple.put(0, DatumFactory.createInt4(i + 1)); - vTuple.put(1, DatumFactory.createInt8(25l)); - vTuple.put(2, DatumFactory.createText("test" + i)); - appender.addTuple(vTuple); - - // find a seek position - if (i % (tupleNum / 3) == 0) { - offsets.add(appender.getOffset()); - } - } - - // end of file - if (!offsets.contains(appender.getOffset())) { - offsets.add(appender.getOffset()); - } - - appender.close(); - if (statsable) { - TableStats stat = appender.getStats(); - assertEquals(tupleNum, stat.getNumRows().longValue()); - } - - FileStatus status = fs.getFileStatus(tablePath); - assertEquals(status.getLen(), appender.getOffset()); - - Scanner scanner; - int tupleCnt = 0; - long prevOffset = 0; - long readBytes = 0; - long readRows = 0; - for (long offset : offsets) { - scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, - new FileFragment("table", tablePath, prevOffset, offset - prevOffset), schema); - scanner.init(); - - while (scanner.next() != null) { - tupleCnt++; - } - - scanner.close(); - if (statsable) { - readBytes += scanner.getInputStats().getNumBytes(); - readRows += scanner.getInputStats().getNumRows(); - } - prevOffset = offset; - } - - assertEquals(tupleNum, tupleCnt); - if (statsable) { - assertEquals(appender.getStats().getNumBytes().longValue(), readBytes); - assertEquals(appender.getStats().getNumRows().longValue(), readRows); - } - } -}
http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/TestTupleComparator.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestTupleComparator.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestTupleComparator.java deleted file mode 100644 index 639ca04..0000000 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestTupleComparator.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class TestTupleComparator { - - @Before - public void setUp() throws Exception { - } - - @After - public void tearDown() throws Exception { - } - - @Test - public final void testCompare() { - Schema schema = new Schema(); - schema.addColumn("col1", Type.INT4); - schema.addColumn("col2", Type.INT4); - schema.addColumn("col3", Type.INT4); - schema.addColumn("col4", Type.INT4); - schema.addColumn("col5", Type.TEXT); - - Tuple tuple1 = new VTuple(5); - Tuple tuple2 = new VTuple(5); - - tuple1.put( - new Datum[] { - DatumFactory.createInt4(9), - DatumFactory.createInt4(3), - DatumFactory.createInt4(33), - DatumFactory.createInt4(4), - DatumFactory.createText("abc")}); - tuple2.put( - new Datum[] { - DatumFactory.createInt4(1), - DatumFactory.createInt4(25), - DatumFactory.createInt4(109), - DatumFactory.createInt4(4), - DatumFactory.createText("abd")}); - - SortSpec sortKey1 = new SortSpec(schema.getColumn("col4"), true, false); - SortSpec sortKey2 = new SortSpec(schema.getColumn("col5"), true, false); - - BaseTupleComparator tc = new BaseTupleComparator(schema, - new SortSpec[] {sortKey1, sortKey2}); - assertEquals(-1, tc.compare(tuple1, tuple2)); - assertEquals(1, tc.compare(tuple2, tuple1)); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java deleted file mode 100644 index 9837fd1..0000000 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - - -import org.junit.Before; -import org.junit.Test; -import org.apache.tajo.datum.DatumFactory; - -import static org.junit.Assert.*; - -public class TestVTuple { - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - - } - - @Test - public void testContain() { - VTuple t1 = new VTuple(260); - t1.put(0, DatumFactory.createInt4(1)); - t1.put(1, DatumFactory.createInt4(1)); - t1.put(27, DatumFactory.createInt4(1)); - t1.put(96, DatumFactory.createInt4(1)); - t1.put(257, DatumFactory.createInt4(1)); - - assertTrue(t1.contains(0)); - assertTrue(t1.contains(1)); - assertFalse(t1.contains(2)); - assertFalse(t1.contains(3)); - assertFalse(t1.contains(4)); - assertTrue(t1.contains(27)); - assertFalse(t1.contains(28)); - assertFalse(t1.contains(95)); - assertTrue(t1.contains(96)); - assertFalse(t1.contains(97)); - assertTrue(t1.contains(257)); - } - - @Test - public void testPut() { - VTuple t1 = new VTuple(260); - t1.put(0, DatumFactory.createText("str")); - t1.put(1, DatumFactory.createInt4(2)); - t1.put(257, DatumFactory.createFloat4(0.76f)); - - assertTrue(t1.contains(0)); - assertTrue(t1.contains(1)); - - assertEquals(t1.getText(0),"str"); - assertEquals(t1.get(1).asInt4(),2); - assertTrue(t1.get(257).asFloat4() == 0.76f); - } - - @Test - public void testEquals() { - Tuple t1 = new VTuple(5); - Tuple t2 = new VTuple(5); - - t1.put(0, DatumFactory.createInt4(1)); - t1.put(1, DatumFactory.createInt4(2)); - t1.put(3, DatumFactory.createInt4(2)); - - t2.put(0, DatumFactory.createInt4(1)); - t2.put(1, DatumFactory.createInt4(2)); - t2.put(3, DatumFactory.createInt4(2)); - - assertEquals(t1,t2); - - Tuple t3 = new VTuple(5); - t2.put(0, DatumFactory.createInt4(1)); - t2.put(1, DatumFactory.createInt4(2)); - t2.put(4, DatumFactory.createInt4(2)); - - assertNotSame(t1,t3); - } - - @Test - public void testHashCode() { - Tuple t1 = new VTuple(5); - Tuple t2 = new VTuple(5); - - t1.put(0, DatumFactory.createInt4(1)); - t1.put(1, DatumFactory.createInt4(2)); - t1.put(3, DatumFactory.createInt4(2)); - t1.put(4, DatumFactory.createText("hyunsik")); - - t2.put(0, DatumFactory.createInt4(1)); - t2.put(1, DatumFactory.createInt4(2)); - t2.put(3, DatumFactory.createInt4(2)); - t2.put(4, DatumFactory.createText("hyunsik")); - - assertEquals(t1.hashCode(),t2.hashCode()); - - Tuple t3 = new VTuple(5); - t3.put(0, DatumFactory.createInt4(1)); - t3.put(1, DatumFactory.createInt4(2)); - t3.put(4, DatumFactory.createInt4(2)); - - assertNotSame(t1.hashCode(),t3.hashCode()); - } - - @Test - public void testPutTuple() { - Tuple t1 = new VTuple(5); - - t1.put(0, DatumFactory.createInt4(1)); - t1.put(1, DatumFactory.createInt4(2)); - t1.put(2, DatumFactory.createInt4(3)); - - Tuple t2 = new VTuple(2); - t2.put(0, DatumFactory.createInt4(4)); - t2.put(1, DatumFactory.createInt4(5)); - - t1.put(3, t2); - - for (int i = 0; i < 5; i++) { - assertEquals(i+1, t1.get(i).asInt4()); - } - } - - @Test - public void testClone() throws CloneNotSupportedException { - Tuple t1 = new VTuple(5); - - t1.put(0, DatumFactory.createInt4(1)); - t1.put(1, DatumFactory.createInt4(2)); - t1.put(3, DatumFactory.createInt4(2)); - t1.put(4, DatumFactory.createText("str")); - - VTuple t2 = (VTuple) t1.clone(); - assertNotSame(t1, t2); - assertEquals(t1, t2); - - assertSame(t1.get(4), t2.get(4)); - - t1.clear(); - assertFalse(t1.equals(t2)); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java b/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java deleted file mode 100644 index 6186e9e..0000000 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.avro; - -import org.apache.avro.Schema; -import org.apache.tajo.HttpFileServer; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.util.FileUtil; -import org.apache.tajo.util.NetUtils; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.URISyntaxException; -import java.net.URL; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -/** - * Tests for {@link org.apache.tajo.storage.avro.AvroUtil}. - */ -public class TestAvroUtil { - private Schema expected; - private URL schemaUrl; - - @Before - public void setUp() throws Exception { - schemaUrl = FileUtil.getResourcePath("testVariousTypes.avsc"); - assertNotNull(schemaUrl); - - File file = new File(schemaUrl.getPath()); - assertTrue(file.exists()); - - expected = new Schema.Parser().parse(file); - } - - @Test - public void testGetSchema() throws IOException, URISyntaxException { - TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO); - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, FileUtil.readTextFile(new File(schemaUrl.getPath()))); - Schema schema = AvroUtil.getAvroSchema(meta, new TajoConf()); - assertEquals(expected, schema); - - meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO); - meta.putOption(StorageConstants.AVRO_SCHEMA_URL, schemaUrl.getPath()); - schema = AvroUtil.getAvroSchema(meta, new TajoConf()); - assertEquals(expected, schema); - - HttpFileServer server = new HttpFileServer(NetUtils.createSocketAddr("127.0.0.1:0")); - try { - server.start(); - InetSocketAddress addr = server.getBindAddress(); - - String url = "http://127.0.0.1:"; + addr.getPort() + schemaUrl.getPath(); - meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO); - meta.putOption(StorageConstants.AVRO_SCHEMA_URL, url); - schema = AvroUtil.getAvroSchema(meta, new TajoConf()); - } finally { - server.stop(); - } - assertEquals(expected, schema); - } - - @Test - public void testGetSchemaFromHttp() throws IOException, URISyntaxException { - HttpFileServer server = new HttpFileServer(NetUtils.createSocketAddr("127.0.0.1:0")); - try { - server.start(); - InetSocketAddress addr = server.getBindAddress(); - - Schema schema = AvroUtil.getAvroSchemaFromHttp("http://127.0.0.1:"; + addr.getPort() + schemaUrl.getPath()); - assertEquals(expected, schema); - } finally { - server.stop(); - } - } - - @Test - public void testGetSchemaFromFileSystem() throws IOException, URISyntaxException { - Schema schema = AvroUtil.getAvroSchemaFromFileSystem(schemaUrl.toString(), new TajoConf()); - - assertEquals(expected, schema); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java deleted file mode 100644 index 39d28b3..0000000 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.hbase; - -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.util.KeyValueSet; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class TestColumnMapping { - @Test - public void testColumnKeyValueMapping() throws Exception { - KeyValueSet keyValueSet = new KeyValueSet(); - keyValueSet.set(HBaseStorageConstants.META_TABLE_KEY, "test"); - keyValueSet.set(HBaseStorageConstants.META_COLUMNS_KEY, ":key,col2:key:,col2:value:#b,col3:"); - - Schema schema = new Schema(); - schema.addColumn("c1", Type.TEXT); - schema.addColumn("c2", Type.TEXT); - schema.addColumn("c3", Type.TEXT); - schema.addColumn("c4", Type.TEXT); - - TableMeta tableMeta = new TableMeta(StoreType.HBASE, keyValueSet); - - ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta); - - List<String> cfNames = columnMapping.getColumnFamilyNames(); - assertEquals(2, cfNames.size()); - assertEquals("col2", cfNames.get(0)); - assertEquals("col3", cfNames.get(1)); - - for (int i = 0; i < columnMapping.getIsBinaryColumns().length; i++) { - if (i == 2) { - assertTrue(columnMapping.getIsBinaryColumns()[i]); - } else { - assertFalse(columnMapping.getIsBinaryColumns()[i]); - } - } - - for (int i = 0; i < columnMapping.getIsRowKeyMappings().length; i++) { - if (i == 0) { - assertTrue(columnMapping.getIsRowKeyMappings()[i]); - } else { - assertFalse(columnMapping.getIsRowKeyMappings()[i]); - } - } - - String[] expectedColumnNames = { null, null, null, null}; - for (int i = 0; i < schema.size(); i++) { - String columnName = columnMapping.getMappingColumns()[i][1] == null ? null : - new String(columnMapping.getMappingColumns()[i][1]); - assertEquals(expectedColumnNames[i], columnName); - } - - for (int i = 0; i < schema.size(); i++) { - if (i == 1) { - assertTrue(columnMapping.getIsColumnKeys()[i]); - } else { - assertFalse(columnMapping.getIsColumnKeys()[i]); - } - } - - for (int i = 0; i < schema.size(); i++) { - if (i == 2) { - assertTrue(columnMapping.getIsColumnValues()[i]); - } else { - assertFalse(columnMapping.getIsColumnValues()[i]); - } - } - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java deleted file mode 100644 index 1fc4065..0000000 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.hbase; - -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.plan.expr.*; -import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.util.Pair; -import org.junit.Test; - -import java.util.List; -import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -public class TestHBaseStorageManager { - @Test - public void testGetIndexPredications() throws Exception { - Column rowkeyColumn = new Column("rk", Type.TEXT); - // where rk >= '020' and rk <= '055' - ScanNode scanNode = new ScanNode(1); - EvalNode evalNode1 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("020"))); - EvalNode evalNode2 = new BinaryEval(EvalType.LEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("055"))); - EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); - scanNode.setQual(evalNodeA); - - HBaseStorageManager storageManager = - (HBaseStorageManager) StorageManager.getStorageManager(new TajoConf(), StoreType.HBASE); - List<Set<EvalNode>> indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); - assertNotNull(indexEvals); - assertEquals(1, indexEvals.size()); - Pair<Datum, Datum> indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); - assertEquals("020", indexPredicateValue.getFirst().asChars()); - assertEquals("055", indexPredicateValue.getSecond().asChars()); - - // where (rk >= '020' and rk <= '055') or rk = '075' - EvalNode evalNode3 = new BinaryEval(EvalType.EQUAL, new FieldEval(rowkeyColumn),new ConstEval(new TextDatum("075"))); - EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3); - scanNode.setQual(evalNodeB); - indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); - assertEquals(2, indexEvals.size()); - indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); - assertEquals("020", indexPredicateValue.getFirst().asChars()); - assertEquals("055", indexPredicateValue.getSecond().asChars()); - - indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(1)); - assertEquals("075", indexPredicateValue.getFirst().asChars()); - assertEquals("075", indexPredicateValue.getSecond().asChars()); - - // where (rk >= '020' and rk <= '055') or (rk >= '072' and rk <= '078') - EvalNode evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("072"))); - EvalNode evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("078"))); - EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); - EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); - scanNode.setQual(evalNodeD); - indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); - assertEquals(2, indexEvals.size()); - - indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); - assertEquals("020", indexPredicateValue.getFirst().asChars()); - assertEquals("055", indexPredicateValue.getSecond().asChars()); - - indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(1)); - assertEquals("072", indexPredicateValue.getFirst().asChars()); - assertEquals("078", indexPredicateValue.getSecond().asChars()); - - // where (rk >= '020' and rk <= '055') or (rk >= '072' and rk <= '078' and rk >= '073') - evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("072"))); - evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("078"))); - evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); - EvalNode evalNode6 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("073"))); - evalNodeD = new BinaryEval(EvalType.AND, evalNodeC, evalNode6); - EvalNode evalNodeE = new BinaryEval(EvalType.OR, evalNodeA, evalNodeD); - scanNode.setQual(evalNodeE); - indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); - assertEquals(2, indexEvals.size()); - - indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); - assertEquals("020", indexPredicateValue.getFirst().asChars()); - assertEquals("055", indexPredicateValue.getSecond().asChars()); - - indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(1)); - assertEquals("073", indexPredicateValue.getFirst().asChars()); - assertEquals("078", indexPredicateValue.getSecond().asChars()); - } -}
