Repository: tajo Updated Branches: refs/heads/branch-0.11.2 ff816003a -> 191faa0fd
TAJO-2092: TestStorages.testNullHandlingTypesWithProjection occasionally fail. Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/191faa0f Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/191faa0f Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/191faa0f Branch: refs/heads/branch-0.11.2 Commit: 191faa0fdef428bfe5febe14ea908c230e0247ab Parents: ff81600 Author: Jinho Kim <[email protected]> Authored: Fri Mar 18 11:15:47 2016 +0900 Committer: Jinho Kim <[email protected]> Committed: Fri Mar 18 11:15:47 2016 +0900 ---------------------------------------------------------------------- CHANGES | 3 ++ .../org/apache/tajo/storage/TestStorages.java | 56 +++++++------------- .../resources/dataset/testVariousTypes.avsc | 3 +- 3 files changed, 24 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/191faa0f/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 8ae6b98..d6ae247 100644 --- a/CHANGES +++ b/CHANGES @@ -24,6 +24,9 @@ Release 0.11.2 - unreleased BUG FIXES + TAJO-2092: TestStorages.testNullHandlingTypesWithProjection occasionally fail. + (jinho) + TAJO-2095: HiveCatalogStore::existIndexesByTable should not throw UnsupportedOperationException. (hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/191faa0f/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java index bd7be4c..b37270b 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -93,8 +93,7 @@ public class TestStorages { " { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" + " { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" + " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" + - " { \"name\": \"col11\", \"type\": \"null\" },\n" + - " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" + + " { \"name\": \"col11\", \"type\": [\"null\", \"bytes\"] }\n" + " ]\n" + "}\n"; @@ -399,9 +398,8 @@ public class TestStorages { schema.addColumn("col8", Type.TEXT); schema.addColumn("col9", Type.BLOB); schema.addColumn("col10", Type.INET4); - schema.addColumn("col11", Type.NULL_TYPE); if (handleProtobuf) { - schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + schema.addColumn("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); } KeyValueSet options = new KeyValueSet(); @@ -420,7 +418,7 @@ public class TestStorages { QueryId queryid = new QueryId("12345", 5); ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName()); - VTuple tuple = new VTuple(11 + (handleProtobuf ? 1 : 0)); + VTuple tuple = new VTuple(10 + (handleProtobuf ? 1 : 0)); tuple.put(new Datum[] { DatumFactory.createBool(true), DatumFactory.createChar("hyunsik"), @@ -432,11 +430,10 @@ public class TestStorages { DatumFactory.createText("hyunsik"), DatumFactory.createBlob("hyunsik".getBytes()), DatumFactory.createInet4("192.168.0.1"), - NullDatum.get() }); if (handleProtobuf) { - tuple.put(11, factory.createDatum(queryid.getProto())); + tuple.put(10, factory.createDatum(queryid.getProto())); } appender.addTuple(tuple); @@ -472,10 +469,9 @@ public class TestStorages { schema.addColumn("col8", Type.TEXT); schema.addColumn("col9", Type.BLOB); schema.addColumn("col10", Type.INET4); - schema.addColumn("col11", Type.NULL_TYPE); if (handleProtobuf) { - schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + schema.addColumn("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); } KeyValueSet options = new KeyValueSet(); @@ -496,7 +492,7 @@ public class TestStorages { QueryId queryid = new QueryId("12345", 5); ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName()); - int columnNum = 11 + (handleProtobuf ? 1 : 0); + int columnNum = 10 + (handleProtobuf ? 1 : 0); VTuple seedTuple = new VTuple(columnNum); seedTuple.put(new Datum[]{ DatumFactory.createBool(true), // 0 @@ -508,12 +504,11 @@ public class TestStorages { DatumFactory.createFloat8(271.9f), // 7 DatumFactory.createText("hyunsik"), // 8 DatumFactory.createBlob("hyunsik".getBytes()),// 9 - DatumFactory.createInet4("192.168.0.1"), // 10 - NullDatum.get(), // 11 + DatumFactory.createInet4("192.168.0.1") // 10 }); if (handleProtobuf) { - seedTuple.put(11, factory.createDatum(queryid.getProto())); // 12 + seedTuple.put(10, factory.createDatum(queryid.getProto())); // 11 } // Making tuples with different null column positions @@ -571,10 +566,9 @@ public class TestStorages { schema.addColumn("col8", Type.TEXT); schema.addColumn("col9", Type.BLOB); schema.addColumn("col10", Type.INET4); - schema.addColumn("col11", Type.NULL_TYPE); if (handleProtobuf) { - schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + schema.addColumn("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); } KeyValueSet options = new KeyValueSet(); @@ -595,7 +589,7 @@ public class TestStorages { QueryId queryid = new QueryId("12345", 5); ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName()); - int columnNum = 11 + (handleProtobuf ? 1 : 0); + int columnNum = 10 + (handleProtobuf ? 1 : 0); VTuple seedTuple = new VTuple(columnNum); seedTuple.put(new Datum[]{ DatumFactory.createBool(true), // 0 @@ -607,12 +601,11 @@ public class TestStorages { DatumFactory.createFloat8(271.9f), // 7 DatumFactory.createText("hyunsik"), // 8 DatumFactory.createBlob("hyunsik".getBytes()),// 9 - DatumFactory.createInet4("192.168.0.1"), // 10 - NullDatum.get(), // 11 + DatumFactory.createInet4("192.168.0.1") // 10 }); if (handleProtobuf) { - seedTuple.put(11, factory.createDatum(queryid.getProto())); // 12 + seedTuple.put(10, factory.createDatum(queryid.getProto())); // 11 } // Making tuples with different null column positions @@ -680,14 +673,13 @@ public class TestStorages { schema.addColumn("col9", Type.TEXT); schema.addColumn("col10", Type.BLOB); schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); meta.putOption(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName()); - Path tablePath = new Path(testDir, "testVariousTypes.data"); + Path tablePath = new Path(testDir, "testRCFileTextSerializeDeserialize.data"); FileTablespace sm = TablespaceManager.getLocalFs(); Appender appender = sm.getAppender(meta, schema, tablePath); appender.enableStats(); @@ -708,7 +700,6 @@ public class TestStorages { DatumFactory.createText("jinho"), DatumFactory.createBlob("hyunsik babo".getBytes()), DatumFactory.createInet4("192.168.0.1"), - NullDatum.get(), factory.createDatum(queryid.getProto()) }); appender.addTuple(tuple); @@ -749,14 +740,13 @@ public class TestStorages { schema.addColumn("col9", Type.TEXT); schema.addColumn("col10", Type.BLOB); schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); meta.putOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName()); - Path tablePath = new Path(testDir, "testVariousTypes.data"); + Path tablePath = new Path(testDir, "testRCFileBinarySerializeDeserialize.data"); FileTablespace sm = TablespaceManager.getLocalFs(); Appender appender = sm.getAppender(meta, schema, tablePath); appender.enableStats(); @@ -777,7 +767,6 @@ public class TestStorages { DatumFactory.createText("jinho"), DatumFactory.createBlob("hyunsik babo".getBytes()), DatumFactory.createInet4("192.168.0.1"), - NullDatum.get(), factory.createDatum(queryid.getProto()) }); appender.addTuple(tuple); @@ -818,14 +807,13 @@ public class TestStorages { schema.addColumn("col9", Type.TEXT); schema.addColumn("col10", Type.BLOB); schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); - Path tablePath = new Path(testDir, "testVariousTypes.data"); + Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserialize.data"); FileTablespace sm = TablespaceManager.getLocalFs(); Appender appender = sm.getAppender(meta, schema, tablePath); appender.enableStats(); @@ -845,7 +833,6 @@ public class TestStorages { DatumFactory.createText("jinho"), DatumFactory.createBlob("hyunsik babo".getBytes()), DatumFactory.createInet4("192.168.0.1"), - NullDatum.get(), ProtobufDatumFactory.createDatum(queryid.getProto()) }); appender.addTuple(tuple); @@ -890,8 +877,7 @@ public class TestStorages { schema.addColumn("col9", Type.TEXT); schema.addColumn("col10", Type.BLOB); schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -905,8 +891,7 @@ public class TestStorages { QueryId queryid = new QueryId("12345", 5); - VTuple tuple = new VTuple(13); - tuple.put(new Datum[] { + VTuple tuple = new VTuple(new Datum[] { DatumFactory.createBool(true), DatumFactory.createBit((byte) 0x99), DatumFactory.createChar("jinho"), @@ -918,7 +903,6 @@ public class TestStorages { DatumFactory.createText("jinho"), DatumFactory.createBlob("hyunsik babo".getBytes()), DatumFactory.createInet4("192.168.0.1"), - NullDatum.get(), ProtobufDatumFactory.createDatum(queryid.getProto()) }); appender.addTuple(tuple); http://git-wip-us.apache.org/repos/asf/tajo/blob/191faa0f/tajo-storage/tajo-storage-hdfs/src/test/resources/dataset/testVariousTypes.avsc ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/resources/dataset/testVariousTypes.avsc b/tajo-storage/tajo-storage-hdfs/src/test/resources/dataset/testVariousTypes.avsc index d4250a9..f71f052 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/resources/dataset/testVariousTypes.avsc +++ b/tajo-storage/tajo-storage-hdfs/src/test/resources/dataset/testVariousTypes.avsc @@ -13,8 +13,7 @@ { "name": "col8", "type": "string" }, { "name": "col9", "type": "bytes" }, { "name": "col10", "type": "bytes" }, - { "name": "col11", "type": "null" }, - { "name": "col12", "type": "bytes" } + { "name": "col11", "type": "bytes" } ] }
