Repository: tajo Updated Branches: refs/heads/master 71d2825cd -> 4aef83a39
http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestLazyTuple.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestLazyTuple.java b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestLazyTuple.java index df5d3b0..76c2ef1 100644 --- a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestLazyTuple.java +++ b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestLazyTuple.java @@ -19,8 +19,9 @@ package org.apache.tajo.storage; +import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; @@ -41,24 +42,23 @@ public class TestLazyTuple { public void setUp() { nullbytes = "\\N".getBytes(); - schema = SchemaFactory.newV1(); - schema.addColumn("col1", TajoDataTypes.Type.BOOLEAN); - schema.addColumn("col2", TajoDataTypes.Type.BIT); - schema.addColumn("col3", TajoDataTypes.Type.CHAR, 7); - schema.addColumn("col4", TajoDataTypes.Type.INT2); - schema.addColumn("col5", TajoDataTypes.Type.INT4); - schema.addColumn("col6", TajoDataTypes.Type.INT8); - schema.addColumn("col7", TajoDataTypes.Type.FLOAT4); - schema.addColumn("col8", TajoDataTypes.Type.FLOAT8); - schema.addColumn("col9", TajoDataTypes.Type.TEXT); - schema.addColumn("col10", TajoDataTypes.Type.BLOB); - schema.addColumn("col11", TajoDataTypes.Type.INET4); - schema.addColumn("col12", TajoDataTypes.Type.INT4); - schema.addColumn("col13", TajoDataTypes.Type.NULL_TYPE); + schema = SchemaBuilder.builder() + .add("col1", TajoDataTypes.Type.BOOLEAN) + .add("col2", CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.CHAR, 7)) + .add("col3", TajoDataTypes.Type.INT2) + .add("col4", TajoDataTypes.Type.INT4) + .add("col5", TajoDataTypes.Type.INT8) + .add("col6", TajoDataTypes.Type.FLOAT4) + .add("col7", TajoDataTypes.Type.FLOAT8) + .add("col8", TajoDataTypes.Type.TEXT) + .add("col9", TajoDataTypes.Type.BLOB) + .add("col10", TajoDataTypes.Type.INET4) + .add("col11", TajoDataTypes.Type.INT4) + .add("col12", TajoDataTypes.Type.NULL_TYPE) + .build(); StringBuilder sb = new StringBuilder(); sb.append(DatumFactory.createBool(true)).append('|'); - sb.append(new String(DatumFactory.createBit((byte) 0x99).asTextBytes())).append('|'); sb.append(DatumFactory.createChar("str")).append('|'); sb.append(DatumFactory.createInt2((short) 17)).append('|'); sb.append(DatumFactory.createInt4(59)).append('|'); @@ -80,18 +80,17 @@ public class TestLazyTuple { LazyTuple t1 = new LazyTuple(schema, textRow, -1, nullbytes, serde); assertEquals(DatumFactory.createBool(true), t1.get(0)); - assertEquals(DatumFactory.createBit((byte) 0x99), t1.get(1)); - assertEquals(DatumFactory.createChar("str"), t1.get(2)); - assertEquals(DatumFactory.createInt2((short) 17), t1.get(3)); - assertEquals(DatumFactory.createInt4(59), t1.get(4)); - assertEquals(DatumFactory.createInt8(23l), t1.get(5)); - assertEquals(DatumFactory.createFloat4(77.9f), t1.get(6)); - assertEquals(DatumFactory.createFloat8(271.9f), t1.get(7)); - assertEquals(DatumFactory.createText("str2"), t1.get(8)); - assertEquals(DatumFactory.createBlob("jinho".getBytes()), t1.get(9)); - assertEquals(DatumFactory.createInet4("192.168.0.1"), t1.get(10)); + assertEquals(DatumFactory.createChar("str"), t1.get(1)); + assertEquals(DatumFactory.createInt2((short) 17), t1.get(2)); + assertEquals(DatumFactory.createInt4(59), t1.get(3)); + assertEquals(DatumFactory.createInt8(23l), t1.get(4)); + assertEquals(DatumFactory.createFloat4(77.9f), t1.get(5)); + assertEquals(DatumFactory.createFloat8(271.9f), t1.get(6)); + assertEquals(DatumFactory.createText("str2"), t1.get(7)); + assertEquals(DatumFactory.createBlob("jinho".getBytes()), t1.get(8)); + assertEquals(DatumFactory.createInet4("192.168.0.1"), t1.get(9)); + assertEquals(NullDatum.get(), t1.get(10)); assertEquals(NullDatum.get(), t1.get(11)); - assertEquals(NullDatum.get(), t1.get(12)); } @Test @@ -115,7 +114,6 @@ public class TestLazyTuple { assertFalse(t1.contains(9)); assertFalse(t1.contains(10)); assertFalse(t1.contains(11)); - assertFalse(t1.contains(12)); } @Test @@ -198,12 +196,13 @@ public class TestLazyTuple { @Test public void testInvalidNumber() { byte[][] bytes = BytesUtils.splitPreserveAllTokens(" 1| |2 ||".getBytes(), '|', 5); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", TajoDataTypes.Type.INT2); - schema.addColumn("col2", TajoDataTypes.Type.INT4); - schema.addColumn("col3", TajoDataTypes.Type.INT8); - schema.addColumn("col4", TajoDataTypes.Type.FLOAT4); - schema.addColumn("col5", TajoDataTypes.Type.FLOAT8); + Schema schema = SchemaBuilder.builder() + .add("col1", TajoDataTypes.Type.INT2) + .add("col2", TajoDataTypes.Type.INT4) + .add("col3", TajoDataTypes.Type.INT8) + .add("col4", TajoDataTypes.Type.FLOAT4) + .add("col5", TajoDataTypes.Type.FLOAT8) + .build(); LazyTuple tuple = new LazyTuple(schema, bytes, 0); assertEquals(bytes.length, tuple.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java index e5d8464..9f0764d 100644 --- a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java +++ b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java @@ -19,7 +19,7 @@ package org.apache.tajo.storage; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.*; @@ -45,12 +45,13 @@ public class TestTupleComparator { @Test public final void testCompare() { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.INT4); - schema.addColumn("col2", Type.INT4); - schema.addColumn("col3", Type.INT4); - schema.addColumn("col4", Type.INT4); - schema.addColumn("col5", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("col1", Type.INT4) + .add("col2", Type.INT4) + .add("col3", Type.INT4) + .add("col4", Type.INT4) + .add("col5", Type.TEXT) + .build(); VTuple tuple1 = new VTuple( new Datum[] { @@ -78,9 +79,10 @@ public class TestTupleComparator { @Test public void testNullFirst() throws Exception { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("name", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("name", Type.TEXT) + .build(); VTuple tuple1 = new VTuple(2); tuple1.put(0, new Int4Datum(1)); @@ -118,9 +120,10 @@ public class TestTupleComparator { @Test public void testNullLast() throws Exception { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("name", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("name", Type.TEXT) + .build(); VTuple tuple1 = new VTuple(2); tuple1.put(0, new Int4Datum(1)); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java index 6713519..b607a33 100644 --- a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java +++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java @@ -19,7 +19,7 @@ package org.apache.tajo.storage.hbase; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.util.KeyValueSet; @@ -36,11 +36,12 @@ public class TestColumnMapping { keyValueSet.set(HBaseStorageConstants.META_TABLE_KEY, "test"); keyValueSet.set(HBaseStorageConstants.META_COLUMNS_KEY, ":key,col2:key:,col2:value:#b,col3:"); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("c1", Type.TEXT); - schema.addColumn("c2", Type.TEXT); - schema.addColumn("c3", Type.TEXT); - schema.addColumn("c4", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("c1", Type.TEXT) + .add("c2", Type.TEXT) + .add("c3", Type.TEXT) + .add("c4", Type.TEXT) + .build(); TableMeta tableMeta = new TableMeta("HBASE", keyValueSet); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java index 4bdb4f4..6e5c551 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java @@ -20,7 +20,7 @@ package org.apache.tajo.storage.parquet; import org.apache.hadoop.conf.Configuration; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; @@ -64,7 +64,8 @@ public class ParquetScanner extends FileScanner { if (targets == null) { targets = schema.toArray(); } - reader = new TajoParquetReader(conf, fragment.getPath(), schema, SchemaFactory.newV1(targets)); + reader = new TajoParquetReader(conf, fragment.getPath(), schema, + SchemaBuilder.builder().addAll(targets).build()); totalRowCount = reader.getTotalRowCount(); currentRowCount = 0; closed = false; http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java index b2f3f34..0719699 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java @@ -25,7 +25,7 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.apache.parquet.schema.Type; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.common.TajoDataTypes; import java.util.ArrayList; @@ -65,7 +65,7 @@ public class TajoSchemaConverter { } Column[] columnsArray = new Column[columns.size()]; columnsArray = columns.toArray(columnsArray); - return SchemaFactory.newV1(columnsArray); + return SchemaBuilder.builder().addAll(columnsArray).build(); } private Column convertField(final Type fieldType) { http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcRecordReader.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcRecordReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcRecordReader.java index df9a7c7..4cfd19d 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcRecordReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcRecordReader.java @@ -26,13 +26,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.io.DiskRange; import org.apache.hadoop.hive.common.io.DiskRangeList; import org.apache.orc.*; -import org.apache.orc.OrcProto; import org.apache.orc.impl.*; -import org.apache.orc.impl.StreamName; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.storage.fragment.FileFragment; @@ -73,7 +70,7 @@ public class OrcRecordReader implements Closeable { public OrcRecordReader(List<StripeInformation> stripes, FileSystem fileSystem, Schema schema, - Column[] target, + Column[] targets, FileFragment fragment, List<OrcProto.Type> types, CompressionCodec codec, @@ -83,7 +80,7 @@ public class OrcRecordReader implements Closeable { Configuration conf, TimeZone timeZone) throws IOException { - result = new VTuple(target.length); + result = new VTuple(targets.length); this.conf = conf; this.path = fragment.getPath(); @@ -91,8 +88,8 @@ public class OrcRecordReader implements Closeable { this.types = types; this.bufferSize = bufferSize; this.included = new boolean[schema.size() + 1]; - included[0] = target.length > 0; // always include root column except when target schema size is 0 - Schema targetSchema = SchemaFactory.newV1(target); + included[0] = targets.length > 0; // always include root column except when target schema size is 0 + Schema targetSchema = SchemaBuilder.builder().addAll(targets).build(); for (int i = 1; i < included.length; i++) { included[i] = targetSchema.contains(schema.getColumn(i - 1)); } @@ -120,9 +117,9 @@ public class OrcRecordReader implements Closeable { firstRow = skippedRows; totalRowCount = rows; - reader = new DatumTreeReader[target.length]; + reader = new DatumTreeReader[targets.length]; for (int i = 0; i < reader.length; i++) { - reader[i] = TreeReaderFactory.createTreeReader(timeZone, schema.getColumnId(target[i].getQualifiedName()), target[i], + reader[i] = TreeReaderFactory.createTreeReader(timeZone, schema.getColumnId(targets[i].getQualifiedName()), targets[i], options.getSkipCorruptRecords()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java index d18961f..f1f1320 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java @@ -31,7 +31,7 @@ import org.apache.orc.OrcConf; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; @@ -113,10 +113,11 @@ public class TestCompressionStorages { } private void storageCompressionTest(String dataFormat, Class<? extends CompressionCodec> codec) throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.FLOAT4); - schema.addColumn("name", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.FLOAT4) + .add("name", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); meta.putProperty("compression.codec", codec.getCanonicalName()); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java index 523a0d0..29d8197 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java @@ -25,7 +25,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; @@ -42,21 +42,23 @@ import static org.junit.Assert.*; public class TestDelimitedTextFile { private static final Log LOG = LogFactory.getLog(TestDelimitedTextFile.class); - private static Schema schema = SchemaFactory.newV1(); + private static Schema schema; private static Tuple baseTuple; static { - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.CHAR, 7); - schema.addColumn("col3", Type.INT2); - schema.addColumn("col4", Type.INT4); - schema.addColumn("col5", Type.INT8); - schema.addColumn("col6", Type.FLOAT4); - schema.addColumn("col7", Type.FLOAT8); - schema.addColumn("col8", Type.TEXT); - schema.addColumn("col9", Type.BLOB); - schema.addColumn("col10", Type.INET4); + schema = SchemaBuilder.builder() + .add("col1", Type.BOOLEAN) + .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7)) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .add("col6", Type.FLOAT4) + .add("col7", Type.FLOAT8) + .add("col8", Type.TEXT) + .add("col9", Type.BLOB) + .add("col10", Type.INET4) + .build(); baseTuple = new VTuple(new Datum[] { DatumFactory.createBool(true), // 0 http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java index 59ff6f6..48740fb 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; @@ -98,10 +98,11 @@ public class TestFileSystems { @Test public void testBlockSplit() throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT4); - schema.addColumn("name", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT4) + .add("name", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta("TEXT"); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java index b8f9893..65453dd 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdfs.*; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; @@ -82,10 +82,11 @@ public class TestFileTablespace { @Test public final void testGetScannerAndAppender() throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age",Type.INT4); - schema.addColumn("name",Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT4) + .add("name", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); @@ -145,10 +146,11 @@ public class TestFileTablespace { TablespaceManager.addTableSpaceForTest(space); assertEquals(fs.getUri(), space.getUri()); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age",Type.INT4); - schema.addColumn("name",Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT4) + .add("name", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); List<Fragment> splits = Lists.newArrayList(); @@ -199,10 +201,11 @@ public class TestFileTablespace { TablespaceManager.addTableSpaceForTest(space); assertEquals(fs.getUri(), space.getUri()); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age",Type.INT4); - schema.addColumn("name",Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT4) + .add("name", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); List<Fragment> splits = Lists.newArrayList(); @@ -242,10 +245,11 @@ public class TestFileTablespace { TablespaceManager.addTableSpaceForTest(sm); assertEquals(fs.getUri(), sm.getUri()); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT4); - schema.addColumn("name", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT4) + .add("name", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); List<Fragment> splits = Lists.newArrayList(); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java index e47741c..b72c98a 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java @@ -27,7 +27,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.compress.DeflateCodec; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; @@ -59,11 +59,12 @@ public class TestLineReader { Path testDir = CommonTestingUtil.getTestDir(TEST_PATH); FileSystem fs = testDir.getFileSystem(conf); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("comment", Type.TEXT); - schema.addColumn("comment2", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .add("comment", Type.TEXT) + .add("comment2", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta("TEXT"); Path tablePath = new Path(testDir, "line.data"); @@ -111,11 +112,12 @@ public class TestLineReader { Path testDir = CommonTestingUtil.getTestDir(TEST_PATH); FileSystem fs = testDir.getFileSystem(conf); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("comment", Type.TEXT); - schema.addColumn("comment2", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .add("comment", Type.TEXT) + .add("comment2", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta("TEXT"); meta.putProperty("compression.codec", DeflateCodec.class.getCanonicalName()); @@ -170,11 +172,12 @@ public class TestLineReader { Path testDir = CommonTestingUtil.getTestDir(TEST_PATH); FileSystem fs = testDir.getFileSystem(conf); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("comment", Type.TEXT); - schema.addColumn("comment2", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .add("comment", Type.TEXT) + .add("comment2", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta("TEXT"); @@ -276,11 +279,12 @@ public class TestLineReader { Path testDir = CommonTestingUtil.getTestDir(TEST_PATH); FileSystem fs = testDir.getFileSystem(conf); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("comment", Type.TEXT); - schema.addColumn("comment2", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .add("comment", Type.TEXT) + .add("comment2", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta("TEXT"); Path tablePath = new Path(testDir, "testSeekableByteBufLineReader.data"); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java index a04867e..ef264e9 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java @@ -23,7 +23,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; @@ -99,11 +99,12 @@ public class TestMergeScanner { @Test public void testMultipleFiles() throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("file", Type.TEXT); - schema.addColumn("name", Type.TEXT); - schema.addColumn("age", Type.INT8); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("file", Type.TEXT) + .add("name", Type.TEXT) + .add("age", Type.INT8) + .build(); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -161,9 +162,10 @@ public class TestMergeScanner { fragment[0] = new FileFragment("tablet1", table1Path, 0, status1.getLen()); fragment[1] = new FileFragment("tablet1", table2Path, 0, status2.getLen()); - Schema targetSchema = SchemaFactory.newV1(); - targetSchema.addColumn(schema.getColumn(0)); - targetSchema.addColumn(schema.getColumn(2)); + Schema targetSchema = SchemaBuilder.builder() + .add(schema.getColumn(0)) + .add(schema.getColumn(2)) + .build(); Scanner scanner = new MergeScanner(conf, schema, meta, Arrays.asList(fragment), targetSchema); assertEquals(isProjectableStorage(meta.getDataFormat()), scanner.isProjectable()); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java index 2e88398..eb07034 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -31,7 +31,7 @@ import org.apache.tajo.QueryId; import org.apache.tajo.TajoIdProtos; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; @@ -186,9 +186,10 @@ public class TestStorages { @Test public void testSplitable() throws IOException { if (splitable) { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); Path tablePath = new Path(testDir, "Splitable.data"); @@ -240,10 +241,11 @@ public class TestStorages { @Test public void testZeroRows() throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("score", Type.FLOAT4); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .add("score", Type.FLOAT4) + .build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); @@ -284,9 +286,10 @@ public class TestStorages { @Test public void testRCFileSplitable() throws IOException { if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); Path tablePath = new Path(testDir, "Splitable.data"); @@ -338,10 +341,11 @@ public class TestStorages { @Test public void testProjection() throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("score", Type.FLOAT4); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .add("score", Type.FLOAT4) + .build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); @@ -369,9 +373,10 @@ public class TestStorages { FileStatus status = fs.getFileStatus(tablePath); FileFragment fragment = new FileFragment("testReadAndWrite", tablePath, 0, status.getLen()); - Schema target = SchemaFactory.newV1(); - target.addColumn("age", Type.INT8); - target.addColumn("score", Type.FLOAT4); + Schema target = SchemaBuilder.builder() + .add("age", Type.INT8) + .add("score", Type.FLOAT4) + .build(); Scanner scanner = sm.getScanner(meta, schema, fragment, target); scanner.init(); int tupleCnt = 0; @@ -401,21 +406,23 @@ public class TestStorages { @Test public void testVariousTypes() throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.CHAR, 7); - schema.addColumn("col3", Type.INT2); - schema.addColumn("col4", Type.INT4); - schema.addColumn("col5", Type.INT8); - schema.addColumn("col6", Type.FLOAT4); - schema.addColumn("col7", Type.FLOAT8); - schema.addColumn("col8", Type.TEXT); - schema.addColumn("col9", Type.BLOB); - schema.addColumn("col10", Type.INET4); + SchemaBuilder schemaBld = SchemaBuilder.builder() + .add("col1", Type.BOOLEAN) + .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7)) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .add("col6", Type.FLOAT4) + .add("col7", Type.FLOAT8) + .add("col8", Type.TEXT) + .add("col9", Type.BLOB) + .add("col10", Type.INET4); if (protoTypeSupport()) { - schema.addColumn("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + schemaBld.add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); } + Schema schema = schemaBld.build(); + KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); @@ -470,22 +477,24 @@ public class TestStorages { @Test public void testNullHandlingTypes() throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.CHAR, 7); - schema.addColumn("col3", Type.INT2); - schema.addColumn("col4", Type.INT4); - schema.addColumn("col5", Type.INT8); - schema.addColumn("col6", Type.FLOAT4); - schema.addColumn("col7", Type.FLOAT8); - schema.addColumn("col8", Type.TEXT); - schema.addColumn("col9", Type.BLOB); - schema.addColumn("col10", Type.INET4); + SchemaBuilder schemaBld = SchemaBuilder.builder() + .add("col1", Type.BOOLEAN) + .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7)) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .add("col6", Type.FLOAT4) + .add("col7", Type.FLOAT8) + .add("col8", Type.TEXT) + .add("col9", Type.BLOB) + .add("col10", Type.INET4); if (protoTypeSupport()) { - schema.addColumn("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + schemaBld.add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); } + Schema schema = schemaBld.build(); + KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); @@ -565,22 +574,24 @@ public class TestStorages { public void testNullHandlingTypesWithProjection() throws IOException { if (internalType) return; - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.CHAR, 7); - schema.addColumn("col3", Type.INT2); - schema.addColumn("col4", Type.INT4); - schema.addColumn("col5", Type.INT8); - schema.addColumn("col6", Type.FLOAT4); - schema.addColumn("col7", Type.FLOAT8); - schema.addColumn("col8", Type.TEXT); - schema.addColumn("col9", Type.BLOB); - schema.addColumn("col10", Type.INET4); + SchemaBuilder schemaBld = SchemaBuilder.builder() + .add("col1", Type.BOOLEAN) + .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7)) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .add("col6", Type.FLOAT4) + .add("col7", Type.FLOAT8) + .add("col8", Type.TEXT) + .add("col9", Type.BLOB) + .add("col10", Type.INET4); if (protoTypeSupport()) { - schema.addColumn("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + schemaBld.add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); } + Schema schema = schemaBld.build(); + KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); @@ -636,14 +647,15 @@ public class TestStorages { // Making projection schema with different column positions - Schema target = SchemaFactory.newV1(); + SchemaBuilder targetBld = SchemaBuilder.builder(); Random random = new Random(); for (int i = 1; i < schema.size(); i++) { int num = random.nextInt(schema.size() - 1) + 1; if (i % num == 0) { - target.addColumn(schema.getColumn(i)); + targetBld.add(schema.getColumn(i)); } } + Schema target = targetBld.build(); FileStatus status = fs.getFileStatus(tablePath); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); @@ -671,19 +683,19 @@ public class TestStorages { public void testRCFileTextSerializeDeserialize() throws IOException { if(!dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) return; - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + Schema schema = SchemaBuilder.builder() + .add("col1", Type.BOOLEAN) + .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7)) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .add("col6", Type.FLOAT4) + .add("col7", Type.FLOAT8) + .add("col8", Type.TEXT) + .add("col9", Type.BLOB) + .add("col10", Type.INET4) + .add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())) + .build(); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -700,7 +712,6 @@ public class TestStorages { VTuple tuple = new VTuple(new Datum[] { DatumFactory.createBool(true), - DatumFactory.createBit((byte) 0x99), DatumFactory.createChar("jinho"), DatumFactory.createInt2((short) 17), DatumFactory.createInt4(59), @@ -738,19 +749,19 @@ public class TestStorages { public void testRCFileBinarySerializeDeserialize() throws IOException { if(!dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) return; - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + Schema schema = SchemaBuilder.builder() + .add("col1", Type.BOOLEAN) + .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7)) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .add("col6", Type.FLOAT4) + .add("col7", Type.FLOAT8) + .add("col8", Type.TEXT) + .add("col9", Type.BLOB) + .add("col10", Type.INET4) + .add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())) + .build(); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -767,7 +778,6 @@ public class TestStorages { VTuple tuple = new VTuple(new Datum[] { DatumFactory.createBool(true), - DatumFactory.createBit((byte) 0x99), DatumFactory.createChar("jinho"), DatumFactory.createInt2((short) 17), DatumFactory.createInt4(59), @@ -805,19 +815,18 @@ public class TestStorages { public void testSequenceFileTextSerializeDeserialize() throws IOException { if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return; - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + Schema schema = SchemaBuilder.builder() + .add("col1", Type.BOOLEAN) + .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7)) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .add("col6", Type.FLOAT4) + .add("col7", Type.FLOAT8) + .add("col8", Type.TEXT) + .add("col9", Type.BLOB) + .add("col10", Type.INET4) + .add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())).build(); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -833,7 +842,6 @@ public class TestStorages { VTuple tuple = new VTuple(new Datum[] { DatumFactory.createBool(true), - DatumFactory.createBit((byte) 0x99), DatumFactory.createChar("jinho"), DatumFactory.createInt2((short) 17), DatumFactory.createInt4(59), @@ -875,19 +883,19 @@ public class TestStorages { public void testSequenceFileBinarySerializeDeserialize() throws IOException { if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return; - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); + Schema schema = SchemaBuilder.builder() + .add("col1", Type.BOOLEAN) + .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7)) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .add("col6", Type.FLOAT4) + .add("col7", Type.FLOAT8) + .add("col8", Type.TEXT) + .add("col9", Type.BLOB) + .add("col10", Type.INET4) + .add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())) + .build(); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -903,7 +911,6 @@ public class TestStorages { VTuple tuple = new VTuple(new Datum[] { DatumFactory.createBool(true), - DatumFactory.createBit((byte) 0x99), DatumFactory.createChar("jinho"), DatumFactory.createInt2((short) 17), DatumFactory.createInt4(59), @@ -946,13 +953,13 @@ public class TestStorages { if (dateTypeSupport() || timeTypeSupport()) { int index = 2; - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.TIMESTAMP); + SchemaBuilder schema = SchemaBuilder.builder() + .add("col1", Type.TIMESTAMP); if (dateTypeSupport()) { - schema.addColumn("col" + index++, Type.DATE); + schema.add("col" + index++, Type.DATE); } if (timeTypeSupport()) { - schema.addColumn("col" + index++, Type.TIME); + schema.add("col" + index++, Type.TIME); } KeyValueSet options = new KeyValueSet(); @@ -960,7 +967,7 @@ public class TestStorages { Path tablePath = new Path(testDir, "testTime.data"); FileTablespace sm = TablespaceManager.getLocalFs(); - Appender appender = sm.getAppender(meta, schema, tablePath); + Appender appender = sm.getAppender(meta, schema.build(), tablePath); appender.init(); VTuple tuple = new VTuple(index - 1); @@ -978,7 +985,7 @@ public class TestStorages { FileStatus status = fs.getFileStatus(tablePath); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); + Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema.build(), fragment, null); scanner.init(); Tuple retrieved; @@ -997,10 +1004,11 @@ public class TestStorages { return; } - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("comment", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .add("comment", Type.TEXT) + .build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); Path tablePath = new Path(testDir, "Seekable.data"); @@ -1072,12 +1080,13 @@ public class TestStorages { @Test public void testMaxValue() throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.FLOAT4); - schema.addColumn("col2", Type.FLOAT8); - schema.addColumn("col3", Type.INT2); - schema.addColumn("col4", Type.INT4); - schema.addColumn("col5", Type.INT8); + Schema schema = SchemaBuilder.builder() + .add("col1", Type.FLOAT4) + .add("col2", Type.FLOAT8) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .build(); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -1139,10 +1148,11 @@ public class TestStorages { return; } - Schema dataSchema = SchemaFactory.newV1(); - dataSchema.addColumn("col1", Type.FLOAT4); - dataSchema.addColumn("col2", Type.FLOAT8); - dataSchema.addColumn("col3", Type.INT2); + Schema dataSchema = SchemaBuilder.builder() + .add("col1", Type.FLOAT4) + .add("col2", Type.FLOAT8) + .add("col3", Type.INT2) + .build(); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -1167,20 +1177,21 @@ public class TestStorages { assertTrue(fs.exists(tablePath)); FileStatus status = fs.getFileStatus(tablePath); - Schema inSchema = SchemaFactory.newV1(); - inSchema.addColumn("col1", Type.FLOAT4); - inSchema.addColumn("col2", Type.FLOAT8); - inSchema.addColumn("col3", Type.INT2); - inSchema.addColumn("col4", Type.INT4); - inSchema.addColumn("col5", Type.INT8); + Schema inSchema = SchemaBuilder.builder() + .add("col1", Type.FLOAT4) + .add("col2", Type.FLOAT8) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .build(); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, inSchema, fragment, null); - Schema target = SchemaFactory.newV1(); - - target.addColumn("col2", Type.FLOAT8); - target.addColumn("col5", Type.INT8); + Schema target = SchemaBuilder.builder() + .add("col2", Type.FLOAT8) + .add("col5", Type.INT8) + .build(); scanner.setTarget(target.toArray()); scanner.init(); @@ -1205,8 +1216,9 @@ public class TestStorages { return; } - Schema dataSchema = SchemaFactory.newV1(); - dataSchema.addColumn("col1", Type.CHAR); + Schema dataSchema = SchemaBuilder.builder() + .add("col1", Type.CHAR) + .build(); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -1248,8 +1260,7 @@ public class TestStorages { return; } - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.TEXT); + Schema schema = SchemaBuilder.builder().add("col1", Type.TEXT).build(); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -1289,10 +1300,11 @@ public class TestStorages { if (internalType) return; - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("score", Type.FLOAT4); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .add("score", Type.FLOAT4) + .build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); @@ -1324,12 +1336,13 @@ public class TestStorages { @Test public void testProgress() throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", Type.FLOAT4); - schema.addColumn("col2", Type.FLOAT8); - schema.addColumn("col3", Type.INT2); - schema.addColumn("col4", Type.INT4); - schema.addColumn("col5", Type.INT8); + Schema schema = SchemaBuilder.builder() + .add("col1", Type.FLOAT4) + .add("col2", Type.FLOAT8) + .add("col3", Type.INT2) + .add("col4", Type.INT4) + .add("col5", Type.INT8) + .build(); KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); @@ -1372,10 +1385,11 @@ public class TestStorages { public void testEmptySchema() throws IOException { if (internalType) return; - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("score", Type.FLOAT4); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("age", Type.INT8) + .add("score", Type.FLOAT4) + .build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); @@ -1410,7 +1424,7 @@ public class TestStorages { } //e,g select count(*) from table - Schema target = SchemaFactory.newV1(); + Schema target = SchemaBuilder.builder().build(); assertEquals(0, target.size()); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java index 18d5012..1eb28b6 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java @@ -60,12 +60,13 @@ public class TestBSTIndex { this.dataFormat = type; conf = new TajoConf(); conf.setVar(TajoConf.ConfVars.ROOT_DIR, TEST_PATH); - schema = SchemaFactory.newV1(); - schema.addColumn(new Column("int", Type.INT4)); - schema.addColumn(new Column("long", Type.INT8)); - schema.addColumn(new Column("double", Type.FLOAT8)); - schema.addColumn(new Column("float", Type.FLOAT4)); - schema.addColumn(new Column("string", Type.TEXT)); + schema = SchemaBuilder.builder() + .add(new Column("int", Type.INT4)) + .add(new Column("long", Type.INT8)) + .add(new Column("double", Type.FLOAT8)) + .add(new Column("float", Type.FLOAT4)) + .add(new Column("string", Type.TEXT)) + .build(); } @@ -111,9 +112,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("long"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("double"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("long", Type.INT8)); - keySchema.addColumn(new Column("double", Type.FLOAT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("long", Type.INT8)) + .add(new Column("double", Type.FLOAT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); @@ -187,9 +189,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("long"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("double"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("long", Type.INT8)); - keySchema.addColumn(new Column("double", Type.FLOAT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("long", Type.INT8)) + .add(new Column("double", Type.FLOAT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); @@ -280,9 +283,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("long"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("double"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("long", Type.INT8)); - keySchema.addColumn(new Column("double", Type.FLOAT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("long", Type.INT8)) + .add(new Column("double", Type.FLOAT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); @@ -353,9 +357,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("int"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("long"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("int", Type.INT4)); - keySchema.addColumn(new Column("long", Type.INT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("int", Type.INT4)) + .add(new Column("long", Type.INT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); @@ -446,9 +451,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("int"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("long"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("int", Type.INT4)); - keySchema.addColumn(new Column("long", Type.INT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("int", Type.INT4)) + .add(new Column("long", Type.INT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); @@ -529,9 +535,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("long"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("double"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("long", Type.INT8)); - keySchema.addColumn(new Column("double", Type.FLOAT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("long", Type.INT8)) + .add(new Column("double", Type.FLOAT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); BSTIndex bst = new BSTIndex(conf); @@ -614,9 +621,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("int"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("long"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("int", Type.INT4)); - keySchema.addColumn(new Column("long", Type.INT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("int", Type.INT4)) + .add(new Column("long", Type.INT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); @@ -721,9 +729,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("int"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("long"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("int", Type.INT4)); - keySchema.addColumn(new Column("long", Type.INT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("int", Type.INT4)) + .add(new Column("long", Type.INT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); @@ -803,9 +812,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("long"), false, false); sortKeys[1] = new SortSpec(schema.getColumn("double"), false, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("long", Type.INT8)); - keySchema.addColumn(new Column("double", Type.FLOAT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("long", Type.INT8)) + .add(new Column("double", Type.FLOAT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); @@ -896,9 +906,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("int"), false, false); sortKeys[1] = new SortSpec(schema.getColumn("long"), false, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("int", Type.INT4)); - keySchema.addColumn(new Column("long", Type.INT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("int", Type.INT4)) + .add(new Column("long", Type.INT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); @@ -996,9 +1007,10 @@ public class TestBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("long"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("double"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("long", Type.INT8)); - keySchema.addColumn(new Column("double", Type.FLOAT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("long", Type.INT8)) + .add(new Column("double", Type.FLOAT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java index 1053853..358be38 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java @@ -56,12 +56,13 @@ public class TestSingleCSVFileBSTIndex { public TestSingleCSVFileBSTIndex() { conf = new TajoConf(); conf.setVar(ConfVars.ROOT_DIR, TEST_PATH); - schema = SchemaFactory.newV1(); - schema.addColumn(new Column("int", Type.INT4)); - schema.addColumn(new Column("long", Type.INT8)); - schema.addColumn(new Column("double", Type.FLOAT8)); - schema.addColumn(new Column("float", Type.FLOAT4)); - schema.addColumn(new Column("string", Type.TEXT)); + schema = SchemaBuilder.builder() + .add(new Column("int", Type.INT4)) + .add(new Column("long", Type.INT8)) + .add(new Column("double", Type.FLOAT8)) + .add(new Column("float", Type.FLOAT4)) + .add(new Column("string", Type.TEXT)) + .build(); } @Before @@ -99,9 +100,10 @@ public class TestSingleCSVFileBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("long"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("double"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("long", Type.INT8)); - keySchema.addColumn(new Column("double", Type.FLOAT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("long", Type.INT8)) + .add(new Column("double", Type.FLOAT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); @@ -190,9 +192,10 @@ public class TestSingleCSVFileBSTIndex { sortKeys[0] = new SortSpec(schema.getColumn("int"), true, false); sortKeys[1] = new SortSpec(schema.getColumn("long"), true, false); - Schema keySchema = SchemaFactory.newV1(); - keySchema.addColumn(new Column("int", Type.INT4)); - keySchema.addColumn(new Column("long", Type.INT8)); + Schema keySchema = SchemaBuilder.builder() + .add(new Column("int", Type.INT4)) + .add(new Column("long", Type.INT8)) + .build(); BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java index c2ce9e7..30367c2 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java @@ -23,14 +23,17 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.storage.*; +import org.apache.tajo.storage.Scanner; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; import org.apache.tajo.storage.fragment.FileFragment; import org.junit.Test; @@ -40,20 +43,22 @@ import java.net.URL; import static org.junit.Assert.*; public class TestJsonSerDe { - private static Schema schema = SchemaFactory.newV1(); + private static Schema schema; static { - schema.addColumn("col1", TajoDataTypes.Type.BOOLEAN); - schema.addColumn("col2", TajoDataTypes.Type.CHAR, 7); - schema.addColumn("col3", TajoDataTypes.Type.INT2); - schema.addColumn("col4", TajoDataTypes.Type.INT4); - schema.addColumn("col5", TajoDataTypes.Type.INT8); - schema.addColumn("col6", TajoDataTypes.Type.FLOAT4); - schema.addColumn("col7", TajoDataTypes.Type.FLOAT8); - schema.addColumn("col8", TajoDataTypes.Type.TEXT); - schema.addColumn("col9", TajoDataTypes.Type.BLOB); - schema.addColumn("col10", TajoDataTypes.Type.INET4); - schema.addColumn("col11", TajoDataTypes.Type.NULL_TYPE); + schema = SchemaBuilder.builder() + .add("col1", TajoDataTypes.Type.BOOLEAN) + .add("col2", CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.CHAR, 7)) + .add("col3", TajoDataTypes.Type.INT2) + .add("col4", TajoDataTypes.Type.INT4) + .add("col5", TajoDataTypes.Type.INT8) + .add("col6", TajoDataTypes.Type.FLOAT4) + .add("col7", TajoDataTypes.Type.FLOAT8) + .add("col8", TajoDataTypes.Type.TEXT) + .add("col9", TajoDataTypes.Type.BLOB) + .add("col10", TajoDataTypes.Type.INET4) + .add("col11", TajoDataTypes.Type.NULL_TYPE) + .build(); } public static Path getResourcePath(String path, String suffix) { @@ -105,10 +110,11 @@ public class TestJsonSerDe { FileStatus status = fs.getFileStatus(tablePath); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", TajoDataTypes.Type.TEXT); - schema.addColumn("col2", TajoDataTypes.Type.TEXT); - schema.addColumn("col3", TajoDataTypes.Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("col1", TajoDataTypes.Type.TEXT) + .add("col2", TajoDataTypes.Type.TEXT) + .add("col3", TajoDataTypes.Type.TEXT) + .build(); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); scanner.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/parquet/TestReadWrite.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/parquet/TestReadWrite.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/parquet/TestReadWrite.java index b39bfce..eb6238b 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/parquet/TestReadWrite.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/parquet/TestReadWrite.java @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; @@ -56,7 +56,6 @@ public class TestReadWrite { private Schema createAllTypesSchema() { List<Column> columns = new ArrayList<>(); columns.add(new Column("myboolean", Type.BOOLEAN)); - columns.add(new Column("mybit", Type.BIT)); columns.add(new Column("mychar", Type.CHAR)); columns.add(new Column("myint2", Type.INT2)); columns.add(new Column("myint4", Type.INT4)); @@ -68,7 +67,7 @@ public class TestReadWrite { columns.add(new Column("mynull", Type.NULL_TYPE)); Column[] columnsArray = new Column[columns.size()]; columnsArray = columns.toArray(columnsArray); - return SchemaFactory.newV1(columnsArray); + return SchemaBuilder.builder().addAll(columnsArray).build(); } @Test @@ -77,16 +76,15 @@ public class TestReadWrite { Schema schema = createAllTypesSchema(); Tuple tuple = new VTuple(schema.size()); tuple.put(0, DatumFactory.createBool(true)); - tuple.put(1, DatumFactory.createBit((byte)128)); - tuple.put(2, DatumFactory.createChar('t')); - tuple.put(3, DatumFactory.createInt2((short)2048)); - tuple.put(4, DatumFactory.createInt4(4096)); - tuple.put(5, DatumFactory.createInt8(8192L)); - tuple.put(6, DatumFactory.createFloat4(0.2f)); - tuple.put(7, DatumFactory.createFloat8(4.1)); - tuple.put(8, DatumFactory.createText(HELLO)); - tuple.put(9, DatumFactory.createBlob(HELLO.getBytes(Charsets.UTF_8))); - tuple.put(10, NullDatum.get()); + tuple.put(1, DatumFactory.createChar('t')); + tuple.put(2, DatumFactory.createInt2((short)2048)); + tuple.put(3, DatumFactory.createInt4(4096)); + tuple.put(4, DatumFactory.createInt8(8192L)); + tuple.put(5, DatumFactory.createFloat4(0.2f)); + tuple.put(6, DatumFactory.createFloat8(4.1)); + tuple.put(7, DatumFactory.createText(HELLO)); + tuple.put(8, DatumFactory.createBlob(HELLO.getBytes(Charsets.UTF_8))); + tuple.put(9, NullDatum.get()); TajoParquetWriter writer = new TajoParquetWriter(file, schema); writer.write(tuple); @@ -97,15 +95,14 @@ public class TestReadWrite { assertNotNull(tuple); assertEquals(true, tuple.getBool(0)); - assertEquals((byte)128, tuple.getByte(1)); - assertTrue(String.valueOf('t').equals(String.valueOf(tuple.getChar(2)))); - assertEquals((short)2048, tuple.getInt2(3)); - assertEquals(4096, tuple.getInt4(4)); - assertEquals(8192L, tuple.getInt8(5)); - assertEquals(new Float(0.2f), new Float(tuple.getFloat4(6))); - assertEquals(new Double(4.1), new Double(tuple.getFloat8(7))); - assertTrue(HELLO.equals(tuple.getText(8))); - assertArrayEquals(HELLO.getBytes(Charsets.UTF_8), tuple.getBytes(9)); - assertTrue(tuple.isBlankOrNull(10)); + assertTrue(String.valueOf('t').equals(String.valueOf(tuple.getChar(1)))); + assertEquals((short)2048, tuple.getInt2(2)); + assertEquals(4096, tuple.getInt4(3)); + assertEquals(8192L, tuple.getInt8(4)); + assertEquals(new Float(0.2f), new Float(tuple.getFloat4(5))); + assertEquals(new Double(4.1), new Double(tuple.getFloat8(6))); + assertTrue(HELLO.equals(tuple.getText(7))); + assertArrayEquals(HELLO.getBytes(Charsets.UTF_8), tuple.getBytes(8)); + assertTrue(tuple.isBlankOrNull(9)); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/parquet/TestSchemaConverter.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/parquet/TestSchemaConverter.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/parquet/TestSchemaConverter.java index 87b5c34..77b5894 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/parquet/TestSchemaConverter.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/parquet/TestSchemaConverter.java @@ -22,13 +22,10 @@ import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaFactory; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.common.TajoDataTypes.Type; import org.junit.Test; -import java.util.ArrayList; -import java.util.List; - import static org.junit.Assert.assertEquals; /** @@ -50,7 +47,6 @@ public class TestSchemaConverter { private static final String CONVERTED_ALL_PARQUET_SCHEMA = "message table_schema {\n" + " optional boolean myboolean;\n" + - " optional int32 mybit;\n" + " optional binary mychar (UTF8);\n" + " optional int32 myint2;\n" + " optional int32 myint4;\n" + @@ -65,38 +61,33 @@ public class TestSchemaConverter { "}\n"; private Schema createAllTypesSchema() { - List<Column> columns = new ArrayList<>(); - columns.add(new Column("myboolean", Type.BOOLEAN)); - columns.add(new Column("mybit", Type.BIT)); - columns.add(new Column("mychar", Type.CHAR)); - columns.add(new Column("myint2", Type.INT2)); - columns.add(new Column("myint4", Type.INT4)); - columns.add(new Column("myint8", Type.INT8)); - columns.add(new Column("myfloat4", Type.FLOAT4)); - columns.add(new Column("myfloat8", Type.FLOAT8)); - columns.add(new Column("mytext", Type.TEXT)); - columns.add(new Column("myblob", Type.BLOB)); - columns.add(new Column("mynull", Type.NULL_TYPE)); - columns.add(new Column("myinet4", Type.INET4)); - columns.add(new Column("myprotobuf", Type.PROTOBUF)); - Column[] columnsArray = new Column[columns.size()]; - columnsArray = columns.toArray(columnsArray); - return SchemaFactory.newV1(columnsArray); + return SchemaBuilder.builder() + .add(new Column("myboolean", Type.BOOLEAN)) + .add(new Column("mychar", Type.CHAR)) + .add(new Column("myint2", Type.INT2)) + .add(new Column("myint4", Type.INT4)) + .add(new Column("myint8", Type.INT8)) + .add(new Column("myfloat4", Type.FLOAT4)) + .add(new Column("myfloat8", Type.FLOAT8)) + .add(new Column("mytext", Type.TEXT)) + .add(new Column("myblob", Type.BLOB)) + .add(new Column("mynull", Type.NULL_TYPE)) + .add(new Column("myinet4", Type.INET4)) + .add(new Column("myprotobuf", Type.PROTOBUF)) + .build(); } private Schema createAllTypesConvertedSchema() { - List<Column> columns = new ArrayList<>(); - columns.add(new Column("myboolean", Type.BOOLEAN)); - columns.add(new Column("myint", Type.INT4)); - columns.add(new Column("mylong", Type.INT8)); - columns.add(new Column("myfloat", Type.FLOAT4)); - columns.add(new Column("mydouble", Type.FLOAT8)); - columns.add(new Column("mybytes", Type.BLOB)); - columns.add(new Column("mystring", Type.TEXT)); - columns.add(new Column("myfixed", Type.BLOB)); - Column[] columnsArray = new Column[columns.size()]; - columnsArray = columns.toArray(columnsArray); - return SchemaFactory.newV1(columnsArray); + return SchemaBuilder.builder() + .add(new Column("myboolean", Type.BOOLEAN)) + .add(new Column("myint", Type.INT4)) + .add(new Column("mylong", Type.INT8)) + .add(new Column("myfloat", Type.FLOAT4)) + .add(new Column("mydouble", Type.FLOAT8)) + .add(new Column("mybytes", Type.BLOB)) + .add(new Column("mystring", Type.TEXT)) + .add(new Column("myfixed", Type.BLOB)) + .build(); } private void testTajoToParquetConversion( http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java index 1a5a41e..01fbc67 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java @@ -127,21 +127,22 @@ public class TestDirectRawFile { } static { - schema = SchemaFactory.newV1(); - schema.addColumn("col0", TajoDataTypes.Type.BOOLEAN); - schema.addColumn("col1", TajoDataTypes.Type.INT2); - schema.addColumn("col2", TajoDataTypes.Type.INT4); - schema.addColumn("col3", TajoDataTypes.Type.INT8); - schema.addColumn("col4", TajoDataTypes.Type.FLOAT4); - schema.addColumn("col5", TajoDataTypes.Type.FLOAT8); - schema.addColumn("col6", TajoDataTypes.Type.TEXT); - schema.addColumn("col7", TajoDataTypes.Type.TIMESTAMP); - schema.addColumn("col8", TajoDataTypes.Type.DATE); - schema.addColumn("col9", TajoDataTypes.Type.TIME); - schema.addColumn("col10", TajoDataTypes.Type.INTERVAL); - schema.addColumn("col11", TajoDataTypes.Type.INET4); - schema.addColumn("col12", - CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, PrimitiveProtos.StringProto.class.getName())); + schema = SchemaBuilder.builder() + .add("col0", TajoDataTypes.Type.BOOLEAN) + .add("col1", TajoDataTypes.Type.INT2) + .add("col2", TajoDataTypes.Type.INT4) + .add("col3", TajoDataTypes.Type.INT8) + .add("col4", TajoDataTypes.Type.FLOAT4) + .add("col5", TajoDataTypes.Type.FLOAT8) + .add("col6", TajoDataTypes.Type.TEXT) + .add("col7", TajoDataTypes.Type.TIMESTAMP) + .add("col8", TajoDataTypes.Type.DATE) + .add("col9", TajoDataTypes.Type.TIME) + .add("col10", TajoDataTypes.Type.INTERVAL) + .add("col11", TajoDataTypes.Type.INET4) + .add("col12", + CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, PrimitiveProtos.StringProto.class.getName())) + .build(); } public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, MemoryRowBlock rowBlock, Path outputFile) http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java index f53c3ed..8f5e7a8 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java @@ -26,7 +26,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.exception.*; +import org.apache.tajo.exception.SQLExceptionUtil; +import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.exception.UndefinedTablespaceException; +import org.apache.tajo.exception.UnsupportedDataTypeException; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; @@ -208,12 +211,12 @@ public abstract class JdbcMetadataProviderBase implements MetadataProvider { }); // transform the pair list into collection for columns - final Schema schema = SchemaFactory.newV1(Collections2.transform(columns, new Function<Pair<Integer,Column>, Column>() { + final Schema schema = SchemaBuilder.builder().addAll(Collections2.transform(columns, new Function<Pair<Integer,Column>, Column>() { @Override public Column apply(@Nullable Pair<Integer, Column> columnPair) { return columnPair.getSecond(); } - })); + })).build(); // fill the table stats
