Repository: tajo Updated Branches: refs/heads/index_support ab0e0048f -> 1071e614b
TAJO-1172: Remove Trevni storage type and its related classes (DaeMyung Kang via hyunsik) Closes #244 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/95770e70 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/95770e70 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/95770e70 Branch: refs/heads/index_support Commit: 95770e70bf4651b64a2ce79bd8f4bcea52d7ac90 Parents: a0625a6 Author: Hyunsik Choi <[email protected]> Authored: Mon Nov 17 04:28:39 2014 -0800 Committer: Hyunsik Choi <[email protected]> Committed: Mon Nov 17 04:28:39 2014 -0800 ---------------------------------------------------------------------- CHANGES | 3 + .../org/apache/tajo/catalog/CatalogUtil.java | 2 - .../src/main/proto/CatalogProtos.proto | 3 +- tajo-storage/pom.xml | 11 +- .../tajo/storage/trevni/TrevniAppender.java | 202 ------------------- .../tajo/storage/trevni/TrevniScanner.java | 193 ------------------ .../src/main/resources/storage-default.xml | 18 +- .../apache/tajo/storage/TestMergeScanner.java | 4 +- .../org/apache/tajo/storage/TestStorages.java | 1 - .../src/test/resources/storage-default.xml | 18 +- 10 files changed, 12 insertions(+), 443 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/95770e70/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index ef1561a..ee40913 100644 --- a/CHANGES +++ b/CHANGES @@ -50,6 +50,9 @@ Release 0.9.1 - unreleased TAJO-1177: Reduce the use of Sun proprietary API. (Jihun Kang via hyunsik) + TAJO-1172: Remove Trevni storage type and its related classes. + (DaeMyung Kang via hyunsik) + BUG FIXES TAJO-1162: to_char() returns "-00" second. (Hyoungjun Kim) http://git-wip-us.apache.org/repos/asf/tajo/blob/95770e70/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 6e66d2a..4171e80 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -270,8 +270,6 @@ public class CatalogUtil { return StoreType.ROWFILE; } else if (typeStr.equalsIgnoreCase(StoreType.RCFILE.name())) { return StoreType.RCFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.TREVNI.name())) { - return StoreType.TREVNI; } else if (typeStr.equalsIgnoreCase(StoreType.PARQUET.name())) { return StoreType.PARQUET; } else if (typeStr.equalsIgnoreCase(StoreType.SEQUENCEFILE.name())) { http://git-wip-us.apache.org/repos/asf/tajo/blob/95770e70/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 31a7446..99f594a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -32,7 +32,6 @@ enum StoreType { RCFILE = 3; ROWFILE = 4; HCFILE = 5; - TREVNI = 6; PARQUET = 7; SEQUENCEFILE = 8; AVRO = 9; @@ -331,4 +330,4 @@ message StaticMethodInvocationDescProto { required string methodName = 2; required string returnClass = 3; repeated string paramClasses = 4; -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/tajo/blob/95770e70/tajo-storage/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml index 680e520..c6877c4 100644 --- a/tajo-storage/pom.xml +++ b/tajo-storage/pom.xml @@ -1,5 +1,5 @@ -<?xml version="1.0" encoding="UTF-8"?> <!-- +<?xml version="1.0" encoding="UTF-8"?> Copyright 2012 Database Lab., Korea Univ. Licensed under the Apache License, Version 2.0 (the "License"); @@ -179,13 +179,8 @@ <dependency> <groupId>org.apache.avro</groupId> - <artifactId>trevni-core</artifactId> - <version>1.7.3</version> - </dependency> - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>trevni-avro</artifactId> - <version>1.7.3</version> + <artifactId>avro-mapred</artifactId> + <version>1.7.7</version> <exclusions> <exclusion> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/tajo/blob/95770e70/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java deleted file mode 100644 index a17fbf3..0000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.trevni; - -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.storage.FileAppender; -import org.apache.tajo.storage.TableStatistics; -import org.apache.tajo.storage.Tuple; -import org.apache.trevni.ColumnFileMetaData; -import org.apache.trevni.ColumnFileWriter; -import org.apache.trevni.ColumnMetaData; -import org.apache.trevni.ValueType; - -import java.io.FileNotFoundException; -import java.io.IOException; - -public class TrevniAppender extends FileAppender { - private FileSystem fs; - private ColumnFileWriter writer; - private FSDataOutputStream fos; - - private TableStatistics stats = null; - private boolean flushed = false; - - public TrevniAppender(Configuration conf, Schema schema, TableMeta meta, Path path) throws IOException { - super(conf, schema, meta, path); - } - - public void init() throws IOException { - fs = path.getFileSystem(conf); - - if (!fs.exists(path.getParent())) { - throw new FileNotFoundException(path.toString()); - } - - fos = fs.create(path); - - ColumnMetaData [] trevniMetas = - new ColumnMetaData[schema.size()]; - int i = 0; - for (Column column : schema.getColumns()) { - trevniMetas[i++] = new ColumnMetaData(column.getSimpleName(), - getType(column.getDataType().getType())); - } - - writer = new ColumnFileWriter(createFileMeta(), trevniMetas); - - if (enabledStats) { - this.stats = new TableStatistics(this.schema); - } - - super.init(); - } - - private ColumnFileMetaData createFileMeta() { - return new ColumnFileMetaData() - .setCodec("null") - .setChecksum("null"); - } - - private static ValueType getType(Type type) { - switch (type) { - case BOOLEAN: - return ValueType.INT; - case BIT: - return ValueType.INT; - case CHAR: - return ValueType.STRING; - case INT2: - return ValueType.INT; - case INT4: - return ValueType.INT; - case INT8: - return ValueType.LONG; - case FLOAT4: - return ValueType.FLOAT; - case FLOAT8: - return ValueType.DOUBLE; - case TEXT: - return ValueType.STRING; - case BLOB: - return ValueType.BYTES; - case INET4: - return ValueType.BYTES; - case INET6: - return ValueType.BYTES; - case PROTOBUF: - return ValueType.BYTES; - case NULL_TYPE: - return ValueType.NULL; - default: - return null; - } - } - - @Override - public long getOffset() throws IOException { - return 0; - } - - @Override - public void addTuple(Tuple t) throws IOException { - Column col; - writer.startRow(); - for (int i = 0; i < schema.size(); i++) { - if (enabledStats) { - stats.analyzeField(i, t.get(i)); - } - - if (!t.isNull(i)) { - col = schema.getColumn(i); - switch (col.getDataType().getType()) { - case NULL_TYPE: - break; - case BOOLEAN: - case BIT: - case INT2: - case INT4: - writer.writeValue(t.get(i).asInt4(), i); - break; - case INT8: - writer.writeValue(t.get(i).asInt8(), i); - break; - case FLOAT4: - writer.writeValue(t.get(i).asFloat4(), i); - break; - case FLOAT8: - writer.writeValue(t.get(i).asFloat8(), i); - break; - case CHAR: - case TEXT: - writer.writeValue(t.get(i).asChars(), i); - break; - case PROTOBUF: - case BLOB: - case INET4: - case INET6: - writer.writeValue(t.get(i).asByteArray(), i); - break; - - default: - break; - } - } - } - writer.endRow(); - - // Statistical section - if (enabledStats) { - stats.incrementRow(); - } - } - - @Override - public void flush() throws IOException { - if (!flushed) { - writer.writeTo(fos); - fos.flush(); - flushed = true; - } - } - - @Override - public void close() throws IOException { - flush(); - IOUtils.closeQuietly(fos); - } - - @Override - public TableStats getStats() { - if (enabledStats) { - return stats.getTableStat(); - } else { - return null; - } - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/95770e70/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java deleted file mode 100644 index 36a589a..0000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java +++ /dev/null @@ -1,193 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.trevni; - -import com.google.protobuf.Message; -import org.apache.hadoop.conf.Configuration; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.datum.BlobDatum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.datum.ProtobufDatumFactory; -import org.apache.tajo.storage.FileScanner; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.VTuple; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.trevni.ColumnFileReader; -import org.apache.trevni.ColumnValues; -import org.apache.trevni.avro.HadoopInput; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import static org.apache.tajo.common.TajoDataTypes.DataType; - -public class TrevniScanner extends FileScanner { - private ColumnFileReader reader; - private int [] projectionMap; - private ColumnValues [] columns; - - public TrevniScanner(Configuration conf, Schema schema, TableMeta meta, FileFragment fragment) throws IOException { - super(conf, schema, meta, fragment); - reader = new ColumnFileReader(new HadoopInput(fragment.getPath(), conf)); - } - - @Override - public void init() throws IOException { - if (targets == null) { - targets = schema.toArray(); - } - - prepareProjection(targets); - - columns = new ColumnValues[projectionMap.length]; - - for (int i = 0; i < projectionMap.length; i++) { - columns[i] = reader.getValues(projectionMap[i]); - } - - super.init(); - } - - private void prepareProjection(Column [] targets) { - projectionMap = new int[targets.length]; - int tid; - for (int i = 0; i < targets.length; i++) { - tid = schema.getColumnId(targets[i].getQualifiedName()); - projectionMap[i] = tid; - } - } - - @Override - public Tuple next() throws IOException { - Tuple tuple = new VTuple(schema.size()); - - if (!columns[0].hasNext()) { - return null; - } - - int tid; // column id of the original input schema - for (int i = 0; i < projectionMap.length; i++) { - tid = projectionMap[i]; - columns[i].startRow(); - DataType dataType = schema.getColumn(tid).getDataType(); - switch (dataType.getType()) { - case BOOLEAN: - tuple.put(tid, - DatumFactory.createBool(((Integer)columns[i].nextValue()).byteValue())); - break; - case BIT: - tuple.put(tid, - DatumFactory.createBit(((Integer) columns[i].nextValue()).byteValue())); - break; - case CHAR: - String str = (String) columns[i].nextValue(); - tuple.put(tid, - DatumFactory.createChar(str)); - break; - - case INT2: - tuple.put(tid, - DatumFactory.createInt2(((Integer) columns[i].nextValue()).shortValue())); - break; - case INT4: - tuple.put(tid, - DatumFactory.createInt4((Integer) columns[i].nextValue())); - break; - - case INT8: - tuple.put(tid, - DatumFactory.createInt8((Long) columns[i].nextValue())); - break; - - case FLOAT4: - tuple.put(tid, - DatumFactory.createFloat4((Float) columns[i].nextValue())); - break; - - case FLOAT8: - tuple.put(tid, - DatumFactory.createFloat8((Double) columns[i].nextValue())); - break; - - case INET4: - tuple.put(tid, - DatumFactory.createInet4(((ByteBuffer) columns[i].nextValue()).array())); - break; - - case TEXT: - tuple.put(tid, - DatumFactory.createText((String) columns[i].nextValue())); - break; - - case PROTOBUF: { - ProtobufDatumFactory factory = ProtobufDatumFactory.get(dataType.getCode()); - Message.Builder builder = factory.newBuilder(); - builder.mergeFrom(((ByteBuffer)columns[i].nextValue()).array()); - tuple.put(tid, factory.createDatum(builder)); - break; - } - - case BLOB: - tuple.put(tid, - new BlobDatum(((ByteBuffer) columns[i].nextValue()))); - break; - - case NULL_TYPE: - tuple.put(tid, NullDatum.get()); - break; - - default: - throw new IOException("Unsupport data type"); - } - } - - return tuple; - } - - @Override - public void reset() throws IOException { - for (int i = 0; i < projectionMap.length; i++) { - columns[i] = reader.getValues(projectionMap[i]); - } - } - - @Override - public void close() throws IOException { - reader.close(); - } - - @Override - public boolean isProjectable() { - return true; - } - - - @Override - public boolean isSelectable() { - return false; - } - - @Override - public boolean isSplittable(){ - return false; - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/95770e70/tajo-storage/src/main/resources/storage-default.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/resources/storage-default.xml b/tajo-storage/src/main/resources/storage-default.xml index f262585..064f250 100644 --- a/tajo-storage/src/main/resources/storage-default.xml +++ b/tajo-storage/src/main/resources/storage-default.xml @@ -35,7 +35,7 @@ <!--- Registered Scanner Handler --> <property> <name>tajo.storage.scanner-handler</name> - <value>textfile,csv,raw,rcfile,row,trevni,parquet,sequencefile,avro</value> + <value>textfile,csv,raw,rcfile,row,parquet,sequencefile,avro</value> </property> <!--- Fragment Class Configurations --> @@ -60,10 +60,6 @@ <value>org.apache.tajo.storage.fragment.FileFragment</value> </property> <property> - <name>tajo.storage.fragment.trevni.class</name> - <value>org.apache.tajo.storage.fragment.FileFragment</value> - </property> - <property> <name>tajo.storage.fragment.parquet.class</name> <value>org.apache.tajo.storage.fragment.FileFragment</value> </property> @@ -103,11 +99,6 @@ </property> <property> - <name>tajo.storage.scanner-handler.trevni.class</name> - <value>org.apache.tajo.storage.trevni.TrevniScanner</value> - </property> - - <property> <name>tajo.storage.scanner-handler.parquet.class</name> <value>org.apache.tajo.storage.parquet.ParquetScanner</value> </property> @@ -125,7 +116,7 @@ <!--- Appender Handler --> <property> <name>tajo.storage.appender-handler</name> - <value>textfile,csv,raw,rcfile,row,trevni,parquet,sequencefile,avro</value> + <value>textfile,csv,raw,rcfile,row,parquet,sequencefile,avro</value> </property> <property> @@ -154,11 +145,6 @@ </property> <property> - <name>tajo.storage.appender-handler.trevni.class</name> - <value>org.apache.tajo.storage.trevni.TrevniAppender</value> - </property> - - <property> <name>tajo.storage.appender-handler.parquet.class</name> <value>org.apache.tajo.storage.parquet.ParquetAppender</value> </property> http://git-wip-us.apache.org/repos/asf/tajo/blob/95770e70/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java index 51c612c..e6714b5 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java @@ -79,7 +79,6 @@ public class TestMergeScanner { {StoreType.CSV}, {StoreType.RAW}, {StoreType.RCFILE}, - {StoreType.TREVNI}, {StoreType.PARQUET}, {StoreType.SEQUENCEFILE}, {StoreType.AVRO}, @@ -92,7 +91,7 @@ public class TestMergeScanner { public void setup() throws Exception { conf = new TajoConf(); conf.setVar(ConfVars.ROOT_DIR, TEST_PATH); - conf.setStrings("tajo.storage.projectable-scanner", "rcfile", "trevni", "parquet", "avro"); + conf.setStrings("tajo.storage.projectable-scanner", "rcfile", "parquet", "avro"); testDir = CommonTestingUtil.getTestDir(TEST_PATH); fs = testDir.getFileSystem(conf); sm = StorageManager.getStorageManager(conf, testDir); @@ -190,7 +189,6 @@ public class TestMergeScanner { private static boolean isProjectableStorage(StoreType type) { switch (type) { case RCFILE: - case TREVNI: case PARQUET: case SEQUENCEFILE: case CSV: http://git-wip-us.apache.org/repos/asf/tajo/blob/95770e70/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java index a3f80cf..6e2bc35 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -281,7 +281,6 @@ public class TestStorages { Tuple tuple; while ((tuple = scanner.next()) != null) { if (storeType == StoreType.RCFILE - || storeType == StoreType.TREVNI || storeType == StoreType.CSV || storeType == StoreType.PARQUET || storeType == StoreType.SEQUENCEFILE http://git-wip-us.apache.org/repos/asf/tajo/blob/95770e70/tajo-storage/src/test/resources/storage-default.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/src/test/resources/storage-default.xml b/tajo-storage/src/test/resources/storage-default.xml index a81f3d6..790d5a8 100644 --- a/tajo-storage/src/test/resources/storage-default.xml +++ b/tajo-storage/src/test/resources/storage-default.xml @@ -28,7 +28,7 @@ <!--- Registered Scanner Handler --> <property> <name>tajo.storage.scanner-handler</name> - <value>textfile,csv,raw,rcfile,row,trevni,parquet,sequencefile,avro</value> + <value>textfile,csv,raw,rcfile,row,parquet,sequencefile,avro</value> </property> <!--- Fragment Class Configurations --> @@ -53,10 +53,6 @@ <value>org.apache.tajo.storage.fragment.FileFragment</value> </property> <property> - <name>tajo.storage.fragment.trevni.class</name> - <value>org.apache.tajo.storage.fragment.FileFragment</value> - </property> - <property> <name>tajo.storage.fragment.parquet.class</name> <value>org.apache.tajo.storage.fragment.FileFragment</value> </property> @@ -96,11 +92,6 @@ </property> <property> - <name>tajo.storage.scanner-handler.trevni.class</name> - <value>org.apache.tajo.storage.trevni.TrevniScanner</value> - </property> - - <property> <name>tajo.storage.scanner-handler.parquet.class</name> <value>org.apache.tajo.storage.parquet.ParquetScanner</value> </property> @@ -118,7 +109,7 @@ <!--- Appender Handler --> <property> <name>tajo.storage.appender-handler</name> - <value>textfile,csv,raw,rcfile,row,trevni,parquet,sequencefile,avro</value> + <value>textfile,csv,raw,rcfile,row,parquet,sequencefile,avro</value> </property> <property> @@ -147,11 +138,6 @@ </property> <property> - <name>tajo.storage.appender-handler.trevni.class</name> - <value>org.apache.tajo.storage.trevni.TrevniAppender</value> - </property> - - <property> <name>tajo.storage.appender-handler.parquet.class</name> <value>org.apache.tajo.storage.parquet.ParquetAppender</value> </property>
