Repository: carbondata Updated Branches: refs/heads/master 9ee0f3579 -> d4a157794
[HOTFIX] [presto] presto integration code cleanup [HOTFIX] [presto] presto integration code cleanup This closes #2615 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d4a15779 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d4a15779 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d4a15779 Branch: refs/heads/master Commit: d4a1577947f746361680e9111884f60b3d804a82 Parents: 9ee0f35 Author: ajantha-bhat <[email protected]> Authored: Mon Aug 6 17:51:03 2018 +0530 Committer: chenliang613 <[email protected]> Committed: Tue Aug 21 22:41:27 2018 +0800 ---------------------------------------------------------------------- integration/presto/pom.xml | 4 +- .../presto/CarbonColumnVectorWrapper.java | 64 +------------ .../presto/CarbondataColumnConstraint.java | 10 +- .../presto/CarbondataColumnHandle.java | 8 +- .../carbondata/presto/CarbondataConnector.java | 3 +- .../presto/CarbondataConnectorFactory.java | 19 ++-- .../presto/CarbondataConnectorId.java | 4 +- .../carbondata/presto/CarbondataMetadata.java | 31 ++++--- .../carbondata/presto/CarbondataModule.java | 8 +- .../carbondata/presto/CarbondataPageSource.java | 3 +- .../presto/CarbondataPageSourceProvider.java | 12 ++- .../carbondata/presto/CarbondataPlugin.java | 3 +- .../carbondata/presto/CarbondataSplit.java | 8 +- .../presto/CarbondataSplitManager.java | 9 +- .../presto/CarbondataTableHandle.java | 10 +- .../presto/CarbondataTableLayoutHandle.java | 7 +- .../presto/CarbondataTransactionHandle.java | 3 +- .../carbondata/presto/PrestoFilterUtil.java | 98 ++++++++++++-------- .../org/apache/carbondata/presto/Types.java | 2 +- .../presto/impl/CarbonLocalInputSplit.java | 11 ++- .../presto/impl/CarbonTableConfig.java | 13 ++- .../presto/impl/CarbonTableReader.java | 47 +++++----- .../readers/DecimalSliceStreamReader.java | 4 +- 23 files changed, 182 insertions(+), 199 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/pom.xml ---------------------------------------------------------------------- diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml index 5ede3e9..bf8c64d 100644 --- a/integration/presto/pom.xml +++ b/integration/presto/pom.xml @@ -486,6 +486,7 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-catalyst_${scala.binary.version}</artifactId> + <scope>test</scope> <version>${spark.version}</version> </dependency> <dependency> @@ -579,9 +580,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> <version>2.17</version> - <configuration> - <skip>true</skip> - </configuration> </plugin> <plugin> <groupId>org.scala-tools</groupId> http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java index b2c4c68..8ad6e62 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java @@ -18,30 +18,12 @@ package org.apache.carbondata.presto; import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.List; import org.apache.carbondata.core.metadata.datatype.DataType; -import org.apache.carbondata.core.metadata.datatype.DataTypes; -import org.apache.carbondata.core.metadata.datatype.StructField; import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; import org.apache.carbondata.core.scan.result.vector.CarbonDictionary; import org.apache.carbondata.core.scan.result.vector.impl.CarbonColumnVectorImpl; -import org.apache.spark.sql.types.ArrayType; -import org.apache.spark.sql.types.BooleanType; -import org.apache.spark.sql.types.DateType; -import org.apache.spark.sql.types.DecimalType; -import org.apache.spark.sql.types.DoubleType; -import org.apache.spark.sql.types.FloatType; -import org.apache.spark.sql.types.IntegerType; -import org.apache.spark.sql.types.LongType; -import org.apache.spark.sql.types.NullType; -import org.apache.spark.sql.types.ShortType; -import org.apache.spark.sql.types.StringType; -import org.apache.spark.sql.types.StructType; -import org.apache.spark.sql.types.TimestampType; - public class CarbonColumnVectorWrapper implements CarbonColumnVector { private CarbonColumnVectorImpl columnVector; @@ -247,7 +229,7 @@ public class CarbonColumnVectorWrapper implements CarbonColumnVector { } @Override public void setDictionary(CarbonDictionary dictionary) { - this.columnVector.setDictionary(dictionary); + this.columnVector.setDictionary(dictionary); } @Override public boolean hasDictionary() { @@ -258,48 +240,4 @@ public class CarbonColumnVectorWrapper implements CarbonColumnVector { return this.columnVector; } - // TODO: this is copied from carbondata-spark-common module, use presto type instead of this - private org.apache.carbondata.core.metadata.datatype.DataType - convertSparkToCarbonDataType(org.apache.spark.sql.types.DataType dataType) { - if (dataType instanceof StringType) { - return DataTypes.STRING; - } else if (dataType instanceof ShortType) { - return DataTypes.SHORT; - } else if (dataType instanceof IntegerType) { - return DataTypes.INT; - } else if (dataType instanceof LongType) { - return DataTypes.LONG; - } else if (dataType instanceof DoubleType) { - return DataTypes.DOUBLE; - } else if (dataType instanceof FloatType) { - return DataTypes.FLOAT; - } else if (dataType instanceof DateType) { - return DataTypes.DATE; - } else if (dataType instanceof BooleanType) { - return DataTypes.BOOLEAN; - } else if (dataType instanceof TimestampType) { - return DataTypes.TIMESTAMP; - } else if (dataType instanceof NullType) { - return DataTypes.NULL; - } else if (dataType instanceof DecimalType) { - DecimalType decimal = (DecimalType) dataType; - return DataTypes.createDecimalType(decimal.precision(), decimal.scale()); - } else if (dataType instanceof ArrayType) { - org.apache.spark.sql.types.DataType elementType = ((ArrayType) dataType).elementType(); - return DataTypes.createArrayType(convertSparkToCarbonDataType(elementType)); - } else if (dataType instanceof StructType) { - StructType structType = (StructType) dataType; - org.apache.spark.sql.types.StructField[] fields = structType.fields(); - List<StructField> carbonFields = new ArrayList<>(); - for (org.apache.spark.sql.types.StructField field : fields) { - carbonFields.add( - new StructField( - field.name(), - convertSparkToCarbonDataType(field.dataType()))); - } - return DataTypes.createStructType(carbonFields); - } else { - throw new UnsupportedOperationException("getting " + dataType + " from presto"); - } - } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnConstraint.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnConstraint.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnConstraint.java index 020e7b2..03351b4 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnConstraint.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnConstraint.java @@ -17,16 +17,18 @@ package org.apache.carbondata.presto; +import java.util.Objects; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + import com.facebook.presto.spi.predicate.Domain; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.Objects; -import java.util.Optional; - import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; + /** * Encapsulating presto Tuple-domain http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java index 9be5560..7152bb4 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java @@ -17,16 +17,18 @@ package org.apache.carbondata.presto; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Objects; - import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; + public class CarbondataColumnHandle implements ColumnHandle { private final String connectorId; http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnector.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnector.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnector.java index 3740ba1..ab56f8d 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnector.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnector.java @@ -17,6 +17,8 @@ package org.apache.carbondata.presto; +import static java.util.Objects.requireNonNull; + import com.facebook.presto.spi.connector.*; import com.facebook.presto.spi.transaction.IsolationLevel; import io.airlift.bootstrap.LifeCycleManager; @@ -24,7 +26,6 @@ import io.airlift.log.Logger; import static com.facebook.presto.spi.transaction.IsolationLevel.READ_COMMITTED; import static com.facebook.presto.spi.transaction.IsolationLevel.checkConnectorSupports; -import static java.util.Objects.requireNonNull; public class CarbondataConnector implements Connector { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java index 2876d1e..6bd52e6 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java @@ -17,6 +17,10 @@ package org.apache.carbondata.presto; +import java.util.Map; + +import static java.util.Objects.requireNonNull; + import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.connector.*; @@ -29,9 +33,6 @@ import io.airlift.bootstrap.Bootstrap; import io.airlift.bootstrap.LifeCycleManager; import io.airlift.json.JsonModule; -import java.util.Map; - -import static java.util.Objects.requireNonNull; /** * Build Carbondata Connector @@ -70,13 +71,13 @@ public class CarbondataConnectorFactory implements ConnectorFactory { LifeCycleManager lifeCycleManager = injector.getInstance(LifeCycleManager.class); ConnectorMetadata metadata = injector.getInstance(CarbondataMetadata.class); ConnectorSplitManager splitManager = injector.getInstance(ConnectorSplitManager.class); - ConnectorPageSourceProvider connectorPageSource = injector.getInstance(ConnectorPageSourceProvider.class); + ConnectorPageSourceProvider connectorPageSource = + injector.getInstance(ConnectorPageSourceProvider.class); - return new CarbondataConnector(lifeCycleManager, new ClassLoaderSafeConnectorMetadata(metadata,classLoader), - new ClassLoaderSafeConnectorSplitManager(splitManager, classLoader), - classLoader, - new ClassLoaderSafeConnectorPageSourceProvider(connectorPageSource, classLoader) - ); + return new CarbondataConnector(lifeCycleManager, + new ClassLoaderSafeConnectorMetadata(metadata, classLoader), + new ClassLoaderSafeConnectorSplitManager(splitManager, classLoader), classLoader, + new ClassLoaderSafeConnectorPageSourceProvider(connectorPageSource, classLoader)); } catch (Exception e) { throw Throwables.propagate(e); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorId.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorId.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorId.java index b4ba1dd..d25e569 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorId.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorId.java @@ -17,12 +17,12 @@ package org.apache.carbondata.presto; -import com.google.inject.Inject; - import java.util.Objects; import static java.util.Objects.requireNonNull; +import com.google.inject.Inject; + public class CarbondataConnectorId { private final String id; http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java index 097d667..f56f517 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java @@ -17,13 +17,16 @@ package org.apache.carbondata.presto; -import javax.inject.Inject; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; + import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; @@ -33,6 +36,8 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; import org.apache.carbondata.presto.impl.CarbonTableReader; +import static org.apache.carbondata.presto.Types.checkType; + import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorSession; @@ -60,8 +65,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; -import static org.apache.carbondata.presto.Types.checkType; public class CarbondataMetadata implements ConnectorMetadata { private final String connectorId; @@ -96,8 +99,9 @@ public class CarbondataMetadata implements ConnectorMetadata { ImmutableList.Builder<SchemaTableName> builder = ImmutableList.builder(); for (String schemaName : schemaNames) { for (String tableName : carbonTableReader.getTableNames(schemaName)) { - if(!tableName.equalsIgnoreCase(".DS_Store")) - builder.add(new SchemaTableName(schemaName, tableName)); + if (!tableName.equalsIgnoreCase(".DS_Store")) { + builder.add(new SchemaTableName(schemaName, tableName)); + } } } return builder.build(); @@ -135,7 +139,8 @@ public class CarbondataMetadata implements ConnectorMetadata { CarbonTable carbonTable = carbonTableReader.getTable(schemaTableName); List<ColumnMetadata> columnsMetaList = new LinkedList<>(); - List<CarbonColumn> carbonColumns = carbonTable.getCreateOrderColumn(schemaTableName.getTableName()); + List<CarbonColumn> carbonColumns = + carbonTable.getCreateOrderColumn(schemaTableName.getTableName()); for (CarbonColumn col : carbonColumns) { //show columns command will return these data ColumnSchema columnSchema = col.getColumnSchema(); @@ -176,18 +181,18 @@ public class CarbondataMetadata implements ConnectorMetadata { Type spiType = carbonDataType2SpiMapper(cs); columnHandles.put(cs.getColumnName(), - new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, column.getSchemaOrdinal(), - column.getKeyOrdinal(), false, - cs.getColumnUniqueId(), cs.isUseInvertedIndex(), cs.getPrecision(), cs.getScale())); + new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, + column.getSchemaOrdinal(), column.getKeyOrdinal(), false, cs.getColumnUniqueId(), + cs.isUseInvertedIndex(), cs.getPrecision(), cs.getScale())); } for (CarbonMeasure measure : cb.getMeasureByTableName(tableName)) { ColumnSchema cs = measure.getColumnSchema(); Type spiType = carbonDataType2SpiMapper(cs); columnHandles.put(cs.getColumnName(), - new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, cs.getSchemaOrdinal(), - measure.getOrdinal(), true, - cs.getColumnUniqueId(), cs.isUseInvertedIndex(), cs.getPrecision(), cs.getScale())); + new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, + cs.getSchemaOrdinal(), measure.getOrdinal(), true, cs.getColumnUniqueId(), + cs.isUseInvertedIndex(), cs.getPrecision(), cs.getScale())); } columnHandleMap = columnHandles.build(); @@ -205,7 +210,7 @@ public class CarbondataMetadata implements ConnectorMetadata { @Override public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) { - return new CarbondataTableHandle(connectorId, tableName); + return new CarbondataTableHandle(connectorId, tableName); } @Override public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session, http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataModule.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataModule.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataModule.java index 7bb864a..f9418a4 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataModule.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataModule.java @@ -17,11 +17,14 @@ package org.apache.carbondata.presto; +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; + import org.apache.carbondata.presto.impl.CarbonTableConfig; import org.apache.carbondata.presto.impl.CarbonTableReader; import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; -import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -31,12 +34,9 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Scopes; -import javax.inject.Inject; - import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.configuration.ConfigBinder.configBinder; -import static java.util.Objects.requireNonNull; public class CarbondataModule implements Module { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java index ad7006a..4f5bb58 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java @@ -20,6 +20,8 @@ package org.apache.carbondata.presto; import java.io.IOException; import java.util.List; +import static java.util.Objects.requireNonNull; + import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.presto.readers.PrestoVectorBlockBuilder; @@ -35,7 +37,6 @@ import com.facebook.presto.spi.block.LazyBlock; import com.facebook.presto.spi.block.LazyBlockLoader; import static com.google.common.base.Preconditions.checkState; -import static java.util.Objects.requireNonNull; /** * Carbondata Page Source class for custom Carbondata RecordSet Iteration. http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java index cc5bf2a..5b15b22 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java @@ -20,6 +20,8 @@ package org.apache.carbondata.presto; import java.io.IOException; import java.util.List; +import static java.util.Objects.requireNonNull; + import org.apache.carbondata.common.CarbonIterator; import org.apache.carbondata.core.datastore.block.TableBlockInfo; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; @@ -39,6 +41,8 @@ import org.apache.carbondata.presto.impl.CarbonLocalMultiBlockSplit; import org.apache.carbondata.presto.impl.CarbonTableCacheModel; import org.apache.carbondata.presto.impl.CarbonTableReader; +import static org.apache.carbondata.presto.Types.checkType; + import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; @@ -55,8 +59,7 @@ import org.apache.hadoop.mapreduce.TaskType; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static java.util.Objects.requireNonNull; -import static org.apache.carbondata.presto.Types.checkType; + /** * Provider Class for Carbondata Page Source class. @@ -101,8 +104,9 @@ public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider try { CarbonIterator iterator = queryExecutor.execute(queryModel); readSupport.initialize(queryModel.getProjectionColumns(), queryModel.getTable()); - PrestoCarbonVectorizedRecordReader reader = new PrestoCarbonVectorizedRecordReader(queryExecutor, queryModel, - (AbstractDetailQueryResultIterator) iterator, readSupport); + PrestoCarbonVectorizedRecordReader reader = + new PrestoCarbonVectorizedRecordReader(queryExecutor, queryModel, + (AbstractDetailQueryResultIterator) iterator, readSupport); reader.setTaskId(carbondataSplit.getIndex()); return reader; } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPlugin.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPlugin.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPlugin.java index 191f13b..d07ec00 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPlugin.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPlugin.java @@ -17,10 +17,11 @@ package org.apache.carbondata.presto; +import org.apache.carbondata.core.datastore.impl.FileFactory; + import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.connector.ConnectorFactory; import com.google.common.collect.ImmutableList; -import org.apache.carbondata.core.datastore.impl.FileFactory; public class CarbondataPlugin implements Plugin { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplit.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplit.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplit.java index 8a3446b..86e9161 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplit.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplit.java @@ -16,8 +16,10 @@ */ package org.apache.carbondata.presto; +import java.util.List; + +import static java.util.Objects.requireNonNull; -import org.apache.carbondata.presto.impl.CarbonLocalInputSplit; import org.apache.carbondata.presto.impl.CarbonLocalMultiBlockSplit; import com.facebook.presto.spi.ColumnHandle; @@ -29,10 +31,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; -import java.util.List; - -import static java.util.Objects.requireNonNull; - public class CarbondataSplit implements ConnectorSplit { private final String connectorId; http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java index 13abd13..e8e1f03 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java @@ -17,10 +17,13 @@ package org.apache.carbondata.presto; -import javax.inject.Inject; import java.util.List; import java.util.Optional; +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; + import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.stats.QueryStatistic; import org.apache.carbondata.core.stats.QueryStatisticsConstants; @@ -30,6 +33,8 @@ import org.apache.carbondata.presto.impl.CarbonLocalMultiBlockSplit; import org.apache.carbondata.presto.impl.CarbonTableCacheModel; import org.apache.carbondata.presto.impl.CarbonTableReader; +import static org.apache.carbondata.presto.Types.checkType; + import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplit; @@ -42,8 +47,6 @@ import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.predicate.TupleDomain; import com.google.common.collect.ImmutableList; -import static java.util.Objects.requireNonNull; -import static org.apache.carbondata.presto.Types.checkType; /** * Build Carbontable splits http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTableHandle.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTableHandle.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTableHandle.java index 0a3c820..6dbd92f 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTableHandle.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTableHandle.java @@ -17,17 +17,17 @@ package org.apache.carbondata.presto; +import java.util.Objects; + +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.SchemaTableName; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; -import java.util.Objects; - -import static java.util.Locale.ENGLISH; -import static java.util.Objects.requireNonNull; - public class CarbondataTableHandle implements ConnectorTableHandle { private final String connectorId; http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTableLayoutHandle.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTableLayoutHandle.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTableLayoutHandle.java index fbaa1d2..5c50ad5 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTableLayoutHandle.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTableLayoutHandle.java @@ -17,16 +17,17 @@ package org.apache.carbondata.presto; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.predicate.TupleDomain; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Objects; - import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; public class CarbondataTableLayoutHandle implements ConnectorTableLayoutHandle { private final CarbondataTableHandle table; http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTransactionHandle.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTransactionHandle.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTransactionHandle.java index 7939d23..40f8692 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTransactionHandle.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataTransactionHandle.java @@ -20,12 +20,13 @@ package org.apache.carbondata.presto; import java.util.Objects; import java.util.UUID; +import static java.util.Objects.requireNonNull; + import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; public class CarbondataTransactionHandle implements ConnectorTransactionHandle { private final UUID uuid; http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java index adefa8b..e96cbf7 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java @@ -27,6 +27,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static java.lang.String.format; +import static java.util.stream.Collectors.toList; + import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; @@ -64,8 +67,6 @@ import io.airlift.slice.Slice; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; -import static java.util.stream.Collectors.toList; /** * PrestoFilterUtil create the carbonData Expression from the presto-domain @@ -74,27 +75,37 @@ public class PrestoFilterUtil { private static Map<Integer, Expression> filterMap = new HashMap<>(); - private final static String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__"; + private static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__"; /** * @param carbondataColumnHandle * @return */ - private static DataType Spi2CarbondataTypeMapper(CarbondataColumnHandle carbondataColumnHandle) { + private static DataType spi2CarbondataTypeMapper(CarbondataColumnHandle carbondataColumnHandle) { Type colType = carbondataColumnHandle.getColumnType(); - if (colType == BooleanType.BOOLEAN) return DataTypes.BOOLEAN; - else if (colType == SmallintType.SMALLINT) return DataTypes.SHORT; - else if (colType == IntegerType.INTEGER) return DataTypes.INT; - else if (colType == BigintType.BIGINT) return DataTypes.LONG; - else if (colType == DoubleType.DOUBLE) return DataTypes.DOUBLE; - else if (colType == VarcharType.VARCHAR) return DataTypes.STRING; - else if (colType == DateType.DATE) return DataTypes.DATE; - else if (colType == TimestampType.TIMESTAMP) return DataTypes.TIMESTAMP; - else if (colType.equals(DecimalType.createDecimalType(carbondataColumnHandle.getPrecision(), - carbondataColumnHandle.getScale()))) return DataTypes - .createDecimalType(carbondataColumnHandle.getPrecision(), - carbondataColumnHandle.getScale()); - else return DataTypes.STRING; + if (colType == BooleanType.BOOLEAN) { + return DataTypes.BOOLEAN; + } else if (colType == SmallintType.SMALLINT) { + return DataTypes.SHORT; + } else if (colType == IntegerType.INTEGER) { + return DataTypes.INT; + } else if (colType == BigintType.BIGINT) { + return DataTypes.LONG; + } else if (colType == DoubleType.DOUBLE) { + return DataTypes.DOUBLE; + } else if (colType == VarcharType.VARCHAR) { + return DataTypes.STRING; + } else if (colType == DateType.DATE) { + return DataTypes.DATE; + } else if (colType == TimestampType.TIMESTAMP) { + return DataTypes.TIMESTAMP; + } else if (colType.equals(DecimalType.createDecimalType(carbondataColumnHandle.getPrecision(), + carbondataColumnHandle.getScale()))) { + return DataTypes.createDecimalType(carbondataColumnHandle.getPrecision(), + carbondataColumnHandle.getScale()); + } else { + return DataTypes.STRING; + } } /** @@ -103,14 +114,16 @@ public class PrestoFilterUtil { * @param originalConstraint * @return */ - public static List<String> getPartitionFilters(CarbonTable carbonTable, TupleDomain<ColumnHandle> originalConstraint) { + public static List<String> getPartitionFilters(CarbonTable carbonTable, + TupleDomain<ColumnHandle> originalConstraint) { List<ColumnSchema> columnSchemas = carbonTable.getPartitionInfo().getColumnSchemaList(); List<String> filter = new ArrayList<>(); for (ColumnHandle columnHandle : originalConstraint.getDomains().get().keySet()) { CarbondataColumnHandle carbondataColumnHandle = (CarbondataColumnHandle) columnHandle; List<ColumnSchema> partitionedColumnSchema = columnSchemas.stream().filter( - columnSchema -> carbondataColumnHandle.getColumnName().equals(columnSchema.getColumnName())).collect(toList()); - if(partitionedColumnSchema.size() != 0) { + columnSchema -> carbondataColumnHandle.getColumnName() + .equals(columnSchema.getColumnName())).collect(toList()); + if (partitionedColumnSchema.size() != 0) { filter.addAll(createPartitionFilters(originalConstraint, carbondataColumnHandle)); } } @@ -130,7 +143,7 @@ public class PrestoFilterUtil { Type type = domain.getType(); if (value == null) { filter.add(carbonDataColumnHandle.getColumnName() + "=" + HIVE_DEFAULT_DYNAMIC_PARTITION); - } else if(carbonDataColumnHandle.getColumnType() instanceof DecimalType) { + } else if (carbonDataColumnHandle.getColumnType() instanceof DecimalType) { int scale = ((DecimalType) carbonDataColumnHandle.getColumnType()).getScale(); if (value instanceof Long) { //create decimal value from Long @@ -138,7 +151,8 @@ public class PrestoFilterUtil { filter.add(carbonDataColumnHandle.getColumnName() + "=" + decimalValue.toString()); } else if (value instanceof Slice) { //create decimal value from Slice - BigDecimal decimalValue = new BigDecimal(Decimals.decodeUnscaledValue((Slice) value), scale); + BigDecimal decimalValue = + new BigDecimal(Decimals.decodeUnscaledValue((Slice) value), scale); filter.add(carbonDataColumnHandle.getColumnName() + "=" + decimalValue.toString()); } } else if (value instanceof Slice) { @@ -153,11 +167,14 @@ public class PrestoFilterUtil { } else if (value instanceof Long && carbonDataColumnHandle.getColumnType() .equals(TimestampType.TIMESTAMP)) { String timeStamp = new Timestamp((Long) value).toString(); - filter.add(carbonDataColumnHandle.getColumnName() + "=" + timeStamp.substring(0,timeStamp.indexOf('.'))); - } else if ((value instanceof Boolean) || (value instanceof Double) || (value instanceof Long)) { + filter.add(carbonDataColumnHandle.getColumnName() + "=" + timeStamp + .substring(0, timeStamp.indexOf('.'))); + } else if ((value instanceof Boolean) || (value instanceof Double) + || (value instanceof Long)) { filter.add(carbonDataColumnHandle.getColumnName() + "=" + value.toString()); } else { - throw new PrestoException(NOT_SUPPORTED, format("Unsupported partition key type: %s", type.getDisplayName())); + throw new PrestoException(NOT_SUPPORTED, + format("Unsupported partition key type: %s", type.getDisplayName())); } } return filter; @@ -173,7 +190,8 @@ public class PrestoFilterUtil { Domain domain; - // final expression for the table, returned by the method after combining all the column filters (colValueExpression). + // final expression for the table, + // returned by the method after combining all the column filters (colValueExpression). Expression finalFilters = null; for (ColumnHandle c : originalConstraint.getDomains().get().keySet()) { @@ -181,27 +199,28 @@ public class PrestoFilterUtil { // Build ColumnExpression for Expression(Carbondata) CarbondataColumnHandle cdch = (CarbondataColumnHandle) c; Type type = cdch.getColumnType(); - DataType coltype = Spi2CarbondataTypeMapper(cdch); + DataType coltype = spi2CarbondataTypeMapper(cdch); Expression colExpression = new ColumnExpression(cdch.getColumnName(), coltype); domain = originalConstraint.getDomains().get().get(c); checkArgument(domain.getType().isOrderable(), "Domain type must be orderable"); List<Object> singleValues = new ArrayList<>(); - // combination of multiple rangeExpression for a single column in case of multiple range Filter - // on single column else this is equal to rangeExpression, combined to create finalFilters + // combination of multiple rangeExpression for a single column, + // in case of multiple range Filter on single column + // else this is equal to rangeExpression, combined to create finalFilters Expression colValueExpression = null; for (Range range : domain.getValues().getRanges().getOrderedRanges()) { if (range.isSingleValue()) { - Object value = ConvertDataByType(range.getLow().getValue(), type); + Object value = convertDataByType(range.getLow().getValue(), type); singleValues.add(value); } else { // generated for each range of column i.e. lessThan, greaterThan, // there can be multiple ranges for a single column. combined to create colValueExpression Expression rangeExpression = null; if (!range.getLow().isLowerUnbounded()) { - Object value = ConvertDataByType(range.getLow().getValue(), type); + Object value = convertDataByType(range.getLow().getValue(), type); switch (range.getLow().getBound()) { case ABOVE: rangeExpression = @@ -220,7 +239,7 @@ public class PrestoFilterUtil { if (!range.getHigh().isUpperUnbounded()) { Expression lessThanExpression; - Object value = ConvertDataByType(range.getHigh().getValue(), type); + Object value = convertDataByType(range.getHigh().getValue(), type); switch (range.getHigh().getBound()) { case ABOVE: throw new IllegalArgumentException("High marker should never use ABOVE bound"); @@ -263,19 +282,20 @@ public class PrestoFilterUtil { return finalFilters; } - private static Object ConvertDataByType(Object rawdata, Type type) { - if (type.equals(IntegerType.INTEGER) || type.equals(SmallintType.SMALLINT)) + private static Object convertDataByType(Object rawdata, Type type) { + if (type.equals(IntegerType.INTEGER) || type.equals(SmallintType.SMALLINT)) { return Integer.valueOf(rawdata.toString()); - // new Integer((rawdata.toString())); - else if (type.equals(BigintType.BIGINT)) return rawdata; - else if (type.equals(VarcharType.VARCHAR)) { + } else if (type.equals(BigintType.BIGINT)) { + return rawdata; + } else if (type.equals(VarcharType.VARCHAR)) { if (rawdata instanceof Slice) { return ((Slice) rawdata).toStringUtf8(); } else { return rawdata; } - } else if (type.equals(BooleanType.BOOLEAN)) return rawdata; - else if (type.equals(DateType.DATE)) { + } else if (type.equals(BooleanType.BOOLEAN)) { + return rawdata; + } else if (type.equals(DateType.DATE)) { Calendar c = Calendar.getInstance(); c.setTime(new Date(0)); c.add(Calendar.DAY_OF_YEAR, ((Long) rawdata).intValue()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/Types.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/Types.java b/integration/presto/src/main/java/org/apache/carbondata/presto/Types.java index cb30907..d4b5e8a 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/Types.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/Types.java @@ -18,9 +18,9 @@ package org.apache.carbondata.presto; import java.util.Locale; +import static java.util.Objects.requireNonNull; import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; public class Types { private Types() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java index af23671..718cb33 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java @@ -17,11 +17,6 @@ package org.apache.carbondata.presto.impl; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.gson.Gson; -import org.apache.hadoop.fs.Path; - import java.io.IOException; import java.util.List; @@ -29,6 +24,12 @@ import org.apache.carbondata.core.indexstore.BlockletDetailInfo; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.hadoop.CarbonInputSplit; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.Gson; + +import org.apache.hadoop.fs.Path; + /** * CarbonLocalInputSplit represents a block, it contains a set of blocklet. */ http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java index f800e59..4d18184 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java @@ -17,10 +17,10 @@ package org.apache.carbondata.presto.impl; -import io.airlift.configuration.Config; - import javax.validation.constraints.NotNull; +import io.airlift.configuration.Config; + /** * Configuration read from etc/catalog/carbondata.properties */ @@ -49,7 +49,8 @@ public class CarbonTableConfig { return dbPath; } - @Config("carbondata-store") public CarbonTableConfig setDbPath(String dbPath) { + @Config("carbondata-store") + public CarbonTableConfig setDbPath(String dbPath) { this.dbPath = dbPath; return this; } @@ -58,7 +59,8 @@ public class CarbonTableConfig { return tablePath; } - @Config("carbondata-store") public CarbonTableConfig setTablePath(String tablePath) { + @Config("carbondata-store") + public CarbonTableConfig setTablePath(String tablePath) { this.tablePath = tablePath; return this; } @@ -67,7 +69,8 @@ public class CarbonTableConfig { return storePath; } - @Config("carbondata-store") public CarbonTableConfig setStorePath(String storePath) { + @Config("carbondata-store") + public CarbonTableConfig setStorePath(String storePath) { this.storePath = storePath; return this; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java index 7441f4b..7916932 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java @@ -32,6 +32,8 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.Objects.requireNonNull; + import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; @@ -79,7 +81,6 @@ import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; import org.apache.thrift.TBase; -import static java.util.Objects.requireNonNull; import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY; import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT; import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY; @@ -119,7 +120,7 @@ public class CarbonTableReader { */ private AtomicReference<HashMap<SchemaTableName, CarbonTableCacheModel>> carbonCache; - private LoadMetadataDetails loadMetadataDetails[]; + private LoadMetadataDetails[] loadMetadataDetails; private String queryId; @@ -132,7 +133,7 @@ public class CarbonTableReader { /** * List Of Schemas */ - private List<String> schemaNames = new ArrayList<>(); + private List<String> schemaNames = new ArrayList<>(); @Inject public CarbonTableReader(CarbonTableConfig config) { this.config = requireNonNull(config, "CarbonTableConfig is null"); @@ -191,8 +192,8 @@ public class CarbonTableReader { } /** - * Get the CarbonFile instance which represents the store path in the configuration, and assign it to - * this.carbonFileList. + * Get the CarbonFile instance which represents the store path in the configuration, + * and assign it to this.carbonFileList. * * @return */ @@ -339,7 +340,8 @@ public class CarbonTableReader { } /** - * Read the metadata of the given table and cache it in this.carbonCache (CarbonTableReader cache). + * Read the metadata of the given table + * and cache it in this.carbonCache (CarbonTableReader cache). * * @param table name of the given table. * @return the CarbonTable instance which contains all the needed metadata for a table. @@ -368,7 +370,8 @@ public class CarbonTableReader { ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() { // TBase is used to read and write thrift objects. // TableInfo is a kind of TBase used to read and write table information. - // TableInfo is generated by thrift, see schema.thrift under format/src/main/thrift for details. + // TableInfo is generated by thrift, + // see schema.thrift under format/src/main/thrift for details. public TBase create() { return new org.apache.carbondata.format.TableInfo(); } @@ -380,10 +383,10 @@ public class CarbonTableReader { (org.apache.carbondata.format.TableInfo) thriftReader.read(); thriftReader.close(); - // Step 3: convert format level TableInfo to code level TableInfo SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl(); - // wrapperTableInfo is the code level information of a table in carbondata core, different from the Thrift TableInfo. + // wrapperTableInfo is the code level information of a table in carbondata core, + // different from the Thrift TableInfo. TableInfo wrapperTableInfo = schemaConverter .fromExternalToWrapperTableInfo(tableInfo, table.getSchemaName(), table.getTableName(), tablePath); @@ -534,22 +537,20 @@ public class CarbonTableReader { addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, config.getEnableUnsafeSort()); addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, config.getEnableQueryStatistics()); } - private void setS3Properties(){ + + private void setS3Properties() { + FileFactory.getConfiguration().set(ACCESS_KEY, Objects.toString(config.getS3A_AcesssKey(), "")); + FileFactory.getConfiguration().set(SECRET_KEY, Objects.toString(config.getS3A_SecretKey())); FileFactory.getConfiguration() - .set(ACCESS_KEY, Objects.toString(config.getS3A_AcesssKey(),"")); + .set(CarbonCommonConstants.S3_ACCESS_KEY, Objects.toString(config.getS3_AcesssKey(), "")); FileFactory.getConfiguration() - .set(SECRET_KEY, Objects.toString(config.getS3A_SecretKey())); - FileFactory.getConfiguration().set(CarbonCommonConstants.S3_ACCESS_KEY, - Objects.toString(config.getS3_AcesssKey(),"")); - FileFactory.getConfiguration().set(CarbonCommonConstants.S3_SECRET_KEY, - Objects.toString(config.getS3_SecretKey())); - FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_ACCESS_KEY, - Objects.toString(config.getS3N_AcesssKey(),"")); - FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_SECRET_KEY, - Objects.toString(config.getS3N_SecretKey(),"")); - FileFactory.getConfiguration().set(ENDPOINT, - Objects.toString(config.getS3EndPoint(),"")); -} + .set(CarbonCommonConstants.S3_SECRET_KEY, Objects.toString(config.getS3_SecretKey())); + FileFactory.getConfiguration() + .set(CarbonCommonConstants.S3N_ACCESS_KEY, Objects.toString(config.getS3N_AcesssKey(), "")); + FileFactory.getConfiguration() + .set(CarbonCommonConstants.S3N_SECRET_KEY, Objects.toString(config.getS3N_SecretKey(), "")); + FileFactory.getConfiguration().set(ENDPOINT, Objects.toString(config.getS3EndPoint(), "")); + } private void addProperty(String propertyName, String propertyValue) { if (propertyValue != null) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d4a15779/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java index ed88343..6860ce0 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java @@ -21,6 +21,8 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.Objects; +import static java.math.RoundingMode.HALF_UP; + import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.scan.result.vector.impl.CarbonColumnVectorImpl; @@ -40,7 +42,7 @@ import static com.facebook.presto.spi.type.Decimals.rescale; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static io.airlift.slice.Slices.utf8Slice; -import static java.math.RoundingMode.HALF_UP; + /** * Reader for DecimalValues
