Repository: incubator-carbondata Updated Branches: refs/heads/master 997af85dc -> 90bc36699
Insert into carbon table new Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5f1abef7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5f1abef7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5f1abef7 Branch: refs/heads/master Commit: 5f1abef794d22110f2b40fc0fabad19c7f215f0b Parents: 997af85 Author: ashok.blend <ashok.bl...@gmail.com> Authored: Mon Nov 28 21:20:45 2016 -0800 Committer: jackylk <jacky.li...@huawei.com> Committed: Thu Dec 1 23:28:40 2016 +0800 ---------------------------------------------------------------------- .../ThriftWrapperSchemaConverterImpl.java | 2 + .../metadata/schema/table/CarbonTable.java | 80 ++++++++- .../schema/table/column/CarbonColumn.java | 14 +- .../schema/table/column/CarbonDimension.java | 7 +- .../schema/table/column/CarbonMeasure.java | 6 +- .../schema/table/column/ColumnSchema.java | 8 + .../core/constants/CarbonCommonConstants.java | 9 + .../carbondata/scan/filter/FilterUtilTest.java | 8 +- format/src/main/thrift/schema.thrift | 7 +- .../execution/command/carbonTableSchema.scala | 15 +- .../spark/sql/CarbonCatalystOperators.scala | 21 +++ .../org/apache/spark/sql/CarbonContext.scala | 2 +- .../spark/sql/CarbonDatasourceRelation.scala | 43 ++++- .../scala/org/apache/spark/sql/CarbonScan.scala | 43 ++--- .../org/apache/spark/sql/CarbonSqlParser.scala | 31 ++-- .../execution/command/carbonTableSchema.scala | 22 +++ .../sql/hive/CarbonPreInsertionCasts.scala | 59 ++++++ .../spark/sql/hive/CarbonStrategies.scala | 4 + .../spark/sql/optimizer/CarbonOptimizer.scala | 1 + .../spark/src/test/resources/shortolap.csv | 5 + .../TestLoadDataWithSingleQuotechar.scala | 8 +- .../InsertIntoCarbonTableTestCase.scala | 178 +++++++++++++++++++ .../TestNoInvertedIndexLoadAndQuery.scala | 2 +- 23 files changed, 500 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java index 8a53895..c1020e3 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java @@ -167,6 +167,7 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { thriftColumnSchema.setColumnProperties(wrapperColumnSchema.getColumnProperties()); thriftColumnSchema.setInvisible(wrapperColumnSchema.isInvisible()); thriftColumnSchema.setColumnReferenceId(wrapperColumnSchema.getColumnReferenceId()); + thriftColumnSchema.setSchemaOrdinal(wrapperColumnSchema.getSchemaOrdinal()); return thriftColumnSchema; } @@ -339,6 +340,7 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { wrapperColumnSchema.setColumnProperties(externalColumnSchema.getColumnProperties()); wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible()); wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId()); + wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal()); return wrapperColumnSchema; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java index c0a7032..d3e2e62 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java @@ -21,7 +21,10 @@ package org.apache.carbondata.core.carbon.metadata.schema.table; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -30,6 +33,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier; import org.apache.carbondata.core.carbon.CarbonTableIdentifier; import org.apache.carbondata.core.carbon.metadata.encoder.Encoding; +import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema; @@ -61,6 +65,7 @@ public class CarbonTable implements Serializable { */ private Map<String, List<CarbonDimension>> tableDimensionsMap; + private Map<String, List<CarbonColumn>> createOrderColumn; /** * table measures list. */ @@ -95,6 +100,7 @@ public class CarbonTable implements Serializable { this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>(); this.tableMeasuresMap = new HashMap<String, List<CarbonMeasure>>(); this.aggregateTablesName = new ArrayList<String>(); + this.createOrderColumn = new HashMap<String, List<CarbonColumn>>(); } /** @@ -113,6 +119,7 @@ public class CarbonTable implements Serializable { new AbsoluteTableIdentifier(tableInfo.getStorePath(), carbontableIdentifier); fillDimensionsAndMeasuresForTables(tableInfo.getFactTable()); + fillCreateOrderColumn(tableInfo.getFactTable().getTableName()); List<TableSchema> aggregateTableList = tableInfo.getAggregateTableList(); for (TableSchema aggTable : aggregateTableList) { this.aggregateTablesName.add(aggTable.getTableName()); @@ -121,6 +128,33 @@ public class CarbonTable implements Serializable { } /** + * fill columns as per user provided order + * @param tableName + */ + private void fillCreateOrderColumn(String tableName) { + List<CarbonColumn> columns = new ArrayList<CarbonColumn>(); + List<CarbonDimension> dimensions = this.tableDimensionsMap.get(tableName); + List<CarbonMeasure> measures = this.tableMeasuresMap.get(tableName); + Iterator<CarbonDimension> dimItr = dimensions.iterator(); + while (dimItr.hasNext()) { + columns.add(dimItr.next()); + } + Iterator<CarbonMeasure> msrItr = measures.iterator(); + while (msrItr.hasNext()) { + columns.add(msrItr.next()); + } + Collections.sort(columns, new Comparator<CarbonColumn>() { + + @Override public int compare(CarbonColumn o1, CarbonColumn o2) { + + return Integer.compare(o1.getSchemaOrdinal(), o2.getSchemaOrdinal()); + } + + }); + this.createOrderColumn.put(tableName, columns); + } + + /** * This method will return the table size. Default table block size will be considered * in case not specified by the user * @@ -166,7 +200,8 @@ public class CarbonTable implements Serializable { if (columnSchema.isDimensionColumn()) { if (columnSchema.getNumberOfChild() > 0) { CarbonDimension complexDimension = - new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, ++complexTypeOrdinal); + new CarbonDimension(columnSchema, dimensionOrdinal++, + columnSchema.getSchemaOrdinal(), -1, -1, ++complexTypeOrdinal); complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild()); dimensions.add(complexDimension); dimensionOrdinal = @@ -176,22 +211,26 @@ public class CarbonTable implements Serializable { complexTypeOrdinal = assignComplexOrdinal(complexDimension, complexTypeOrdinal); } else { if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) { - dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1)); + dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++, + columnSchema.getSchemaOrdinal(), -1, -1, -1)); } else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY) && columnSchema.getColumnGroupId() == -1) { dimensions - .add(new CarbonDimension(columnSchema, dimensionOrdinal++, keyOrdinal++, -1, -1)); + .add(new CarbonDimension(columnSchema, dimensionOrdinal++, + columnSchema.getSchemaOrdinal(), keyOrdinal++, -1, -1)); } else { columnGroupOrdinal = previousColumnGroupId == columnSchema.getColumnGroupId() ? ++columnGroupOrdinal : 0; previousColumnGroupId = columnSchema.getColumnGroupId(); - dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++, keyOrdinal++, + dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++, + columnSchema.getSchemaOrdinal(), keyOrdinal++, columnGroupOrdinal, -1)); } } } else { - measures.add(new CarbonMeasure(columnSchema, measureOrdinal++)); + measures.add(new CarbonMeasure(columnSchema, measureOrdinal++, + columnSchema.getSchemaOrdinal())); } } } @@ -213,7 +252,8 @@ public class CarbonTable implements Serializable { if (columnSchema.isDimensionColumn()) { if (columnSchema.getNumberOfChild() > 0) { CarbonDimension complexDimension = - new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1); + new CarbonDimension(columnSchema, dimensionOrdinal++, + columnSchema.getSchemaOrdinal(), -1, -1, -1); complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild()); parentDimension.getListOfChildDimensions().add(complexDimension); dimensionOrdinal = @@ -221,7 +261,8 @@ public class CarbonTable implements Serializable { listOfColumns, complexDimension); } else { parentDimension.getListOfChildDimensions() - .add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1)); + .add(new CarbonDimension(columnSchema, dimensionOrdinal++, + columnSchema.getSchemaOrdinal(), -1, -1, -1)); } } } @@ -336,6 +377,15 @@ public class CarbonTable implements Serializable { } /** + * This will give user created order column + * + * @return + */ + public List<CarbonColumn> getCreateOrderColumn(String tableName) { + return createOrderColumn.get(tableName); + } + + /** * to get particular measure from a table * * @param tableName @@ -370,6 +420,22 @@ public class CarbonTable implements Serializable { } /** + * @param tableName + * @param columnName + * @return + */ + public CarbonColumn getColumnByName(String tableName, String columnName) { + List<CarbonColumn> columns = createOrderColumn.get(tableName); + Iterator<CarbonColumn> colItr = columns.iterator(); + while (colItr.hasNext()) { + CarbonColumn col = colItr.next(); + if (col.getColName().equalsIgnoreCase(columnName)) { + return col; + } + } + return null; + } + /** * gets all children dimension for complex type * * @param dimName http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java index 2cef374..9cf3601 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java @@ -45,6 +45,10 @@ public class CarbonColumn implements Serializable { protected int ordinal; /** + * order in which user has created table + */ + protected int schemaOrdinal; + /** * default value for in case of restructuring will be used when older * segment does not have particular column */ @@ -55,13 +59,13 @@ public class CarbonColumn implements Serializable { */ protected ColumnIdentifier columnIdentifier; - public CarbonColumn(ColumnSchema columnSchema, int ordinal) { + public CarbonColumn(ColumnSchema columnSchema, int ordinal, int schemaOrdinal) { this.columnSchema = columnSchema; this.ordinal = ordinal; + this.schemaOrdinal = schemaOrdinal; this.columnIdentifier = - new ColumnIdentifier(getColumnId(), getColumnProperties(), getDataType()); + new ColumnIdentifier(getColumnId(), getColumnProperties(), getDataType()); } - /** * @return columnar or row based */ @@ -171,4 +175,8 @@ public class CarbonColumn implements Serializable { public ColumnIdentifier getColumnIdentifier() { return this.columnIdentifier; } + + public int getSchemaOrdinal() { + return this.schemaOrdinal; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java index c81cea4..214d990 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java @@ -54,8 +54,13 @@ public class CarbonDimension extends CarbonColumn { private int complexTypeOrdinal; public CarbonDimension(ColumnSchema columnSchema, int ordinal, int keyOrdinal, + int columnGroupOrdinal, int complexTypeOrdinal) { + this(columnSchema, ordinal, 0, keyOrdinal, columnGroupOrdinal, complexTypeOrdinal); + } + + public CarbonDimension(ColumnSchema columnSchema, int ordinal, int schemaOrdinal, int keyOrdinal, int columnGroupOrdinal, int complexTypeOrdinal) { - super(columnSchema, ordinal); + super(columnSchema, ordinal, schemaOrdinal); this.keyOrdinal = keyOrdinal; this.columnGroupOrdinal = columnGroupOrdinal; this.complexTypeOrdinal = complexTypeOrdinal; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java index 2c8c11e..ae0a25a 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java @@ -45,7 +45,11 @@ public class CarbonMeasure extends CarbonColumn { private int precision; public CarbonMeasure(ColumnSchema columnSchema, int ordinal) { - super(columnSchema, ordinal); + this(columnSchema, ordinal, 0); + } + + public CarbonMeasure(ColumnSchema columnSchema, int ordinal, int schemaOrdinal) { + super(columnSchema, ordinal, schemaOrdinal); this.scale = columnSchema.getScale(); this.precision = columnSchema.getPrecision(); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java index 0431ad5..c0b2569 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java @@ -92,6 +92,7 @@ public class ColumnSchema implements Serializable { private int precision; + private int schemaOrdinal; /** * Nested fields. Since thrift does not support nested fields, * the nesting is flattened to a single list by a depth-first traversal. @@ -415,4 +416,11 @@ public class ColumnSchema implements Serializable { this.columnReferenceId = columnReferenceId; } + public int getSchemaOrdinal() { + return schemaOrdinal; + } + + public void setSchemaOrdinal(int schemaOrdinal) { + this.schemaOrdinal = schemaOrdinal; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index a9e0442..443c8c4 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -910,6 +910,15 @@ public final class CarbonCommonConstants { * data file version header */ public static final String CARBON_DATA_VERSION_HEADER = "CARBONDATAVERSION#"; + /** + * Maximum no of column supported + */ + public static int DEFAULT_MAX_NUMBER_OF_COLUMNS = 20000; + + /** + * maximum length of column + */ + public static final int DEFAULT_COLUMN_LENGTH = 100000; private CarbonCommonConstants() { } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java index ba4c46a..23e2d10 100644 --- a/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java @@ -264,7 +264,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest { dimColumn.setColumnUniqueId(UUID.randomUUID().toString()); dimColumn.setDataType(DataType.STRING); dimColumn.setDimensionColumn(true); - final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal); + final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal, -1); new MockUp<ColumnExpression>() { @Mock public CarbonColumn getCarbonColumn() { return carbonColumn; @@ -295,7 +295,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest { dimColumn.setColumnUniqueId(UUID.randomUUID().toString()); dimColumn.setDataType(DataType.STRING); dimColumn.setDimensionColumn(true); - final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal); + final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal, -1); new MockUp<ColumnExpression>() { @Mock public CarbonColumn getCarbonColumn() { return carbonColumn; @@ -318,7 +318,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest { String defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL; int defaultSurrogate = 1; int ordinal = 1; - final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal); + final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal, -1); new MockUp<ColumnExpression>() { @Mock public CarbonColumn getCarbonColumn() { return carbonColumn; @@ -385,7 +385,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest { } }; - final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal); + final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal, -1); new MockUp<ColumnExpression>() { @Mock public CarbonColumn getCarbonColumn() { return carbonColumn; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/format/src/main/thrift/schema.thrift ---------------------------------------------------------------------- diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift index 33bb14e..885beba 100644 --- a/format/src/main/thrift/schema.thrift +++ b/format/src/main/thrift/schema.thrift @@ -97,8 +97,11 @@ struct ColumnSchema{ /** * column reference id */ - 15: optional string columnReferenceId; - + 15: optional string columnReferenceId; + /** + * It will have column order which user has provided + */ + 16: optional i32 schemaOrdinal } /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala index 9360ad8..7416c90 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala @@ -55,6 +55,7 @@ case class TableModel( case class Field(column: String, var dataType: Option[String], name: Option[String], children: Option[List[Field]], parent: String = null, storeType: Option[String] = Some("columnar"), + var schemaOrdinal: Int = -1, var precision: Int = 0, var scale: Int = 0) case class ColumnProperty(key: String, value: String) @@ -126,7 +127,8 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) { val columnSchema: ColumnSchema = getColumnSchema( DataTypeConverterUtil.convertToCarbonType(field.dataType.getOrElse("")), field.name.getOrElse(field.column), index, - isCol = true, encoders, isDimensionCol = true, rowGroup, field.precision, field.scale) + isCol = true, encoders, isDimensionCol = true, rowGroup, field.precision, field.scale, + field.schemaOrdinal) allColumns ++= Seq(columnSchema) index = index + 1 rowGroup = rowGroup + 1 @@ -141,7 +143,7 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) { def getColumnSchema(dataType: DataType, colName: String, index: Integer, isCol: Boolean, encoders: java.util.List[Encoding], isDimensionCol: Boolean, - colGroup: Integer, precision: Integer, scale: Integer): ColumnSchema = { + colGroup: Integer, precision: Integer, scale: Integer, schemaOrdinal: Int): ColumnSchema = { val columnSchema = new ColumnSchema() columnSchema.setDataType(dataType) columnSchema.setColumnName(colName) @@ -169,6 +171,7 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) { columnSchema.setColumnGroup(colGroup) columnSchema.setPrecision(precision) columnSchema.setScale(scale) + columnSchema.setSchemaOrdinal(schemaOrdinal) // TODO: Need to fill RowGroupID, converted type // & Number of Children after DDL finalization columnSchema @@ -191,7 +194,8 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) { isDimensionCol = true, -1, field.precision, - field.scale) + field.scale, + field.schemaOrdinal) allColumns ++= Seq(columnSchema) index = index + 1 if (field.children.isDefined && field.children.get != null) { @@ -211,7 +215,8 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) { isDimensionCol = false, -1, field.precision, - field.scale) + field.scale, + field.schemaOrdinal) val measureCol = columnSchema allColumns ++= Seq(measureCol) @@ -274,7 +279,7 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) { true, encoders, false, - -1, 0, 0) + -1, 0, 0, schemaOrdinal = -1) columnSchema.setInvisible(true) val measureColumn = columnSchema measures += measureColumn http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala index d394bfa..01904ba 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala @@ -105,3 +105,24 @@ case class DropDatabase(dbName: String, isCascade: Boolean, sql: String) Seq() } } + +/** + * A logical plan representing insertion into Hive table. + * This plan ignores nullability of ArrayType, MapType, StructType unlike InsertIntoTable + * because Hive table doesn't have nullability for ARRAY, MAP, STRUCT types. + */ +case class InsertIntoCarbonTable( + table: CarbonDatasourceRelation, + partition: Map[String, Option[String]], + child: LogicalPlan, + overwrite: Boolean, + ifNotExists: Boolean) + extends LogicalPlan with Command { + + override def children: Seq[LogicalPlan] = child :: Nil + override def output: Seq[Attribute] = Seq.empty + + // This is the expected schema of the table prepared to be inserted into, + // including dynamic partition columns. + val tableOutput = table.carbonRelation.output +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala index de43319..e5d3d6b 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala @@ -78,7 +78,7 @@ class CarbonContext( override val extendedResolutionRules = catalog.ParquetConversions :: catalog.CreateTables :: - catalog.PreInsertionCasts :: + CarbonPreInsertionCasts :: ExtractPythonUDFs :: ResolveHiveWindowFunction :: PreInsertCastAndRename :: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala index 681c0c8..a534ab0 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql import java.util.LinkedHashSet import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer import scala.language.implicitConversions import org.apache.hadoop.fs.Path @@ -32,7 +33,7 @@ import org.apache.spark.sql.hive.{CarbonMetaData, CarbonMetastoreTypes} import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.{DataType, StructType} -import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension +import org.apache.carbondata.core.carbon.metadata.schema.table.column.{CarbonColumn, CarbonDimension} import org.apache.carbondata.core.carbon.path.CarbonStorePath import org.apache.carbondata.core.datastorage.store.impl.FileFactory import org.apache.carbondata.lcm.status.SegmentStatusManager @@ -154,7 +155,7 @@ private[sql] case class CarbonDatasourceRelation( case class CarbonRelation( databaseName: String, tableName: String, - metaData: CarbonMetaData, + var metaData: CarbonMetaData, tableMeta: TableMeta, alias: Option[String])(@transient sqlContext: SQLContext) extends LeafNode with MultiInstanceRelation { @@ -245,8 +246,40 @@ case class CarbonRelation( nullable = true)(qualifiers = tableName +: alias.toSeq)) } - override val output = dimensionsAttr ++ measureAttr - + override val output = { + val factTable = tableMeta.carbonTable.getFactTableName + var columns = tableMeta.carbonTable.getCreateOrderColumn(tableMeta.carbonTable.getFactTableName) + .asScala + columns.filter(!_.getColumnSchema.isInvisible).map { column => + if (column.isDimesion()) { + val output: DataType = column.getDataType.toString.toLowerCase match { + case "array" => + CarbonMetastoreTypes.toDataType(s"array<${getArrayChildren(column.getColName)}>") + case "struct" => + CarbonMetastoreTypes.toDataType(s"struct<${getStructChildren(column.getColName)}>") + case dType => + val dataType = addDecimalScaleAndPrecision(column, dType) + CarbonMetastoreTypes.toDataType(dataType) + } + AttributeReference(column.getColName, output, + nullable = true + )(qualifiers = tableName +: alias.toSeq) + } else { + AttributeReference(column.getColName, CarbonMetastoreTypes.toDataType( + column.getDataType.toString + .toLowerCase match { + case "int" => "long" + case "short" => "long" + case "decimal" => "decimal(" + column.getColumnSchema.getPrecision + "," + column + .getColumnSchema.getScale + ")" + case others => others + } + ), + nullable = true + )(qualifiers = tableName +: alias.toSeq) + } + } + } // TODO: Use data from the footers. override lazy val statistics = Statistics(sizeInBytes = this.sizeInBytes) @@ -258,7 +291,7 @@ case class CarbonRelation( } } - def addDecimalScaleAndPrecision(dimval: CarbonDimension, dataType: String): String = { + def addDecimalScaleAndPrecision(dimval: CarbonColumn, dataType: String): String = { var dType = dataType if (dimval.getDataType == org.apache.carbondata.core.carbon.metadata.datatype.DataType.DECIMAL) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala index 848e752..6f3b5df 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala @@ -88,39 +88,28 @@ case class CarbonScan( columnProjection = attributeOut } - val dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName) - val measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName) - val dimAttr = new Array[Attribute](dimensions.size()) - val msrAttr = new Array[Attribute](measures.size()) + val columns = carbonTable.getCreateOrderColumn(carbonTable.getFactTableName) + val colAttr = new Array[Attribute](columns.size()) columnProjection.foreach { attr => - val carbonDimension = - carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name) - if(carbonDimension != null) { - dimAttr(dimensions.indexOf(carbonDimension)) = attr - } else { - val carbonMeasure = - carbonTable.getMeasureByName(carbonTable.getFactTableName, attr.name) - if(carbonMeasure != null) { - msrAttr(measures.indexOf(carbonMeasure)) = attr - } - } + val column = + carbonTable.getColumnByName(carbonTable.getFactTableName, attr.name) + if(column != null) { + colAttr(columns.indexOf(column)) = attr + } } - columnProjection = dimAttr.filter(f => f != null) ++ msrAttr.filter(f => f != null) + columnProjection = colAttr.filter(f => f != null) var queryOrder: Integer = 0 columnProjection.foreach { attr => - val carbonDimension = - carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name) - if (carbonDimension != null) { - val dim = new QueryDimension(attr.name) - dim.setQueryOrder(queryOrder) - queryOrder = queryOrder + 1 - selectedDims += dim - } else { - val carbonMeasure = - carbonTable.getMeasureByName(carbonTable.getFactTableName, attr.name) - if (carbonMeasure != null) { + val carbonColumn = carbonTable.getColumnByName(carbonTable.getFactTableName, attr.name) + if (carbonColumn != null) { + if (carbonColumn.isDimesion()) { + val dim = new QueryDimension(attr.name) + dim.setQueryOrder(queryOrder) + queryOrder = queryOrder + 1 + selectedDims += dim + } else { val m1 = new QueryMeasure(attr.name) m1.setQueryOrder(queryOrder) queryOrder = queryOrder + 1 http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala index 47fa82e..aa9f95a 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala @@ -505,6 +505,9 @@ class CarbonSqlParser() extends AbstractSparkSQLParser { tableProperties: Map[String, String]): TableModel = { + fields.zipWithIndex.foreach { x => + x._1.schemaOrdinal = x._2 + } val (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields( fields, tableProperties) if (dims.isEmpty) { @@ -1154,36 +1157,36 @@ class CarbonSqlParser() extends AbstractSparkSQLParser { val dataType = field.dataType.getOrElse("NIL") dataType match { case "string" => Field(field.column, Some("String"), field.name, Some(null), field.parent, - field.storeType + field.storeType, field.schemaOrdinal ) case "integer" | "int" => Field(field.column, Some("Integer"), field.name, Some(null), - field.parent, field.storeType + field.parent, field.storeType, field.schemaOrdinal ) case "long" => Field(field.column, Some("Long"), field.name, Some(null), field.parent, - field.storeType + field.storeType, field.schemaOrdinal ) case "double" => Field(field.column, Some("Double"), field.name, Some(null), field.parent, - field.storeType + field.storeType, field.schemaOrdinal ) case "timestamp" => Field(field.column, Some("Timestamp"), field.name, Some(null), - field.parent, field.storeType + field.parent, field.storeType, field.schemaOrdinal ) case "numeric" => Field(field.column, Some("Numeric"), field.name, Some(null), field.parent, - field.storeType + field.storeType, field.schemaOrdinal ) case "array" => Field(field.column, Some("Array"), field.name, field.children.map(f => f.map(normalizeType(_))), - field.parent, field.storeType + field.parent, field.storeType, field.schemaOrdinal ) case "struct" => Field(field.column, Some("Struct"), field.name, field.children.map(f => f.map(normalizeType(_))), - field.parent, field.storeType + field.parent, field.storeType, field.schemaOrdinal ) case "bigint" => Field(field.column, Some("BigInt"), field.name, Some(null), field.parent, - field.storeType + field.storeType, field.schemaOrdinal ) case "decimal" => Field(field.column, Some("Decimal"), field.name, Some(null), field.parent, - field.storeType, field.precision, field.scale + field.storeType, field.schemaOrdinal, field.precision, field.scale ) // checking if the nested data type contains the child type as decimal(10,0), // if it is present then extracting the precision and scale. resetting the data type @@ -1195,7 +1198,7 @@ class CarbonSqlParser() extends AbstractSparkSQLParser { field.name, Some(null), field.parent, - field.storeType, precision, + field.storeType, field.schemaOrdinal, precision, scale ) case _ => @@ -1207,10 +1210,10 @@ class CarbonSqlParser() extends AbstractSparkSQLParser { field.dataType.getOrElse("NIL") match { case "Array" => Field(field.column, Some("Array"), field.name, field.children.map(f => f.map(appendParentForEachChild(_, field.column))), field.parent, - field.storeType) + field.storeType, field.schemaOrdinal) case "Struct" => Field(field.column, Some("Struct"), field.name, field.children.map(f => f.map(appendParentForEachChild(_, field.column))), field.parent, - field.storeType) + field.storeType, field.schemaOrdinal) case _ => field } } @@ -1243,7 +1246,7 @@ class CarbonSqlParser() extends AbstractSparkSQLParser { Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName) case "Decimal" => Field(parentName + "." + field.column, Some("Decimal"), Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName, - field.storeType, field.precision, field.scale) + field.storeType, field.schemaOrdinal, field.precision, field.scale) case _ => field } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala index 9353a92..1b939f7 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala @@ -26,6 +26,7 @@ import scala.language.implicitConversions import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Cast, Literal} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.{RunnableCommand, SparkPlan} import org.apache.spark.sql.hive.CarbonMetastore import org.apache.spark.sql.types.TimestampType @@ -331,6 +332,27 @@ object LoadTable { } +private[sql] case class LoadTableByInsert(relation: CarbonDatasourceRelation, + child: LogicalPlan) extends RunnableCommand { + val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) + def run(sqlContext: SQLContext): Seq[Row] = { + val df = new DataFrame(sqlContext, child) + val header = relation.carbonRelation.output.map(_.name).mkString(",") + val load = LoadTable( + Some(relation.carbonRelation.databaseName), + relation.carbonRelation.tableName, + null, + Seq(), + scala.collection.immutable.Map(("fileheader" -> header)), + false, + null, + Some(df)).run(sqlContext) + // updating relation metadata. This is in case of auto detect high cardinality + relation.carbonRelation.metaData = + CarbonSparkUtil.createSparkMeta(relation.carbonRelation.tableMeta.carbonTable) + load + } +} case class LoadTable( databaseNameOp: Option[String], tableName: String, http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonPreInsertionCasts.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonPreInsertionCasts.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonPreInsertionCasts.scala new file mode 100644 index 0000000..381697b --- /dev/null +++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonPreInsertionCasts.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.hive + +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.expressions.Alias +import org.apache.spark.sql.catalyst.expressions.AttributeReference +import org.apache.spark.sql.catalyst.expressions.Cast +import org.apache.spark.sql.catalyst.plans.logical +import org.apache.spark.sql.catalyst.plans.logical.InsertIntoTable +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules._ +import org.apache.spark.sql.execution.datasources.LogicalRelation + +import org.apache.carbondata.core.constants.CarbonCommonConstants + +/** + * Insert into carbon table from other source + */ +object CarbonPreInsertionCasts extends Rule[LogicalPlan] { + def apply(plan: LogicalPlan): LogicalPlan = plan.transform { + // Wait until children are resolved. + case p: LogicalPlan if !p.childrenResolved => p + + case p @ InsertIntoTable(relation: LogicalRelation, _, child, _, _) + if relation.relation.isInstanceOf[CarbonDatasourceRelation] => + castChildOutput(p, relation.relation.asInstanceOf[CarbonDatasourceRelation], child) + } + + def castChildOutput(p: InsertIntoTable, relation: CarbonDatasourceRelation, child: LogicalPlan) + : LogicalPlan = { + if (relation.carbonRelation.output.size > CarbonCommonConstants + .DEFAULT_MAX_NUMBER_OF_COLUMNS) { + sys + .error("Maximum supported column by carbon is:" + CarbonCommonConstants + .DEFAULT_MAX_NUMBER_OF_COLUMNS + ) + } + if (child.output.size >= relation.carbonRelation.output.size ) { + InsertIntoCarbonTable(relation, p.partition, p.child, p.overwrite, p.ifNotExists) + } else { + sys.error("Cannot insert into target table because column number are different") + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala index 44dbd6b..91c96af 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala @@ -67,6 +67,10 @@ class CarbonStrategies(sqlContext: SQLContext) extends QueryPlanner[SparkPlan] { } else { carbonRawScan(projectList, predicates, l)(sqlContext) :: Nil } + case InsertIntoCarbonTable(relation: CarbonDatasourceRelation, + _, child: LogicalPlan, _, _) => + ExecutedCommand(LoadTableByInsert(relation, + child)) :: Nil case CarbonDictionaryCatalystDecoder(relations, profile, aliasMap, _, child) => CarbonDictionaryDecoder(relations, profile, http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala index 499be5c..1d52762 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala @@ -92,6 +92,7 @@ class ResolveCarbonFunctions(relations: Seq[CarbonDecoderRelation]) def isOptimized(plan: LogicalPlan): Boolean = { plan find { case cd: CarbonDictionaryCatalystDecoder => true + case ic: InsertIntoCarbonTable => true case other => false } isDefined } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/test/resources/shortolap.csv ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/resources/shortolap.csv b/integration/spark/src/test/resources/shortolap.csv new file mode 100644 index 0000000..1c237b8 --- /dev/null +++ b/integration/spark/src/test/resources/shortolap.csv @@ -0,0 +1,5 @@ +imei0,2147,9279,100.05,100.055,2016-05-01 12:25:36,aa,11 +imei1,-2148,-9807,10.05,100.05,2016-05-02 19:25:15,bb,22 +imei2,2147,9279,100.05,100.055,2016-05-01 12:25:36,cc,33 +imei3,-217,-9206,100.005,100.05,2016-05-02 19:25:15,dd,44 +imei4,10,0,15.5,45,2016-05-02 19:25:15,ee,55 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala index bc39df3..1d8eea5 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala @@ -42,10 +42,10 @@ class TestLoadDataWithSingleQuotechar extends QueryTest with BeforeAndAfterAll { "carbontable OPTIONS('DELIMITER'= ',')") checkAnswer( sql("SELECT * from carbontable"), - Seq(Row("Tom",1), - Row("Tony\n3,Lily",2), - Row("Games\"",4), - Row("prival\"\n6,\"hello\"",5) + Seq(Row(1,"Tom"), + Row(2,"Tony\n3,Lily"), + Row(4,"Games\""), + Row(5,"prival\"\n6,\"hello\"") ) ) } catch { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala new file mode 100644 index 0000000..4199062 --- /dev/null +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.spark.testsuite.allqueries + +import org.apache.spark.sql.Row +import org.apache.spark.sql.common.util.CarbonHiveContext._ +import org.apache.spark.sql.common.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties + +class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll { + override def beforeAll { + sql("drop table if exists THive") + sql("create table THive (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions st ring, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','") + sql("LOAD DATA local INPATH './src/test/resources/100_olap.csv' INTO TABLE THive") + } + test("insert from hive") { + val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) + + sql("drop table if exists TCarbon") + sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'") + sql("insert into TCarbon select * from THive") + checkAnswer( + sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from THive order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Late st_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription"), + sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from TCarbon order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,La test_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription") + ) + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig) + } + test("insert from hive-sum expression") { + sql("drop table if exists TCarbon") + sql("create table TCarbon (MAC string,deviceInformationIdSum int) STORED BY 'org.apache.carbondata.format'") + sql("insert into TCarbon select MAC,sum(deviceInformationId+ 10) as a from THive group by MAC") + checkAnswer( + sql("select MAC,deviceInformationIdSum from TCarbon order by MAC"), + sql("select MAC,sum(deviceInformationId+ 10) as a from THive group by MAC order by MAC") + ) + } + test("insert from carbon-select columns") { + val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) + + sql("drop table if exists TCarbonSource") + sql("drop table if exists TCarbon") + sql("create table TCarbonSource (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVe rsions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'") + sql("LOAD DATA INPATH './src/test/resources/100_olap.csv' INTO table TCarbonSource options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVer Number,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointDescription,gamePointId,contractNumber')") + sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'") + sql("insert into TCarbon select * from TCarbonSource") + checkAnswer( + sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from TCarbonSource order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNum ber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription"), + sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from TCarbon order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,La test_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription") + ) + val result = sql("show segments for table TCarbon").collect()(0).get(1).toString() + if(!"Success".equalsIgnoreCase(result)) { + assert(false) + } + + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig) + } + test("insert from carbon-select columns-source table has more column then target column") { + val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) + + sql("drop table if exists load") + sql("drop table if exists inser") + sql("CREATE TABLE load(imei string,age int,task bigint,num double,level decimal(10,3),productdate timestamp,name string,point int)STORED BY 'org.apache.carbondata.format'") + sql("LOAD DATA INPATH './src/test/resources/shortolap.csv' INTO TABLE load options ('DELIMITER'=',', 'QUOTECHAR'='\"','FILEHEADER' = 'imei,age,task,num,level,productdate,name,point')") + sql("CREATE TABLE inser(imei string,age int,task bigint,num double,level decimal(10,3),productdate timestamp)STORED BY 'org.apache.carbondata.format'") + sql("insert into inser select * from load") + checkAnswer( + sql("select * from inser"), + sql("select imei,age,task,num,level,productdate from load") + ) + sql("drop table if exists load") + sql("drop table if exists inser") + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig) + } + test("insert from carbon-select * columns") { + val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) + + sql("drop table if exists TCarbonSource") + sql("drop table if exists TCarbon") + sql("create table TCarbonSource (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVe rsions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'") + sql("LOAD DATA INPATH './src/test/resources/100_olap.csv' INTO table TCarbonSource options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVer Number,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointDescription,gamePointId,contractNumber')") + sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'") + sql("insert into TCarbon select * from TCarbonSource") + checkAnswer( + sql("select * from TCarbonSource"), + sql("select * from TCarbon") + ) + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig) + } + test("insert->hive column more than carbon column->success") { + sql("drop table if exists TCarbon") + sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'") + + sql("insert into TCarbon select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber,device_backColor,modelId,CUPAudit,CPIClocked from THive") + checkAnswer( + sql("select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber from THive"), + sql("select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber from TCarbon") + ) + } + test("insert->carbon column is more then hive-fails") { + sql("drop table if exists TCarbon") + sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'") + try { + sql("insert into TCarbon select imei,deviceInformationId,MAC,deviceColor,gamePointId from THive") + assert(false) + } catch { + case ex: Exception => assert(true) + } + } + test("insert->insert wrong data types-pass") { + sql("drop table if exists TCarbon") + sql("create table TCarbon (imei string,deviceInformationId int,MAC string) STORED BY 'org.apache.carbondata.format'") + sql("drop table if exists TCarbonLocal") + sql("create table TCarbonLocal (imei string,deviceInformationId int,MAC string) STORED BY 'org.apache.carbondata.format'") + sql("LOAD DATA INPATH './src/test/resources/100_olap.csv' INTO table TCarbonLocal options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,MAC,deviceInformationId')") + sql("insert into TCarbon select imei,MAC,deviceInformationId from THive") + checkAnswer( + sql("select imei,deviceInformationId,MAC from TCarbon"), + sql("select imei,deviceInformationId,MAC from TCarbonLocal") + ) + } + test("insert->insert empty data -pass") { + sql("drop table if exists TCarbon") + sql("create table TCarbon (imei string,deviceInformationId int,MAC string) STORED BY 'org.apache.carbondata.format'") + sql("insert into TCarbon select imei,deviceInformationId,MAC from THive where MAC='wrongdata'") + val result = sql("select imei,deviceInformationId,MAC from TCarbon where MAC='wrongdata'").collect() + checkAnswer( + sql("select imei,deviceInformationId,MAC from THive where MAC='wrongdata'"), + sql("select imei,deviceInformationId,MAC from TCarbon where MAC='wrongdata'") + ) + } + test("insert into existing load-pass") { + val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) + + sql("drop table if exists TCarbon") + sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'") + sql("LOAD DATA INPATH './src/test/resources/100_olap.csv' INTO table TCarbon options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber ,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointDescription,gamePointId,contractNumber')") + sql("insert into TCarbon select * from THive") + sql("LOAD DATA local INPATH './src/test/resources/100_olap.csv' INTO TABLE THive") + checkAnswer( + sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from THive order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Late st_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription"), + sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from TCarbon order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,La test_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription") + ) + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig) + } + override def afterAll { + sql("DROP TABLE IF EXISTS THive") + sql("DROP TABLE IF EXISTS TCarbon") + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala index 8bcbbbc..17fe053 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala @@ -63,7 +63,7 @@ class TestNoInvertedIndexLoadAndQuery extends QueryTest with BeforeAndAfterAll{ sql(""" SELECT * FROM index1 WHERE city = "Bangalore" """), - Seq(Row("Emily", "Bangalore", 19.0))) + Seq(Row(19.0,"Emily", "Bangalore" ))) }