http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala b/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala index 03ec3a1..563206f 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala @@ -17,7 +17,10 @@ package org.apache.spark.sql +import java.util.Date + import org.apache.spark.sql.test.util.QueryTest + import org.apache.carbondata.core.constants.CarbonCommonConstants import org.scalatest.BeforeAndAfterAll @@ -59,7 +62,7 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll { .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) || row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE)) assert(res1.length == 2) - res1.foreach(row => assert(row.getString(1).trim.toLong > 0)) + res1.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0)) } test("get data size and index size after major compaction") { @@ -73,7 +76,7 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll { .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) || row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE)) assert(res2.length == 2) - res2.foreach(row => assert(row.getString(1).trim.toLong > 0)) + res2.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0)) } test("get data size and index size after minor compaction") { @@ -91,7 +94,7 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll { .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) || row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE)) assert(res3.length == 2) - res3.foreach(row => assert(row.getString(1).trim.toLong > 0)) + res3.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0)) } test("get data size and index size after insert into") { @@ -105,7 +108,7 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll { .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) || row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE)) assert(res4.length == 2) - res4.foreach(row => assert(row.getString(1).trim.toLong > 0)) + res4.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0)) } test("get data size and index size after insert overwrite") { @@ -119,7 +122,7 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll { .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) || row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE)) assert(res5.length == 2) - res5.foreach(row => assert(row.getString(1).trim.toLong > 0)) + res5.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0)) } test("get data size and index size for empty table") { @@ -128,15 +131,14 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll { .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) || row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE)) assert(res6.length == 2) - res6.foreach(row => assert(row.getString(1).trim.toLong == 0)) + res6.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble == 0)) } test("get last update time for empty table") { sql("CREATE TABLE tableSize9 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'") val res7 = sql("DESCRIBE FORMATTED tableSize9").collect() - .filter(row => row.getString(0).contains(CarbonCommonConstants.LAST_UPDATE_TIME)) + .filter(row => row.getString(0).contains("Last Update")) assert(res7.length == 1) - res7.foreach(row => assert(row.getString(1).trim.toLong == 0)) } test("get last update time for unempty table") { @@ -144,9 +146,8 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll { sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize10 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""") val res8 = sql("DESCRIBE FORMATTED tableSize10").collect() - .filter(row => row.getString(0).contains(CarbonCommonConstants.LAST_UPDATE_TIME)) + .filter(row => row.getString(0).contains("Last Update")) assert(res8.length == 1) - res8.foreach(row => assert(row.getString(1).trim.toLong > 0)) } test("index and datasize for update scenario") { @@ -160,13 +161,13 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll { .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) || row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE)) assert(res9.length == 2) - res9.foreach(row => assert(row.getString(1).trim.toLong > 0)) + res9.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0)) sql("update tableSize11 set (empno) = (234)").show() val res10 = sql("DESCRIBE FORMATTED tableSize11").collect() .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) || row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE)) assert(res10.length == 2) - res10.foreach(row => assert(row.getString(1).trim.toLong > 0)) + res10.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0)) } }
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java index bcca915..4926cd8 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java @@ -25,6 +25,7 @@ import org.apache.carbondata.common.CarbonIterator; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.constants.CarbonLoadOptionConstants; +import org.apache.carbondata.core.constants.SortScopeOptions; import org.apache.carbondata.core.datastore.TableSpec; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataTypes; @@ -37,7 +38,6 @@ import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants; import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException; import org.apache.carbondata.processing.loading.model.CarbonLoadModel; -import org.apache.carbondata.processing.loading.sort.SortScopeOptions; import org.apache.carbondata.processing.loading.steps.CarbonRowDataWriterProcessorStepImpl; import org.apache.carbondata.processing.loading.steps.DataConverterProcessorStepImpl; import org.apache.carbondata.processing.loading.steps.DataWriterBatchProcessorStepImpl; http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java index 03b2645..f69f0af 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java @@ -31,6 +31,7 @@ import org.apache.carbondata.common.constants.LoggerAction; import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.constants.SortScopeOptions; import org.apache.carbondata.core.datastore.compression.CompressorFactory; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; @@ -38,7 +39,6 @@ import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants; import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat; -import org.apache.carbondata.processing.loading.sort.SortScopeOptions; import org.apache.carbondata.processing.util.CarbonBadRecordUtil; import org.apache.carbondata.processing.util.TableOptionConstant; http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java deleted file mode 100644 index 7b3c3df..0000000 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.processing.loading.sort; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; - -/** - * Sort scope options - */ -public class SortScopeOptions { - - public static SortScope getSortScope(String sortScope) { - if (sortScope == null) { - sortScope = CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT; - } - switch (sortScope.toUpperCase()) { - case "BATCH_SORT": - return SortScope.BATCH_SORT; - case "LOCAL_SORT": - return SortScope.LOCAL_SORT; - case "GLOBAL_SORT": - return SortScope.GLOBAL_SORT; - case "NO_SORT": - return SortScope.NO_SORT; - default: - return SortScope.LOCAL_SORT; - } - } - - public enum SortScope { - NO_SORT, BATCH_SORT, LOCAL_SORT, GLOBAL_SORT - } -} - http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java index aad9083..af57759 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.constants.SortScopeOptions; import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration; import org.apache.carbondata.processing.loading.sort.impl.ParallelReadMergeSorterImpl; http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java index 4012774..92c48bc 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.constants.SortScopeOptions; import org.apache.carbondata.core.datastore.TableSpec; import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.keygenerator.KeyGenerator; @@ -48,7 +49,6 @@ import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration; import org.apache.carbondata.processing.loading.DataField; import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants; import org.apache.carbondata.processing.loading.model.CarbonLoadModel; -import org.apache.carbondata.processing.loading.sort.SortScopeOptions; import org.apache.carbondata.processing.util.CarbonDataProcessorUtil; import org.apache.log4j.Logger; http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java index 48993c5..f168796 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java @@ -45,7 +45,7 @@ import org.apache.carbondata.processing.store.writer.AbstractFactDataWriter; import static org.apache.carbondata.core.constants.CarbonCommonConstants.TABLE_BLOCKLET_SIZE; import static org.apache.carbondata.core.constants.CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB; import static org.apache.carbondata.core.constants.CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE; -import static org.apache.carbondata.processing.loading.sort.SortScopeOptions.SortScope.NO_SORT; +import static org.apache.carbondata.core.constants.SortScopeOptions.SortScope.NO_SORT; import org.apache.log4j.Logger; http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java index b34ed13..437f628 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java @@ -33,6 +33,7 @@ import org.apache.carbondata.common.CarbonIterator; import org.apache.carbondata.common.constants.LoggerAction; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.constants.SortScopeOptions; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; @@ -52,7 +53,6 @@ import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration; import org.apache.carbondata.processing.loading.DataField; import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants; import org.apache.carbondata.processing.loading.model.CarbonDataLoadSchema; -import org.apache.carbondata.processing.loading.sort.SortScopeOptions; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils;