Repository: carbondata Updated Branches: refs/heads/master b8a02f391 -> b0d4de94e
[CARBONDATA-1872][API] Clean up unused constant in CarbonCommonConstant This closes #1631 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b0d4de94 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b0d4de94 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b0d4de94 Branch: refs/heads/master Commit: b0d4de94ebf6bd9f5851c14f0d4a11bf2d74da23 Parents: b8a02f3 Author: Jacky Li <[email protected]> Authored: Thu Dec 7 19:40:45 2017 +0800 Committer: ravipesala <[email protected]> Committed: Tue Dec 19 23:08:58 2017 +0530 ---------------------------------------------------------------------- .../core/constants/CarbonCommonConstants.java | 161 +------------------ .../core/mutate/CarbonUpdateUtil.java | 1 - .../core/statusmanager/LoadMetadataDetails.java | 19 --- .../sql/test/Spark2TestQueryExecutor.scala | 6 +- .../store/CarbonFactDataHandlerColumnar.java | 53 ------ 5 files changed, 6 insertions(+), 234 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index f67b0c5..b8a998c 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -46,14 +46,6 @@ public final class CarbonCommonConstants { * byte to KB conversion factor */ public static final int BYTE_TO_KB_CONVERSION_FACTOR = 1024; - /** - * BYTE_ENCODING - */ - public static final String BYTE_ENCODING = "ISO-8859-1"; - /** - * measure meta data file name - */ - public static final String MEASURE_METADATA_FILE_NAME = "/msrMetaData_"; /** * set the segment ids to query from the table @@ -179,22 +171,6 @@ public final class CarbonCommonConstants { public static final int BYTEBUFFER_SIZE = 24 * 1024; /** - * Average constant - */ - public static final String AVERAGE = "avg"; - /** - * Count constant - */ - public static final String COUNT = "count"; - /** - * SUM - */ - public static final String SUM = "sum"; - /** - * DUMMY aggregation function - */ - public static final String DUMMY = "dummy"; - /** * MEMBER_DEFAULT_VAL */ public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!"; @@ -234,18 +210,10 @@ public final class CarbonCommonConstants { */ public static final String CARBON_BADRECORDS_LOC_DEFAULT_VAL = ""; /** - * HIERARCHY_FILE_EXTENSION - */ - public static final String HIERARCHY_FILE_EXTENSION = ".hierarchy"; - /** * SORT_TEMP_FILE_LOCATION */ public static final String SORT_TEMP_FILE_LOCATION = "sortrowtmp"; /** - * CARBON_RESULT_SIZE_DEFAULT - */ - public static final String LEVEL_FILE_EXTENSION = ".level"; - /** * FACT_FILE_EXT */ public static final String FACT_FILE_EXT = ".carbondata"; @@ -260,35 +228,6 @@ public final class CarbonCommonConstants { */ public static final String UPDATE_DELTA_FILE_EXT = FACT_FILE_EXT; /** - * MEASUREMETADATA_FILE_EXT - */ - public static final String MEASUREMETADATA_FILE_EXT = ".msrmetadata"; - /** - * GRAPH_ROWSET_SIZE - */ - @CarbonProperty - public static final String GRAPH_ROWSET_SIZE = "carbon.graph.rowset.size"; - /** - * GRAPH_ROWSET_SIZE_DEFAULT - */ - public static final String GRAPH_ROWSET_SIZE_DEFAULT = "500"; - /** - * Comment for <code>TYPE_MYSQL</code> - */ - public static final String TYPE_MYSQL = "MYSQL"; - /** - * Comment for <code>TYPE_MSSQL</code> - */ - public static final String TYPE_MSSQL = "MSSQL"; - /** - * Comment for <code>TYPE_ORACLE</code> - */ - public static final String TYPE_ORACLE = "ORACLE"; - /** - * Comment for <code>TYPE_SYBASE</code> - */ - public static final String TYPE_SYBASE = "SYBASE"; - /** * SORT_INTERMEDIATE_FILES_LIMIT */ @CarbonProperty @@ -298,10 +237,6 @@ public final class CarbonCommonConstants { */ public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20"; /** - * BAD_RECORD_KEY_VALUE - */ - public static final String BAD_RECORD_KEY = "BADRECORD"; - /** * MERGERD_EXTENSION */ public static final String MERGERD_EXTENSION = ".merge"; @@ -311,15 +246,6 @@ public final class CarbonCommonConstants { @CarbonProperty public static final String SORT_FILE_BUFFER_SIZE = "carbon.sort.file.buffer.size"; /** - * no.of records after which counter to be printed - */ - @CarbonProperty - public static final String DATA_LOAD_LOG_COUNTER = "carbon.load.log.counter"; - /** - * DATA_LOAD_LOG_COUNTER_DEFAULT_COUNTER - */ - public static final String DATA_LOAD_LOG_COUNTER_DEFAULT_COUNTER = "500000"; - /** * SORT_FILE_WRITE_BUFFER_SIZE */ @CarbonProperty @@ -405,18 +331,6 @@ public final class CarbonCommonConstants { */ public static final String SEMICOLON_SPC_CHARACTER = ";#!@:SEMIC:@!#;"; /** - * AMPERSAND_SPC_CHARATER - */ - public static final String AMPERSAND_SPC_CHARACTER = "&#!@:AMPER:@!#&"; - /** - * ATTHERATE_SPC_CHARATER - */ - public static final String COMA_SPC_CHARACTER = ",#!:COMA:!#,"; - /** - * HYPHEN_SPC_CHARACTER - */ - public static final String HYPHEN_SPC_CHARACTER = "-#!:HYPHEN:!#-"; - /** * SORT_TEMP_FILE_EXT */ public static final String SORT_TEMP_FILE_EXT = ".sorttemp"; @@ -485,27 +399,10 @@ public final class CarbonCommonConstants { @CarbonProperty public static final String STORE_LOCATION_TEMP_PATH = "carbon.tempstore.location"; /** - * IS_COLUMNAR_STORAGE_DEFAULTVALUE - */ - public static final String IS_COLUMNAR_STORAGE_DEFAULTVALUE = "true"; - /** - * DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE - */ - public static final String DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE = "1"; - /** * IS_FULLY_FILLED_BITS_DEFAULT_VALUE */ public static final String IS_FULLY_FILLED_BITS_DEFAULT_VALUE = "true"; /** - * IS_INT_BASED_INDEXER - */ - @CarbonProperty - public static final String AGGREAGATE_COLUMNAR_KEY_BLOCK = "aggregate.columnar.keyblock"; - /** - * IS_INT_BASED_INDEXER_DEFAULTVALUE - */ - public static final String AGGREAGATE_COLUMNAR_KEY_BLOCK_DEFAULTVALUE = "true"; - /** * ENABLE_QUERY_STATISTICS */ @CarbonProperty @@ -541,10 +438,6 @@ public final class CarbonCommonConstants { */ public static final String UNDERSCORE = "_"; /** - * DASH - */ - public static final String DASH = "-"; - /** * POINT */ public static final String POINT = "."; @@ -621,16 +514,6 @@ public final class CarbonCommonConstants { public static final String TIMESTAMP = "Timestamp"; public static final String ARRAY = "array"; public static final String STRUCT = "struct"; - public static final String FROM = "from"; - /** - * FACT_UPDATE_EXTENSION. - */ - public static final String FACT_UPDATE_EXTENSION = ".carbondata_update"; - public static final String FACT_DELETE_EXTENSION = "_delete"; - /** - * MARKED_FOR_UPDATION - */ - public static final String FACT_FILE_UPDATED = "update"; /** * MAX_QUERY_EXECUTION_TIME */ @@ -663,17 +546,11 @@ public final class CarbonCommonConstants { */ @CarbonProperty public static final String CARBON_MAX_DRIVER_LRU_CACHE_SIZE = "carbon.max.driver.lru.cache.size"; - public static final String POSITION_REFERENCE = "positionReference"; /** * implicit column which will be added to each carbon table */ public static final String POSITION_ID = "positionId"; /** - * max driver lru cache size upto which lru cache will be loaded in memory - */ - @CarbonProperty - public static final String CARBON_MAX_LEVEL_CACHE_SIZE = "carbon.max.level.cache.size"; - /** * max executor lru cache size upto which lru cache will be loaded in memory */ @CarbonProperty @@ -1027,16 +904,6 @@ public final class CarbonCommonConstants { public static final String UPDATE_INDEX_FILE_EXT = ".carbonindex"; /** - * Key word for true - */ - public static final String KEYWORD_TRUE = "TRUE"; - - /** - * Key word for false - */ - public static final String KEYWORD_FALSE = "FALSE"; - - /** * hyphen */ public static final String HYPHEN = "-"; @@ -1047,11 +914,6 @@ public final class CarbonCommonConstants { public static final String UPDATED_COL_EXTENSION = "-updatedColumn"; /** - * appending the key to differentiate the update flow with insert flow. - */ - public static final String RDDUTIL_UPDATE_KEY = "UPDATE_"; - - /** * to determine to use the rdd persist or not. */ @CarbonProperty @@ -1099,10 +961,6 @@ public final class CarbonCommonConstants { public static final String CARBON_DATA_FILE_DEFAULT_VERSION = "V3"; /** - * data file version header - */ - public static final String CARBON_DATA_VERSION_HEADER = "CARBONDATAVERSION#"; - /** * Maximum no of column supported */ public static final int DEFAULT_MAX_NUMBER_OF_COLUMNS = 20000; @@ -1200,12 +1058,13 @@ public final class CarbonCommonConstants { @CarbonProperty public static final String LOAD_BATCH_SORT_SIZE_INMB = "carbon.load.batch.sort.size.inmb"; public static final String LOAD_BATCH_SORT_SIZE_INMB_DEFAULT = "0"; - @CarbonProperty + /** * The Number of partitions to use when shuffling data for sort. If user don't configurate or * configurate it less than 1, it uses the number of map tasks as reduce tasks. In general, we * recommend 2-3 tasks per CPU core in your cluster. */ + @CarbonProperty public static final String LOAD_GLOBAL_SORT_PARTITIONS = "carbon.load.global.sort.partitions"; public static final String LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT = "0"; @@ -1232,11 +1091,6 @@ public final class CarbonCommonConstants { public static final String IS_DRIVER_INSTANCE = "is.driver.instance"; /** - * maximum length of column - */ - public static final int DEFAULT_COLUMN_LENGTH = 100000; - - /** * property for enabling unsafe based query processing */ @CarbonProperty @@ -1248,17 +1102,6 @@ public final class CarbonCommonConstants { public static final String ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE = "false"; /** - * property for offheap based processing - */ - @CarbonProperty - public static final String USE_OFFHEAP_IN_QUERY_PROCSSING = "use.offheap.in.query.processing"; - - /** - * default value of offheap based processing - */ - public static final String USE_OFFHEAP_IN_QUERY_PROCSSING_DEFAULT = "true"; - - /** * whether to prefetch data while loading. */ @CarbonProperty http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java index f4566ac..aeaa205 100644 --- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java @@ -231,7 +231,6 @@ public class CarbonUpdateUtil { // if this call is coming from the delete delta flow then the time stamp // String will come empty then no need to write into table status file. if (isTimestampUpdationRequired) { - loadMetadata.setIsDeleted(CarbonCommonConstants.KEYWORD_TRUE); // if in case of update flow. if (loadMetadata.getUpdateDeltaStartTimestamp().isEmpty()) { // this means for first time it is getting updated . http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java index 85602bc..e2bcc8a 100644 --- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java +++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java @@ -63,7 +63,6 @@ public class LoadMetadataDetails implements Serializable { // partition count of this segment private String partitionCount; - private String isDeleted = CarbonCommonConstants.KEYWORD_FALSE; private String dataSize; private String indexSize; @@ -339,24 +338,6 @@ public class LoadMetadataDetails implements Serializable { } /** - * To get isDeleted property. - * - * @return isDeleted - */ - public String getIsDeleted() { - return isDeleted; - } - - /** - * To set isDeleted property. - * - * @param isDeleted - */ - public void setIsDeleted(String isDeleted) { - this.isDeleted = isDeleted; - } - - /** * To get the update delta end timestamp * * @return updateDeltaEndTimestamp http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala index 3fed15d..e668de7 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala @@ -40,11 +40,13 @@ class Spark2TestQueryExecutor extends TestQueryExecutorRegister { } object Spark2TestQueryExecutor { + + private val STORE_LOCATION_TEMP_PATH = "carbon.tempstore.location" + private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) LOGGER.info("use TestQueryExecutorImplV2") CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH, - System.getProperty("java.io.tmpdir")) + .addProperty(STORE_LOCATION_TEMP_PATH, System.getProperty("java.io.tmpdir")) .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FORCE") http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java index 75fcea3..ac31fcb 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java @@ -38,8 +38,6 @@ import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel; import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException; import org.apache.carbondata.core.datastore.row.CarbonRow; import org.apache.carbondata.core.keygenerator.KeyGenException; -import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter; -import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthEquiSplitGenerator; import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.metadata.datatype.DataType; @@ -47,7 +45,6 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.CarbonThreadFactory; -import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.processing.datatypes.GenericDataType; import org.apache.carbondata.processing.loading.sort.SortScopeOptions; import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo; @@ -371,22 +368,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { } LOGGER.info("Number of rows per column blocklet " + pageSize); dataRows = new ArrayList<>(this.pageSize); - int dimSet = - Integer.parseInt(CarbonCommonConstants.DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE); - // if at least one dimension is present then initialize column splitter otherwise null - int noOfColStore = colGrpModel.getNoOfColumnStore(); - int[] keyBlockSize = new int[noOfColStore + getExpandedComplexColsCount()]; - - if (model.getDimLens().length > 0) { - //Using Variable length variable split generator - //This will help in splitting mdkey to columns. variable split is required because all - // columns which are part of - //row store will be in single column store - //e.g if {0,1,2,3,4,5} is dimension and {0,1,2) is row store dimension - //than below splitter will return column as {0,1,2}{3}{4}{5} - ColumnarSplitter columnarSplitter = model.getSegmentProperties().getFixedLengthKeySplitter(); - System.arraycopy(columnarSplitter.getBlockKeySize(), 0, keyBlockSize, 0, noOfColStore); - } // agg type List<Integer> otherMeasureIndexList = @@ -411,46 +392,12 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { customMeasureIndex[i] = customMeasureIndexList.get(i); } setComplexMapSurrogateIndex(model.getDimensionCount()); - int[] blockKeySize = getBlockKeySizeWithComplexTypes(new MultiDimKeyVarLengthEquiSplitGenerator( - CarbonUtil.getIncrementedCardinalityFullyFilled(model.getDimLens().clone()), (byte) dimSet) - .getBlockKeySize()); - System.arraycopy(blockKeySize, noOfColStore, keyBlockSize, noOfColStore, - blockKeySize.length - noOfColStore); this.dataWriter = getFactDataWriter(); // initialize the channel; this.dataWriter.initializeWriter(); } /** - * This method combines primitive dimensions with complex metadata columns - * - * @param primitiveBlockKeySize - * @return all dimensions cardinality including complex dimension metadata column - */ - private int[] getBlockKeySizeWithComplexTypes(int[] primitiveBlockKeySize) { - int allColsCount = getExpandedComplexColsCount(); - int[] blockKeySizeWithComplexTypes = - new int[this.colGrpModel.getNoOfColumnStore() + allColsCount]; - - List<Integer> blockKeySizeWithComplex = - new ArrayList<Integer>(blockKeySizeWithComplexTypes.length); - int dictDimensionCount = model.getDimensionCount(); - for (int i = 0; i < dictDimensionCount; i++) { - GenericDataType complexDataType = model.getComplexIndexMap().get(i); - if (complexDataType != null) { - complexDataType.fillBlockKeySize(blockKeySizeWithComplex, primitiveBlockKeySize); - } else { - blockKeySizeWithComplex.add(primitiveBlockKeySize[i]); - } - } - for (int i = 0; i < blockKeySizeWithComplexTypes.length; i++) { - blockKeySizeWithComplexTypes[i] = blockKeySizeWithComplex.get(i); - } - - return blockKeySizeWithComplexTypes; - } - - /** * Below method will be used to get the fact data writer instance * * @return data writer instance
