Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2914#discussion_r241631889
--- Diff:
core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
---
@@ -40,1372 +41,1024 @@ private CarbonCommonConstants() {
@CarbonProperty
public static final String STORE_LOCATION = "carbon.storelocation";
- /**
- * blocklet size in carbon file
- */
- @CarbonProperty
- public static final String BLOCKLET_SIZE = "carbon.blocklet.size";
-
- /**
- * default blocklet size
- */
- public static final String BLOCKLET_SIZE_DEFAULT_VAL = "120000";
-
- /**
- * min blocklet size
- */
- public static final int BLOCKLET_SIZE_MIN_VAL = 2000;
-
- /**
- * max blocklet size
- */
- public static final int BLOCKLET_SIZE_MAX_VAL = 12000000;
-
- /**
- * min block size in MB
- */
- public static final int BLOCK_SIZE_MIN_VAL = 1;
-
- /**
- * max block size in MB
- */
- public static final int BLOCK_SIZE_MAX_VAL = 2048;
-
- /**
- * carbon properties file path
- */
- @CarbonProperty
- public static final String CARBON_PROPERTIES_FILE_PATH =
"carbon.properties.filepath";
-
- /**
- * default carbon properties file path
- */
- public static final String CARBON_PROPERTIES_FILE_PATH_DEFAULT =
- "../../../conf/carbon.properties";
-
- /**
- * CARBON_DDL_BASE_HDFS_URL
- */
- @CarbonProperty
- public static final String CARBON_DDL_BASE_HDFS_URL =
"carbon.ddl.base.hdfs.url";
-
- /**
- * CARBON_BADRECORDS_LOCATION
- */
- @CarbonProperty
- public static final String CARBON_BADRECORDS_LOC =
"carbon.badRecords.location";
-
- /**
- * CARBON_BADRECORDS_LOCATION_DEFAULT
- */
- public static final String CARBON_BADRECORDS_LOC_DEFAULT_VAL = "";
-
- /**
- * Property for specifying the format of TIMESTAMP data type column.
- * e.g. yyyy/MM/dd HH:mm:ss, or using default value
- */
- @CarbonProperty
- public static final String CARBON_TIMESTAMP_FORMAT =
"carbon.timestamp.format";
-
- /**
- * default value
- */
- public static final String CARBON_TIMESTAMP_DEFAULT_FORMAT = "yyyy-MM-dd
HH:mm:ss";
-
- /**
- * CARBON_TIMESTAMP
- */
- public static final String CARBON_TIMESTAMP = "dd-MM-yyyy HH:mm:ss";
-
- /**
- * CARBON_TIMESTAMP
- */
- public static final String CARBON_TIMESTAMP_MILLIS = "dd-MM-yyyy
HH:mm:ss:SSS";
-
- /**
- * Property for specifying the format of DATE data type column.
- * e.g. yyyy/MM/dd , or using default value
- */
- @CarbonProperty
- public static final String CARBON_DATE_FORMAT = "carbon.date.format";
-
- /**
- * default value
- */
- public static final String CARBON_DATE_DEFAULT_FORMAT = "yyyy-MM-dd";
-
- /**
- * compressor for writing/reading CarbonData file
- */
- @CarbonProperty
- public static final String COMPRESSOR = "carbon.column.compressor";
-
- /**
- * default compressor is snappy
- */
- public static final String DEFAULT_COMPRESSOR = "snappy";
-
- /**
- * ZOOKEEPER_ENABLE_LOCK if this is set to true then zookeeper
- * will be used to handle locking
- * mechanism of carbon
- */
- @CarbonProperty
- public static final String LOCK_TYPE = "carbon.lock.type";
-
- /**
- * ZOOKEEPER_ENABLE_DEFAULT the default value for zookeeper will be true
for carbon
- */
- public static final String LOCK_TYPE_DEFAULT = "LOCALLOCK";
-
- /**
- * Specifies the path where the lock files have to be created.
- * By default, lock files are created in table path.
- */
- @CarbonProperty
- public static final String LOCK_PATH = "carbon.lock.path";
-
- public static final String LOCK_PATH_DEFAULT = "";
-
- /**
- * ZOOKEEPER_LOCATION this is the location in zookeeper file system
where locks are created.
- * mechanism of carbon
- */
- public static final String ZOOKEEPER_LOCATION = "/CarbonLocks";
-
- /**
- * xxhash algorithm property for hashmap
- */
- @CarbonProperty
- public static final String ENABLE_XXHASH = "carbon.enableXXHash";
-
- /**
- * xxhash algorithm property for hashmap Default value false
- */
- public static final String ENABLE_XXHASH_DEFAULT = "true";
-
- /**
- * System property to enable or disable local dictionary generation
- */
- @CarbonProperty
- public static final String LOCAL_DICTIONARY_SYSTEM_ENABLE =
"carbon.local.dictionary.enable";
-
- /**
- * System property to enable or disable decoder based local dictionary
fallback
- */
- @CarbonProperty
- public static final String LOCAL_DICTIONARY_DECODER_BASED_FALLBACK =
- "carbon.local.dictionary.decoder.fallback";
-
- /**
- * System property to enable or disable decoder based local dictionary
fallback default value
- */
- public static final String
LOCAL_DICTIONARY_DECODER_BASED_FALLBACK_DEFAULT = "true";
-
- /**
- * zookeeper url key
- */
- @CarbonProperty
- public static final String ZOOKEEPER_URL = "spark.deploy.zookeeper.url";
-
- /**
- * carbon data file version property
- */
- @CarbonProperty
- public static final String CARBON_DATA_FILE_VERSION =
"carbon.data.file.version";
-
- @CarbonProperty
- public static final String SPARK_SCHEMA_STRING_LENGTH_THRESHOLD =
- "spark.sql.sources.schemaStringLengthThreshold";
-
- public static final int SPARK_SCHEMA_STRING_LENGTH_THRESHOLD_DEFAULT =
4000;
-
- @CarbonProperty
- public static final String CARBON_BAD_RECORDS_ACTION =
"carbon.bad.records.action";
-
- /**
- * FAIL action will fail the load in case of bad records in loading data
- */
- public static final String CARBON_BAD_RECORDS_ACTION_DEFAULT = "FAIL";
-
- @CarbonProperty
- public static final String ENABLE_HIVE_SCHEMA_META_STORE =
"spark.carbon.hive.schema.store";
-
- public static final String ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT =
"false";
-
- /**
- * There is more often that in production uses different drivers for
load and queries. So in case
- * of load driver user should set this property to enable loader
specific clean up.
- */
- @CarbonProperty
- public static final String DATA_MANAGEMENT_DRIVER =
"spark.carbon.datamanagement.driver";
-
- public static final String DATA_MANAGEMENT_DRIVER_DEFAULT = "true";
-
- @CarbonProperty
- public static final String CARBON_SESSIONSTATE_CLASSNAME =
"spark.carbon.sessionstate.classname";
-
- /**
- * This property will be used to configure the sqlastbuilder class.
- */
- @CarbonProperty
- public static final String CARBON_SQLASTBUILDER_CLASSNAME =
- "spark.carbon.sqlastbuilder.classname";
-
- public static final String CARBON_SQLASTBUILDER_CLASSNAME_DEFAULT =
- "org.apache.spark.sql.hive.CarbonSqlAstBuilder";
-
- @CarbonProperty
- public static final String CARBON_LEASE_RECOVERY_RETRY_COUNT =
- "carbon.lease.recovery.retry.count";
-
- public static final String CARBON_LEASE_RECOVERY_RETRY_COUNT_DEFAULT =
"5";
-
- public static final int CARBON_LEASE_RECOVERY_RETRY_COUNT_MIN = 1;
-
- public static final int CARBON_LEASE_RECOVERY_RETRY_COUNT_MAX = 50;
-
- @CarbonProperty
- public static final String CARBON_LEASE_RECOVERY_RETRY_INTERVAL =
- "carbon.lease.recovery.retry.interval";
-
- public static final String CARBON_LEASE_RECOVERY_RETRY_INTERVAL_DEFAULT
= "1000";
-
- public static final int CARBON_LEASE_RECOVERY_RETRY_INTERVAL_MIN = 1000;
-
- public static final int CARBON_LEASE_RECOVERY_RETRY_INTERVAL_MAX = 10000;
-
- @CarbonProperty
- public static final String CARBON_SECURE_DICTIONARY_SERVER =
- "carbon.secure.dictionary.server";
-
- public static final String CARBON_SECURE_DICTIONARY_SERVER_DEFAULT =
"true";
-
- /**
- * ENABLE_CALCULATE_DATA_INDEX_SIZE
- */
- @CarbonProperty
- public static final String ENABLE_CALCULATE_SIZE =
"carbon.enable.calculate.size";
-
- /**
- * default value of ENABLE_CALCULATE_DATA_INDEX_SIZE
- */
- public static final String DEFAULT_ENABLE_CALCULATE_SIZE = "true";
-
- /**
- * this will be used to skip / ignore empty lines while loading
- */
- @CarbonProperty
- public static final String CARBON_SKIP_EMPTY_LINE =
"carbon.skip.empty.line";
-
- public static final String CARBON_SKIP_EMPTY_LINE_DEFAULT = "false";
-
- /**
- * Currently the segment lock files are not deleted immediately when
unlock,
- * this value indicates the number of hours the segment lock files will
be preserved.
- */
- @CarbonProperty
- public static final String CARBON_SEGMENT_LOCK_FILES_PRESERVE_HOURS =
- "carbon.segment.lock.files.preserve.hours";
-
- /**
- * default value is 2 days
- */
- public static final String
CARBON_SEGMENT_LOCK_FILES_PRESERVE_HOURS_DEFAULT = "48";
-
- /**
- * The number of invisible segment info which will be preserved in
tablestatus file,
- * if it exceeds this value, they will be removed and write to
tablestatus.history file.
- */
- @CarbonProperty
- public static final String CARBON_INVISIBLE_SEGMENTS_PRESERVE_COUNT =
- "carbon.invisible.segments.preserve.count";
-
- /**
- * default value is 200, it means that it will preserve 200 invisible
segment info
- * in tablestatus file.
- * The size of one segment info is about 500 bytes, so the size of
tablestatus file
- * will remain at 100KB.
- */
- public static final String
CARBON_INVISIBLE_SEGMENTS_PRESERVE_COUNT_DEFAULT = "200";
-
- /**
- * System older location to store system level data like datamap schema
and status files.
- */
- @CarbonProperty
- public static final String CARBON_SYSTEM_FOLDER_LOCATION =
"carbon.system.folder.location";
-
- /**
- * It is internal configuration and used only for test purpose.
- * It will merge the carbon index files with in the segment to single
segment.
- */
- @CarbonProperty
- public static final String CARBON_MERGE_INDEX_IN_SEGMENT =
- "carbon.merge.index.in.segment";
-
- public static final String CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT =
"true";
-
- /**
- * property to be used for specifying the max byte limit for
string/varchar data type till
- * where storing min/max in data file will be considered
- */
- @CarbonProperty
- public static final String CARBON_MINMAX_ALLOWED_BYTE_COUNT =
- "carbon.minmax.allowed.byte.count";
-
- public static final String CARBON_MINMAX_ALLOWED_BYTE_COUNT_DEFAULT =
"200";
-
-
//////////////////////////////////////////////////////////////////////////////////////////
- // Table level property start here
-
//////////////////////////////////////////////////////////////////////////////////////////
- // Table level property is the table property for Carbon table
-
- // Flat folder support on table. when it is true all CarbonData files
store directly under table
- // path instead of sub folders.
- public static final String FLAT_FOLDER = "flat_folder";
-
- /**
- * FLAT_FOLDER default value
- */
- public static final String DEFAULT_FLAT_FOLDER = "false";
-
- public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
-
- public static final String DICTIONARY_INCLUDE = "dictionary_include";
-
- public static final String LONG_STRING_COLUMNS = "long_string_columns";
-
- /**
- * Table property to enable or disable local dictionary generation
- */
- public static final String LOCAL_DICTIONARY_ENABLE =
"local_dictionary_enable";
-
- /**
- * default value for local dictionary generation
- */
- public static final String LOCAL_DICTIONARY_ENABLE_DEFAULT = "true";
-
- /**
- * Threshold value for local dictionary
- */
- public static final String LOCAL_DICTIONARY_THRESHOLD =
"local_dictionary_threshold";
-
- /**
- * default value for local dictionary
- */
- public static final String LOCAL_DICTIONARY_THRESHOLD_DEFAULT = "10000";
-
- /**
- * max dictionary threshold
- */
- public static final int LOCAL_DICTIONARY_MAX = 100000;
-
- /**
- * min dictionary threshold
- */
- public static final int LOCAL_DICTIONARY_MIN = 1000;
-
- /**
- * Table property to specify the columns for which local dictionary
needs to be generated.
- */
- public static final String LOCAL_DICTIONARY_INCLUDE =
"local_dictionary_include";
-
- /**
- * Table property to specify the columns for which local dictionary
should not be to be generated.
- */
- public static final String LOCAL_DICTIONARY_EXCLUDE =
"local_dictionary_exclude";
-
- /**
- * DMPROPERTY for Index DataMap, like lucene, bloomfilter DataMap,
- * to indicate a list of column name to be indexed
- */
- public static final String INDEX_COLUMNS = "INDEX_COLUMNS";
-
- /**
- * key for dictionary path
- */
- public static final String DICTIONARY_PATH = "dictionary_path";
- public static final String SORT_COLUMNS = "sort_columns";
- public static final String PARTITION_TYPE = "partition_type";
- public static final String NUM_PARTITIONS = "num_partitions";
- public static final String RANGE_INFO = "range_info";
- public static final String LIST_INFO = "list_info";
- public static final String COLUMN_PROPERTIES = "columnproperties";
- // table block size in MB
- public static final String TABLE_BLOCKSIZE = "table_blocksize";
-
- // default block size in MB
- public static final String TABLE_BLOCK_SIZE_DEFAULT = "1024";
-
- // table blocklet size in MB
- public static final String TABLE_BLOCKLET_SIZE = "table_blocklet_size";
-
- // default blocklet size value in MB
- public static final String TABLE_BLOCKLET_SIZE_DEFAULT = "64";
-
- /**
- * set in column level to disable inverted index
- * @Deprecated :This property is deprecated, it is kept just for
compatibility
- */
- public static final String NO_INVERTED_INDEX = "no_inverted_index";
- // set in column level to enable inverted index
- public static final String INVERTED_INDEX = "inverted_index";
- // table property name of major compaction size
- public static final String TABLE_MAJOR_COMPACTION_SIZE =
"major_compaction_size";
- // table property name of auto load merge
- public static final String TABLE_AUTO_LOAD_MERGE = "auto_load_merge";
- // table property name of compaction level threshold
- public static final String TABLE_COMPACTION_LEVEL_THRESHOLD =
"compaction_level_threshold";
- // table property name of preserve segments numbers while compaction
- public static final String TABLE_COMPACTION_PRESERVE_SEGMENTS =
"compaction_preserve_segments";
- // table property name of allowed compaction days while compaction
- public static final String TABLE_ALLOWED_COMPACTION_DAYS =
"allowed_compaction_days";
-
- /**
- * property to be specified for caching min/max of required columns
- */
- public static final String COLUMN_META_CACHE = "column_meta_cache";
-
- /**
- * property to be specified for caching level (Block/Blocket)
- */
- public static final String CACHE_LEVEL = "cache_level";
-
- /**
- * default value for cache level
- */
- public static final String CACHE_LEVEL_DEFAULT_VALUE = "BLOCK";
-
-
//////////////////////////////////////////////////////////////////////////////////////////
- // Data loading parameter start here
-
//////////////////////////////////////////////////////////////////////////////////////////
-
- /**
- * Number of cores to be used while loading
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String NUM_CORES_LOADING =
"carbon.number.of.cores.while.loading";
-
- /**
- * default value of number of cores to be used
- */
- public static final String NUM_CORES_DEFAULT_VAL = "2";
-
- /**
- * Number of cores to be used while compacting
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String NUM_CORES_COMPACTING =
"carbon.number.of.cores.while.compacting";
-
- /**
- * Number of cores to be used while alter partition
- */
- @CarbonProperty
- public static final String NUM_CORES_ALT_PARTITION =
"carbon.number.of.cores.while.altPartition";
-
- /**
- * BYTEBUFFER_SIZE
- */
- public static final int BYTEBUFFER_SIZE = 24 * 1024;
-
- /**
- * SORT_TEMP_FILE_LOCATION
- */
- public static final String SORT_TEMP_FILE_LOCATION = "sortrowtmp";
-
- /**
- * SORT_INTERMEDIATE_FILES_LIMIT
- */
- @CarbonProperty
- public static final String SORT_INTERMEDIATE_FILES_LIMIT =
"carbon.sort.intermediate.files.limit";
-
- /**
- * SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE
- */
- public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE =
"20";
-
- public static final int SORT_INTERMEDIATE_FILES_LIMIT_MIN = 2;
-
- public static final int SORT_INTERMEDIATE_FILES_LIMIT_MAX = 50;
-
- /**
- * SORT_FILE_WRITE_BUFFER_SIZE
- */
- @CarbonProperty
- public static final String CARBON_SORT_FILE_WRITE_BUFFER_SIZE =
- "carbon.sort.file.write.buffer.size";
-
- /**
- * SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE
- */
- public static final String
CARBON_SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE = "16384";
-
- public static final int CARBON_SORT_FILE_WRITE_BUFFER_SIZE_MIN = 10240;
-
- public static final int CARBON_SORT_FILE_WRITE_BUFFER_SIZE_MAX =
10485760;
-
- /**
- * CSV_READ_BUFFER_SIZE
- */
- @CarbonProperty
- public static final String CSV_READ_BUFFER_SIZE =
"carbon.csv.read.buffersize.byte";
-
- /**
- * CSV_READ_BUFFER_SIZE
- * default value is 1mb
- */
- public static final String CSV_READ_BUFFER_SIZE_DEFAULT = "1048576";
-
- /**
- * min value for csv read buffer size, 10 kb
- */
- public static final int CSV_READ_BUFFER_SIZE_MIN = 10240;
-
- /**
- * max value for csv read buffer size, 10 mb
- */
- public static final int CSV_READ_BUFFER_SIZE_MAX = 10485760;
-
- /**
- * CARBON_MERGE_SORT_READER_THREAD
- */
- @CarbonProperty
- public static final String CARBON_MERGE_SORT_READER_THREAD =
"carbon.merge.sort.reader.thread";
-
- /**
- * CARBON_MERGE_SORT_READER_THREAD DEFAULT value
- */
- public static final String CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE
= "3";
-
- /**
- * TIME_STAT_UTIL_TYPE
- */
- @CarbonProperty
- public static final String ENABLE_DATA_LOADING_STATISTICS =
"enable.data.loading.statistics";
-
- /**
- * TIME_STAT_UTIL_TYPE_DEFAULT
- */
-
- public static final String ENABLE_DATA_LOADING_STATISTICS_DEFAULT =
"false";
-
- /**
- * NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK
- */
- @CarbonProperty
- public static final String NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK =
- "carbon.concurrent.lock.retries";
-
- /**
- * NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK.
- *
- * Because we want concurrent loads to be completed even if they have to
wait for the lock
- * therefore taking the default as 100.
- *
- * Example: Concurrent loads will use this to wait to acquire the table
status lock.
- */
- public static final int NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK_DEFAULT =
100;
-
- /**
- * MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK
- */
- @CarbonProperty
- public static final String MAX_TIMEOUT_FOR_CONCURRENT_LOCK =
- "carbon.concurrent.lock.retry.timeout.sec";
-
- /**
- * MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK
- *
- * Example: Concurrent loads will use this to wait to acquire the table
status lock.
- */
- public static final int MAX_TIMEOUT_FOR_CONCURRENT_LOCK_DEFAULT = 1;
-
- /**
- * NUMBER_OF_TRIES_FOR_CARBON_LOCK
- */
- @CarbonProperty
- public static final String NUMBER_OF_TRIES_FOR_CARBON_LOCK =
- "carbon.lock.retries";
-
- /**
- * NUMBER_OF_TRIES_FOR_CARBON_LOCK
- */
- public static final int NUMBER_OF_TRIES_FOR_CARBON_LOCK_DEFAULT = 3;
-
- /**
- * MAX_TIMEOUT_FOR_CARBON_LOCK
- */
- @CarbonProperty
- public static final String MAX_TIMEOUT_FOR_CARBON_LOCK =
- "carbon.lock.retry.timeout.sec";
-
- /**
- * MAX_TIMEOUT_FOR_CARBON_LOCK
- */
- public static final int MAX_TIMEOUT_FOR_CARBON_LOCK_DEFAULT = 5;
-
- /**
- * CARBON_PREFETCH_BUFFERSIZE
- */
- @CarbonProperty
- public static final String CARBON_PREFETCH_BUFFERSIZE =
"carbon.prefetch.buffersize";
-
- /**
- * CARBON_PREFETCH_BUFFERSIZE DEFAULT VALUE
- */
- public static final String CARBON_PREFETCH_BUFFERSIZE_DEFAULT = "1000";
-
- /**
- * CARBON_PREFETCH_IN_MERGE
- */
- public static final boolean CARBON_PREFETCH_IN_MERGE_VALUE = false;
-
- /**
- * ENABLE_AUTO_LOAD_MERGE
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String ENABLE_AUTO_LOAD_MERGE =
"carbon.enable.auto.load.merge";
-
- /**
- * DEFAULT value of ENABLE_AUTO_LOAD_MERGE
- */
- public static final String DEFAULT_ENABLE_AUTO_LOAD_MERGE = "false";
-
- /**
- * maximum dictionary chunk size that can be kept in memory while
writing dictionary file
- */
- @CarbonProperty
- public static final String DICTIONARY_ONE_CHUNK_SIZE =
"carbon.dictionary.chunk.size";
-
- /**
- * dictionary chunk default size
- */
- public static final String DICTIONARY_ONE_CHUNK_SIZE_DEFAULT = "10000";
-
- /**
- * Dictionary Server Worker Threads
- */
- @CarbonProperty
- public static final String DICTIONARY_WORKER_THREADS =
"dictionary.worker.threads";
-
- /**
- * Dictionary Server Worker Threads
- */
- public static final String DICTIONARY_WORKER_THREADS_DEFAULT = "1";
-
- /**
- * Size of Major Compaction in MBs
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String CARBON_MAJOR_COMPACTION_SIZE =
"carbon.major.compaction.size";
-
- /**
- * By default size of major compaction in MBs.
- */
- public static final String DEFAULT_CARBON_MAJOR_COMPACTION_SIZE = "1024";
-
- /**
- * This property is used to tell how many segments to be preserved from
merging.
- */
- @CarbonProperty
- public static final java.lang.String PRESERVE_LATEST_SEGMENTS_NUMBER =
- "carbon.numberof.preserve.segments";
-
- /**
- * If preserve property is enabled then 2 segments will be preserved.
- */
- public static final String DEFAULT_PRESERVE_LATEST_SEGMENTS_NUMBER = "0";
-
- /**
- * This property will determine the loads of how many days can be
compacted.
- */
- @CarbonProperty
- public static final java.lang.String DAYS_ALLOWED_TO_COMPACT =
"carbon.allowed.compaction.days";
-
- /**
- * Default value of 1 day loads can be compacted
- */
- public static final String DEFAULT_DAYS_ALLOWED_TO_COMPACT = "0";
-
- /**
- * space reserved for writing block meta data in carbon data file
- */
- @CarbonProperty
- public static final String CARBON_BLOCK_META_RESERVED_SPACE =
- "carbon.block.meta.size.reserved.percentage";
-
- /**
- * default value for space reserved for writing block meta data in
carbon data file
- */
- public static final String CARBON_BLOCK_META_RESERVED_SPACE_DEFAULT =
"10";
-
- /**
- * this variable is to enable/disable prefetch of data during merge sort
while
- * reading data from sort temp files
- */
- @CarbonProperty
- public static final String CARBON_MERGE_SORT_PREFETCH =
"carbon.merge.sort.prefetch";
-
- public static final String CARBON_MERGE_SORT_PREFETCH_DEFAULT = "true";
-
- /**
- * If we are executing insert into query from source table using select
statement
- * & loading the same source table concurrently, when select happens on
source table
- * during the data load , it gets new record for which dictionary is not
generated,
- * So there will be inconsistency. To avoid this condition we can
persist the dataframe
- * into MEMORY_AND_DISK and perform insert into operation. By default
this value
- * will be false because no need to persist the dataframe in all cases.
If user want
- * to run load and insert queries on source table concurrently then user
can enable this flag
- */
- @InterfaceStability.Evolving
- @CarbonProperty
- public static final String CARBON_INSERT_PERSIST_ENABLED =
"carbon.insert.persist.enable";
-
- /**
- * by default rdd will not be persisted in the insert case.
- */
- public static final String CARBON_INSERT_PERSIST_ENABLED_DEFAULT =
"false";
-
- /**
- * Which storage level to persist dataset when insert into data
- * with 'carbon.insert.persist.enable'='true'
- */
- @InterfaceStability.Evolving
- @CarbonProperty
- public static final String CARBON_INSERT_STORAGE_LEVEL =
- "carbon.insert.storage.level";
-
- /**
- * The default value(MEMORY_AND_DISK) is the same as the default storage
level of Dataset.
- * Unlike `RDD.cache()`, the default storage level is set to be
`MEMORY_AND_DISK` because
- * recomputing the in-memory columnar representation of the underlying
table is expensive.
- *
- * if user's executor has less memory, set the
CARBON_INSERT_STORAGE_LEVEL
- * to MEMORY_AND_DISK_SER or other storage level to correspond to
different environment.
- * You can get more recommendations about storage level in spark website:
- *
http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence.
- */
- public static final String CARBON_INSERT_STORAGE_LEVEL_DEFAULT =
"MEMORY_AND_DISK";
-
- /**
- * Number of unmerged segments to be merged.
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String COMPACTION_SEGMENT_LEVEL_THRESHOLD =
- "carbon.compaction.level.threshold";
-
- /**
- * Default count for Number of segments to be merged in levels is 4,3
- */
- public static final String DEFAULT_SEGMENT_LEVEL_THRESHOLD = "4,3";
-
- /**
- * Number of Update Delta files which is the Threshold for IUD
compaction.
- * Only accepted Range is 0 - 10000. Outside this range system will pick
default value.
- */
- @CarbonProperty
- public static final String
UPDATE_DELTAFILE_COUNT_THRESHOLD_IUD_COMPACTION =
- "carbon.horizontal.update.compaction.threshold";
-
- /**
- * Default count of segments which act as a threshold for IUD compaction
merge.
- */
- public static final String
DEFAULT_UPDATE_DELTAFILE_COUNT_THRESHOLD_IUD_COMPACTION = "1";
-
- /**
- * Number of Delete Delta files which is the Threshold for IUD
compaction.
- * Only accepted Range is 0 - 10000. Outside this range system will pick
default value.
- */
- @CarbonProperty
- public static final String
DELETE_DELTAFILE_COUNT_THRESHOLD_IUD_COMPACTION =
- "carbon.horizontal.delete.compaction.threshold";
-
- /**
- * Default count of segments which act as a threshold for IUD compaction
merge.
- */
- public static final String
DEFAULT_DELETE_DELTAFILE_COUNT_THRESHOLD_IUD_COMPACTION = "1";
-
- /**
- * @Deprecated : This property has been deprecated.
- * Property for enabling system level compaction lock.1 compaction can
run at once.
- */
- @CarbonProperty
- public static final String ENABLE_CONCURRENT_COMPACTION =
"carbon.concurrent.compaction";
-
- /**
- * Default value of Property for enabling system level compaction lock.1
compaction can run
- * at once.
- */
- public static final String DEFAULT_ENABLE_CONCURRENT_COMPACTION = "true";
-
- /**
- * This batch size is used to send rows from load step to another step
in batches.
- */
- @CarbonProperty
- public static final String DATA_LOAD_BATCH_SIZE = "DATA_LOAD_BATCH_SIZE";
-
- /**
- * Default size of data load batch size.
- */
- public static final String DATA_LOAD_BATCH_SIZE_DEFAULT = "1000";
-
- /**
- * to determine to use the rdd persist or not.
- */
- @InterfaceStability.Evolving
- @CarbonProperty
- public static final String CARBON_UPDATE_PERSIST_ENABLE =
"carbon.update.persist.enable";
-
- /**
- * by default rdd will be persisted in the update case.
- */
- public static final String CARBON_UPDATE_PERSIST_ENABLE_DEFAULT = "true";
-
- /**
- * for enabling or disabling Horizontal Compaction.
- */
- @CarbonProperty
- public static final String CARBON_HORIZONTAL_COMPACTION_ENABLE =
- "carbon.horizontal.compaction.enable";
-
- /**
- * Default value for HorizontalCompaction is true.
- */
- public static final String CARBON_HORIZONTAL_COMPACTION_ENABLE_DEFAULT =
"true";
-
- /**
- * Which storage level to persist dataset when updating data
- * with 'carbon.update.persist.enable'='true'
- */
- @InterfaceStability.Evolving
- @CarbonProperty
- public static final String CARBON_UPDATE_STORAGE_LEVEL =
- "carbon.update.storage.level";
-
- /**
- * The default value(MEMORY_AND_DISK) is the same as the default storage
level of Dataset.
- * Unlike `RDD.cache()`, the default storage level is set to be
`MEMORY_AND_DISK` because
- * recomputing the in-memory columnar representation of the underlying
table is expensive.
- *
- * if user's executor has less memory, set the
CARBON_UPDATE_STORAGE_LEVEL
- * to MEMORY_AND_DISK_SER or other storage level to correspond to
different environment.
- * You can get more recommendations about storage level in spark website:
- *
http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence.
- */
- public static final String CARBON_UPDATE_STORAGE_LEVEL_DEFAULT =
"MEMORY_AND_DISK";
-
- /**
- * to enable unsafe column page
- */
- @CarbonProperty
- public static final String ENABLE_UNSAFE_COLUMN_PAGE =
"enable.unsafe.columnpage";
-
- /**
- * default value of ENABLE_UNSAFE_COLUMN_PAGE
- */
- public static final String ENABLE_UNSAFE_COLUMN_PAGE_DEFAULT = "true";
-
- /**
- * to enable offheap sort
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String ENABLE_UNSAFE_SORT = "enable.unsafe.sort";
-
- /**
- * to enable offheap sort
- */
- public static final String ENABLE_UNSAFE_SORT_DEFAULT = "true";
-
- /**
- * to enable offheap sort
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String ENABLE_OFFHEAP_SORT = "enable.offheap.sort";
-
- /**
- * to enable offheap sort
- */
- public static final String ENABLE_OFFHEAP_SORT_DEFAULT = "true";
-
- @CarbonProperty
- public static final String ENABLE_INMEMORY_MERGE_SORT =
"enable.inmemory.merge.sort";
-
- public static final String ENABLE_INMEMORY_MERGE_SORT_DEFAULT = "false";
-
- @CarbonProperty
- public static final String OFFHEAP_SORT_CHUNK_SIZE_IN_MB =
"offheap.sort.chunk.size.inmb";
-
- public static final String OFFHEAP_SORT_CHUNK_SIZE_IN_MB_DEFAULT = "64";
-
- @CarbonProperty
- public static final String UNSAFE_WORKING_MEMORY_IN_MB =
"carbon.unsafe.working.memory.in.mb";
-
- public static final String UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT = "512";
-
- @CarbonProperty
- public static final String UNSAFE_DRIVER_WORKING_MEMORY_IN_MB =
- "carbon.unsafe.driver.working.memory.in.mb";
-
- /**
- * Sorts the data in batches and writes the batch data to store with
index file.
- */
- @CarbonProperty
- public static final String LOAD_SORT_SCOPE = "carbon.load.sort.scope";
-
- /**
- * If set to BATCH_SORT, the sorting scope is smaller and more index
tree will be created,
- * thus loading is faster but query maybe slower.
- * If set to LOCAL_SORT, the sorting scope is bigger and one index tree
per data node will be
- * created, thus loading is slower but query is faster.
- * If set to GLOBAL_SORT, the sorting scope is bigger and one index tree
per task will be
- * created, thus loading is slower but query is faster.
- */
- public static final String LOAD_SORT_SCOPE_DEFAULT = "LOCAL_SORT";
-
- /**
- * Size of batch data to keep in memory, as a thumb rule it supposed
- * to be less than 45% of sort.inmemory.size.inmb otherwise it may spill
intermediate data to disk
- */
- @CarbonProperty
- public static final String LOAD_BATCH_SORT_SIZE_INMB =
"carbon.load.batch.sort.size.inmb";
-
- public static final String LOAD_BATCH_SORT_SIZE_INMB_DEFAULT = "0";
-
- /**
- * The Number of partitions to use when shuffling data for sort. If user
don't configurate or
- * configurate it less than 1, it uses the number of map tasks as reduce
tasks. In general, we
- * recommend 2-3 tasks per CPU core in your cluster.
- */
- @CarbonProperty
- public static final String LOAD_GLOBAL_SORT_PARTITIONS =
"carbon.load.global.sort.partitions";
-
- public static final String LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT = "0";
-
- /*
- * carbon dictionary server port
- */
- @CarbonProperty
- public static final String DICTIONARY_SERVER_PORT =
"carbon.dictionary.server.port";
-
- /**
- * Default carbon dictionary server port
- */
- public static final String DICTIONARY_SERVER_PORT_DEFAULT = "2030";
- /**
- * whether to prefetch data while loading.
- */
- @CarbonProperty
- public static final String USE_PREFETCH_WHILE_LOADING =
"carbon.loading.prefetch";
-
- /**
- * default value for prefetch data while loading.
- */
- public static final String USE_PREFETCH_WHILE_LOADING_DEFAULT = "false";
-
- /**
- * for loading, whether to use yarn's local dir the main purpose is to
avoid single disk hot spot
- */
- @CarbonProperty
- public static final String CARBON_LOADING_USE_YARN_LOCAL_DIR =
"carbon.use.local.dir";
-
- /**
- * default value for whether to enable carbon use yarn local dir
- */
- public static final String CARBON_LOADING_USE_YARN_LOCAL_DIR_DEFAULT =
"true";
+ public static final Property BLOCKLET_SIZE = Property.buildIntProperty()
+ .key("carbon.blocklet.size")
+ .defaultValue(120000)
+ .minValue(2000)
+ .maxValue(12000000)
+ .doc("blocklet size in carbon file")
+ .build();
/**
- * name of compressor to compress sort temp files
+ * min block size in MB
*/
- @CarbonProperty
- public static final String CARBON_SORT_TEMP_COMPRESSOR =
"carbon.sort.temp.compressor";
+ public static final int BLOCK_SIZE_MIN_VAL = 1;
/**
- * The optional values are 'SNAPPY','GZIP','BZIP2','LZ4','ZSTD' and
empty.
- * Specially, empty means that Carbondata will not compress the sort
temp files.
+ * max block size in MB
*/
- public static final String CARBON_SORT_TEMP_COMPRESSOR_DEFAULT =
"SNAPPY";
+ public static final int BLOCK_SIZE_MAX_VAL = 2048;
- /**
- * Which storage level to persist rdd when sort_scope=global_sort
- */
- @InterfaceStability.Evolving
- @CarbonProperty
- public static final String CARBON_GLOBAL_SORT_RDD_STORAGE_LEVEL =
- "carbon.global.sort.rdd.storage.level";
+ public static final Property CARBON_PROPERTIES_FILE_PATH =
Property.buildStringProperty()
+ .key("carbon.properties.filepath")
+ .defaultValue("../../../conf/carbon.properties")
+ .doc("carbon properties file path")
+ .build();
+
+ public static final Property CARBON_DDL_BASE_HDFS_URL =
Property.buildStringProperty()
+ .key("carbon.ddl.base.hdfs.url")
+ .defaultValue("")
+ .doc("CARBON_DDL_BASE_HDFS_URL")
+ .build();
+
+ public static final Property CARBON_BADRECORDS_LOC =
Property.buildStringProperty()
+ .key("carbon.badRecords.location")
+ .defaultValue("")
+ .doc("CARBON_BADRECORDS_LOCATION")
+ .build();
+
+ public static final Property CARBON_TIMESTAMP_FORMAT =
+ Property.buildStringProperty()
+ .key("carbon.timestamp.format")
+ .defaultValue("yyyy-MM-dd HH:mm:ss")
+ .doc("Property for specifying the format of TIMESTAMP data type
column. " +
+ "e.g. yyyy/MM/dd HH:mm:ss, or using default value")
+ .build();
/**
- * The default value(MEMORY_ONLY) is designed for executors with big
memory, if user's executor
- * has less memory, set the CARBON_GLOBAL_SORT_RDD_STORAGE_LEVEL to
MEMORY_AND_DISK_SER or
- * other storage level to correspond to different environment.
- * You can get more recommendations about storage level in spark website:
- *
http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence.
+ * CARBON_TIMESTAMP
*/
- public static final String CARBON_GLOBAL_SORT_RDD_STORAGE_LEVEL_DEFAULT
= "MEMORY_ONLY";
+ public static final String CARBON_TIMESTAMP = "dd-MM-yyyy HH:mm:ss";
/**
- * property for configuring parallelism per segment when doing an
update. Increase this
- * value will avoid data screw problem for a large segment.
- * Refer to CarbonData-1373 for more details.
+ * CARBON_TIMESTAMP
*/
- @CarbonProperty
- public static final String CARBON_UPDATE_SEGMENT_PARALLELISM =
- "carbon.update.segment.parallelism";
+ public static final String CARBON_TIMESTAMP_MILLIS = "dd-MM-yyyy
HH:mm:ss:SSS";
- /**
- * In default we will not optimize the update
- */
- public static final String CARBON_UPDATE_SEGMENT_PARALLELISM_DEFAULT =
"1";
- /**
- * The property to configure the mdt file folder path, earlier it was
pointing to the
- * fixed carbon store path. This is needed in case of the federation
setup when user removes
- * the fixed store path nameService
- */
- @CarbonProperty
- public static final String CARBON_UPDATE_SYNC_FOLDER =
"carbon.update.sync.folder";
+ public static final Property CARBON_DATE_FORMAT =
Property.buildStringProperty()
+ .key("carbon.date.format")
+ .defaultValue("yyyy-MM-dd")
+ .doc("Property for specifying the format of DATE data type column." +
+ " e.g. yyyy/MM/dd , or using default value")
+ .build();
- public static final String CARBON_UPDATE_SYNC_FOLDER_DEFAULT =
"/tmp/carbondata";
+ public static final Property COMPRESSOR = Property.buildStringProperty()
+ .key("carbon.column.compressor")
+ .defaultValue("snappy")
+ .doc("compressor for writing/reading CarbonData file")
+ .build();
- /**
- * Configures the parser/writer to limit the length of displayed
contents being parsed/written
- * in the exception message when an error occurs.
- * Here {@code 0} means no exceptions will include the content being
manipulated in their
- * attributes.
- */
- public static final int CARBON_ERROR_CONTENT_LENGTH = 0;
+ public static final Property LOCK_TYPE = Property.buildStringProperty()
+ .key("carbon.lock.type")
+ .defaultValue("LOCALLOCK")
+ .doc("ZOOKEEPER_ENABLE_LOCK if this is set to true then zookeeper
will" +
+ " be used to handle locking mechanism of carbon")
+ .build();
- /**
- * if the byte size of streaming segment reach this value,
- * the system will create a new stream segment
- */
- @CarbonProperty
- public static final String HANDOFF_SIZE =
"carbon.streaming.segment.max.size";
+ public static final Property LOCK_PATH = Property.buildStringProperty()
+ .key("carbon.lock.path")
+ .defaultValue("")
+ .doc("Specifies the path where the lock files have to be created. " +
+ "By default, lock files are created in table path.")
+ .build();
/**
- * the default handoff size of streaming segment, the unit is byte
+ * ZOOKEEPER_LOCATION this is the location in zookeeper file system
where locks are created.
+ * mechanism of carbon
*/
- public static final long HANDOFF_SIZE_DEFAULT = 1024L * 1024 * 1024;
+ public static final String ZOOKEEPER_LOCATION = "/CarbonLocks";
--- End diff --
ok, done
---
