[
https://issues.apache.org/jira/browse/CARBONDATA-1914?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Venkata Ramana G resolved CARBONDATA-1914.
------------------------------------------
Resolution: Fixed
Assignee: sounak chakraborty
Fix Version/s: 1.3.0
> Dictionary Cache Access Count Maintenance
> -----------------------------------------
>
> Key: CARBONDATA-1914
> URL: https://issues.apache.org/jira/browse/CARBONDATA-1914
> Project: CarbonData
> Issue Type: Bug
> Reporter: sounak chakraborty
> Assignee: sounak chakraborty
> Fix For: 1.3.0
>
> Time Spent: 3h 20m
> Remaining Estimate: 0h
>
> When dictionary cache is being accessed the access count is being atomicaly
> increased and after the access it has to be decremented. This access count
> prevent it from removal from cache when cache overflow scenario comes into
> picture.
> There are some code places where access is incremented but not decremented
> after usage is completed. This is a resource leak.
> Test Case
> =======
> Precondition
> ----------------
> (CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "1")
> (CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, "1")
> spark.sql("drop table if exists carbon_new6").show(200,false)
> spark.sql("drop table if exists carbon_new7").show(200,false)
> spark.sql("drop table if exists carbon_new8").show(200,false)
> // spark.sql("CREATE TABLE carbon_new6 (CUST_ID INT,CUST_NAME
> STRING,ACTIVE_EMUI_VERSION STRING, DOB TIMESTAMP, DOJ TIMESTAMP,
> BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 decimal(30,10),
> DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2
> double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format'
> TBLPROPERTIES ('dictionary_include'='CUST_NAME')").show(200,false)
> // spark.sql("LOAD DATA INPATH '/home/root1/data_2000.csv' INTO TABLE
> carbon_new6 OPTIONS('DELIMITER'=',' ,
> 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,
> INTEGER_COLUMN1')").show(200,false)
> spark.sql("CREATE TABLE carbon_new7 (CUST_ID INT,CUST_NAME
> STRING,ACTIVE_EMUI_VERSION STRING, DOB TIMESTAMP, DOJ TIMESTAMP,
> BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 decimal(30,10),
> DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2
> double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format'
> TBLPROPERTIES ('dictionary_include'='CUST_NAME,
> ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1,
> Double_COLUMN2')").show(200,false)
> spark.sql("LOAD DATA INPATH '/home/root1/data_2000.csv' INTO TABLE
> carbon_new7 OPTIONS('DELIMITER'=',' ,
> 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,
> INTEGER_COLUMN1')").show(200,false)
> spark.sql("LOAD DATA INPATH '/home/root1/data_2000.csv' INTO TABLE
> carbon_new7 OPTIONS('DELIMITER'=',' ,
> 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,
> INTEGER_COLUMN1')").show(200,false)
> spark.sql("CREATE TABLE carbon_new8 (CUST_ID INT,CUST_NAME
> STRING,ACTIVE_EMUI_VERSION STRING, DOB TIMESTAMP, DOJ TIMESTAMP,
> BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 decimal(30,10),
> DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2
> double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format'
> TBLPROPERTIES ('dictionary_include'='CUST_NAME,
> ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1,
> Double_COLUMN2')").show(200,false)
> spark.sql("LOAD DATA INPATH '/home/root1/data_2000.csv' INTO TABLE
> carbon_new8 OPTIONS('DELIMITER'=',' ,
> 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,
> INTEGER_COLUMN1')").show(200,false)
> spark.sql("LOAD DATA INPATH '/home/root1/data_2000.csv' INTO TABLE
> carbon_new8 OPTIONS('DELIMITER'=',' ,
> 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,
> INTEGER_COLUMN1')").show(200,false)
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
