[jira] [Created] (CARBONDATA-1479) Tree exception displays to user in spark 1.6
manoj mathpal created CARBONDATA-1479:
-
Summary: Tree exception displays to user in spark 1.6
Key: CARBONDATA-1479
URL: https://issues.apache.org/jira/browse/CARBONDATA-1479
Project: CarbonData
Issue Type: Bug
Components: sql
Affects Versions: 1.1.1
Environment: spark 1.6
Reporter: manoj mathpal
Attachments: 2000_UniqData.csv, 3000_1_UniqData.csv,
3000_UniqData.csv, 4000_UniqData.csv, 5000_UniqData.csv, 6000_UniqData.csv,
7000_UniqData.csv
Step to reproduce:
1:Create a table in carbon data :
CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,
DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2
bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2
decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1
int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ("TABLE_BLOCKSIZE"=
"256 MB")
2: Load Data into the table:
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/2000_UniqData_tabdelm.csv'
into table uniqdata OPTIONS('DELIMITER'='/' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/3000_UniqDatatdelm.csv' into
table uniqdata OPTIONS('DELIMITER'='\t' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/5000_UniqData!delm.csv' into
table uniqdata OPTIONS('DELIMITER'='!' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/6000_UniqData|delm.csv' into
table uniqdata OPTIONS('DELIMITER'='|' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/7000_UniqData;delm.csv' into
table uniqdata OPTIONS('DELIMITER'=';' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/3000_1_UniqData.csv' into
table uniqdata OPTIONS('DELIMITER'=',' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
3: Select Query:
select
cust_id,cust_name,to_date(doj),quarter(dob),month(doj),day(dob),hour(doj),minute(dob),second(doj),weekofyear(dob),datediff(doj,current_date),date_add(dob,4),date_sub(doj,1),to_utc_timestamp(doj,current_date),add_months(dob,5),last_day(doj),months_between(doj,current_date),date_format(dob,current_date)
from uniqdata where substr(cust_name,0,4)='CUST' and
length(cust_name)in(15,14,13,16) and ACTIVE_EMUI_VERSION rlike 'ACTIVE' and
month(dob)=01 and minute(dob)=0 group by
doj,cust_id,cust_name,ACTIVE_EMUI_VERSION,dob having max(cust_id)=10830 and
count(distinct(cust_id))<=2001 and max(cust_name) not like '%def%' order by
cust_id,cust_name
Result:
execute, tree: CarbonDictionaryDecoder
[CarbonDecoderRelation(Map(decimal_column1#17584 -> decimal_column1#17584,
dob#17580 -> dob#17580, double_column1#17586 -> double_column1#17586,
cust_name#17578 -> cust_name#17578, integer_column1#17588 ->
integer_column1#17588, bigint_column2#17583L -> bigint_column2#17583L,
bigint_column1#17582L -> bigint_column1#17582L, decimal_column2#17585 ->
decimal_column2#17585, active_emui_version#17579 -> active_emui_version#17579,
double_column2#17587 -> double_column2#17587, cust_id#17577 -> cust_id#17577,
doj#17581 -> doj#17581),CarbonDatasourceRelation(`default`.`uniqdata`,None))],
ExcludeProfile(ArrayBuffer(ACTIVE_EMUI_VERSION#17579, cust_name#17578, #19968,
dob#17580)), CarbonAliasDecoderRelation() +- Sort [cust_id#17577
ASC,cust_name#17578 ASC], true, 0+- ConvertToUnsafe +- Exchange
rangepartitioning(cust_id#17577 ASC,cust_name#17578 ASC,200), None +-
ConvertToSafe +- Project
[cust_id#17577,cust_name#17578,_c2#19946,_c3#19947,_c4#19948,_c5#19949,_c6#19943,_c7#19944,_c8#19945,_c9#19950,_c10#19951,_c11#19952,_c12#19953,_c13#19954,_c14#19955,_c15#19956,_c16#19957,_c17#19958]
+- Filter
[jira] [Created] (CARBONDATA-1069) Compaction is not working
manoj mathpal created CARBONDATA-1069:
-
Summary: Compaction is not working
Key: CARBONDATA-1069
URL: https://issues.apache.org/jira/browse/CARBONDATA-1069
Project: CarbonData
Issue Type: Bug
Components: sql
Affects Versions: 1.1.0
Environment: Spark 2.1
Reporter: manoj mathpal
Reproduce bug
I have checked this query on cluster
Create table:
CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,
DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2
bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2
decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1
int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ("TABLE_BLOCKSIZE"=
"256 MB") ;
Load data into table:
1-LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/2000_UniqData.csv' into
table uniqdata OPTIONS('DELIMITER'=',' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
;
2-LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/3000_UniqData.csv' into
table uniqdata OPTIONS('DELIMITER'=',' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
;
3-LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/4000_UniqData.csv' into
table uniqdata OPTIONS('DELIMITER'=',' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
;
4- LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/5000_UniqData.csv' into
table uniqdata OPTIONS('DELIMITER'=',' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
;
5-LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/5000_UniqData.csv' into
table uniqdata OPTIONS('DELIMITER'=',' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
;
alter table uniqdata compact 'minor'
Result:
Compaction failed. Please check logs for more info. Exception in compaction
java.lang.Exception: Compaction Failure in Merger Rdd.
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
[jira] [Created] (CARBONDATA-1009) Select statement is not working with empty string in where clause.
manoj mathpal created CARBONDATA-1009:
-
Summary: Select statement is not working with empty string in
where clause.
Key: CARBONDATA-1009
URL: https://issues.apache.org/jira/browse/CARBONDATA-1009
Project: CarbonData
Issue Type: Bug
Components: sql
Affects Versions: 1.1.0
Environment: SPARK 2.1
Reporter: manoj mathpal
Priority: Trivial
Steps to reproduces:
1- Create table in hive:
CREATE TABLE uniqdata_h (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION
string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2
bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2
decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1
int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' ;
2:Load data in Hive:
load data local inpath '/home/manoj/Downloads/2000_UniqData.csv' into table
uniqdata_h ;
load data local inpath '/home/manoj/Downloads/4000_UniqData.csv' into table
uniqdata_h ;
load data local inpath '/home/manoj/Downloads/6000_UniqData.csv' into table
uniqdata_h ;
load data local inpath '/home/manoj/Downloads/7000_UniqData.csv' into table
uniqdata_h ;
load data local inpath '/home/manoj/3000_1_UniqData.csv' into table uniqdata_h ;
3: Execute Selet queries:
select
CUST_ID,CUST_NAME,DOB,BIGINT_COLUMN1,DECIMAL_COLUMN1,Double_COLUMN1,INTEGER_COLUMN1
from uniqdata_h where CUST_ID in
('10020','10030','10032','10035','10040','10060','',NULL,' ') and
BIGINT_COLUMN1 in
(123372037874,123372037884,123372037886,123372037889,'',NULL,' ') ;
Result:
+--+--++-+-+--+--+--+
| CUST_ID |CUST_NAME | DOB | BIGINT_COLUMN1 |
DECIMAL_COLUMN1 |Double_COLUMN1| INTEGER_COLUMN1 |
+--+--++-+-+--+--+--+
| 10020| CUST_NAME_01020 | 1972-10-17 01:00:03.0 | 123372037874|
12345679921.123400 | 1.12345674897976E10 | 1021 |
| 10030| CUST_NAME_01030 | 1972-10-27 01:00:03.0 | 123372037884|
12345679931.123400 | 1.12345674897976E10 | 1031 |
| 10032| CUST_NAME_01032 | 1972-10-29 01:00:03.0 | 123372037886|
12345679933.123400 | 1.12345674897976E10 | 1033 |
| 10035| CUST_NAME_01035 | 1972-11-01 01:00:03.0 | 123372037889|
12345679936.123400 | 1.12345674897976E10 | 1036 |
+--+--++-+-+--+--+--+
Create table in Carbondata..
1: create table:
CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,
DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2
bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2
decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1
int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ("TABLE_BLOCKSIZE"=
"256 MB")
2:Load data in table:
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/2000_UniqData_tabdelm.csv'
into table uniqdata OPTIONS('DELIMITER'='/' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
;
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/3000_UniqDatatdelm.csv' into
table uniqdata OPTIONS('DELIMITER'='\t' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
;
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/4000_UniqDataquotedelm.csv'
into table uniqdata OPTIONS('DELIMITER'='"' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
;
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/5000_UniqData!delm.csv' into
table uniqdata OPTIONS('DELIMITER'='!' ,
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
;
LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/6000_UniqData|delm.csv' into
table uniqdata OPTIONS('DELIMITER'='|' ,
