CREATE TABLE `table_abc` ( `col_2` STRING NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_1` BIGINT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_3` BIGINT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_4` BIGINT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_5` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_6` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_7` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_8` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_9` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_10` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_11` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_12` BIGINT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_13` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_14` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_15` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_16` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_17` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_18` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_19` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_20` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_21` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_22` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_23` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_24` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_25` BIGINT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_26` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_27` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_28` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_29` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_30` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_31` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_32` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_33` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_34` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_35` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_36` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_37` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_38` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_39` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_40` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_41` BIGINT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_42` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_43` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_44` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_45` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_46` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_47` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_48` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_49` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_50` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_51` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_52` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_53` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_54` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_55` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_56` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_57` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_58` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_59` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_60` BIGINT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_61` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_62` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_63` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_64` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_65` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_66` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_67` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_68` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_69` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_70` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_71` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_72` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_73` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_74` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_75` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_76` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_77` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_78` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_79` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_80` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_81` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_82` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_83` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_84` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_85` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_86` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_87` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_88` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_89` BIGINT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_90` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_91` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_92` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_93` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_94` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_95` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_96` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_97` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_98` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_99` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_100` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_101` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_102` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_103` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_104` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_105` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_106` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_107` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_108` BIGINT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_109` DOUBLE NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_110` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_111` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_112` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_113` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_114` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_115` STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_116` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, `col_117` INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, PRIMARY KEY (col_2, col_1, col_3, col_4) ) PARTITION BY HASH (col_2) PARTITIONS 4, HASH (col_3) PARTITIONS 64, RANGE (col_1) (PARTITION 1622505600000000000 <= VALUES < 1625097599999999999) STORED AS KUDU TBLPROPERTIES ('kudu.master_addresses'='node-1:8051,node-2:8051,node-3:8051,node-4:8051,node-5:8051', 'kudu.num_tablet_replicas'='3', 'numFiles'='0', 'totalSize'='0') Thank you! From: Quanlong Huang Hi Sumudu, Could you share the DDL of the table, i.e. "show create table default.table_abc"? Just want to check the primary keys. > We have achieved the performance as we expected in Kudu level, however, in Impala we haven’t reached to the performance we expected. The TopN operator in the query also takes a significant amount of time. How do you test the performance in Kudu level? It'd be helpful if you could share the test codes. Another question is about the versions. The profile doesn't show the exact Impala version. What's the kudu client version used in your Impala? What's the version of your kudu cluster? Regards, Quanlong On Mon, May 2, 2022 at 5:31 PM Sumudu Madushanka <ngsmadushanka...@cse.mrt.ac.lk> wrote: > > Hi, > Here is the profile of the query > > > > Thank you! > > > > Best Regards > > Sumudu Madushanka > > > > From: Tim Armstrong > Sent: Friday, April 29, 2022 8:55 PM > To: user@impala.apache.org > Subject: Re: Impala Performance Issue with huge Kudu Data load > > > > Hi Sumudu, > > We might be able to help more if you attach a query profile. You can get it from impala-shell with the "profile" command or from the Impala debug web UI "queries" tab. > > > > - Tim > > > > On Fri, 29 Apr 2022 at 07:07, Sumudu Madushanka <ngsmadushanka...@cse.mrt.ac.lk> wrote: > > Hi Team, > > This is regarding a performance issue me and my team have on a huge data load in Kudu. We are hoping you can guide us on a solution to the below mentioned concerns. > > We have 212 million data loads in Kudu. Currently for such a data load, when loading through impala, 47 seconds are spent for query processing and loading overall. We have used default configurations in Kudu and Impala with 6 node clusters to get these numbers. We have achieved the performance as we expected in Kudu level, however, in Impala we haven’t reached to the performance we expected. > > > What can we do to reduce the time spent for loading 212 million data loads from 47 seconds to 10 seconds through impala? > > We would be much obliged if you can provide us with some solutions. > > Thank You! > > > Best Regards > > Sumudu Madushanka > > |
- Impala Performance Issue with huge Kudu Data load Sumudu Madushanka
- Re: Impala Performance Issue with huge Kudu Data lo... Tim Armstrong
- RE: Impala Performance Issue with huge Kudu Dat... Sumudu Madushanka
- Re: Impala Performance Issue with huge Kudu... Quanlong Huang
- RE: Impala Performance Issue with huge ... Sumudu Madushanka
- Re: Impala Performance Issue with ... Sumudu Madushanka
