[
https://issues.apache.org/jira/browse/IMPALA-13192?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Quanlong Huang updated IMPALA-13192:
------------------------------------
Priority: Critical (was: Major)
> Impala Coordinator stuck and Full GC when execute query from nested temporary
> table.
> ------------------------------------------------------------------------------------
>
> Key: IMPALA-13192
> URL: https://issues.apache.org/jira/browse/IMPALA-13192
> Project: IMPALA
> Issue Type: Bug
> Components: Frontend
> Environment: impalad version 4.3.0-RELEASE RELEASE (build
> 14bb13e67e48742df72f9e1dd73be15ec7ba31bd)
> Reporter: LiuYuan
> Priority: Critical
>
> 1.Create a table as below:
>
> {code:java}
> CREATE TABLE trunck_info (
> user_id BIGINT ,
> truck_length DOUBLE,
> length_type STRING,
> point_km DOUBLE,
> estimate_mileage DOUBLE,
> dep_rate DOUBLE,
> line_day_cnt_01 BIGINT,
> line_ly_cnt_01 BIGINT,
> line_day_cnt_30 BIGINT,
> line_ly_cnt_30 BIGINT,
> line_day_cnt_60 BIGINT,
> line_ly_cnt_60 BIGINT,
> num_all_60 BIGINT,
> num_est_60 BIGINT,
> num_est_order_60 BIGINT,
> num_act_60 BIGINT,
> num_inh_60 BIGINT,
> num_all_30 BIGINT,
> num_est_30 BIGINT,
> num_est_order_30 BIGINT,
> num_act_30 BIGINT,
> num_inh_30 BIGINT,
> conn_num_60 BIGINT,
> conn_num_30 BIGINT,
> hp_num_60 INT,
> hp_num_30 INT,
> bzj_num INT,
> feidan8_num_60 BIGINT,
> feidan8_num_30 INT,
> ts_num_60 BIGINT,
> ts_num_30 INT,
> new_mile_point_60 BIGINT,
> new_mile_point_30 BIGINT
> )
> WITH SERDEPROPERTIES ('serialization.format'='1')
> STORED AS TEXTFILE {code}
>
> 2.Query from nested temporary table, we can see coordinator hung and full gc
>
>
>
> {panel:title=hung.sql}
> with t1
> as
> (
> select user_id
> ,nvl(num_inh_60,0)+nvl(conn_num_60,0)+nvl(new_mile_point_60,0) as
> score_all
> , nvl(conn_num_60,0)+nvl(new_mile_point_60,0) as
> score_noinh
> from trunck_info
> )
> ,t2
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t1
> where score_noinh > 0
> ) a
> )
> ,t3
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t2
> where score_noinh > 0
> ) a
> )
> ,t4
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t3
> where score_noinh > 0
> ) a
> )
> ,t5
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t4
> where score_noinh > 0
> ) a
> )
> ,t6
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t5
> where score_noinh > 0
> ) a
> )
> ,t7
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t6
> where score_noinh > 0
> ) a
> )
> ,t8
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t7
> where score_noinh > 0
> ) a
> )
> ,t9
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t8
> where score_noinh > 0
> ) a
> )
> ,t10
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t9
> where score_noinh > 0
> ) a
> )
> ,t11
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t10
> where score_noinh > 0
> ) a
> )
> ,t12
> as
> (
> select user_id
> ,score_noinh + score_inh as score_all
> ,score_noinh
> from
> (
> select user_id
> ,score_noinh
> ,case when score_all >= 800 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 600 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 450 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all >= 300 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> when score_all > 0 then if(score_all*0.5 >=
> 450,450,score_all*0.5)
> end as score_inh
> from t11
> where score_noinh > 0
> ) a
> )
>
>
> select user_id
> ,score_all
> ,case when score_all >= 800 then 'V6'
> when score_all >= 600 then 'V5'
> when score_all >= 450 then 'V4'
> when score_all >= 300 then 'V3'
> when score_all > 0 then 'V2'
> else 'V1'
> end as tag
> from t12
> order by 1 desc{panel}
> after execute SQL coordinator will be hung and full GC, get jstack:
> {code:java}
> "Thread-24" #62 prio=5 os_prio=0 cpu=16381.52ms elapsed=2102.84s
> tid=0x000000000b8b4800 nid=0x8da9a runnable [0x00007efdf2f16000]
> java.lang.Thread.State: RUNNABLE
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1238)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.FunctionCallExpr.<init>(FunctionCallExpr.java:205)
> at
> org.apache.impala.analysis.FunctionCallExpr.clone(FunctionCallExpr.java:819)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.ArithmeticExpr.<init>(ArithmeticExpr.java:100)
> at
> org.apache.impala.analysis.ArithmeticExpr.clone(ArithmeticExpr.java:296)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at org.apache.impala.analysis.CastExpr.<init>(CastExpr.java:121)
> at org.apache.impala.analysis.CastExpr.clone(CastExpr.java:460)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.ArithmeticExpr.<init>(ArithmeticExpr.java:100)
> at
> org.apache.impala.analysis.ArithmeticExpr.clone(ArithmeticExpr.java:296)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at org.apache.impala.analysis.Predicate.<init>(Predicate.java:51)
> at
> org.apache.impala.analysis.BinaryPredicate.<init>(BinaryPredicate.java:156)
> at
> org.apache.impala.analysis.BinaryPredicate.clone(BinaryPredicate.java:418)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.FunctionCallExpr.<init>(FunctionCallExpr.java:205)
> at
> org.apache.impala.analysis.FunctionCallExpr.clone(FunctionCallExpr.java:819)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at org.apache.impala.analysis.CaseExpr.<init>(CaseExpr.java:169)
> at org.apache.impala.analysis.CaseExpr.clone(CaseExpr.java:457)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.ArithmeticExpr.<init>(ArithmeticExpr.java:100)
> at
> org.apache.impala.analysis.ArithmeticExpr.clone(ArithmeticExpr.java:296)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.ArithmeticExpr.<init>(ArithmeticExpr.java:100)
> at
> org.apache.impala.analysis.ArithmeticExpr.clone(ArithmeticExpr.java:296)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at org.apache.impala.analysis.Predicate.<init>(Predicate.java:51)
> at
> org.apache.impala.analysis.BinaryPredicate.<init>(BinaryPredicate.java:156)
> at
> org.apache.impala.analysis.BinaryPredicate.clone(BinaryPredicate.java:418)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.FunctionCallExpr.<init>(FunctionCallExpr.java:205)
> at
> org.apache.impala.analysis.FunctionCallExpr.clone(FunctionCallExpr.java:819)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at org.apache.impala.analysis.CaseExpr.<init>(CaseExpr.java:169)
> at org.apache.impala.analysis.CaseExpr.clone(CaseExpr.java:457)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.ArithmeticExpr.<init>(ArithmeticExpr.java:100)
> at
> org.apache.impala.analysis.ArithmeticExpr.clone(ArithmeticExpr.java:296)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.ArithmeticExpr.<init>(ArithmeticExpr.java:100)
> at
> org.apache.impala.analysis.ArithmeticExpr.clone(ArithmeticExpr.java:296)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at org.apache.impala.analysis.Predicate.<init>(Predicate.java:51)
> at
> org.apache.impala.analysis.BinaryPredicate.<init>(BinaryPredicate.java:156)
> at
> org.apache.impala.analysis.BinaryPredicate.clone(BinaryPredicate.java:418)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.FunctionCallExpr.<init>(FunctionCallExpr.java:205)
> at
> org.apache.impala.analysis.FunctionCallExpr.clone(FunctionCallExpr.java:819)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at org.apache.impala.analysis.CaseExpr.<init>(CaseExpr.java:169)
> at org.apache.impala.analysis.CaseExpr.clone(CaseExpr.java:457)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.ArithmeticExpr.<init>(ArithmeticExpr.java:100)
> at
> org.apache.impala.analysis.ArithmeticExpr.clone(ArithmeticExpr.java:296)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at org.apache.impala.analysis.Predicate.<init>(Predicate.java:51)
> at
> org.apache.impala.analysis.BinaryPredicate.<init>(BinaryPredicate.java:156)
> at
> org.apache.impala.analysis.BinaryPredicate.clone(BinaryPredicate.java:418)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at org.apache.impala.analysis.CaseExpr.<init>(CaseExpr.java:169)
> at org.apache.impala.analysis.CaseExpr.clone(CaseExpr.java:457)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.ArithmeticExpr.<init>(ArithmeticExpr.java:100)
> at
> org.apache.impala.analysis.ArithmeticExpr.clone(ArithmeticExpr.java:296)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.ArithmeticExpr.<init>(ArithmeticExpr.java:100)
> at
> org.apache.impala.analysis.ArithmeticExpr.clone(ArithmeticExpr.java:296)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at
> org.apache.impala.analysis.FunctionCallExpr.<init>(FunctionCallExpr.java:205)
> at
> org.apache.impala.analysis.FunctionCallExpr.clone(FunctionCallExpr.java:819)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at org.apache.impala.analysis.Expr.<init>(Expr.java:459)
> at org.apache.impala.analysis.CaseExpr.<init>(CaseExpr.java:169)
> at org.apache.impala.analysis.CaseExpr.clone(CaseExpr.java:457)
> at org.apache.impala.analysis.Expr.cloneList(Expr.java:1237)
> at
> org.apache.impala.analysis.ExprSubstitutionMap.clone(ExprSubstitutionMap.java:191)
> at org.apache.impala.analysis.InlineViewRef.<init>(InlineViewRef.java:143)
> at org.apache.impala.analysis.InlineViewRef.clone(InlineViewRef.java:600)
> at org.apache.impala.analysis.FromClause.clone(FromClause.java:167)
> at org.apache.impala.analysis.SelectStmt.<init>(SelectStmt.java:1638)
> at org.apache.impala.analysis.SelectStmt.clone(SelectStmt.java:1733)
> at org.apache.impala.analysis.SelectStmt.clone(SelectStmt.java:68)
> at org.apache.impala.analysis.InlineViewRef.<init>(InlineViewRef.java:113)
> at org.apache.impala.analysis.Analyzer.resolveTableRef(Analyzer.java:935)
> at org.apache.impala.analysis.FromClause.analyze(FromClause.java:90)
> at
> org.apache.impala.analysis.SelectStmt$SelectAnalyzer.analyze(SelectStmt.java:330)
> at
> org.apache.impala.analysis.SelectStmt$SelectAnalyzer.access$100(SelectStmt.java:282)
> at org.apache.impala.analysis.SelectStmt.analyze(SelectStmt.java:274)
> at
> org.apache.impala.analysis.InlineViewRef.analyze(InlineViewRef.java:240)
> at org.apache.impala.analysis.FromClause.analyze(FromClause.java:93)
> at
> org.apache.impala.analysis.SelectStmt$SelectAnalyzer.analyze(SelectStmt.java:330)
> at
> org.apache.impala.analysis.SelectStmt$SelectAnalyzer.access$100(SelectStmt.java:282)
> at org.apache.impala.analysis.SelectStmt.analyze(SelectStmt.java:274)
> at org.apache.impala.analysis.WithClause.analyze(WithClause.java:83)
> at org.apache.impala.analysis.QueryStmt.analyze(QueryStmt.java:168)
> at org.apache.impala.analysis.SelectStmt.analyze(SelectStmt.java:273)
> at
> org.apache.impala.analysis.AnalysisContext.analyze(AnalysisContext.java:530)
> at
> org.apache.impala.analysis.AnalysisContext.analyzeAndAuthorize(AnalysisContext.java:477)
> at
> org.apache.impala.service.Frontend.doCreateExecRequest(Frontend.java:2341)
> at org.apache.impala.service.Frontend.getTExecRequest(Frontend.java:2089)
> at
> org.apache.impala.service.Frontend.createExecRequest(Frontend.java:1860)
> at
> org.apache.impala.service.JniFrontend.createExecRequest(JniFrontend.java:165)
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
