Manish Kumar created SPARK-15666:
------------------------------------
Summary: Join on two tables generated from a same table throwing
query analyzer issue
Key: SPARK-15666
URL: https://issues.apache.org/jira/browse/SPARK-15666
Project: Spark
Issue Type: Bug
Components: Spark Core
Affects Versions: 1.6.1
Environment: AWS EMR
Reporter: Manish Kumar
Priority: Blocker
If two dataframes which are created by performimg some opeartions on a single
dataframe are joined then we are getting some analyzer issue:
leftdataframe schema
{noformat}
root
|-- affinity_monitor_copay: string (nullable = true)
|-- affinity_monitor_digital_pull: string (nullable = true)
|-- affinity_monitor_digital_push: string (nullable = true)
|-- affinity_monitor_direct: string (nullable = true)
|-- affinity_monitor_peer: string (nullable = true)
|-- affinity_monitor_peer_interaction: string (nullable = true)
|-- affinity_monitor_personal_f2f: string (nullable = true)
|-- affinity_monitor_personal_remote: string (nullable = true)
|-- affinity_monitor_sample: string (nullable = true)
|-- affinity_monitor_voucher: string (nullable = true)
|-- afltn_id: string (nullable = true)
|-- attribute_2_value: string (nullable = true)
|-- brand: string (nullable = true)
|-- city: string (nullable = true)
|-- cycle_time_id: integer (nullable = true)
|-- full_name: string (nullable = true)
|-- hcp: string (nullable = true)
|-- like17_mg17_metric114_aggregated: double (nullable = true)
|-- like17_mg17_metric118_aggregated: double (nullable = true)
|-- metric_group_sk: integer (nullable = true)
|-- metrics: array (nullable = true)
| |-- element: struct (containsNull = true)
| | |-- hcp: string (nullable = true)
| | |-- brand: string (nullable = true)
| | |-- rep: string (nullable = true)
| | |-- month: string (nullable = true)
| | |-- metric117: string (nullable = true)
| | |-- metric114: string (nullable = true)
| | |-- metric118: string (nullable = true)
| | |-- specialty_1: string (nullable = true)
| | |-- full_name: string (nullable = true)
| | |-- pri_st: string (nullable = true)
| | |-- city: string (nullable = true)
| | |-- zip_code: string (nullable = true)
| | |-- prsn_id: string (nullable = true)
| | |-- afltn_id: string (nullable = true)
| | |-- npi_id: string (nullable = true)
| | |-- affinity_monitor_sample: string (nullable = true)
| | |-- affinity_monitor_personal_f2f: string (nullable = true)
| | |-- affinity_monitor_peer: string (nullable = true)
| | |-- affinity_monitor_copay: string (nullable = true)
| | |-- affinity_monitor_digital_push: string (nullable = true)
| | |-- affinity_monitor_voucher: string (nullable = true)
| | |-- affinity_monitor_direct: string (nullable = true)
| | |-- affinity_monitor_peer_interaction: string (nullable = true)
| | |-- affinity_monitor_digital_pull: string (nullable = true)
| | |-- affinity_monitor_personal_remote: string (nullable = true)
| | |-- attribute_2_value: string (nullable = true)
| | |-- metric211: double (nullable = false)
|-- mg17_metric117_3: double (nullable = true)
|-- mg17_metric117_3_actual_metric: double (nullable = true)
|-- mg17_metric117_3_planned_metric: double (nullable = true)
|-- mg17_metric117_D_suggestion_id: integer (nullable = true)
|-- mg17_metric117_D_suggestion_text: string (nullable = true)
|-- mg17_metric117_D_suggestion_text_raw: string (nullable = true)
|-- mg17_metric117_exp_score: integer (nullable = true)
|-- mg17_metric117_severity_index: double (nullable = true)
|-- mg17_metric117_test: integer (nullable = true)
|-- mg17_metric211_P_suggestion_id: integer (nullable = true)
|-- mg17_metric211_P_suggestion_text: string (nullable = true)
|-- mg17_metric211_P_suggestion_text_raw: string (nullable = true)
|-- mg17_metric211_aggregated: double (nullable = false)
|-- mg17_metric211_deviationfrompeers_p_value: double (nullable = true)
|-- mg17_metric211_deviationfromtrend_current_mu: double (nullable = true)
|-- mg17_metric211_deviationfromtrend_p_value: double (nullable = true)
|-- mg17_metric211_deviationfromtrend_previous_mu: double (nullable = true)
|-- mg17_metric211_exp_score: integer (nullable = true)
|-- mg17_metric211_mahal: double (nullable = true)
|-- mg17_metric211_segment_mu: double (nullable = true)
|-- mg17_metric211_segment_stddev: double (nullable = false)
|-- mg17_metric211_self_test_type: string (nullable = true)
|-- mg17_metric211_severity_index: double (nullable = true)
|-- mg17_metric211_test: integer (nullable = true)
|-- mg17_metric211_threshold: double (nullable = true)
|-- npi_id: string (nullable = true)
|-- pri_st: string (nullable = true)
|-- prsn_id: string (nullable = true)
|-- rep: string (nullable = true)
|-- seg13_1_pri_st: string (nullable = true)
|-- seg13_2_specialty_1: string (nullable = true)
|-- specialty_1: string (nullable = true)
{noformat}
leftdataframe queryplan
{noformat}
== Physical Plan ==
Union
:- Project
[affinity_monitor_copay#308,affinity_monitor_digital_pull#314,affinity_monitor_digital_push#302,affinity_monitor_direct#295,affinity_monitor_peer#309,affinity_monitor_peer_interaction#311,affinity_monitor_personal_f2f#317,affinity_monitor_personal_remote#300,affinity_monitor_sample#310,affinity_monitor_voucher#294,afltn_id#306,attribute_2_value#322,brand#320,city#304,cycle_time_id#318,full_name#296,hcp#312,like17_mg17_metric114_aggregated#316,like17_mg17_metric118_aggregated#297,metric_group_sk#293,metrics#303,null
AS mg17_metric117_3#658,null AS mg17_metric117_3_actual_metric#659,null AS
mg17_metric117_3_planned_metric#660,CASE WHEN true THEN -1 AS
mg17_metric117_D_suggestion_id#450,UDF(mg17_metric211_segment_mu#313) AS
mg17_metric117_D_suggestion_text#451,null AS
mg17_metric117_D_suggestion_text_raw#661,CASE WHEN true THEN -2 AS
mg17_metric117_exp_score#453,CASE WHEN true THEN -2.0 AS
mg17_metric117_severity_index#452,CASE WHEN true THEN 3 AS
mg17_metric117_test#449,mg17_metric211_P_suggestion_id#332,mg17_metric211_P_suggestion_text#334,mg17_metric211_P_suggestion_text_raw#333,mg17_metric211_aggregated#323,mg17_metric211_deviationfrompeers_p_value#383,mg17_metric211_deviationfromtrend_current_mu#324,mg17_metric211_deviationfromtrend_p_value#326,mg17_metric211_deviationfromtrend_previous_mu#325,mg17_metric211_exp_score#331,mg17_metric211_mahal#384,mg17_metric211_segment_mu#313,mg17_metric211_segment_stddev#315,mg17_metric211_self_test_type#328,mg17_metric211_severity_index#329,mg17_metric211_test#330,mg17_metric211_threshold#327,npi_id#301,pri_st#319,prsn_id#299,rep#307,seg13_1_pri_st#321,seg13_2_specialty_1#298,specialty_1#305]
: +- SortMergeJoin [hcp#312], [hcp#426]
: :- Sort [hcp#312 ASC], false, 0
: : +- TungstenExchange hashpartitioning(hcp#312,200), None
: : +- Union
: : :- Project
[affinity_monitor_copay#308,affinity_monitor_digital_pull#314,affinity_monitor_digital_push#302,affinity_monitor_direct#295,affinity_monitor_peer#309,affinity_monitor_peer_interaction#311,affinity_monitor_personal_f2f#317,affinity_monitor_personal_remote#300,affinity_monitor_sample#310,affinity_monitor_voucher#294,afltn_id#306,attribute_2_value#322,brand#320,city#304,cycle_time_id#318,full_name#296,hcp#312,like17_mg17_metric114_aggregated#316,like17_mg17_metric118_aggregated#297,metric_group_sk#293,metrics#303,mg17_metric211_P_suggestion_id#332,mg17_metric211_P_suggestion_text#334,mg17_metric211_P_suggestion_text_raw#333,mg17_metric211_aggregated#323,null
AS
mg17_metric211_deviationfrompeers_p_value#383,mg17_metric211_deviationfromtrend_current_mu#324,mg17_metric211_deviationfromtrend_p_value#326,mg17_metric211_deviationfromtrend_previous_mu#325,mg17_metric211_exp_score#331,null
AS
mg17_metric211_mahal#384,mg17_metric211_segment_mu#313,mg17_metric211_segment_stddev#315,mg17_metric211_self_test_type#328,mg17_metric211_severity_index#329,mg17_metric211_test#330,mg17_metric211_threshold#327,npi_id#301,pri_st#319,prsn_id#299,rep#307,seg13_1_pri_st#321,seg13_2_specialty_1#298,specialty_1#305]
: : : +- Filter (cast(rep#307 as double) = -2.0)
: : : +- Scan
ExistingRDD[metric_group_sk#293,affinity_monitor_voucher#294,affinity_monitor_direct#295,full_name#296,like17_mg17_metric118_aggregated#297,seg13_2_specialty_1#298,prsn_id#299,affinity_monitor_personal_remote#300,npi_id#301,affinity_monitor_digital_push#302,metrics#303,city#304,specialty_1#305,afltn_id#306,rep#307,affinity_monitor_copay#308,affinity_monitor_peer#309,affinity_monitor_sample#310,affinity_monitor_peer_interaction#311,hcp#312,mg17_metric211_segment_mu#313,affinity_monitor_digital_pull#314,mg17_metric211_segment_stddev#315,like17_mg17_metric114_aggregated#316,affinity_monitor_personal_f2f#317,cycle_time_id#318,pri_st#319,brand#320,seg13_1_pri_st#321,attribute_2_value#322,mg17_metric211_aggregated#323,mg17_metric211_deviationfromtrend_current_mu#324,mg17_metric211_deviationfromtrend_previous_mu#325,mg17_metric211_deviationfromtrend_p_value#326,mg17_metric211_threshold#327,mg17_metric211_self_test_type#328,mg17_metric211_severity_index#329,mg17_metric211_test#330,mg17_metric211_exp_score#331,mg17_metric211_P_suggestion_id#332,mg17_metric211_P_suggestion_text_raw#333,mg17_metric211_P_suggestion_text#334]
: : :- Project
[affinity_monitor_copay#352,affinity_monitor_digital_pull#358,affinity_monitor_digital_push#346,affinity_monitor_direct#339,affinity_monitor_peer#353,affinity_monitor_peer_interaction#355,affinity_monitor_personal_f2f#361,affinity_monitor_personal_remote#344,affinity_monitor_sample#354,affinity_monitor_voucher#338,afltn_id#350,attribute_2_value#366,brand#364,city#348,cycle_time_id#362,full_name#340,hcp#356,like17_mg17_metric114_aggregated#360,like17_mg17_metric118_aggregated#341,metric_group_sk#337,metrics#347,CASE
WHEN ((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = -1)) THEN CASE WHEN true THEN 110 WHEN
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = 1)) THEN CASE WHEN true THEN 109 ELSE CASE WHEN true THEN -1 AS
mg17_metric211_P_suggestion_id#378,CASE WHEN
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = -1)) THEN CASE WHEN true THEN CIALIS percent split in managed
care advantage plan is lower as compared to peer group. WHEN
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = 1)) THEN CASE WHEN true THEN CIALIS percent split in managed care
advantage plan is higher as compared to peer group. ELSE CASE WHEN true THEN
AS mg17_metric211_P_suggestion_text#380,CASE WHEN
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = -1)) THEN CASE WHEN true THEN CIALIS percent split in managed
care advantage plan is lower as compared to peer group. WHEN
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = 1)) THEN CASE WHEN true THEN CIALIS percent split in managed care
advantage plan is higher as compared to peer group. ELSE CASE WHEN true THEN
AS
mg17_metric211_P_suggestion_text_raw#379,mg17_metric211_aggregated#367,mg17_metric211_deviationfrompeers_p_value#375,mg17_metric211_deviationfromtrend_current_mu#368,mg17_metric211_deviationfromtrend_p_value#370,mg17_metric211_deviationfromtrend_previous_mu#369,CASE
WHEN (mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 AS
mg17_metric211_exp_score#377,mg17_metric211_mahal#374,mg17_metric211_segment_mu#357,mg17_metric211_segment_stddev#359,mg17_metric211_self_test_type#372,CASE
WHEN (mg17_metric211_deviationfrompeers_p_value#375 > 0.9) THEN
(mg17_metric211_deviationfrompeers_p_value#375 - 0.9) ELSE CASE WHEN true THEN
0.0 AS
mg17_metric211_severity_index#376,mg17_metric211_test#373,mg17_metric211_threshold#371,npi_id#345,pri_st#363,prsn_id#343,rep#351,seg13_1_pri_st#365,seg13_2_specialty_1#342,specialty_1#349]
: : : +- Filter (((mg17_metric211_deviationfrompeers_p_value#375 >
0.9) && (cast(rep#351 as double) = -2.0)) && (cast(rep#351 as double) = -2.0))
: : : +- Scan
ExistingRDD[metric_group_sk#337,affinity_monitor_voucher#338,affinity_monitor_direct#339,full_name#340,like17_mg17_metric118_aggregated#341,seg13_2_specialty_1#342,prsn_id#343,affinity_monitor_personal_remote#344,npi_id#345,affinity_monitor_digital_push#346,metrics#347,city#348,specialty_1#349,afltn_id#350,rep#351,affinity_monitor_copay#352,affinity_monitor_peer#353,affinity_monitor_sample#354,affinity_monitor_peer_interaction#355,hcp#356,mg17_metric211_segment_mu#357,affinity_monitor_digital_pull#358,mg17_metric211_segment_stddev#359,like17_mg17_metric114_aggregated#360,affinity_monitor_personal_f2f#361,cycle_time_id#362,pri_st#363,brand#364,seg13_1_pri_st#365,attribute_2_value#366,mg17_metric211_aggregated#367,mg17_metric211_deviationfromtrend_current_mu#368,mg17_metric211_deviationfromtrend_previous_mu#369,mg17_metric211_deviationfromtrend_p_value#370,mg17_metric211_threshold#371,mg17_metric211_self_test_type#372,mg17_metric211_test#373,mg17_metric211_mahal#374,mg17_metric211_deviationfrompeers_p_value#375]
: : +- Project
[affinity_monitor_copay#89,affinity_monitor_digital_pull#94,affinity_monitor_digital_push#90,affinity_monitor_direct#92,affinity_monitor_peer#88,affinity_monitor_peer_interaction#93,affinity_monitor_personal_f2f#87,affinity_monitor_personal_remote#95,affinity_monitor_sample#86,affinity_monitor_voucher#91,afltn_id#84,attribute_2_value#96,brand#77,city#82,cycle_time_id#99,full_name#80,hcp#76,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,metric_group_sk#98,metrics#97,CASE
WHEN true THEN -1 AS
mg17_metric211_P_suggestion_id#385,UDF(metric_group_sk#98) AS
mg17_metric211_P_suggestion_text#386,null AS
mg17_metric211_P_suggestion_text_raw#398,mg17_metric211_aggregated#102,null AS
mg17_metric211_deviationfrompeers_p_value#399,null AS
mg17_metric211_deviationfromtrend_current_mu#400,null AS
mg17_metric211_deviationfromtrend_p_value#401,null AS
mg17_metric211_deviationfromtrend_previous_mu#402,CASE WHEN true THEN -2 AS
mg17_metric211_exp_score#388,null AS
mg17_metric211_mahal#403,mg17_metric211_segment_mu#111,mg17_metric211_segment_stddev#112,null
AS mg17_metric211_self_test_type#404,CASE WHEN true THEN -2.0 AS
mg17_metric211_severity_index#387,null AS mg17_metric211_test#405,null AS
mg17_metric211_threshold#406,npi_id#85,pri_st#81,prsn_id#83,rep#78,seg13_1_pri_st#103,seg13_2_specialty_1#104,specialty_1#79]
: : +- SortMergeJoin
[seg13_1_pri_st#103,seg13_2_specialty_1#104],
[seg13_1_pri_st#135,seg13_2_specialty_1#123]
: : :- Sort [seg13_1_pri_st#103 ASC,seg13_2_specialty_1#104
ASC], false, 0
: : : +- TungstenExchange
hashpartitioning(seg13_1_pri_st#103,seg13_2_specialty_1#104,200), None
: : : +- Union
: : : :- Project
[affinity_monitor_copay#89,affinity_monitor_digital_pull#94,affinity_monitor_digital_push#90,affinity_monitor_direct#92,affinity_monitor_peer#88,affinity_monitor_peer_interaction#93,affinity_monitor_personal_f2f#87,affinity_monitor_personal_remote#95,affinity_monitor_sample#86,affinity_monitor_voucher#91,afltn_id#84,attribute_2_value#96,brand#77,city#82,cycle_time_id#99,full_name#80,hcp#76,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,metric_group_sk#98,metrics#97,coalesce((like17_mg17_metric114_aggregated#100
/ like17_mg17_metric118_aggregated#101),0.0) AS
mg17_metric211_aggregated#102,npi_id#85,pri_st#81,prsn_id#83,rep#78,UDF(hcp#76)
AS seg13_1_pri_st#103,UDF(hcp#76) AS seg13_2_specialty_1#104,specialty_1#79]
: : : : +- Filter ((NOT (cast(rep#78 as double) =
-2.0) && (cast(rep#78 as double) = -2.0)) && (cast(rep#78 as double) = -2.0))
: : : : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: : : +- Project
[affinity_monitor_copay#89,affinity_monitor_digital_pull#94,affinity_monitor_digital_push#90,affinity_monitor_direct#92,affinity_monitor_peer#88,affinity_monitor_peer_interaction#93,affinity_monitor_personal_f2f#87,affinity_monitor_personal_remote#95,affinity_monitor_sample#86,affinity_monitor_voucher#91,afltn_id#84,attribute_2_value#96,brand#77,city#82,cycle_time_id#99,full_name#80,hcp#76,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,metric_group_sk#98,metrics#97,mg17_metric211_aggregated#102,npi_id#85,pri_st#81,prsn_id#83,rep#78,cast(seg13_1_pri_st#105L
as string) AS seg13_1_pri_st#109,cast(cast(_we0#108 as bigint) as string) AS
seg13_2_specialty_1#110,specialty_1#79]
: : : +- Filter ((NOT (cast(rep#78 as double) =
-2.0) && (cast(rep#78 as double) = -2.0)) && (cast(rep#78 as double) = -2.0))
: : : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,seg13_1_pri_st#105L],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(specialty_1#79)
windowspecdefinition(seg13_1_pri_st#105L,specialty_1#79 ASC,ROWS BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS _we0#108],
[seg13_1_pri_st#105L], [specialty_1#79 ASC]
: : : +- Sort [seg13_1_pri_st#105L
ASC,specialty_1#79 ASC], false, 0
: : : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,cast(_we0#106
as bigint) AS seg13_1_pri_st#105L]
: : : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(pri_st#81)
windowspecdefinition(pri_st#81 ASC,ROWS BETWEEN UNBOUNDED PRECEDING AND
UNBOUNDED FOLLOWING) AS _we0#106], [pri_st#81 ASC]
: : : +- Sort [pri_st#81 ASC], false, 0
: : : +- TungstenExchange
SinglePartition, None
: : : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,coalesce((like17_mg17_metric114_aggregated#100
/ like17_mg17_metric118_aggregated#101),0.0) AS mg17_metric211_aggregated#102]
: : : +- Filter (cast(rep#78
as double) = -2.0)
: : : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: : +- Sort [seg13_1_pri_st#135 ASC,seg13_2_specialty_1#123
ASC], false, 0
: : +-
TungstenAggregate(key=[seg13_1_pri_st#135,seg13_2_specialty_1#123],
functions=[(avg(mg17_metric211_aggregated#102),mode=Final,isDistinct=false),(stddev_samp(mg17_metric211_aggregated#102,0,0),mode=Final,isDistinct=false)],
output=[seg13_1_pri_st#135,seg13_2_specialty_1#123,mg17_metric211_segment_mu#111,mg17_metric211_segment_stddev#112])
: : +- TungstenExchange
hashpartitioning(seg13_1_pri_st#135,seg13_2_specialty_1#123,200), None
: : +-
TungstenAggregate(key=[seg13_1_pri_st#135,seg13_2_specialty_1#123],
functions=[(avg(mg17_metric211_aggregated#102),mode=Partial,isDistinct=false),(stddev_samp(mg17_metric211_aggregated#102,0,0),mode=Partial,isDistinct=false)],
output=[seg13_1_pri_st#135,seg13_2_specialty_1#123,sum#140,count#141L,M0#130,M1#131,M2#132,M3#133,M4#134])
: : +- Union
: : :- Project [UDF(hcp#76) AS
seg13_1_pri_st#135,UDF(hcp#76) AS
seg13_2_specialty_1#123,coalesce((like17_mg17_metric114_aggregated#100 /
like17_mg17_metric118_aggregated#101),0.0) AS mg17_metric211_aggregated#102]
: : : +- Filter NOT (cast(rep#78 as double) =
-2.0)
: : : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: : +- Project [cast(seg13_1_pri_st#105L as
string) AS seg13_1_pri_st#109,cast(cast(_we0#108 as bigint) as string) AS
seg13_2_specialty_1#110,mg17_metric211_aggregated#102]
: : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,seg13_1_pri_st#105L],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(specialty_1#79)
windowspecdefinition(seg13_1_pri_st#105L,specialty_1#79 ASC,ROWS BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS _we0#108],
[seg13_1_pri_st#105L], [specialty_1#79 ASC]
: : +- Sort [seg13_1_pri_st#105L
ASC,specialty_1#79 ASC], false, 0
: : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,cast(_we0#106
as bigint) AS seg13_1_pri_st#105L]
: : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(pri_st#81)
windowspecdefinition(pri_st#81 ASC,ROWS BETWEEN UNBOUNDED PRECEDING AND
UNBOUNDED FOLLOWING) AS _we0#106], [pri_st#81 ASC]
: : +- Sort [pri_st#81 ASC],
false, 0
: : +- TungstenExchange
SinglePartition, None
: : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,coalesce((like17_mg17_metric114_aggregated#100
/ like17_mg17_metric118_aggregated#101),0.0) AS mg17_metric211_aggregated#102]
: : +- Filter
(cast(rep#78 as double) = -2.0)
: : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: +- Sort [hcp#426 ASC], false, 0
: +- TungstenAggregate(key=[hcp#426], functions=[], output=[hcp#426])
: +- TungstenExchange hashpartitioning(hcp#426,200), None
: +- TungstenAggregate(key=[hcp#426], functions=[],
output=[hcp#426])
: +- Union
: :- Project [hcp#426]
: : +- Filter (cast(rep#421 as double) = -2.0)
: : +- Scan
ExistingRDD[metric_group_sk#407,affinity_monitor_voucher#408,affinity_monitor_direct#409,full_name#410,like17_mg17_metric118_aggregated#411,seg13_2_specialty_1#412,prsn_id#413,affinity_monitor_personal_remote#414,npi_id#415,affinity_monitor_digital_push#416,metrics#417,city#418,specialty_1#419,afltn_id#420,rep#421,affinity_monitor_copay#422,affinity_monitor_peer#423,affinity_monitor_sample#424,affinity_monitor_peer_interaction#425,hcp#426,mg17_metric211_segment_mu#427,affinity_monitor_digital_pull#428,mg17_metric211_segment_stddev#429,like17_mg17_metric114_aggregated#430,affinity_monitor_personal_f2f#431,cycle_time_id#432,pri_st#433,brand#434,seg13_1_pri_st#435,attribute_2_value#436,mg17_metric211_aggregated#437,mg17_metric211_deviationfromtrend_current_mu#438,mg17_metric211_deviationfromtrend_previous_mu#439,mg17_metric211_deviationfromtrend_p_value#440,mg17_metric211_threshold#441,mg17_metric211_self_test_type#442,mg17_metric211_severity_index#443,mg17_metric211_test#444,mg17_metric211_exp_score#445,mg17_metric211_P_suggestion_id#446,mg17_metric211_P_suggestion_text_raw#447,mg17_metric211_P_suggestion_text#448]
: :- Project [hcp#356]
: : +- Filter
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (cast(rep#351 as
double) = -2.0))
: : +- Scan
ExistingRDD[metric_group_sk#337,affinity_monitor_voucher#338,affinity_monitor_direct#339,full_name#340,like17_mg17_metric118_aggregated#341,seg13_2_specialty_1#342,prsn_id#343,affinity_monitor_personal_remote#344,npi_id#345,affinity_monitor_digital_push#346,metrics#347,city#348,specialty_1#349,afltn_id#350,rep#351,affinity_monitor_copay#352,affinity_monitor_peer#353,affinity_monitor_sample#354,affinity_monitor_peer_interaction#355,hcp#356,mg17_metric211_segment_mu#357,affinity_monitor_digital_pull#358,mg17_metric211_segment_stddev#359,like17_mg17_metric114_aggregated#360,affinity_monitor_personal_f2f#361,cycle_time_id#362,pri_st#363,brand#364,seg13_1_pri_st#365,attribute_2_value#366,mg17_metric211_aggregated#367,mg17_metric211_deviationfromtrend_current_mu#368,mg17_metric211_deviationfromtrend_previous_mu#369,mg17_metric211_deviationfromtrend_p_value#370,mg17_metric211_threshold#371,mg17_metric211_self_test_type#372,mg17_metric211_test#373,mg17_metric211_mahal#374,mg17_metric211_deviationfrompeers_p_value#375]
: +- Project [hcp#76]
: +- SortMergeJoin
[seg13_1_pri_st#103,seg13_2_specialty_1#104],
[seg13_1_pri_st#135,seg13_2_specialty_1#123]
: :- Sort [seg13_1_pri_st#103
ASC,seg13_2_specialty_1#104 ASC], false, 0
: : +- TungstenExchange
hashpartitioning(seg13_1_pri_st#103,seg13_2_specialty_1#104,200), None
: : +- Union
: : :- Project [UDF(hcp#76) AS
seg13_1_pri_st#103,hcp#76,UDF(hcp#76) AS seg13_2_specialty_1#104]
: : : +- Filter (NOT (cast(rep#78 as double) =
-2.0) && (cast(rep#78 as double) = -2.0))
: : : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: : +- Project [cast(seg13_1_pri_st#105L as
string) AS seg13_1_pri_st#109,hcp#76,cast(cast(_we0#108 as bigint) as string)
AS seg13_2_specialty_1#110]
: : +- Filter (NOT (cast(rep#78 as double) =
-2.0) && (cast(rep#78 as double) = -2.0))
: : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,seg13_1_pri_st#105L],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(specialty_1#79)
windowspecdefinition(seg13_1_pri_st#105L,specialty_1#79 ASC,ROWS BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS _we0#108],
[seg13_1_pri_st#105L], [specialty_1#79 ASC]
: : +- Sort [seg13_1_pri_st#105L
ASC,specialty_1#79 ASC], false, 0
: : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,cast(_we0#106
as bigint) AS seg13_1_pri_st#105L]
: : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(pri_st#81)
windowspecdefinition(pri_st#81 ASC,ROWS BETWEEN UNBOUNDED PRECEDING AND
UNBOUNDED FOLLOWING) AS _we0#106], [pri_st#81 ASC]
: : +- Sort [pri_st#81 ASC],
false, 0
: : +- TungstenExchange
SinglePartition, None
: : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,coalesce((like17_mg17_metric114_aggregated#100
/ like17_mg17_metric118_aggregated#101),0.0) AS mg17_metric211_aggregated#102]
: : +- Filter
(cast(rep#78 as double) = -2.0)
: : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: +- Sort [seg13_1_pri_st#135
ASC,seg13_2_specialty_1#123 ASC], false, 0
: +- Project
[seg13_2_specialty_1#123,seg13_1_pri_st#135]
: +-
TungstenAggregate(key=[seg13_1_pri_st#135,seg13_2_specialty_1#123],
functions=[], output=[seg13_1_pri_st#135,seg13_2_specialty_1#123])
: +- TungstenExchange
hashpartitioning(seg13_1_pri_st#135,seg13_2_specialty_1#123,200), None
: +-
TungstenAggregate(key=[seg13_1_pri_st#135,seg13_2_specialty_1#123],
functions=[], output=[seg13_1_pri_st#135,seg13_2_specialty_1#123])
: +- Union
: :- Project [UDF(hcp#76) AS
seg13_1_pri_st#135,UDF(hcp#76) AS seg13_2_specialty_1#123]
: : +- Filter NOT (cast(rep#78 as
double) = -2.0)
: : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: +- Project
[cast(seg13_1_pri_st#105L as string) AS seg13_1_pri_st#109,cast(cast(_we0#108
as bigint) as string) AS seg13_2_specialty_1#110]
: +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,seg13_1_pri_st#105L],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(specialty_1#79)
windowspecdefinition(seg13_1_pri_st#105L,specialty_1#79 ASC,ROWS BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS _we0#108],
[seg13_1_pri_st#105L], [specialty_1#79 ASC]
: +- Sort [seg13_1_pri_st#105L
ASC,specialty_1#79 ASC], false, 0
: +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,cast(_we0#106
as bigint) AS seg13_1_pri_st#105L]
: +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(pri_st#81)
windowspecdefinition(pri_st#81 ASC,ROWS BETWEEN UNBOUNDED PRECEDING AND
UNBOUNDED FOLLOWING) AS _we0#106], [pri_st#81 ASC]
: +- Sort [pri_st#81
ASC], false, 0
: +-
TungstenExchange SinglePartition, None
: +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,coalesce((like17_mg17_metric114_aggregated#100
/ like17_mg17_metric118_aggregated#101),0.0) AS mg17_metric211_aggregated#102]
: +- Filter
(cast(rep#78 as double) = -2.0)
: +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
+- Project
[affinity_monitor_copay#623,affinity_monitor_digital_pull#632,affinity_monitor_digital_push#614,affinity_monitor_direct#606,affinity_monitor_peer#625,affinity_monitor_peer_interaction#628,affinity_monitor_personal_f2f#636,affinity_monitor_personal_remote#611,affinity_monitor_sample#626,affinity_monitor_voucher#605,afltn_id#619,attribute_2_value#641,brand#639,city#616,cycle_time_id#637,full_name#607,hcp#629,like17_mg17_metric114_aggregated#635,like17_mg17_metric118_aggregated#608,metric_group_sk#602,metrics#615,mg17_metric117_3#646,mg17_metric117_3_actual_metric#647,mg17_metric117_3_planned_metric#648,mg17_metric117_D_suggestion_id#650,mg17_metric117_D_suggestion_text#652,mg17_metric117_D_suggestion_text_raw#651,mg17_metric117_exp_score#649,CASE
WHEN (mg17_metric117_exp_score#649 = 1) THEN (100.0 - mg17_metric117_3#646)
WHEN (mg17_metric117_exp_score#649 = -1) THEN (mg17_metric117_3#646 - -50.0) AS
mg17_metric117_severity_index#653,mg17_metric117_test#645,mg17_metric211_P_suggestion_id#604,mg17_metric211_P_suggestion_text#617,mg17_metric211_P_suggestion_text_raw#627,mg17_metric211_aggregated#643,mg17_metric211_deviationfrompeers_p_value#620,mg17_metric211_deviationfromtrend_current_mu#633,mg17_metric211_deviationfromtrend_p_value#622,mg17_metric211_deviationfromtrend_previous_mu#630,mg17_metric211_exp_score#624,mg17_metric211_mahal#631,mg17_metric211_segment_mu#601,mg17_metric211_segment_stddev#642,mg17_metric211_self_test_type#603,mg17_metric211_severity_index#612,mg17_metric211_test#644,mg17_metric211_threshold#634,npi_id#613,pri_st#638,prsn_id#610,rep#621,seg13_1_pri_st#640,seg13_2_specialty_1#609,specialty_1#618]
+- Filter (cast(rep#621 as double) = -2.0)
+- Scan
ExistingRDD[mg17_metric211_segment_mu#601,metric_group_sk#602,mg17_metric211_self_test_type#603,mg17_metric211_P_suggestion_id#604,affinity_monitor_voucher#605,affinity_monitor_direct#606,full_name#607,like17_mg17_metric118_aggregated#608,seg13_2_specialty_1#609,prsn_id#610,affinity_monitor_personal_remote#611,mg17_metric211_severity_index#612,npi_id#613,affinity_monitor_digital_push#614,metrics#615,city#616,mg17_metric211_P_suggestion_text#617,specialty_1#618,afltn_id#619,mg17_metric211_deviationfrompeers_p_value#620,rep#621,mg17_metric211_deviationfromtrend_p_value#622,affinity_monitor_copay#623,mg17_metric211_exp_score#624,affinity_monitor_peer#625,affinity_monitor_sample#626,mg17_metric211_P_suggestion_text_raw#627,affinity_monitor_peer_interaction#628,hcp#629,mg17_metric211_deviationfromtrend_previous_mu#630,mg17_metric211_mahal#631,affinity_monitor_digital_pull#632,mg17_metric211_deviationfromtrend_current_mu#633,mg17_metric211_threshold#634,like17_mg17_metric114_aggregated#635,affinity_monitor_personal_f2f#636,cycle_time_id#637,pri_st#638,brand#639,seg13_1_pri_st#640,attribute_2_value#641,mg17_metric211_segment_stddev#642,mg17_metric211_aggregated#643,mg17_metric211_test#644,mg17_metric117_test#645,mg17_metric117_3#646,mg17_metric117_3_actual_metric#647,mg17_metric117_3_planned_metric#648,mg17_metric117_exp_score#649,mg17_metric117_D_suggestion_id#650,mg17_metric117_D_suggestion_text_raw#651,mg17_metric117_D_suggestion_text#652]
{noformat}
rightdataframe schema
{noformat}
root
|-- affinity_monitor_copay: string (nullable = true)
|-- affinity_monitor_digital_pull: string (nullable = true)
|-- affinity_monitor_digital_push: string (nullable = true)
|-- affinity_monitor_direct: string (nullable = true)
|-- affinity_monitor_peer: string (nullable = true)
|-- affinity_monitor_peer_interaction: string (nullable = true)
|-- affinity_monitor_personal_f2f: string (nullable = true)
|-- affinity_monitor_personal_remote: string (nullable = true)
|-- affinity_monitor_sample: string (nullable = true)
|-- affinity_monitor_voucher: string (nullable = true)
|-- afltn_id: string (nullable = true)
|-- attribute_2_value: string (nullable = true)
|-- brand: string (nullable = true)
|-- city: string (nullable = true)
|-- cycle_time_id: integer (nullable = true)
|-- full_name: string (nullable = true)
|-- hcp: string (nullable = true)
|-- like17_mg17_metric114_aggregated: double (nullable = true)
|-- like17_mg17_metric118_aggregated: double (nullable = true)
|-- metric_group_sk: integer (nullable = true)
|-- metrics: array (nullable = true)
| |-- element: struct (containsNull = true)
| | |-- hcp: string (nullable = true)
| | |-- brand: string (nullable = true)
| | |-- rep: string (nullable = true)
| | |-- month: string (nullable = true)
| | |-- metric117: string (nullable = true)
| | |-- metric114: string (nullable = true)
| | |-- metric118: string (nullable = true)
| | |-- specialty_1: string (nullable = true)
| | |-- full_name: string (nullable = true)
| | |-- pri_st: string (nullable = true)
| | |-- city: string (nullable = true)
| | |-- zip_code: string (nullable = true)
| | |-- prsn_id: string (nullable = true)
| | |-- afltn_id: string (nullable = true)
| | |-- npi_id: string (nullable = true)
| | |-- affinity_monitor_sample: string (nullable = true)
| | |-- affinity_monitor_personal_f2f: string (nullable = true)
| | |-- affinity_monitor_peer: string (nullable = true)
| | |-- affinity_monitor_copay: string (nullable = true)
| | |-- affinity_monitor_digital_push: string (nullable = true)
| | |-- affinity_monitor_voucher: string (nullable = true)
| | |-- affinity_monitor_direct: string (nullable = true)
| | |-- affinity_monitor_peer_interaction: string (nullable = true)
| | |-- affinity_monitor_digital_pull: string (nullable = true)
| | |-- affinity_monitor_personal_remote: string (nullable = true)
| | |-- attribute_2_value: string (nullable = true)
| | |-- metric211: double (nullable = false)
|-- mg17_metric117_3: double (nullable = true)
|-- mg17_metric117_3_actual_metric: double (nullable = true)
|-- mg17_metric117_3_planned_metric: double (nullable = true)
|-- mg17_metric117_D_suggestion_id: integer (nullable = true)
|-- mg17_metric117_D_suggestion_text: string (nullable = true)
|-- mg17_metric117_D_suggestion_text_raw: string (nullable = true)
|-- mg17_metric117_exp_score: integer (nullable = true)
|-- mg17_metric117_severity_index: double (nullable = true)
|-- mg17_metric117_test: integer (nullable = true)
|-- mg17_metric211_P_suggestion_id: integer (nullable = true)
|-- mg17_metric211_P_suggestion_text: string (nullable = true)
|-- mg17_metric211_P_suggestion_text_raw: string (nullable = true)
|-- mg17_metric211_aggregated: double (nullable = false)
|-- mg17_metric211_deviationfrompeers_p_value: double (nullable = true)
|-- mg17_metric211_deviationfromtrend_current_mu: double (nullable = true)
|-- mg17_metric211_deviationfromtrend_p_value: double (nullable = true)
|-- mg17_metric211_deviationfromtrend_previous_mu: double (nullable = true)
|-- mg17_metric211_exp_score: integer (nullable = true)
|-- mg17_metric211_mahal: double (nullable = true)
|-- mg17_metric211_segment_mu: double (nullable = true)
|-- mg17_metric211_segment_stddev: double (nullable = false)
|-- mg17_metric211_self_test_type: string (nullable = true)
|-- mg17_metric211_severity_index: double (nullable = true)
|-- mg17_metric211_test: integer (nullable = true)
|-- mg17_metric211_threshold: double (nullable = true)
|-- npi_id: string (nullable = true)
|-- pri_st: string (nullable = true)
|-- prsn_id: string (nullable = true)
|-- rep: string (nullable = true)
|-- seg13_1_pri_st: string (nullable = true)
|-- seg13_2_specialty_1: string (nullable = true)
|-- specialty_1: string (nullable = true)
{noformat}
rightdataframe queryplan
{noformat}
== Physical Plan ==
Union
:- Project
[affinity_monitor_copay#308,affinity_monitor_digital_pull#314,affinity_monitor_digital_push#302,affinity_monitor_direct#295,affinity_monitor_peer#309,affinity_monitor_peer_interaction#311,affinity_monitor_personal_f2f#317,affinity_monitor_personal_remote#300,affinity_monitor_sample#310,affinity_monitor_voucher#294,afltn_id#306,attribute_2_value#322,brand#320,city#304,cycle_time_id#318,full_name#296,hcp#312,like17_mg17_metric114_aggregated#316,like17_mg17_metric118_aggregated#297,metric_group_sk#293,metrics#303,null
AS mg17_metric117_3#658,null AS mg17_metric117_3_actual_metric#659,null AS
mg17_metric117_3_planned_metric#660,CASE WHEN true THEN -1 AS
mg17_metric117_D_suggestion_id#450,UDF(mg17_metric211_segment_mu#313) AS
mg17_metric117_D_suggestion_text#451,null AS
mg17_metric117_D_suggestion_text_raw#661,CASE WHEN true THEN -2 AS
mg17_metric117_exp_score#453,CASE WHEN true THEN -2.0 AS
mg17_metric117_severity_index#452,CASE WHEN true THEN 3 AS
mg17_metric117_test#449,mg17_metric211_P_suggestion_id#332,mg17_metric211_P_suggestion_text#334,mg17_metric211_P_suggestion_text_raw#333,mg17_metric211_aggregated#323,mg17_metric211_deviationfrompeers_p_value#383,mg17_metric211_deviationfromtrend_current_mu#324,mg17_metric211_deviationfromtrend_p_value#326,mg17_metric211_deviationfromtrend_previous_mu#325,mg17_metric211_exp_score#331,mg17_metric211_mahal#384,mg17_metric211_segment_mu#313,mg17_metric211_segment_stddev#315,mg17_metric211_self_test_type#328,mg17_metric211_severity_index#329,mg17_metric211_test#330,mg17_metric211_threshold#327,npi_id#301,pri_st#319,prsn_id#299,rep#307,seg13_1_pri_st#321,seg13_2_specialty_1#298,specialty_1#305]
: +- SortMergeJoin [hcp#312], [hcp#426]
: :- Sort [hcp#312 ASC], false, 0
: : +- TungstenExchange hashpartitioning(hcp#312,200), None
: : +- Union
: : :- Project
[affinity_monitor_copay#308,affinity_monitor_digital_pull#314,affinity_monitor_digital_push#302,affinity_monitor_direct#295,affinity_monitor_peer#309,affinity_monitor_peer_interaction#311,affinity_monitor_personal_f2f#317,affinity_monitor_personal_remote#300,affinity_monitor_sample#310,affinity_monitor_voucher#294,afltn_id#306,attribute_2_value#322,brand#320,city#304,cycle_time_id#318,full_name#296,hcp#312,like17_mg17_metric114_aggregated#316,like17_mg17_metric118_aggregated#297,metric_group_sk#293,metrics#303,mg17_metric211_P_suggestion_id#332,mg17_metric211_P_suggestion_text#334,mg17_metric211_P_suggestion_text_raw#333,mg17_metric211_aggregated#323,null
AS
mg17_metric211_deviationfrompeers_p_value#383,mg17_metric211_deviationfromtrend_current_mu#324,mg17_metric211_deviationfromtrend_p_value#326,mg17_metric211_deviationfromtrend_previous_mu#325,mg17_metric211_exp_score#331,null
AS
mg17_metric211_mahal#384,mg17_metric211_segment_mu#313,mg17_metric211_segment_stddev#315,mg17_metric211_self_test_type#328,mg17_metric211_severity_index#329,mg17_metric211_test#330,mg17_metric211_threshold#327,npi_id#301,pri_st#319,prsn_id#299,rep#307,seg13_1_pri_st#321,seg13_2_specialty_1#298,specialty_1#305]
: : : +- Filter ((cast(rep#307 as double) = -2.0) && NOT
(cast(rep#307 as double) = -2.0))
: : : +- Scan
ExistingRDD[metric_group_sk#293,affinity_monitor_voucher#294,affinity_monitor_direct#295,full_name#296,like17_mg17_metric118_aggregated#297,seg13_2_specialty_1#298,prsn_id#299,affinity_monitor_personal_remote#300,npi_id#301,affinity_monitor_digital_push#302,metrics#303,city#304,specialty_1#305,afltn_id#306,rep#307,affinity_monitor_copay#308,affinity_monitor_peer#309,affinity_monitor_sample#310,affinity_monitor_peer_interaction#311,hcp#312,mg17_metric211_segment_mu#313,affinity_monitor_digital_pull#314,mg17_metric211_segment_stddev#315,like17_mg17_metric114_aggregated#316,affinity_monitor_personal_f2f#317,cycle_time_id#318,pri_st#319,brand#320,seg13_1_pri_st#321,attribute_2_value#322,mg17_metric211_aggregated#323,mg17_metric211_deviationfromtrend_current_mu#324,mg17_metric211_deviationfromtrend_previous_mu#325,mg17_metric211_deviationfromtrend_p_value#326,mg17_metric211_threshold#327,mg17_metric211_self_test_type#328,mg17_metric211_severity_index#329,mg17_metric211_test#330,mg17_metric211_exp_score#331,mg17_metric211_P_suggestion_id#332,mg17_metric211_P_suggestion_text_raw#333,mg17_metric211_P_suggestion_text#334]
: : :- Project
[affinity_monitor_copay#352,affinity_monitor_digital_pull#358,affinity_monitor_digital_push#346,affinity_monitor_direct#339,affinity_monitor_peer#353,affinity_monitor_peer_interaction#355,affinity_monitor_personal_f2f#361,affinity_monitor_personal_remote#344,affinity_monitor_sample#354,affinity_monitor_voucher#338,afltn_id#350,attribute_2_value#366,brand#364,city#348,cycle_time_id#362,full_name#340,hcp#356,like17_mg17_metric114_aggregated#360,like17_mg17_metric118_aggregated#341,metric_group_sk#337,metrics#347,CASE
WHEN ((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = -1)) THEN CASE WHEN true THEN 110 WHEN
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = 1)) THEN CASE WHEN true THEN 109 ELSE CASE WHEN true THEN -1 AS
mg17_metric211_P_suggestion_id#378,CASE WHEN
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = -1)) THEN CASE WHEN true THEN CIALIS percent split in managed
care advantage plan is lower as compared to peer group. WHEN
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = 1)) THEN CASE WHEN true THEN CIALIS percent split in managed care
advantage plan is higher as compared to peer group. ELSE CASE WHEN true THEN
AS mg17_metric211_P_suggestion_text#380,CASE WHEN
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = -1)) THEN CASE WHEN true THEN CIALIS percent split in managed
care advantage plan is lower as compared to peer group. WHEN
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (CASE WHEN
(mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 = 1)) THEN CASE WHEN true THEN CIALIS percent split in managed care
advantage plan is higher as compared to peer group. ELSE CASE WHEN true THEN
AS
mg17_metric211_P_suggestion_text_raw#379,mg17_metric211_aggregated#367,mg17_metric211_deviationfrompeers_p_value#375,mg17_metric211_deviationfromtrend_current_mu#368,mg17_metric211_deviationfromtrend_p_value#370,mg17_metric211_deviationfromtrend_previous_mu#369,CASE
WHEN (mg17_metric211_mahal#374 > 0.0) THEN CASE WHEN true THEN 1 WHEN
(mg17_metric211_mahal#374 < 0.0) THEN CASE WHEN true THEN -1 ELSE CASE WHEN
true THEN 0 AS
mg17_metric211_exp_score#377,mg17_metric211_mahal#374,mg17_metric211_segment_mu#357,mg17_metric211_segment_stddev#359,mg17_metric211_self_test_type#372,CASE
WHEN (mg17_metric211_deviationfrompeers_p_value#375 > 0.9) THEN
(mg17_metric211_deviationfrompeers_p_value#375 - 0.9) ELSE CASE WHEN true THEN
0.0 AS
mg17_metric211_severity_index#376,mg17_metric211_test#373,mg17_metric211_threshold#371,npi_id#345,pri_st#363,prsn_id#343,rep#351,seg13_1_pri_st#365,seg13_2_specialty_1#342,specialty_1#349]
: : : +- Filter (((mg17_metric211_deviationfrompeers_p_value#375 >
0.9) && (cast(rep#351 as double) = -2.0)) && NOT (cast(rep#351 as double) =
-2.0))
: : : +- Scan
ExistingRDD[metric_group_sk#337,affinity_monitor_voucher#338,affinity_monitor_direct#339,full_name#340,like17_mg17_metric118_aggregated#341,seg13_2_specialty_1#342,prsn_id#343,affinity_monitor_personal_remote#344,npi_id#345,affinity_monitor_digital_push#346,metrics#347,city#348,specialty_1#349,afltn_id#350,rep#351,affinity_monitor_copay#352,affinity_monitor_peer#353,affinity_monitor_sample#354,affinity_monitor_peer_interaction#355,hcp#356,mg17_metric211_segment_mu#357,affinity_monitor_digital_pull#358,mg17_metric211_segment_stddev#359,like17_mg17_metric114_aggregated#360,affinity_monitor_personal_f2f#361,cycle_time_id#362,pri_st#363,brand#364,seg13_1_pri_st#365,attribute_2_value#366,mg17_metric211_aggregated#367,mg17_metric211_deviationfromtrend_current_mu#368,mg17_metric211_deviationfromtrend_previous_mu#369,mg17_metric211_deviationfromtrend_p_value#370,mg17_metric211_threshold#371,mg17_metric211_self_test_type#372,mg17_metric211_test#373,mg17_metric211_mahal#374,mg17_metric211_deviationfrompeers_p_value#375]
: : +- Project
[affinity_monitor_copay#89,affinity_monitor_digital_pull#94,affinity_monitor_digital_push#90,affinity_monitor_direct#92,affinity_monitor_peer#88,affinity_monitor_peer_interaction#93,affinity_monitor_personal_f2f#87,affinity_monitor_personal_remote#95,affinity_monitor_sample#86,affinity_monitor_voucher#91,afltn_id#84,attribute_2_value#96,brand#77,city#82,cycle_time_id#99,full_name#80,hcp#76,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,metric_group_sk#98,metrics#97,CASE
WHEN true THEN -1 AS
mg17_metric211_P_suggestion_id#385,UDF(metric_group_sk#98) AS
mg17_metric211_P_suggestion_text#386,null AS
mg17_metric211_P_suggestion_text_raw#398,mg17_metric211_aggregated#102,null AS
mg17_metric211_deviationfrompeers_p_value#399,null AS
mg17_metric211_deviationfromtrend_current_mu#400,null AS
mg17_metric211_deviationfromtrend_p_value#401,null AS
mg17_metric211_deviationfromtrend_previous_mu#402,CASE WHEN true THEN -2 AS
mg17_metric211_exp_score#388,null AS
mg17_metric211_mahal#403,mg17_metric211_segment_mu#111,mg17_metric211_segment_stddev#112,null
AS mg17_metric211_self_test_type#404,CASE WHEN true THEN -2.0 AS
mg17_metric211_severity_index#387,null AS mg17_metric211_test#405,null AS
mg17_metric211_threshold#406,npi_id#85,pri_st#81,prsn_id#83,rep#78,seg13_1_pri_st#103,seg13_2_specialty_1#104,specialty_1#79]
: : +- SortMergeJoin
[seg13_1_pri_st#103,seg13_2_specialty_1#104],
[seg13_1_pri_st#135,seg13_2_specialty_1#123]
: : :- Sort [seg13_1_pri_st#103 ASC,seg13_2_specialty_1#104
ASC], false, 0
: : : +- TungstenExchange
hashpartitioning(seg13_1_pri_st#103,seg13_2_specialty_1#104,200), None
: : : +- Union
: : : :- Project
[affinity_monitor_copay#89,affinity_monitor_digital_pull#94,affinity_monitor_digital_push#90,affinity_monitor_direct#92,affinity_monitor_peer#88,affinity_monitor_peer_interaction#93,affinity_monitor_personal_f2f#87,affinity_monitor_personal_remote#95,affinity_monitor_sample#86,affinity_monitor_voucher#91,afltn_id#84,attribute_2_value#96,brand#77,city#82,cycle_time_id#99,full_name#80,hcp#76,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,metric_group_sk#98,metrics#97,coalesce((like17_mg17_metric114_aggregated#100
/ like17_mg17_metric118_aggregated#101),0.0) AS
mg17_metric211_aggregated#102,npi_id#85,pri_st#81,prsn_id#83,rep#78,UDF(hcp#76)
AS seg13_1_pri_st#103,UDF(hcp#76) AS seg13_2_specialty_1#104,specialty_1#79]
: : : : +- Filter ((NOT (cast(rep#78 as double) =
-2.0) && (cast(rep#78 as double) = -2.0)) && NOT (cast(rep#78 as double) =
-2.0))
: : : : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: : : +- Project
[affinity_monitor_copay#89,affinity_monitor_digital_pull#94,affinity_monitor_digital_push#90,affinity_monitor_direct#92,affinity_monitor_peer#88,affinity_monitor_peer_interaction#93,affinity_monitor_personal_f2f#87,affinity_monitor_personal_remote#95,affinity_monitor_sample#86,affinity_monitor_voucher#91,afltn_id#84,attribute_2_value#96,brand#77,city#82,cycle_time_id#99,full_name#80,hcp#76,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,metric_group_sk#98,metrics#97,mg17_metric211_aggregated#102,npi_id#85,pri_st#81,prsn_id#83,rep#78,cast(seg13_1_pri_st#105L
as string) AS seg13_1_pri_st#109,cast(cast(_we0#108 as bigint) as string) AS
seg13_2_specialty_1#110,specialty_1#79]
: : : +- Filter ((NOT (cast(rep#78 as double) =
-2.0) && (cast(rep#78 as double) = -2.0)) && NOT (cast(rep#78 as double) =
-2.0))
: : : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,seg13_1_pri_st#105L],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(specialty_1#79)
windowspecdefinition(seg13_1_pri_st#105L,specialty_1#79 ASC,ROWS BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS _we0#108],
[seg13_1_pri_st#105L], [specialty_1#79 ASC]
: : : +- Sort [seg13_1_pri_st#105L
ASC,specialty_1#79 ASC], false, 0
: : : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,cast(_we0#106
as bigint) AS seg13_1_pri_st#105L]
: : : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(pri_st#81)
windowspecdefinition(pri_st#81 ASC,ROWS BETWEEN UNBOUNDED PRECEDING AND
UNBOUNDED FOLLOWING) AS _we0#106], [pri_st#81 ASC]
: : : +- Sort [pri_st#81 ASC], false, 0
: : : +- TungstenExchange
SinglePartition, None
: : : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,coalesce((like17_mg17_metric114_aggregated#100
/ like17_mg17_metric118_aggregated#101),0.0) AS mg17_metric211_aggregated#102]
: : : +- Filter (cast(rep#78
as double) = -2.0)
: : : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: : +- Sort [seg13_1_pri_st#135 ASC,seg13_2_specialty_1#123
ASC], false, 0
: : +-
TungstenAggregate(key=[seg13_1_pri_st#135,seg13_2_specialty_1#123],
functions=[(avg(mg17_metric211_aggregated#102),mode=Final,isDistinct=false),(stddev_samp(mg17_metric211_aggregated#102,0,0),mode=Final,isDistinct=false)],
output=[seg13_1_pri_st#135,seg13_2_specialty_1#123,mg17_metric211_segment_mu#111,mg17_metric211_segment_stddev#112])
: : +- TungstenExchange
hashpartitioning(seg13_1_pri_st#135,seg13_2_specialty_1#123,200), None
: : +-
TungstenAggregate(key=[seg13_1_pri_st#135,seg13_2_specialty_1#123],
functions=[(avg(mg17_metric211_aggregated#102),mode=Partial,isDistinct=false),(stddev_samp(mg17_metric211_aggregated#102,0,0),mode=Partial,isDistinct=false)],
output=[seg13_1_pri_st#135,seg13_2_specialty_1#123,sum#140,count#141L,M0#130,M1#131,M2#132,M3#133,M4#134])
: : +- Union
: : :- Project [UDF(hcp#76) AS
seg13_1_pri_st#135,UDF(hcp#76) AS
seg13_2_specialty_1#123,coalesce((like17_mg17_metric114_aggregated#100 /
like17_mg17_metric118_aggregated#101),0.0) AS mg17_metric211_aggregated#102]
: : : +- Filter NOT (cast(rep#78 as double) =
-2.0)
: : : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: : +- Project [cast(seg13_1_pri_st#105L as
string) AS seg13_1_pri_st#109,cast(cast(_we0#108 as bigint) as string) AS
seg13_2_specialty_1#110,mg17_metric211_aggregated#102]
: : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,seg13_1_pri_st#105L],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(specialty_1#79)
windowspecdefinition(seg13_1_pri_st#105L,specialty_1#79 ASC,ROWS BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS _we0#108],
[seg13_1_pri_st#105L], [specialty_1#79 ASC]
: : +- Sort [seg13_1_pri_st#105L
ASC,specialty_1#79 ASC], false, 0
: : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,cast(_we0#106
as bigint) AS seg13_1_pri_st#105L]
: : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(pri_st#81)
windowspecdefinition(pri_st#81 ASC,ROWS BETWEEN UNBOUNDED PRECEDING AND
UNBOUNDED FOLLOWING) AS _we0#106], [pri_st#81 ASC]
: : +- Sort [pri_st#81 ASC],
false, 0
: : +- TungstenExchange
SinglePartition, None
: : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,coalesce((like17_mg17_metric114_aggregated#100
/ like17_mg17_metric118_aggregated#101),0.0) AS mg17_metric211_aggregated#102]
: : +- Filter
(cast(rep#78 as double) = -2.0)
: : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: +- Sort [hcp#426 ASC], false, 0
: +- TungstenAggregate(key=[hcp#426], functions=[], output=[hcp#426])
: +- TungstenExchange hashpartitioning(hcp#426,200), None
: +- TungstenAggregate(key=[hcp#426], functions=[],
output=[hcp#426])
: +- Union
: :- Project [hcp#426]
: : +- Filter (cast(rep#421 as double) = -2.0)
: : +- Scan
ExistingRDD[metric_group_sk#407,affinity_monitor_voucher#408,affinity_monitor_direct#409,full_name#410,like17_mg17_metric118_aggregated#411,seg13_2_specialty_1#412,prsn_id#413,affinity_monitor_personal_remote#414,npi_id#415,affinity_monitor_digital_push#416,metrics#417,city#418,specialty_1#419,afltn_id#420,rep#421,affinity_monitor_copay#422,affinity_monitor_peer#423,affinity_monitor_sample#424,affinity_monitor_peer_interaction#425,hcp#426,mg17_metric211_segment_mu#427,affinity_monitor_digital_pull#428,mg17_metric211_segment_stddev#429,like17_mg17_metric114_aggregated#430,affinity_monitor_personal_f2f#431,cycle_time_id#432,pri_st#433,brand#434,seg13_1_pri_st#435,attribute_2_value#436,mg17_metric211_aggregated#437,mg17_metric211_deviationfromtrend_current_mu#438,mg17_metric211_deviationfromtrend_previous_mu#439,mg17_metric211_deviationfromtrend_p_value#440,mg17_metric211_threshold#441,mg17_metric211_self_test_type#442,mg17_metric211_severity_index#443,mg17_metric211_test#444,mg17_metric211_exp_score#445,mg17_metric211_P_suggestion_id#446,mg17_metric211_P_suggestion_text_raw#447,mg17_metric211_P_suggestion_text#448]
: :- Project [hcp#356]
: : +- Filter
((mg17_metric211_deviationfrompeers_p_value#375 > 0.9) && (cast(rep#351 as
double) = -2.0))
: : +- Scan
ExistingRDD[metric_group_sk#337,affinity_monitor_voucher#338,affinity_monitor_direct#339,full_name#340,like17_mg17_metric118_aggregated#341,seg13_2_specialty_1#342,prsn_id#343,affinity_monitor_personal_remote#344,npi_id#345,affinity_monitor_digital_push#346,metrics#347,city#348,specialty_1#349,afltn_id#350,rep#351,affinity_monitor_copay#352,affinity_monitor_peer#353,affinity_monitor_sample#354,affinity_monitor_peer_interaction#355,hcp#356,mg17_metric211_segment_mu#357,affinity_monitor_digital_pull#358,mg17_metric211_segment_stddev#359,like17_mg17_metric114_aggregated#360,affinity_monitor_personal_f2f#361,cycle_time_id#362,pri_st#363,brand#364,seg13_1_pri_st#365,attribute_2_value#366,mg17_metric211_aggregated#367,mg17_metric211_deviationfromtrend_current_mu#368,mg17_metric211_deviationfromtrend_previous_mu#369,mg17_metric211_deviationfromtrend_p_value#370,mg17_metric211_threshold#371,mg17_metric211_self_test_type#372,mg17_metric211_test#373,mg17_metric211_mahal#374,mg17_metric211_deviationfrompeers_p_value#375]
: +- Project [hcp#76]
: +- SortMergeJoin
[seg13_1_pri_st#103,seg13_2_specialty_1#104],
[seg13_1_pri_st#135,seg13_2_specialty_1#123]
: :- Sort [seg13_1_pri_st#103
ASC,seg13_2_specialty_1#104 ASC], false, 0
: : +- TungstenExchange
hashpartitioning(seg13_1_pri_st#103,seg13_2_specialty_1#104,200), None
: : +- Union
: : :- Project [UDF(hcp#76) AS
seg13_1_pri_st#103,hcp#76,UDF(hcp#76) AS seg13_2_specialty_1#104]
: : : +- Filter (NOT (cast(rep#78 as double) =
-2.0) && (cast(rep#78 as double) = -2.0))
: : : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: : +- Project [cast(seg13_1_pri_st#105L as
string) AS seg13_1_pri_st#109,hcp#76,cast(cast(_we0#108 as bigint) as string)
AS seg13_2_specialty_1#110]
: : +- Filter (NOT (cast(rep#78 as double) =
-2.0) && (cast(rep#78 as double) = -2.0))
: : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,seg13_1_pri_st#105L],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(specialty_1#79)
windowspecdefinition(seg13_1_pri_st#105L,specialty_1#79 ASC,ROWS BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS _we0#108],
[seg13_1_pri_st#105L], [specialty_1#79 ASC]
: : +- Sort [seg13_1_pri_st#105L
ASC,specialty_1#79 ASC], false, 0
: : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,cast(_we0#106
as bigint) AS seg13_1_pri_st#105L]
: : +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(pri_st#81)
windowspecdefinition(pri_st#81 ASC,ROWS BETWEEN UNBOUNDED PRECEDING AND
UNBOUNDED FOLLOWING) AS _we0#106], [pri_st#81 ASC]
: : +- Sort [pri_st#81 ASC],
false, 0
: : +- TungstenExchange
SinglePartition, None
: : +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,coalesce((like17_mg17_metric114_aggregated#100
/ like17_mg17_metric118_aggregated#101),0.0) AS mg17_metric211_aggregated#102]
: : +- Filter
(cast(rep#78 as double) = -2.0)
: : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: +- Sort [seg13_1_pri_st#135
ASC,seg13_2_specialty_1#123 ASC], false, 0
: +- Project
[seg13_2_specialty_1#123,seg13_1_pri_st#135]
: +-
TungstenAggregate(key=[seg13_1_pri_st#135,seg13_2_specialty_1#123],
functions=[], output=[seg13_1_pri_st#135,seg13_2_specialty_1#123])
: +- TungstenExchange
hashpartitioning(seg13_1_pri_st#135,seg13_2_specialty_1#123,200), None
: +-
TungstenAggregate(key=[seg13_1_pri_st#135,seg13_2_specialty_1#123],
functions=[], output=[seg13_1_pri_st#135,seg13_2_specialty_1#123])
: +- Union
: :- Project [UDF(hcp#76) AS
seg13_1_pri_st#135,UDF(hcp#76) AS seg13_2_specialty_1#123]
: : +- Filter NOT (cast(rep#78 as
double) = -2.0)
: : +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
: +- Project
[cast(seg13_1_pri_st#105L as string) AS seg13_1_pri_st#109,cast(cast(_we0#108
as bigint) as string) AS seg13_2_specialty_1#110]
: +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,seg13_1_pri_st#105L],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(specialty_1#79)
windowspecdefinition(seg13_1_pri_st#105L,specialty_1#79 ASC,ROWS BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS _we0#108],
[seg13_1_pri_st#105L], [specialty_1#79 ASC]
: +- Sort [seg13_1_pri_st#105L
ASC,specialty_1#79 ASC], false, 0
: +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102,cast(_we0#106
as bigint) AS seg13_1_pri_st#105L]
: +- Window
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,mg17_metric211_aggregated#102],
[HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank(pri_st#81)
windowspecdefinition(pri_st#81 ASC,ROWS BETWEEN UNBOUNDED PRECEDING AND
UNBOUNDED FOLLOWING) AS _we0#106], [pri_st#81 ASC]
: +- Sort [pri_st#81
ASC], false, 0
: +-
TungstenExchange SinglePartition, None
: +- Project
[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101,coalesce((like17_mg17_metric114_aggregated#100
/ like17_mg17_metric118_aggregated#101),0.0) AS mg17_metric211_aggregated#102]
: +- Filter
(cast(rep#78 as double) = -2.0)
: +- Scan
ExistingRDD[hcp#76,brand#77,rep#78,specialty_1#79,full_name#80,pri_st#81,city#82,prsn_id#83,afltn_id#84,npi_id#85,affinity_monitor_sample#86,affinity_monitor_personal_f2f#87,affinity_monitor_peer#88,affinity_monitor_copay#89,affinity_monitor_digital_push#90,affinity_monitor_voucher#91,affinity_monitor_direct#92,affinity_monitor_peer_interaction#93,affinity_monitor_digital_pull#94,affinity_monitor_personal_remote#95,attribute_2_value#96,metrics#97,metric_group_sk#98,cycle_time_id#99,like17_mg17_metric114_aggregated#100,like17_mg17_metric118_aggregated#101]
+- Project
[affinity_monitor_copay#623,affinity_monitor_digital_pull#632,affinity_monitor_digital_push#614,affinity_monitor_direct#606,affinity_monitor_peer#625,affinity_monitor_peer_interaction#628,affinity_monitor_personal_f2f#636,affinity_monitor_personal_remote#611,affinity_monitor_sample#626,affinity_monitor_voucher#605,afltn_id#619,attribute_2_value#641,brand#639,city#616,cycle_time_id#637,full_name#607,hcp#629,like17_mg17_metric114_aggregated#635,like17_mg17_metric118_aggregated#608,metric_group_sk#602,metrics#615,mg17_metric117_3#646,mg17_metric117_3_actual_metric#647,mg17_metric117_3_planned_metric#648,mg17_metric117_D_suggestion_id#650,mg17_metric117_D_suggestion_text#652,mg17_metric117_D_suggestion_text_raw#651,mg17_metric117_exp_score#649,CASE
WHEN (mg17_metric117_exp_score#649 = 1) THEN (100.0 - mg17_metric117_3#646)
WHEN (mg17_metric117_exp_score#649 = -1) THEN (mg17_metric117_3#646 - -50.0) AS
mg17_metric117_severity_index#653,mg17_metric117_test#645,mg17_metric211_P_suggestion_id#604,mg17_metric211_P_suggestion_text#617,mg17_metric211_P_suggestion_text_raw#627,mg17_metric211_aggregated#643,mg17_metric211_deviationfrompeers_p_value#620,mg17_metric211_deviationfromtrend_current_mu#633,mg17_metric211_deviationfromtrend_p_value#622,mg17_metric211_deviationfromtrend_previous_mu#630,mg17_metric211_exp_score#624,mg17_metric211_mahal#631,mg17_metric211_segment_mu#601,mg17_metric211_segment_stddev#642,mg17_metric211_self_test_type#603,mg17_metric211_severity_index#612,mg17_metric211_test#644,mg17_metric211_threshold#634,npi_id#613,pri_st#638,prsn_id#610,rep#621,seg13_1_pri_st#640,seg13_2_specialty_1#609,specialty_1#618]
+- Filter NOT (cast(rep#621 as double) = -2.0)
+- Scan
ExistingRDD[mg17_metric211_segment_mu#601,metric_group_sk#602,mg17_metric211_self_test_type#603,mg17_metric211_P_suggestion_id#604,affinity_monitor_voucher#605,affinity_monitor_direct#606,full_name#607,like17_mg17_metric118_aggregated#608,seg13_2_specialty_1#609,prsn_id#610,affinity_monitor_personal_remote#611,mg17_metric211_severity_index#612,npi_id#613,affinity_monitor_digital_push#614,metrics#615,city#616,mg17_metric211_P_suggestion_text#617,specialty_1#618,afltn_id#619,mg17_metric211_deviationfrompeers_p_value#620,rep#621,mg17_metric211_deviationfromtrend_p_value#622,affinity_monitor_copay#623,mg17_metric211_exp_score#624,affinity_monitor_peer#625,affinity_monitor_sample#626,mg17_metric211_P_suggestion_text_raw#627,affinity_monitor_peer_interaction#628,hcp#629,mg17_metric211_deviationfromtrend_previous_mu#630,mg17_metric211_mahal#631,affinity_monitor_digital_pull#632,mg17_metric211_deviationfromtrend_current_mu#633,mg17_metric211_threshold#634,like17_mg17_metric114_aggregated#635,affinity_monitor_personal_f2f#636,cycle_time_id#637,pri_st#638,brand#639,seg13_1_pri_st#640,attribute_2_value#641,mg17_metric211_segment_stddev#642,mg17_metric211_aggregated#643,mg17_metric211_test#644,mg17_metric117_test#645,mg17_metric117_3#646,mg17_metric117_3_actual_metric#647,mg17_metric117_3_planned_metric#648,mg17_metric117_exp_score#649,mg17_metric117_D_suggestion_id#650,mg17_metric117_D_suggestion_text_raw#651,mg17_metric117_D_suggestion_text#652]
{noformat}
Join syntax which is throwing error:
{noformat}
leftDf.join(rightDf, joiningExp, "left_outer")
{noformat}
Actual error:
{noformat}
Exception in thread "main" org.apache.spark.sql.AnalysisException: resolved
attribute(s)
mg17_metric211_deviationfrompeers_p_value#1008,mg17_metric211_mahal#1009
missing from
affinity_monitor_personal_f2f#431,cycle_time_id#432,rep#421,brand#434,attribute_2_value#436,seg13_2_specialty_1#412,mg17_metric211_P_suggestion_text#448,affinity_monitor_direct#409,mg17_metric211_aggregated#437,mg17_metric211_deviationfromtrend_previous_mu#439,affinity_monitor_copay#422,affinity_monitor_digital_push#416,affinity_monitor_personal_remote#414,hcp#426,full_name#410,mg17_metric211_deviationfrompeers_p_value#383,pri_st#433,mg17_metric211_severity_index#443,city#418,prsn_id#413,mg17_metric211_P_suggestion_id#446,npi_id#415,specialty_1#419,metric_group_sk#407,mg17_metric211_deviationfromtrend_current_mu#438,affinity_monitor_digital_pull#428,like17_mg17_metric118_aggregated#411,mg17_metric211_segment_mu#427,mg17_metric211_mahal#384,affinity_monitor_peer#423,mg17_metric211_self_test_type#442,mg17_metric211_threshold#441,affinity_monitor_voucher#408,mg17_metric211_deviationfromtrend_p_value#440,affinity_monitor_sample#424,like17_mg17_metric114_aggregated#430,mg17_metric211_test#444,affinity_monitor_peer_interaction#425,mg17_metric211_P_suggestion_text_raw#447,mg17_metric211_exp_score#445,mg17_metric211_segment_stddev#429,afltn_id#420,metrics#417,seg13_1_pri_st#435
in operator !Project
[affinity_monitor_copay#422,affinity_monitor_digital_pull#428,affinity_monitor_digital_push#416,affinity_monitor_direct#409,affinity_monitor_peer#423,affinity_monitor_peer_interaction#425,affinity_monitor_personal_f2f#431,affinity_monitor_personal_remote#414,affinity_monitor_sample#424,affinity_monitor_voucher#408,afltn_id#420,attribute_2_value#436,brand#434,city#418,cycle_time_id#432,full_name#410,hcp#426,like17_mg17_metric114_aggregated#430,like17_mg17_metric118_aggregated#411,metric_group_sk#407,metrics#417,mg17_metric211_P_suggestion_id#446,mg17_metric211_P_suggestion_text#448,mg17_metric211_P_suggestion_text_raw#447,mg17_metric211_aggregated#437,mg17_metric211_deviationfrompeers_p_value#1008,mg17_metric211_deviationfromtrend_current_mu#438,mg17_metric211_deviationfromtrend_p_value#440,mg17_metric211_deviationfromtrend_previous_mu#439,mg17_metric211_exp_score#445,mg17_metric211_mahal#1009,mg17_metric211_segment_mu#427,mg17_metric211_segment_stddev#429,mg17_metric211_self_test_type#442,mg17_metric211_severity_index#443,mg17_metric211_test#444,mg17_metric211_threshold#441,npi_id#415,pri_st#433,prsn_id#413,rep#421,seg13_1_pri_st#435,seg13_2_specialty_1#412,specialty_1#419];
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:38)
at
org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:44)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:183)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:50)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:105)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:104)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:104)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:50)
at
org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:44)
at
org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:34)
at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:133)
at
org.apache.spark.sql.DataFrame.org$apache$spark$sql$DataFrame$$withPlan(DataFrame.scala:2165)
at org.apache.spark.sql.DataFrame.join(DataFrame.scala:585)
at
com.zsassociates.se.statements.ExceptStatement.applyInternal(ExceptStatement.scala:45)
at com.zsassociates.se.statements.Statement.apply(Statement.scala:33)
at
com.zsassociates.se.statements.UnionStatement$$anonfun$applyInternal$1.apply(UnionStatement.scala:28)
at
com.zsassociates.se.statements.UnionStatement$$anonfun$applyInternal$1.apply(UnionStatement.scala:24)
at
scala.collection.TraversableOnce$$anonfun$foldLeft$1.apply(TraversableOnce.scala:144)
at
scala.collection.TraversableOnce$$anonfun$foldLeft$1.apply(TraversableOnce.scala:144)
at scala.collection.immutable.Set$Set3.foreach(Set.scala:115)
at
scala.collection.TraversableOnce$class.foldLeft(TraversableOnce.scala:144)
at scala.collection.AbstractTraversable.foldLeft(Traversable.scala:105)
at
com.zsassociates.se.statements.UnionStatement.applyInternal(UnionStatement.scala:24)
at com.zsassociates.se.statements.Statement.apply(Statement.scala:33)
at
com.zsassociates.se.statements.Statement.previousStatementOutput(Statement.scala:28)
at com.zsassociates.se.statements.Statement.apply(Statement.scala:32)
at
com.zsassociates.se.statements.Statement.previousStatementOutput(Statement.scala:28)
at com.zsassociates.se.statements.Statement.apply(Statement.scala:32)
at
com.zsassociates.se.statements.Statement.previousStatementOutput(Statement.scala:28)
at com.zsassociates.se.statements.Statement.apply(Statement.scala:32)
at
com.zsassociates.se.statements.Statement.previousStatementOutput(Statement.scala:28)
at com.zsassociates.se.statements.Statement.apply(Statement.scala:32)
at
com.zsassociates.se.statements.SequenceStatement.applyInternal(SequenceStatement.scala:20)
at com.zsassociates.se.statements.Statement.apply(Statement.scala:33)
at
com.zsassociates.se.statements.SequenceStatement.applyInternal(SequenceStatement.scala:20)
at com.zsassociates.se.statements.Statement.apply(Statement.scala:33)
at
com.zsassociates.se.statements.SequenceStatement.applyInternal(SequenceStatement.scala:20)
at com.zsassociates.se.statements.Statement.apply(Statement.scala:33)
at com.zsassociates.se.SuggestionEngine$.run(SuggestionEngine.scala:50)
at com.zsassociates.se.SuggestionEngine$.run(SuggestionEngine.scala:36)
at com.zsassociates.se.SERunner2$.main(SERunner2.scala:14)
at com.zsassociates.se.SERunner2.main(SERunner2.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144)
16/05/26 19:14:31 INFO SparkContext: Invoking stop() from shutdown hook
16/05/26 19:14:31 INFO SparkUI: Stopped Spark web UI at http://10.29.30.196:4041
16/05/26 19:14:31 INFO MapOutputTrackerMasterEndpoint:
MapOutputTrackerMasterEndpoint stopped!
16/05/26 19:14:31 INFO MemoryStore: MemoryStore cleared
16/05/26 19:14:31 INFO BlockManager: BlockManager stopped
16/05/26 19:14:31 INFO BlockManagerMaster: BlockManagerMaster stopped
16/05/26 19:14:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:
OutputCommitCoordinator stopped!
16/05/26 19:14:31 INFO SparkContext: Successfully stopped SparkContext
16/05/26 19:14:31 INFO ShutdownHookManager: Shutdown hook called
16/05/26 19:14:31 INFO ShutdownHookManager: Deleting directory
C:\Temp\spark-5f813d2e-7f59-487f-b06c-783ecc1e2d94
16/05/26 19:14:31 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down
remote daemon.
16/05/26 19:14:31 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon
shut down; proceeding with flushing remote transports.
16/05/26 19:14:31 INFO RemoteActorRefProvider$RemotingTerminator: Remoting shut
down.
16/05/26 19:14:31 ERROR ShutdownHookManager: Exception while deleting Spark
temp dir: C:\Temp\spark-5f813d2e-7f59-487f-b06c-783ecc1e2d94
java.io.IOException: Failed to delete:
C:\Temp\spark-5f813d2e-7f59-487f-b06c-783ecc1e2d94
at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:928)
at
org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:65)
at
org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:62)
at
scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108)
at
org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:62)
at
org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:267)
at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:239)
at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:239)
at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:239)
at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1741)
at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:239)
at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:239)
at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:239)
at scala.util.Try$.apply(Try.scala:161)
at
org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:239)
at
org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:218)
at
org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54)
{noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
