[ https://issues.apache.org/jira/browse/SPARK-13800?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Weizhong updated SPARK-13800: ----------------------------- Description: {color:red}connect to Hive MetaStore service as we have set hive.metastore.uris in hive-site.xml{color} 1. start ThriftServer 2. beeline 1 connect to TS, then run {code:sql} create database if not exists hive_bin_partitioned_orc_3; use hive_bin_partitioned_orc_3; set hive.exec.dynamic.partition.mode=nonstrict; set hive.exec.max.dynamic.partitions=10000; set spark.sql.autoBroadcastJoinThreshold=-1; drop table if exists store_returns; create table store_returns( sr_returned_date_sk int , sr_return_time_sk int , sr_item_sk int , sr_customer_sk int , sr_cdemo_sk int , sr_hdemo_sk int , sr_addr_sk int , sr_store_sk int , sr_reason_sk int , sr_ticket_number int , sr_return_quantity int , sr_return_amt float , sr_return_tax float , sr_return_amt_inc_tax float , sr_fee float , sr_return_ship_cost float , sr_refunded_cash float , sr_reversed_charge float , sr_store_credit float , sr_net_loss float ) partitioned by (sr_returned_date string) stored as orc; insert overwrite table store_returns partition (sr_returned_date) select sr.sr_returned_date_sk, sr.sr_return_time_sk, sr.sr_item_sk, sr.sr_customer_sk, sr.sr_cdemo_sk, sr.sr_hdemo_sk, sr.sr_addr_sk, sr.sr_store_sk, sr.sr_reason_sk, sr.sr_ticket_number, sr.sr_return_quantity, sr.sr_return_amt, sr.sr_return_tax, sr.sr_return_amt_inc_tax, sr.sr_fee, sr.sr_return_ship_cost, sr.sr_refunded_cash, sr.sr_reversed_charge, sr.sr_store_credit, sr.sr_net_loss, dd.d_date as sr_returned_date from tpcds_text_3.store_returns sr join tpcds_text_3.date_dim dd on (sr.sr_returned_date_sk = dd.d_date_sk); {code} 3. beeline 2 connect to TS, then run {code:sql} show tables; {code} *INSERT ... SELECT failed as hive.exec.max.dynamic.partitions have been modified to default value(1000).* {noformat} Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.spark.sql.hive.client.Shim_v1_2.loadDynamicPartitions(HiveShim.scala:602) at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$loadDynamicPartitions$1.apply$mcV$sp(ClientWrapper.scala:895) at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$loadDynamicPartitions$1.apply(ClientWrapper.scala:895) at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$loadDynamicPartitions$1.apply(ClientWrapper.scala:895) at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$withHiveState$1.apply(ClientWrapper.scala:322) at org.apache.spark.sql.hive.client.ClientWrapper.liftedTree1$1(ClientWrapper.scala:269) at org.apache.spark.sql.hive.client.ClientWrapper.retryLocked(ClientWrapper.scala:268) at org.apache.spark.sql.hive.client.ClientWrapper.withHiveState(ClientWrapper.scala:311) at org.apache.spark.sql.hive.client.ClientWrapper.loadDynamicPartitions(ClientWrapper.scala:894) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anon$1.run(InsertIntoHiveTable.scala:228) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anon$1.run(InsertIntoHiveTable.scala:226) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1711) ... 25 more Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Number of dynamic partitions created is 1823, which is more than 1000. To solve this try to set hive.exec.max.dynamic.partitions to at least 1823. at org.apache.hadoop.hive.ql.metadata.Hive.loadDynamicPartitions(Hive.java:1584) ... 43 more {noformat} We know, run "SET x=y" will set current SessionState's conf, but the conf on metadataHive's client(Hive object) don't be updated as it's cached by clientLoader {code:java} def client: Hive = { if (clientLoader.cachedHive != null) { clientLoader.cachedHive.asInstanceOf[Hive] } else { val c = Hive.get(conf) clientLoader.cachedHive = c c } } {code} Maybe we need to get Hive object from current SessionState. {code:java} def client: Hive = { Hive.get(state.getConf) } {code} was: {color:red}connect to Hive MetaStore service as we have set hive.metastore.uris in hive-site.xml{color} 1. start ThriftServer 2. beeline 1 connect to TS, then run {code:sql} create database if not exists hive_bin_partitioned_orc_3; use hive_bin_partitioned_orc_3; set hive.exec.dynamic.partition.mode=nonstrict; set hive.exec.max.dynamic.partitions=10000; set spark.sql.autoBroadcastJoinThreshold=-1; drop table if exists store_returns; create table store_returns( sr_returned_date_sk int , sr_return_time_sk int , sr_item_sk int , sr_customer_sk int , sr_cdemo_sk int , sr_hdemo_sk int , sr_addr_sk int , sr_store_sk int , sr_reason_sk int , sr_ticket_number int , sr_return_quantity int , sr_return_amt float , sr_return_tax float , sr_return_amt_inc_tax float , sr_fee float , sr_return_ship_cost float , sr_refunded_cash float , sr_reversed_charge float , sr_store_credit float , sr_net_loss float ) partitioned by (sr_returned_date string) stored as orc; insert overwrite table store_returns partition (sr_returned_date) select sr.sr_returned_date_sk, sr.sr_return_time_sk, sr.sr_item_sk, sr.sr_customer_sk, sr.sr_cdemo_sk, sr.sr_hdemo_sk, sr.sr_addr_sk, sr.sr_store_sk, sr.sr_reason_sk, sr.sr_ticket_number, sr.sr_return_quantity, sr.sr_return_amt, sr.sr_return_tax, sr.sr_return_amt_inc_tax, sr.sr_fee, sr.sr_return_ship_cost, sr.sr_refunded_cash, sr.sr_reversed_charge, sr.sr_store_credit, sr.sr_net_loss, dd.d_date as sr_returned_date from tpcds_text_3.store_returns sr join tpcds_text_3.date_dim dd on (sr.sr_returned_date_sk = dd.d_date_sk); {code} 3. beeline 2 connect to TS, then run {code:sql} show tables; {code} *INSERT ... SELECT failed as hive.exec.max.dynamic.partitions have been modified to default value(1000).* {noformat} Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.spark.sql.hive.client.Shim_v1_2.loadDynamicPartitions(HiveShim.scala:602) at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$loadDynamicPartitions$1.apply$mcV$sp(ClientWrapper.scala:895) at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$loadDynamicPartitions$1.apply(ClientWrapper.scala:895) at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$loadDynamicPartitions$1.apply(ClientWrapper.scala:895) at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$withHiveState$1.apply(ClientWrapper.scala:322) at org.apache.spark.sql.hive.client.ClientWrapper.liftedTree1$1(ClientWrapper.scala:269) at org.apache.spark.sql.hive.client.ClientWrapper.retryLocked(ClientWrapper.scala:268) at org.apache.spark.sql.hive.client.ClientWrapper.withHiveState(ClientWrapper.scala:311) at org.apache.spark.sql.hive.client.ClientWrapper.loadDynamicPartitions(ClientWrapper.scala:894) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anon$1.run(InsertIntoHiveTable.scala:228) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anon$1.run(InsertIntoHiveTable.scala:226) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1711) ... 25 more Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Number of dynamic partitions created is 1823, which is more than 1000. To solve this try to set hive.exec.max.dynamic.partitions to at least 1823. at org.apache.hadoop.hive.ql.metadata.Hive.loadDynamicPartitions(Hive.java:1584) ... 43 more {noformat} We know, run "SET x=y" will set current SessionState's conf, but the conf on metadataHive's client(Hive object) don't be updated as it's cached by clientLoader {code:java} def client: Hive = { if (clientLoader.cachedHive != null) { clientLoader.cachedHive.asInstanceOf[Hive] } else { val c = Hive.get(conf) clientLoader.cachedHive = c c } } {code} Maybe we need to get Hive object from current SessionState. > Hive conf will be modified on multi-beeline connect to thriftserver > ------------------------------------------------------------------- > > Key: SPARK-13800 > URL: https://issues.apache.org/jira/browse/SPARK-13800 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 1.5.1 > Reporter: Weizhong > Priority: Minor > > {color:red}connect to Hive MetaStore service as we have set > hive.metastore.uris in hive-site.xml{color} > 1. start ThriftServer > 2. beeline 1 connect to TS, then run > {code:sql} > create database if not exists hive_bin_partitioned_orc_3; > use hive_bin_partitioned_orc_3; > set hive.exec.dynamic.partition.mode=nonstrict; > set hive.exec.max.dynamic.partitions=10000; > set spark.sql.autoBroadcastJoinThreshold=-1; > drop table if exists store_returns; > create table store_returns( > sr_returned_date_sk int , > sr_return_time_sk int , > sr_item_sk int , > sr_customer_sk int , > sr_cdemo_sk int , > sr_hdemo_sk int , > sr_addr_sk int , > sr_store_sk int , > sr_reason_sk int , > sr_ticket_number int , > sr_return_quantity int , > sr_return_amt float , > sr_return_tax float , > sr_return_amt_inc_tax float , > sr_fee float , > sr_return_ship_cost float , > sr_refunded_cash float , > sr_reversed_charge float , > sr_store_credit float , > sr_net_loss float > ) > partitioned by (sr_returned_date string) > stored as orc; > insert overwrite table store_returns partition (sr_returned_date) > select > sr.sr_returned_date_sk, > sr.sr_return_time_sk, > sr.sr_item_sk, > sr.sr_customer_sk, > sr.sr_cdemo_sk, > sr.sr_hdemo_sk, > sr.sr_addr_sk, > sr.sr_store_sk, > sr.sr_reason_sk, > sr.sr_ticket_number, > sr.sr_return_quantity, > sr.sr_return_amt, > sr.sr_return_tax, > sr.sr_return_amt_inc_tax, > sr.sr_fee, > sr.sr_return_ship_cost, > sr.sr_refunded_cash, > sr.sr_reversed_charge, > sr.sr_store_credit, > sr.sr_net_loss, > dd.d_date as sr_returned_date > from tpcds_text_3.store_returns sr > join tpcds_text_3.date_dim dd > on (sr.sr_returned_date_sk = dd.d_date_sk); > {code} > 3. beeline 2 connect to TS, then run > {code:sql} > show tables; > {code} > *INSERT ... SELECT failed as hive.exec.max.dynamic.partitions have been > modified to default value(1000).* > {noformat} > Caused by: java.lang.reflect.InvocationTargetException > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.apache.spark.sql.hive.client.Shim_v1_2.loadDynamicPartitions(HiveShim.scala:602) > at > org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$loadDynamicPartitions$1.apply$mcV$sp(ClientWrapper.scala:895) > at > org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$loadDynamicPartitions$1.apply(ClientWrapper.scala:895) > at > org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$loadDynamicPartitions$1.apply(ClientWrapper.scala:895) > at > org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$withHiveState$1.apply(ClientWrapper.scala:322) > at > org.apache.spark.sql.hive.client.ClientWrapper.liftedTree1$1(ClientWrapper.scala:269) > at > org.apache.spark.sql.hive.client.ClientWrapper.retryLocked(ClientWrapper.scala:268) > at > org.apache.spark.sql.hive.client.ClientWrapper.withHiveState(ClientWrapper.scala:311) > at > org.apache.spark.sql.hive.client.ClientWrapper.loadDynamicPartitions(ClientWrapper.scala:894) > at > org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anon$1.run(InsertIntoHiveTable.scala:228) > at > org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anon$1.run(InsertIntoHiveTable.scala:226) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:422) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1711) > ... 25 more > Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Number of > dynamic partitions created is 1823, which is more than 1000. To solve this > try to set hive.exec.max.dynamic.partitions to at least 1823. > at > org.apache.hadoop.hive.ql.metadata.Hive.loadDynamicPartitions(Hive.java:1584) > ... 43 more > {noformat} > We know, run "SET x=y" will set current SessionState's conf, but the conf on > metadataHive's client(Hive object) don't be updated as it's cached by > clientLoader > {code:java} > def client: Hive = { > if (clientLoader.cachedHive != null) { > clientLoader.cachedHive.asInstanceOf[Hive] > } else { > val c = Hive.get(conf) > clientLoader.cachedHive = c > c > } > } > {code} > Maybe we need to get Hive object from current SessionState. > {code:java} > def client: Hive = { > Hive.get(state.getConf) > } > {code} -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org