Repository: spark Updated Branches: refs/heads/master ee575f12f -> 1904bac38
[SPARK-3392] [SQL] Show value spark.sql.shuffle.partitions for mapred.reduce.tasks This is a tiny fix for getting the value of "mapred.reduce.tasks", which make more sense for the hive user. As well as the command "set -v", which should output verbose information for all of the key/values. Author: Cheng Hao <[email protected]> Closes #2261 from chenghao-intel/set_mapreduce_tasks and squashes the following commits: 653858a [Cheng Hao] show value spark.sql.shuffle.partitions for mapred.reduce.tasks Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1904bac3 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1904bac3 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1904bac3 Branch: refs/heads/master Commit: 1904bac38d97df5ae9fb193e92a83c7f8ff6d255 Parents: ee575f1 Author: Cheng Hao <[email protected]> Authored: Thu Sep 4 19:16:12 2014 -0700 Committer: Michael Armbrust <[email protected]> Committed: Thu Sep 4 19:16:12 2014 -0700 ---------------------------------------------------------------------- .../org/apache/spark/sql/execution/commands.scala | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/1904bac3/sql/core/src/main/scala/org/apache/spark/sql/execution/commands.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/commands.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/commands.scala index 286c6d2..94543fc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/commands.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/commands.scala @@ -60,10 +60,10 @@ case class SetCommand( logWarning(s"Property ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS} is deprecated, " + s"automatically converted to ${SQLConf.SHUFFLE_PARTITIONS} instead.") context.setConf(SQLConf.SHUFFLE_PARTITIONS, v) - Array(Row(s"${SQLConf.SHUFFLE_PARTITIONS}=$v")) + Seq(Row(s"${SQLConf.SHUFFLE_PARTITIONS}=$v")) } else { context.setConf(k, v) - Array(Row(s"$k=$v")) + Seq(Row(s"$k=$v")) } // Query the value bound to key k. @@ -78,11 +78,19 @@ case class SetCommand( "hive-hwi-0.12.0.jar", "hive-0.12.0.jar").mkString(":") - Array( + context.getAllConfs.map { case (k, v) => + Row(s"$k=$v") + }.toSeq ++ Seq( Row("system:java.class.path=" + hiveJars), Row("system:sun.java.command=shark.SharkServer2")) } else { - Array(Row(s"$k=${context.getConf(k, "<undefined>")}")) + if (k == SQLConf.Deprecated.MAPRED_REDUCE_TASKS) { + logWarning(s"Property ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS} is deprecated, " + + s"showing ${SQLConf.SHUFFLE_PARTITIONS} instead.") + Seq(Row(s"${SQLConf.SHUFFLE_PARTITIONS}=${context.numShufflePartitions}")) + } else { + Seq(Row(s"$k=${context.getConf(k, "<undefined>")}")) + } } // Query all key-value pairs that are set in the SQLConf of the context. --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
