[
https://issues.apache.org/jira/browse/SPARK-9374?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14643761#comment-14643761
]
Adrian Wang commented on SPARK-9374:
------------------------------------
[~chenghao][~cloud_fan][~jameszhouyi]
UnixTimestamp is a non-deterministic expression, because when we pass zero
argument to this function, it means the same with current_timestamp.
And there is a determistic version of this function in hive, namely
to_unix_timstamp. We could use that temporarily. After SPARK-8174 resolved, we
would be able to tell whether the use of unix_timestamp is deterministic or not.
> [Spark SQL] Throw out erorr of "AnalysisException: nondeterministic
> expressions are only allowed in Project or Filter" during the spark sql parse
> phase
> -------------------------------------------------------------------------------------------------------------------------------------------------------
>
> Key: SPARK-9374
> URL: https://issues.apache.org/jira/browse/SPARK-9374
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 1.5.0
> Reporter: Yi Zhou
> Priority: Blocker
>
> #################Spark SQL Query########
> INSERT INTO TABLE TEST_QUERY_0_result
> SELECT w_state, i_item_id,
> SUM(
> CASE WHEN (unix_timestamp(d_date,'yyyy-MM-dd') <
> unix_timestamp('2001-03-16','yyyy-MM-dd'))
> THEN ws_sales_price - COALESCE(wr_refunded_cash,0)
> ELSE 0.0 END
> ) AS sales_before,
> SUM(
> CASE WHEN (unix_timestamp(d_date,'yyyy-MM-dd') >=
> unix_timestamp('2001-03-16','yyyy-MM-dd'))
> THEN ws_sales_price - coalesce(wr_refunded_cash,0)
> ELSE 0.0 END
> ) AS sales_after
> FROM (
> SELECT *
> FROM web_sales ws
> LEFT OUTER JOIN web_returns wr ON (ws.ws_order_number = wr.wr_order_number
> AND ws.ws_item_sk = wr.wr_item_sk)
> ) a1
> JOIN item i ON a1.ws_item_sk = i.i_item_sk
> JOIN warehouse w ON a1.ws_warehouse_sk = w.w_warehouse_sk
> JOIN date_dim d ON a1.ws_sold_date_sk = d.d_date_sk
> AND unix_timestamp(d.d_date, 'yyyy-MM-dd') >= unix_timestamp('2001-03-16',
> 'yyyy-MM-dd') - 30*24*60*60 --subtract 30 days in seconds
> AND unix_timestamp(d.d_date, 'yyyy-MM-dd') <= unix_timestamp('2001-03-16',
> 'yyyy-MM-dd') + 30*24*60*60 --add 30 days in seconds
> GROUP BY w_state,i_item_id
> CLUSTER BY w_state,i_item_id
> ############Error Message##################
> org.apache.spark.sql.AnalysisException: nondeterministic expressions are only
> allowed in Project or Filter, found:
> (((ws_sold_date_sk = d_date_sk) &&
> (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date,yyyy-MM-dd)
> >=
> (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd)
> - CAST((((30 * 24) * 60) * 60), LongType)))) &&
> (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date,yyyy-MM-dd)
> <=
> (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd)
> + CAST((((30 * 24) * 60) * 60), LongType))))
> in operator Join Inner, Some((((ws_sold_date_sk#289L = d_date_sk#383L) &&
> (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date#385,yyyy-MM-dd)
> >=
> (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd)
> - CAST((((30 * 24) * 60) * 60), LongType)))) &&
> (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date#385,yyyy-MM-dd)
> <=
> (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd)
> + CAST((((30 * 24) * 60) * 60), LongType)))))
> ;
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:37)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:43)
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:148)
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:49)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:103)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:49)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:43)
> at
> org.apache.spark.sql.SQLContext$QueryExecution.assertAnalyzed(SQLContext.scala:976)
> at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:131)
> at org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:52)
> at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:792)
> at
> org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:61)
> at
> org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:284)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:423)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:359)
> at
> org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:456)
> at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:466)
> at
> org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:163)
> at
> org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:666)
> at
> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:178)
> at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:203)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:118)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
