Daniel Thomas created SPARK-12180:
-------------------------------------
Summary: DataFrame.join() in PySpark gives misleading exception
when column name exists on both side
Key: SPARK-12180
URL: https://issues.apache.org/jira/browse/SPARK-12180
Project: Spark
Issue Type: Bug
Components: PySpark
Affects Versions: 1.5.2
Reporter: Daniel Thomas
When joining two DataFrames on a column 'session_uuid' I got the following
exception, because both DataFrames hat a column called 'at'. The exception is
misleading in the cause and in the column causing the problem. Renaming the
column fixed the exception.
---------------------------------------------------------------------------
Py4JJavaError Traceback (most recent call last)
/Applications/spark-1.5.2-bin-hadoop2.4/python/pyspark/sql/utils.py in deco(*a,
**kw)
35 try:
---> 36 return f(*a, **kw)
37 except py4j.protocol.Py4JJavaError as e:
/Applications/spark-1.5.2-bin-hadoop2.4/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py
in get_return_value(answer, gateway_client, target_id, name)
299 'An error occurred while calling {0}{1}{2}.\n'.
--> 300 format(target_id, '.', name), value)
301 else:
Py4JJavaError: An error occurred while calling o484.join.
: org.apache.spark.sql.AnalysisException: resolved attribute(s)
session_uuid#3278 missing from
uuid_x#9078,total_session_sec#9115L,at#3248,session_uuid#9114,uuid#9117,at#9084
in operator !Join Inner, Some((uuid_x#9078 = session_uuid#3278));
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:37)
at
org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:44)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:154)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:49)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:103)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:49)
at
org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:44)
at
org.apache.spark.sql.SQLContext$QueryExecution.assertAnalyzed(SQLContext.scala:914)
at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:132)
at
org.apache.spark.sql.DataFrame.org$apache$spark$sql$DataFrame$$logicalPlanToDataFrame(DataFrame.scala:154)
at org.apache.spark.sql.DataFrame.join(DataFrame.scala:553)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
at py4j.Gateway.invoke(Gateway.java:259)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:207)
at java.lang.Thread.run(Thread.java:745)
During handling of the above exception, another exception occurred:
AnalysisException Traceback (most recent call last)
<ipython-input-49-8fc6876649b7> in <module>()
1 sel_starts = starts.select('uuid', 'at').withColumnRenamed('uuid',
'uuid_x')#.withColumnRenamed('at', 'at_x')
2 sel_closes = closes.select('uuid', 'at', 'session_uuid',
'total_session_sec')
----> 3 start_close = sel_starts.join(sel_closes, sel_starts['uuid_x'] ==
sel_closes['session_uuid'])
4 start_close.cache()
5 start_close.take(1)
/Applications/spark-1.5.2-bin-hadoop2.4/python/pyspark/sql/dataframe.py in
join(self, other, on, how)
579 on = on[0]
580 if how is None:
--> 581 jdf = self._jdf.join(other._jdf, on._jc, "inner")
582 else:
583 assert isinstance(how, basestring), "how should be
basestring"
/Applications/spark-1.5.2-bin-hadoop2.4/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py
in __call__(self, *args)
536 answer = self.gateway_client.send_command(command)
537 return_value = get_return_value(answer, self.gateway_client,
--> 538 self.target_id, self.name)
539
540 for temp_arg in temp_args:
/Applications/spark-1.5.2-bin-hadoop2.4/python/pyspark/sql/utils.py in deco(*a,
**kw)
38 s = e.java_exception.toString()
39 if s.startswith('org.apache.spark.sql.AnalysisException: '):
---> 40 raise AnalysisException(s.split(': ', 1)[1])
41 if s.startswith('java.lang.IllegalArgumentException: '):
42 raise IllegalArgumentException(s.split(': ', 1)[1])
AnalysisException: resolved attribute(s) session_uuid#3278 missing from
uuid_x#9078,total_session_sec#9115L,at#3248,session_uuid#9114,uuid#9117,at#9084
in operator !Join Inner, Some((uuid_x#9078 = session_uuid#3278));
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
