Daniel Thomas created SPARK-12180: ------------------------------------- Summary: DataFrame.join() in PySpark gives misleading exception when column name exists on both side Key: SPARK-12180 URL: https://issues.apache.org/jira/browse/SPARK-12180 Project: Spark Issue Type: Bug Components: PySpark Affects Versions: 1.5.2 Reporter: Daniel Thomas
When joining two DataFrames on a column 'session_uuid' I got the following exception, because both DataFrames hat a column called 'at'. The exception is misleading in the cause and in the column causing the problem. Renaming the column fixed the exception. --------------------------------------------------------------------------- Py4JJavaError Traceback (most recent call last) /Applications/spark-1.5.2-bin-hadoop2.4/python/pyspark/sql/utils.py in deco(*a, **kw) 35 try: ---> 36 return f(*a, **kw) 37 except py4j.protocol.Py4JJavaError as e: /Applications/spark-1.5.2-bin-hadoop2.4/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name) 299 'An error occurred while calling {0}{1}{2}.\n'. --> 300 format(target_id, '.', name), value) 301 else: Py4JJavaError: An error occurred while calling o484.join. : org.apache.spark.sql.AnalysisException: resolved attribute(s) session_uuid#3278 missing from uuid_x#9078,total_session_sec#9115L,at#3248,session_uuid#9114,uuid#9117,at#9084 in operator !Join Inner, Some((uuid_x#9078 = session_uuid#3278)); at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:37) at org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:44) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:154) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:49) at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:103) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:49) at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:44) at org.apache.spark.sql.SQLContext$QueryExecution.assertAnalyzed(SQLContext.scala:914) at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:132) at org.apache.spark.sql.DataFrame.org$apache$spark$sql$DataFrame$$logicalPlanToDataFrame(DataFrame.scala:154) at org.apache.spark.sql.DataFrame.join(DataFrame.scala:553) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379) at py4j.Gateway.invoke(Gateway.java:259) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.GatewayConnection.run(GatewayConnection.java:207) at java.lang.Thread.run(Thread.java:745) During handling of the above exception, another exception occurred: AnalysisException Traceback (most recent call last) <ipython-input-49-8fc6876649b7> in <module>() 1 sel_starts = starts.select('uuid', 'at').withColumnRenamed('uuid', 'uuid_x')#.withColumnRenamed('at', 'at_x') 2 sel_closes = closes.select('uuid', 'at', 'session_uuid', 'total_session_sec') ----> 3 start_close = sel_starts.join(sel_closes, sel_starts['uuid_x'] == sel_closes['session_uuid']) 4 start_close.cache() 5 start_close.take(1) /Applications/spark-1.5.2-bin-hadoop2.4/python/pyspark/sql/dataframe.py in join(self, other, on, how) 579 on = on[0] 580 if how is None: --> 581 jdf = self._jdf.join(other._jdf, on._jc, "inner") 582 else: 583 assert isinstance(how, basestring), "how should be basestring" /Applications/spark-1.5.2-bin-hadoop2.4/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py in __call__(self, *args) 536 answer = self.gateway_client.send_command(command) 537 return_value = get_return_value(answer, self.gateway_client, --> 538 self.target_id, self.name) 539 540 for temp_arg in temp_args: /Applications/spark-1.5.2-bin-hadoop2.4/python/pyspark/sql/utils.py in deco(*a, **kw) 38 s = e.java_exception.toString() 39 if s.startswith('org.apache.spark.sql.AnalysisException: '): ---> 40 raise AnalysisException(s.split(': ', 1)[1]) 41 if s.startswith('java.lang.IllegalArgumentException: '): 42 raise IllegalArgumentException(s.split(': ', 1)[1]) AnalysisException: resolved attribute(s) session_uuid#3278 missing from uuid_x#9078,total_session_sec#9115L,at#3248,session_uuid#9114,uuid#9117,at#9084 in operator !Join Inner, Some((uuid_x#9078 = session_uuid#3278)); -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org