[ https://issues.apache.org/jira/browse/SPARK-12180?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15059816#comment-15059816 ]
Daniel Thomas commented on SPARK-12180: --------------------------------------- Here is the code. Without renaming the columns it was throwing the exception. {code} sel_starts = starts.select('uuid', 'at').withColumnRenamed('uuid', 'uuid_x').withColumnRenamed('at', 'at_x') sel_closes = closes.select('uuid', 'at', 'session_uuid', 'total_session_sec') start_close = sel_starts.join(sel_closes, sel_starts['uuid_x'] == sel_closes['session_uuid']) start_close.cache() start_close.take(1) {code} > DataFrame.join() in PySpark gives misleading exception when column name > exists on both side > ------------------------------------------------------------------------------------------- > > Key: SPARK-12180 > URL: https://issues.apache.org/jira/browse/SPARK-12180 > Project: Spark > Issue Type: Bug > Components: PySpark > Affects Versions: 1.5.2 > Reporter: Daniel Thomas > > When joining two DataFrames on a column 'session_uuid' I got the following > exception, because both DataFrames hat a column called 'at'. The exception is > misleading in the cause and in the column causing the problem. Renaming the > column fixed the exception. > --------------------------------------------------------------------------- > Py4JJavaError Traceback (most recent call last) > /Applications/spark-1.5.2-bin-hadoop2.4/python/pyspark/sql/utils.py in > deco(*a, **kw) > 35 try: > ---> 36 return f(*a, **kw) > 37 except py4j.protocol.Py4JJavaError as e: > /Applications/spark-1.5.2-bin-hadoop2.4/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py > in get_return_value(answer, gateway_client, target_id, name) > 299 'An error occurred while calling {0}{1}{2}.\n'. > --> 300 format(target_id, '.', name), value) > 301 else: > Py4JJavaError: An error occurred while calling o484.join. > : org.apache.spark.sql.AnalysisException: resolved attribute(s) > session_uuid#3278 missing from > uuid_x#9078,total_session_sec#9115L,at#3248,session_uuid#9114,uuid#9117,at#9084 > in operator !Join Inner, Some((uuid_x#9078 = session_uuid#3278)); > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:37) > at > org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:44) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:154) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:49) > at > org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:103) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:49) > at > org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:44) > at > org.apache.spark.sql.SQLContext$QueryExecution.assertAnalyzed(SQLContext.scala:914) > at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:132) > at > org.apache.spark.sql.DataFrame.org$apache$spark$sql$DataFrame$$logicalPlanToDataFrame(DataFrame.scala:154) > at org.apache.spark.sql.DataFrame.join(DataFrame.scala:553) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231) > at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379) > at py4j.Gateway.invoke(Gateway.java:259) > at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133) > at py4j.commands.CallCommand.execute(CallCommand.java:79) > at py4j.GatewayConnection.run(GatewayConnection.java:207) > at java.lang.Thread.run(Thread.java:745) > During handling of the above exception, another exception occurred: > AnalysisException Traceback (most recent call last) > <ipython-input-49-8fc6876649b7> in <module>() > 1 sel_starts = starts.select('uuid', 'at').withColumnRenamed('uuid', > 'uuid_x')#.withColumnRenamed('at', 'at_x') > 2 sel_closes = closes.select('uuid', 'at', 'session_uuid', > 'total_session_sec') > ----> 3 start_close = sel_starts.join(sel_closes, sel_starts['uuid_x'] == > sel_closes['session_uuid']) > 4 start_close.cache() > 5 start_close.take(1) > /Applications/spark-1.5.2-bin-hadoop2.4/python/pyspark/sql/dataframe.py in > join(self, other, on, how) > 579 on = on[0] > 580 if how is None: > --> 581 jdf = self._jdf.join(other._jdf, on._jc, "inner") > 582 else: > 583 assert isinstance(how, basestring), "how should be > basestring" > /Applications/spark-1.5.2-bin-hadoop2.4/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py > in __call__(self, *args) > 536 answer = self.gateway_client.send_command(command) > 537 return_value = get_return_value(answer, self.gateway_client, > --> 538 self.target_id, self.name) > 539 > 540 for temp_arg in temp_args: > /Applications/spark-1.5.2-bin-hadoop2.4/python/pyspark/sql/utils.py in > deco(*a, **kw) > 38 s = e.java_exception.toString() > 39 if s.startswith('org.apache.spark.sql.AnalysisException: > '): > ---> 40 raise AnalysisException(s.split(': ', 1)[1]) > 41 if s.startswith('java.lang.IllegalArgumentException: '): > 42 raise IllegalArgumentException(s.split(': ', 1)[1]) > AnalysisException: resolved attribute(s) session_uuid#3278 missing from > uuid_x#9078,total_session_sec#9115L,at#3248,session_uuid#9114,uuid#9117,at#9084 > in operator !Join Inner, Some((uuid_x#9078 = session_uuid#3278)); -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org