Xuxiang Mao created SPARK-20053:
-----------------------------------
Summary: Can't select col when the dot (.) in col name
Key: SPARK-20053
URL: https://issues.apache.org/jira/browse/SPARK-20053
Project: Spark
Issue Type: Bug
Components: Java API
Affects Versions: 2.1.0
Environment: mac OX
Reporter: Xuxiang Mao
I use java API read a csv file as Dataframe and try to do
Dataframe.select("column name").show(). This operation can successfully done
when the column name contains no ".", but it will fail when the column name has
".". ERROR:
Exception in thread "main" org.apache.spark.sql.AnalysisException: cannot
resolve '`r_2_shape_1.8`' given input columns: [z_2.1.1, z_2.1.11, ....
r_1.34.2, r_1.14.2, r_2_shape_1.8, z_1.2.39];;
'Project ['r_2_shape_1.8]
+- TypedFilter
com.amazon.recommerce.pricing.forecasting.postProcessing.utils.RawFileUtils$1@a03529c,
interface org.apache.spark.sql.Row, [StructField(lp__,IntegerType,true),
StructField(b.1,DoubleType,true), StructField(temp_Intercept,DoubleType,true),
StructField(b_shape.1,DoubleType,true), StructField(sd_1.1,DoubleType,true),
StructField(sd_1_2,DoubleType,true), StructField(z_1.1.1,DoubleType,true),
StructField(z_1.2.1,DoubleType,true), StructField(z_1.1.2,DoubleType,true),
StructField(z_1.2.2,DoubleType,true), StructField(z_1.1.3,DoubleType,true),
StructField(z_1.2.3,DoubleType,true), StructField(z_1.1.4,DoubleType,true),
StructField(z_1.2.4,DoubleType,true), StructField(z_1.1.5,DoubleType,true),
StructField(z_1.2.5,DoubleType,true), StructField(z_1.1.6,DoubleType,true),
StructField(z_1.2.6,DoubleType,true), StructField(z_1.1.7,DoubleType,true),
StructField(z_1.2.7,DoubleType,true), StructField(z_1.1.8,DoubleType,true),
StructField(z_1.2.8,DoubleType,true), StructField(z_1.1.9,DoubleType,true),
StructField(z_1.2.9,DoubleType,true), ... 294 more fields],
createexternalrow(lp__#0, b.1#1, temp_Intercept#2, b_shape.1#3, sd_1.1#4,
sd_1_2#5, z_1.1.1#6, z_1.2.1#7, z_1.1.2#8, z_1.2.2#9, z_1.1.3#10, z_1.2.3#11,
z_1.1.4#12, z_1.2.4#13, z_1.1.5#14, z_1.2.5#15, z_1.1.6#16, z_1.2.6#17,
z_1.1.7#18, z_1.2.7#19, z_1.1.8#20, z_1.2.8#21, z_1.1.9#22, z_1.2.9#23, ... 612
more fields)
+-
Relation[lp__#0,b.1#1,temp_Intercept#2,b_shape.1#3,sd_1.1#4,sd_1_2#5,z_1.1.1#6,z_1.2.1#7,z_1.1.2#8,z_1.2.2#9,z_1.1.3#10,z_1.2.3#11,z_1.1.4#12,z_1.2.4#13,z_1.1.5#14,z_1.2.5#15,z_1.1.6#16,z_1.2.6#17,z_1.1.7#18,z_1.2.7#19,z_1.1.8#20,z_1.2.8#21,z_1.1.9#22,z_1.2.9#23,...
294 more fields] csv
at
org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:77)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:74)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:310)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:310)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:309)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionUp$1(QueryPlan.scala:282)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2(QueryPlan.scala:292)
at
org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2$1.apply(QueryPlan.scala:296)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.AbstractTraversable.map(Traversable.scala:104)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2(QueryPlan.scala:296)
at
org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$7.apply(QueryPlan.scala:301)
at
org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:188)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:301)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:74)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:67)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:128)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:67)
at
org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:57)
at
org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:48)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:63)
at
org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$withPlan(Dataset.scala:2822)
at org.apache.spark.sql.Dataset.select(Dataset.scala:1121)
at org.apache.spark.sql.Dataset.select(Dataset.scala:1139)
at org.apache.spark.sql.Dataset.select(Dataset.scala:1139)
at
com.amazon.recommerce.pricing.forecasting.postProcessing.PostProcessing.main(PostProcessing.java:47)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
