Lishu Liu created SPARK-5632: -------------------------------- Summary: not able to resolve dot('.') in field name Key: SPARK-5632 URL: https://issues.apache.org/jira/browse/SPARK-5632 Project: Spark Issue Type: Bug Components: SQL Affects Versions: 1.2.0 Environment: Spark cluster: EC2 m1.small + Spark 1.2.0 Cassandra cluster: EC2 m3.xlarge + Cassandra 2.1.2 Reporter: Lishu Liu
My cassandra table task_trace has a field sm.result which contains dot in the name. So SQL tried to look up sm instead of full name 'sm.result'. Here is my code: scala> import org.apache.spark.sql.cassandra.CassandraSQLContext scala> val cc = new CassandraSQLContext(sc) scala> val task_trace = cc.jsonFile("/task_trace.json") scala> task_trace.registerTempTable("task_trace") scala> cc.setKeyspace("cerberus_data_v4") scala> val res = cc.sql("SELECT received_datetime, task_body.cerberus_id, task_body.sm.result FROM task_trace WHERE task_id = 'fff7304e-9984-4b45-b10c-0423a96745ce'") res: org.apache.spark.sql.SchemaRDD = SchemaRDD[57] at RDD at SchemaRDD.scala:108 == Query Plan == == Physical Plan == java.lang.RuntimeException: No such struct field sm in cerberus_batch_id, cerberus_id, couponId, coupon_code, created, description, domain, expires, message_id, neverShowAfter, neverShowBefore, offerTitle, screenshots, sm.result, sm.task, startDate, task_id, url, uuid, validationDateTime, validity The full schema look like this: scala> task_trace.printSchema() root |-- received_datetime: long (nullable = true) |-- task_body: struct (nullable = true) | |-- cerberus_batch_id: string (nullable = true) | |-- cerberus_id: string (nullable = true) | |-- couponId: integer (nullable = true) | |-- coupon_code: string (nullable = true) | |-- created: string (nullable = true) | |-- description: string (nullable = true) | |-- domain: string (nullable = true) | |-- expires: string (nullable = true) | |-- message_id: string (nullable = true) | |-- neverShowAfter: string (nullable = true) | |-- neverShowBefore: string (nullable = true) | |-- offerTitle: string (nullable = true) | |-- screenshots: array (nullable = true) | | |-- element: string (containsNull = false) | |-- sm.result: struct (nullable = true) | | |-- cerberus_batch_id: string (nullable = true) | | |-- cerberus_id: string (nullable = true) | | |-- code: string (nullable = true) | | |-- couponId: integer (nullable = true) | | |-- created: string (nullable = true) | | |-- description: string (nullable = true) | | |-- domain: string (nullable = true) | | |-- expires: string (nullable = true) | | |-- message_id: string (nullable = true) | | |-- neverShowAfter: string (nullable = true) | | |-- neverShowBefore: string (nullable = true) | | |-- offerTitle: string (nullable = true) | | |-- result: struct (nullable = true) | | | |-- post: struct (nullable = true) | | | | |-- alchemy_out_of_stock: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: boolean (nullable = true) | | | | |-- meta: struct (nullable = true) | | | | | |-- None_tx_value: array (nullable = true) | | | | | | |-- element: string (containsNull = false) | | | | | |-- exceptions: array (nullable = true) | | | | | | |-- element: string (containsNull = false) | | | | | |-- no_input_value: array (nullable = true) | | | | | | |-- element: string (containsNull = false) | | | | | |-- not_mapped: array (nullable = true) | | | | | | |-- element: string (containsNull = false) | | | | | |-- not_transformed: array (nullable = true) | | | | | | |-- element: array (containsNull = false) | | | | | | | |-- element: string (containsNull = false) | | | | |-- now_price_checkout: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: double (nullable = true) | | | | |-- shipping_price: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: double (nullable = true) | | | | |-- tax: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: double (nullable = true) | | | | |-- total: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: double (nullable = true) | | | |-- pre: struct (nullable = true) | | | | |-- alchemy_out_of_stock: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: boolean (nullable = true) | | | | |-- meta: struct (nullable = true) | | | | | |-- None_tx_value: array (nullable = true) | | | | | | |-- element: string (containsNull = false) | | | | | |-- exceptions: array (nullable = true) | | | | | | |-- element: string (containsNull = false) | | | | | |-- no_input_value: array (nullable = true) | | | | | | |-- element: string (containsNull = false) | | | | | |-- not_mapped: array (nullable = true) | | | | | | |-- element: string (containsNull = false) | | | | | |-- not_transformed: array (nullable = true) | | | | | | |-- element: array (containsNull = false) | | | | | | | |-- element: string (containsNull = false) | | | | |-- now_price_checkout: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: double (nullable = true) | | | | |-- shipping_price: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: double (nullable = true) | | | | |-- tax: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: double (nullable = true) | | | | |-- total: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: double (nullable = true) | | |-- startDate: string (nullable = true) | | |-- status: string (nullable = true) | | |-- task_id: string (nullable = true) | | |-- url: string (nullable = true) | | |-- uuid: string (nullable = true) | | |-- validationDateTime: string (nullable = true) | | |-- validity: string (nullable = true) | |-- sm.task: struct (nullable = true) | | |-- cerberus_batch_id: string (nullable = true) | | |-- cerberus_id: string (nullable = true) | | |-- couponId: integer (nullable = true) | | |-- coupon_code: string (nullable = true) | | |-- created: string (nullable = true) | | |-- description: string (nullable = true) | | |-- domain: string (nullable = true) | | |-- expires: string (nullable = true) | | |-- message_id: string (nullable = true) | | |-- neverShowAfter: string (nullable = true) | | |-- neverShowBefore: string (nullable = true) | | |-- offerTitle: string (nullable = true) | | |-- result: struct (nullable = true) | | | |-- post: struct (nullable = true) | | | | |-- final_price: struct (nullable = true) | | | | | |-- ci: integer (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- merchant_exception: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: boolean (nullable = true) | | | | |-- now_price_checkout: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- orig_price: struct (nullable = true) | | | | | |-- ci: integer (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- out_of_stock: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: boolean (nullable = true) | | | | |-- sale_price: struct (nullable = true) | | | | | |-- ci: integer (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- shipping_price: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- tax: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- total: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: string (nullable = true) | | | |-- pre: struct (nullable = true) | | | | |-- final_price: struct (nullable = true) | | | | | |-- ci: integer (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- merchant_exception: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: boolean (nullable = true) | | | | |-- now_price_checkout: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- orig_price: struct (nullable = true) | | | | | |-- ci: integer (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- out_of_stock: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: boolean (nullable = true) | | | | |-- sale_price: struct (nullable = true) | | | | | |-- ci: integer (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- shipping_price: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- tax: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: string (nullable = true) | | | | |-- total: struct (nullable = true) | | | | | |-- ci: double (nullable = true) | | | | | |-- value: string (nullable = true) | | |-- screenshots: array (nullable = true) | | | |-- element: string (containsNull = false) | | |-- startDate: string (nullable = true) | | |-- status: string (nullable = true) | | |-- task_id: string (nullable = true) | | |-- url: string (nullable = true) | | |-- uuid: string (nullable = true) | | |-- validationDateTime: string (nullable = true) | | |-- validity: string (nullable = true) | |-- startDate: string (nullable = true) | |-- task_id: string (nullable = true) | |-- url: string (nullable = true) | |-- uuid: string (nullable = true) | |-- validationDateTime: string (nullable = true) | |-- validity: string (nullable = true) |-- task_id: string (nullable = true) |-- task_step: integer (nullable = true) |-- task_type: integer (nullable = true) I also tried val sqlContext = new org.apache.spark.sql.SQLContext(sc), but it give the same exception. -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org