root |-- adultbasefare: long (nullable = true) |-- adultcommission: long (nullable = true) |-- adultservicetax: long (nullable = true) |-- adultsurcharge: long (nullable = true) |-- airline: string (nullable = true) |-- arrdate: string (nullable = true) |-- arrtime: string (nullable = true) |-- cafecommission: long (nullable = true) |-- carrierid: string (nullable = true) |-- class: string (nullable = true) |-- depdate: string (nullable = true) |-- deptime: string (nullable = true) |-- destination: string (nullable = true) |-- discount: long (nullable = true) |-- duration: string (nullable = true) |-- fare: struct (nullable = true) | |-- A: long (nullable = true) | |-- C: long (nullable = true) | |-- I: long (nullable = true) | |-- adultairlinetxncharge: long (nullable = true) | |-- adultairporttax: long (nullable = true) | |-- adultbasefare: long (nullable = true) | |-- adultcommission: double (nullable = true) | |-- adultsurcharge: long (nullable = true) | |-- adulttotalfare: long (nullable = true) | |-- childairlinetxncharge: long (nullable = true) | |-- childairporttax: long (nullable = true) | |-- childbasefare: long (nullable = true) | |-- childcommission: double (nullable = true) | |-- childsurcharge: long (nullable = true) | |-- childtotalfare: long (nullable = true) | |-- discount: long (nullable = true) | |-- infantairlinetxncharge: long (nullable = true) | |-- infantairporttax: long (nullable = true) | |-- infantbasefare: long (nullable = true) | |-- infantcommission: long (nullable = true) | |-- infantsurcharge: long (nullable = true) | |-- infanttotalfare: long (nullable = true) | |-- servicetax: long (nullable = true) | |-- totalbasefare: long (nullable = true) | |-- totalcommission: double (nullable = true) | |-- totalfare: long (nullable = true) | |-- totalsurcharge: long (nullable = true) | |-- transactionfee: long (nullable = true) |-- farebasis: string (nullable = true) |-- farerule: string (nullable = true) |-- flightcode: string (nullable = true) |-- flightno: string (nullable = true) |-- k: string (nullable = true) |-- onwardflights: array (nullable = true) | |-- element: string (containsNull = true) |-- origin: string (nullable = true) |-- promocode: string (nullable = true) |-- promodiscount: long (nullable = true) |-- promotionText: string (nullable = true) |-- stops: string (nullable = true) |-- tickettype: string (nullable = true) |-- totalbasefare: long (nullable = true) |-- totalcommission: long (nullable = true) |-- totalfare: long (nullable = true) |-- totalpriceamount: long (nullable = true) |-- totalsurcharge: long (nullable = true) |-- transactionfee: long (nullable = true) |-- viacharges: long (nullable = true) |-- warnings: string (nullable = true)
Now i want to flatten it so that the fare field will be removed and everything will be flatten For this i used explode. But i am getting an error: org.apache.spark.sql.AnalysisException: cannot resolve 'explode(fare)' due to data type mismatch: input to function explode should be array or map type, not StructType(StructField(A,LongType,true), StructField(C,LongType,true), StructField(I,LongType,true), StructField(adultairlinetxncharge,LongType,true), StructField(adultairporttax,LongType,true), StructField(adultbasefare,LongType,true), StructField(adultcommission,DoubleType,true), StructField(adultsurcharge,LongType,true), StructField(adulttotalfare,LongType,true), StructField(childairlinetxncharge,LongType,true), StructField(childairporttax,LongType,true), StructField(childbasefare,LongType,true), StructField(childcommission,DoubleType,true), StructField(childsurcharge,LongType,true), StructField(childtotalfare,LongType,true), StructField(discount,LongType,true), StructField(infantairlinetxncharge,LongType,true), StructField(infantairporttax,LongType,true), StructField(infantbasefare,LongType,true), StructField(infantcommission,LongType,true), StructField(infantsurcharge,LongType,true), StructField(infanttotalfare,LongType,true), StructField(servicetax,LongType,true), StructField(totalbasefare,LongType,true), StructField(totalcommission,DoubleType,true), StructField(totalfare,LongType,true), StructField(totalsurcharge,LongType,true), StructField(transactionfee,LongType,true)); If not explode how can i flatten it.Your help will be appreciated. Thanks -- View this message in context: http://apache-spark-user-list.1001560.n3.nabble.com/how-to-flatten-the-dataframe-tp26411.html Sent from the Apache Spark User List mailing list archive at Nabble.com. --------------------------------------------------------------------- To unsubscribe, e-mail: user-unsubscr...@spark.apache.org For additional commands, e-mail: user-h...@spark.apache.org
