Rahul Kumar created CARBONDATA-2490:
---------------------------------------
Summary: Even Bad records action is FORCE , SDK writer failed with
ClassCastException
Key: CARBONDATA-2490
URL: https://issues.apache.org/jira/browse/CARBONDATA-2490
Project: CarbonData
Issue Type: Bug
Reporter: Rahul Kumar
Assignee: Rahul Kumar
Test Step :
def myStruct(): Unit = {
val myaSchema=
"""
{
| "namespace": "com.apache.schema",
| "type": "record",
| "name": "StudentActivity",
| "fields": [
| {
| "name": "id",
| "type": "int"
| },
| {
| "name": "name",
| "type": "string"
| },
| {
| "name": "age",
| "type": "double"
| },
|
| {
| "name": "course_details",
| "type": {
| "name": "Activity",
| "type": "record",
| "fields": [
| {
| "name": "course_id",
| "type": "int"
| }
| ]
| }
| },
| {
| "name":"teachers",
| "type":
| {
| "type":"array",
| "items": "string"
|
| }
| },
| {
| "name":"salary",
| "type":
| {
| "type":"array",
| "items": "double"
|
| }
| }
|
| ]
| }
""".stripMargin
var json = "{\"id\": 101,\"name\": \"babu\",\"age\": 12.1,\"course_details\":
" +
"\{\"course_id\": 11},\"teachers\": [\"PK\", \"DK\", \"MK\",
\"SK\"],\"salary\": " +
"[12.12,25.12,65]}"
val nn = new org.apache.avro.Schema.Parser().parse(myaSchema)
val converter = new JsonAvroConverter
var record = converter
.convertToGenericDataRecord(json.getBytes(CharEncoding.UTF_8), nn)
val fields = new Array[Field](6)
fields(0) = new Field("id", DataTypes.INT)
fields(1) = new Field("name", DataTypes.INT)
fields(2) = new Field("age", DataTypes.DOUBLE)
// fields[1] = new Field("age", DataTypes.INT);
val fld = new java.util.ArrayList[StructField]
fld.add(new StructField("course_id", DataTypes.INT))
fields(3) = new Field("course_details", "struct", fld)
val arr = new java.util.ArrayList[StructField]
arr.add(new StructField("teachers",DataTypes.STRING))
fields(4) = new Field("teachers", "array",arr)
val arr1 = new java.util.ArrayList[StructField]
arr1.add(new StructField("salary",DataTypes.DOUBLE))
fields(5) = new Field("salary", "array",arr)
import scala.collection.JavaConverters._
var options = Map("bad_records_action" -> "FORCE").asJava
try {
val writer = CarbonWriter.builder.withSchema(new
Schema(fields)).sortBy(Array("name","id"))
.outputPath("D:/Hadoop/carbon_1_3/sdk/carbon_c1/").isTransactionalTable(false).withLoadOptions(options).buildWriterForAvroInput
for(i <-0 to 100){
val json1 = "{\"id\": 101,\"name\": \"babu\",\"age\": 12.1,\"course_details\":
" +
"\{\"course_id\": 11},\"teachers\": [\"PK\", \"DK\", \"MK\",
\"SK\"],\"salary\": " +
"[12.12,25.12,"+i+"]}";
val record1 = converter
.convertToGenericDataRecord(json1.getBytes(CharEncoding.UTF_8), nn)
writer.write(record1)
}
json = "{\"id\": 501,\"name\": \"babu\",\"age\": 12.1,\"course_details\": " +
"\{\"course_id\": 11},\"teachers\": [\"PK\", \"DK\", \"MK\",
\"SK\",\"\",\"null\"],\"salary\": " +
"[12.12,25.12]}";
record = converter
.convertToGenericDataRecord(json.getBytes(CharEncoding.UTF_8), nn)
writer.write(record)
writer.close()
}
catch {
case e: Exception => {
e.printStackTrace()
}
}
}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
