spyzzz edited a comment on issue #2175:
URL: https://github.com/apache/hudi/issues/2175#issuecomment-713434086
sorry for delay;
Here some tips about what i did :
```
val df = SparkUtils.getStream(sparkSession,config).option("subscribe",
table.name).load()
val df2 = df
.withColumn("deser_value",Deserializer.deser(table.name,config.getRegistryProps)(col("value")))
.withColumn("parsed_value",from_json(col("deser_value"),sru.getLastestSchema(table.name).dataType))
val df_upsert = df2.select("parsed_value")
.select(
col("parsed_value.after.*"),
col("parsed_value.ts_ms"),
col("parsed_value.op"),
col("parsed_value.before."+table.pk).as("id_before")
)
.withColumn(table.pk,when(col("op") ===
'd',col("id_before")).otherwise(col(table.pk))).drop("id_before")
.filter(col(table.pk).isNotNull)
.withColumn("_hoodie_is_deleted",when(col("op") === 'd' ,
true).otherwise(false))
df_upsert.writeStream
.trigger(Trigger.ProcessingTime("120 seconds"))
.foreachBatch((df,id) => {
df.write.format("org.apache.hudi")
.options(HudiUtils.getHudiOptions(table))
.options(HudiUtils.getHiveSyncOptions(table.name))
.options(HudiUtils.getCompactionOptions)
.mode(SaveMode.Append)
.save(config.pathConf.outputPath + "/out/" + table.name )
})
.option("checkpointLocation",config.pathConf.outputPath +
"/checkpoint/" + table.name)
.start()
```
```
object HudiUtils {
def getHudiOptions(table:Table) : Map[String,String] ={
Map(
TABLE_TYPE_OPT_KEY -> "MERGE_ON_READ",
PRECOMBINE_FIELD_OPT_KEY -> "ts_ms",
RECORDKEY_FIELD_OPT_KEY -> table.pk,
OPERATION_OPT_KEY -> "upsert",
KEYGENERATOR_CLASS_OPT_KEY->
"org.apache.hudi.keygen.NonpartitionedKeyGenerator",
TABLE_NAME_OPT_KEY -> ("hudi_" + table.name),
"hoodie.table.name" -> ("hudi_" + table.name),
"hoodie.upsert.shuffle.parallelism"-> "6",
"hoodie.insert.shuffle.parallelism"-> "6",
"hoodie.bulkinsert.shuffle.parallelism"-> "6",
"hoodie.parquet.small.file.limit" -> "4194304"
)
}
def getCompactionOptions : Map[String,String] = {
Map(
"hoodie.compact.inline" -> "true",
"hoodie.compact.inline.max.delta.commits" -> "10",
"hoodie.cleaner.commits.retained" -> "10",
"hoodie.cleaner.fileversions.retained" -> "10",
"hoodie.keep.min.commits" -> "12",
"hoodie.keep.max.commits" -> "13"
//"hoodie.clean.async" -> "false",
//"hoodie.clean.automatic" ->"true",
//"hoodie.parquet.compression.codec" -> "snappy"
)
}
def getHiveSyncOptions(tableName:String) : Map[String,String] = {
Map(
HIVE_SYNC_ENABLED_OPT_KEY -> "true",
HIVE_USE_JDBC_OPT_KEY -> "false",
HIVE_DATABASE_OPT_KEY -> "raw_eu_hudi",
HIVE_URL_OPT_KEY -> "thrift://x",
HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY->
"org.apache.hudi.hive.NonPartitionedExtractor",
HIVE_TABLE_OPT_KEY -> tableName.split("\\.").drop(1).mkString("_")
)
}
}
```
```
object Deserializer extends Serializable {
def deser(topic:String,props:Map[String,String]) = udf((input:
Array[Byte]) => deserializeMessage(props,topic,input))
val valueDeserializer = new KafkaAvroDeserializer()
private def
deserializeMessage(props:Map[String,String],topic:String,input: Array[Byte]):
String = {
try {
valueDeserializer.configure(props.asJava,false)
valueDeserializer.deserialize(topic,
input).asInstanceOf[GenericRecord].toString
} catch {
case e: Exception => {
e.printStackTrace()
null
}
}
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
