boneanxs commented on issue #5083:
URL: https://github.com/apache/hudi/issues/5083#issuecomment-1300351564
Here is the schema, and it's quite large:
```json
"extraMetadata" : {
"schema" :
"{\"type\":\"record\",\"name\":\"spark_application_hudi_record\",\"namespace\":\"hoodie.spark_application_hudi\",\"fields\":[{\"name\":\"uuid\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"grass_date\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"applicationId\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"applicationName\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"attemptId\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"startTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"endTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"lastUpdated\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"duration\",\"type\":\"long\"},{\"name\":\"sparkUser\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"completed\",\"type\":\"boolean\"},{\"name\":\"appSparkVersion\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"startTimeEpoc
h\",\"type\":\"long\"},{\"name\":\"endTimeEpoch\",\"type\":\"long\"},{\"name\":\"lastUpdatedEpoch\",\"type\":\"long\"},{\"name\":\"yarnTags\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"yarnTags\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record\",\"fields\":[{\"name\":\"env\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"pic\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"pri\",\"type\":\"int\"},{\"name\":\"proj\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"tic\",\"type\":[\"null\",\"string\"],\"default\":null}]}],\"default\":null},{\"name\":\"yarnAggregatedResources\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"yarnAggregatedResources\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record\",\"fields\":[{\"name\":\"memory\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"cpu\",\"type\":[\"null\",\"string\"],\"default\":null}]}],\"default\":null},{\"name\":\"ti
melineInfo\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"timelineInfo\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record\",\"fields\":[{\"name\":\"YARN_APPLICATION_CALLER_CONTEXT\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"YARN_APPLICATION_DIAGNOSTICS_INFO\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"YARN_APPLICATION_FINAL_STATUS\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"YARN_APPLICATION_NAME\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"YARN_APPLICATION_USER\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"YARN_APPLICATION_UNMANAGED_APPLICATION\",\"type\":\"boolean\"},{\"name\":\"FROM_ID\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"UID\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"YARN_APPLICATION_VIEW_ACLS\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"YARN_APPLICATION_SUBMITTED_TIME\",\"type\":[\"null\",\"str
ing\"],\"default\":null},{\"name\":\"YARN_AM_CONTAINER_LAUNCH_COMMAND\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[\"string\",\"null\"]}],\"default\":null},{\"name\":\"YARN_APPLICATION_QUEUE\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"YARN_APPLICATION_TYPE\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"YARN_APPLICATION_PRIORITY\",\"type\":\"int\"},{\"name\":\"YARN_APPLICATION_LATEST_APP_ATTEMPT\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"YARN_APPLICATION_TAGS\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[\"string\",\"null\"]}],\"default\":null},{\"name\":\"YARN_APPLICATION_STATE\",\"type\":[\"null\",\"string\"],\"default\":null}]}],\"default\":null},{\"name\":\"jobs\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"record\",\"name\":\"jobs\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record\",\"fields\":[{\"name\":\"jobId\",\"type\":\"int\"},{\"name\":\"name\",\"type\":[\"null\
",\"string\"],\"default\":null},{\"name\":\"description\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"submissionTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"completionTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"stageIds\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null},{\"name\":\"jobGroup\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"status\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"numTasks\",\"type\":\"int\"},{\"name\":\"numActiveTasks\",\"type\":\"int\"},{\"name\":\"numCompletedTasks\",\"type\":\"int\"},{\"name\":\"numSkippedTasks\",\"type\":\"int\"},{\"name\":\"numFailedTasks\",\"type\":\"int\"},{\"name\":\"numKilledTasks\",\"type\":\"int\"},{\"name\":\"numCompletedIndices\",\"type\":\"int\"},{\"name\":\"numActiveStages\",\"type\":\"int\"},{\"name\":\"numCompletedStages\",\"type\":\"int\"},{\"name\":\"numSkippedStages\",\"type\":\"int\"},{\"name\":\"
numFailedStages\",\"type\":\"int\"},{\"name\":\"killedTasksSummary\",\"type\":[\"null\",{\"type\":\"map\",\"values\":\"int\"}],\"default\":null}]},\"null\"]}],\"default\":null},{\"name\":\"stages\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"record\",\"name\":\"stages\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record\",\"fields\":[{\"name\":\"status\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"stageId\",\"type\":\"int\"},{\"name\":\"attemptId\",\"type\":\"int\"},{\"name\":\"numTasks\",\"type\":\"int\"},{\"name\":\"numActiveTasks\",\"type\":\"int\"},{\"name\":\"numCompleteTasks\",\"type\":\"int\"},{\"name\":\"numFailedTasks\",\"type\":\"int\"},{\"name\":\"numKilledTasks\",\"type\":\"int\"},{\"name\":\"numCompletedIndices\",\"type\":\"int\"},{\"name\":\"submissionTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"firstTaskLaunchedTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"completi
onTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"failureReason\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"executorDeserializeTime\",\"type\":\"long\"},{\"name\":\"executorDeserializeCpuTime\",\"type\":\"long\"},{\"name\":\"executorRunTime\",\"type\":\"long\"},{\"name\":\"executorCpuTime\",\"type\":\"long\"},{\"name\":\"resultSize\",\"type\":\"long\"},{\"name\":\"jvmGcTime\",\"type\":\"long\"},{\"name\":\"resultSerializationTime\",\"type\":\"long\"},{\"name\":\"memoryBytesSpilled\",\"type\":\"long\"},{\"name\":\"diskBytesSpilled\",\"type\":\"long\"},{\"name\":\"peakExecutionMemory\",\"type\":\"long\"},{\"name\":\"inputBytes\",\"type\":\"long\"},{\"name\":\"inputRecords\",\"type\":\"long\"},{\"name\":\"outputBytes\",\"type\":\"long\"},{\"name\":\"outputRecords\",\"type\":\"long\"},{\"name\":\"shuffleRemoteBlocksFetched\",\"type\":\"long\"},{\"name\":\"shuffleLocalBlocksFetched\",\"type\":\"long\"},{\"name\":\"shuffleFetchWaitTime\",\"type\":\"l
ong\"},{\"name\":\"shuffleRemoteBytesRead\",\"type\":\"long\"},{\"name\":\"shuffleRemoteBytesReadToDisk\",\"type\":\"long\"},{\"name\":\"shuffleLocalBytesRead\",\"type\":\"long\"},{\"name\":\"shuffleReadBytes\",\"type\":\"long\"},{\"name\":\"shuffleReadRecords\",\"type\":\"long\"},{\"name\":\"shuffleWriteBytes\",\"type\":\"long\"},{\"name\":\"shuffleWriteTime\",\"type\":\"long\"},{\"name\":\"shuffleWriteRecords\",\"type\":\"long\"},{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"description\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"details\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"schedulingPool\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"rddIds\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null},{\"name\":\"accumulatorUpdates\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"record\",\"name\":\"accumulatorUpdates\",\"namespace\":\"hoodie.spark_applic
ation_hudi.spark_application_hudi_record.stages\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"update\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"value\",\"type\":[\"null\",\"string\"],\"default\":null}]},\"null\"]}],\"default\":null},{\"name\":\"tasks\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[{\"type\":\"record\",\"name\":\"tasks\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages\",\"fields\":[{\"name\":\"taskId\",\"type\":\"long\"},{\"name\":\"index\",\"type\":\"int\"},{\"name\":\"attempt\",\"type\":\"int\"},{\"name\":\"launchTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"resultFetchStart\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"duration\",\"type\":[\"null\",\"long\"],\"default\":null},{\"name\":\"executorId\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"host\",\"type\":[\"null\",
\"string\"],\"default\":null},{\"name\":\"status\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"taskLocality\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"speculative\",\"type\":\"boolean\"},{\"name\":\"accumulatorUpdates\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"record\",\"name\":\"accumulatorUpdates\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages.tasks\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"update\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"value\",\"type\":[\"null\",\"string\"],\"default\":null}]},\"null\"]}],\"default\":null},{\"name\":\"errorMessage\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"taskMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"taskMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages.tasks\",\
"fields\":[{\"name\":\"executorDeserializeTime\",\"type\":\"long\"},{\"name\":\"executorDeserializeCpuTime\",\"type\":\"long\"},{\"name\":\"executorRunTime\",\"type\":\"long\"},{\"name\":\"executorCpuTime\",\"type\":\"long\"},{\"name\":\"resultSize\",\"type\":\"long\"},{\"name\":\"jvmGcTime\",\"type\":\"long\"},{\"name\":\"resultSerializationTime\",\"type\":\"long\"},{\"name\":\"memoryBytesSpilled\",\"type\":\"long\"},{\"name\":\"diskBytesSpilled\",\"type\":\"long\"},{\"name\":\"peakExecutionMemory\",\"type\":\"long\"},{\"name\":\"inputMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"inputMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages.tasks.taskMetrics\",\"fields\":[{\"name\":\"bytesRead\",\"type\":\"long\"},{\"name\":\"recordsRead\",\"type\":\"long\"}]}],\"default\":null},{\"name\":\"outputMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"outputMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_applicat
ion_hudi_record.stages.tasks.taskMetrics\",\"fields\":[{\"name\":\"bytesWritten\",\"type\":\"long\"},{\"name\":\"recordsWritten\",\"type\":\"long\"}]}],\"default\":null},{\"name\":\"shuffleReadMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"shuffleReadMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages.tasks.taskMetrics\",\"fields\":[{\"name\":\"remoteBlocksFetched\",\"type\":\"long\"},{\"name\":\"localBlocksFetched\",\"type\":\"long\"},{\"name\":\"fetchWaitTime\",\"type\":\"long\"},{\"name\":\"remoteBytesRead\",\"type\":\"long\"},{\"name\":\"remoteBytesReadToDisk\",\"type\":\"long\"},{\"name\":\"localBytesRead\",\"type\":\"long\"},{\"name\":\"recordsRead\",\"type\":\"long\"}]}],\"default\":null},{\"name\":\"shuffleWriteMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"shuffleWriteMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages.tasks.taskMetrics\",\"fields\":[{\"name\":\"byt
esWritten\",\"type\":\"long\"},{\"name\":\"writeTime\",\"type\":\"long\"},{\"name\":\"recordsWritten\",\"type\":\"long\"}]}],\"default\":null}]}],\"default\":null},{\"name\":\"executorLogs\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"string\",\"null\"]}],\"default\":null},{\"name\":\"schedulerDelay\",\"type\":\"long\"},{\"name\":\"gettingResultTime\",\"type\":\"long\"}]},\"null\"]}],\"default\":null},{\"name\":\"executorSummary\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[{\"type\":\"record\",\"name\":\"executorSummary\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages\",\"fields\":[{\"name\":\"taskTime\",\"type\":\"long\"},{\"name\":\"failedTasks\",\"type\":\"int\"},{\"name\":\"succeededTasks\",\"type\":\"int\"},{\"name\":\"killedTasks\",\"type\":\"int\"},{\"name\":\"inputBytes\",\"type\":\"long\"},{\"name\":\"inputRecords\",\"type\":\"long\"},{\"name\":\"outputBytes\",\"type\":\"long\"},{\"name\":\"outputRecords\",\"type\":\"long\"},
{\"name\":\"shuffleRead\",\"type\":\"long\"},{\"name\":\"shuffleReadRecords\",\"type\":\"long\"},{\"name\":\"shuffleWrite\",\"type\":\"long\"},{\"name\":\"shuffleWriteRecords\",\"type\":\"long\"},{\"name\":\"memoryBytesSpilled\",\"type\":\"long\"},{\"name\":\"diskBytesSpilled\",\"type\":\"long\"},{\"name\":\"isBlacklistedForStage\",\"type\":\"boolean\"},{\"name\":\"peakMemoryMetrics\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[{\"type\":\"array\",\"items\":\"double\"},\"null\"]}],\"default\":null},{\"name\":\"isExcludedForStage\",\"type\":\"boolean\"}]},\"null\"]}],\"default\":null},{\"name\":\"killedTasksSummary\",\"type\":[\"null\",{\"type\":\"map\",\"values\":\"int\"}],\"default\":null},{\"name\":\"resourceProfileId\",\"type\":\"int\"},{\"name\":\"peakExecutorMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"peakExecutorMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages\",\"fields\":[{\"name\":\"JVMHeapMemory\",\"type\":\
"long\"},{\"name\":\"JVMOffHeapMemory\",\"type\":\"long\"},{\"name\":\"OnHeapExecutionMemory\",\"type\":\"long\"},{\"name\":\"OffHeapExecutionMemory\",\"type\":\"long\"},{\"name\":\"OnHeapStorageMemory\",\"type\":\"long\"},{\"name\":\"OffHeapStorageMemory\",\"type\":\"long\"},{\"name\":\"OnHeapUnifiedMemory\",\"type\":\"long\"},{\"name\":\"OffHeapUnifiedMemory\",\"type\":\"long\"},{\"name\":\"DirectPoolMemory\",\"type\":\"long\"},{\"name\":\"MappedPoolMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreeJVMVMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreeJVMRSSMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreePythonVMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreePythonRSSMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreeOtherVMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreeOtherRSSMemory\",\"type\":\"long\"},{\"name\":\"MinorGCCount\",\"type\":\"long\"},{\"name\":\"MinorGCTime\",\"type\":\"long\"},{\"name\":\"MajorGCCount\",\"type\":\"long\"},{\"name\":\"MajorGCTime\",\"ty
pe\":\"long\"}]}],\"default\":null},{\"name\":\"totalTaskExecutorMetrics\",\"type\":[\"null\",{\"type\":\"map\",\"values\":\"long\"}],\"default\":null},{\"name\":\"taskMetricsDistributions\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"taskMetricsDistributions\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages\",\"fields\":[{\"name\":\"quantiles\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"duration\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"executorDeserializeTime\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"executorDeserializeCpuTime\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"executorRunTime\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"executorCpuTime\",\"type\":[\"null\",{\"type\":\"array\",\"item
s\":\"double\"}],\"default\":null},{\"name\":\"resultSize\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"jvmGcTime\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"resultSerializationTime\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"gettingResultTime\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"schedulerDelay\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"peakExecutionMemory\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"memoryBytesSpilled\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"diskBytesSpilled\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"inputMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":
\"inputMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages.taskMetricsDistributions\",\"fields\":[{\"name\":\"bytesRead\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"recordsRead\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null}]}],\"default\":null},{\"name\":\"outputMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"outputMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages.taskMetricsDistributions\",\"fields\":[{\"name\":\"bytesWritten\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"recordsWritten\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null}]}],\"default\":null},{\"name\":\"shuffleReadMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"shuffleReadMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi
_record.stages.taskMetricsDistributions\",\"fields\":[{\"name\":\"readBytes\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"readRecords\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"remoteBlocksFetched\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"localBlocksFetched\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"fetchWaitTime\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"remoteBytesRead\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"remoteBytesReadToDisk\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"totalBlocksFetched\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null}]}],\"default\":null},{\"name\":\"shuffleWriteMetrics\",\"ty
pe\":[\"null\",{\"type\":\"record\",\"name\":\"shuffleWriteMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages.taskMetricsDistributions\",\"fields\":[{\"name\":\"writeBytes\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"writeRecords\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"writeTime\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null}]}],\"default\":null}]}],\"default\":null},{\"name\":\"executorMetricsDistributions\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"executorMetricsDistributions\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.stages\",\"fields\":[{\"name\":\"quantiles\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"taskTime\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"failedT
asks\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"succeededTasks\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"killedTasks\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"inputBytes\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"inputRecords\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"outputBytes\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"outputRecords\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"shuffleRead\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"shuffleReadRecords\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"shuffleWrite\",\"type\":[\"null\",{\"t
ype\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"shuffleWriteRecords\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"memoryBytesSpilled\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"diskBytesSpilled\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"double\"}],\"default\":null},{\"name\":\"peakMemoryMetrics\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[{\"type\":\"array\",\"items\":\"double\"},\"null\"]}],\"default\":null}]}],\"default\":null}]},\"null\"]}],\"default\":null},{\"name\":\"executors\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"record\",\"name\":\"executors\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record\",\"fields\":[{\"name\":\"id\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"hostPort\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"isActive\",\"type\":\"boolean\"},{\"na
me\":\"rddBlocks\",\"type\":\"int\"},{\"name\":\"memoryUsed\",\"type\":\"long\"},{\"name\":\"diskUsed\",\"type\":\"long\"},{\"name\":\"totalCores\",\"type\":\"int\"},{\"name\":\"maxTasks\",\"type\":\"int\"},{\"name\":\"activeTasks\",\"type\":\"int\"},{\"name\":\"failedTasks\",\"type\":\"int\"},{\"name\":\"completedTasks\",\"type\":\"int\"},{\"name\":\"totalTasks\",\"type\":\"int\"},{\"name\":\"totalDuration\",\"type\":\"long\"},{\"name\":\"totalGCTime\",\"type\":\"long\"},{\"name\":\"totalInputBytes\",\"type\":\"long\"},{\"name\":\"totalShuffleRead\",\"type\":\"long\"},{\"name\":\"totalShuffleWrite\",\"type\":\"long\"},{\"name\":\"isBlacklisted\",\"type\":\"boolean\"},{\"name\":\"maxMemory\",\"type\":\"long\"},{\"name\":\"addTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"removeTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"removeReason\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"executorLogs\",\"type\":[\"null\",{\"type\":
\"map\",\"values\":[\"string\",\"null\"]}],\"default\":null},{\"name\":\"memoryMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"memoryMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.executors\",\"fields\":[{\"name\":\"usedOnHeapStorageMemory\",\"type\":\"long\"},{\"name\":\"usedOffHeapStorageMemory\",\"type\":\"long\"},{\"name\":\"totalOnHeapStorageMemory\",\"type\":\"long\"},{\"name\":\"totalOffHeapStorageMemory\",\"type\":\"long\"}]}],\"default\":null},{\"name\":\"blacklistedInStages\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null},{\"name\":\"peakMemoryMetrics\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"peakMemoryMetrics\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.executors\",\"fields\":[{\"name\":\"JVMHeapMemory\",\"type\":\"long\"},{\"name\":\"JVMOffHeapMemory\",\"type\":\"long\"},{\"name\":\"OnHeapExecutionMemory\",\"type\":\"long\"},{\"name\":\"OffHeapExe
cutionMemory\",\"type\":\"long\"},{\"name\":\"OnHeapStorageMemory\",\"type\":\"long\"},{\"name\":\"OffHeapStorageMemory\",\"type\":\"long\"},{\"name\":\"OnHeapUnifiedMemory\",\"type\":\"long\"},{\"name\":\"OffHeapUnifiedMemory\",\"type\":\"long\"},{\"name\":\"DirectPoolMemory\",\"type\":\"long\"},{\"name\":\"MappedPoolMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreeJVMVMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreeJVMRSSMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreePythonVMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreePythonRSSMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreeOtherVMemory\",\"type\":\"long\"},{\"name\":\"ProcessTreeOtherRSSMemory\",\"type\":\"long\"},{\"name\":\"MinorGCCount\",\"type\":\"long\"},{\"name\":\"MinorGCTime\",\"type\":\"long\"},{\"name\":\"MajorGCCount\",\"type\":\"long\"},{\"name\":\"MajorGCTime\",\"type\":\"long\"}]}],\"default\":null},{\"name\":\"attributes\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"string\",\"null\"]}],\"d
efault\":null},{\"name\":\"resources\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[{\"type\":\"record\",\"name\":\"resources\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.executors\",\"fields\":[{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"addresses\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[\"string\",\"null\"]}],\"default\":null}]},\"null\"]}],\"default\":null},{\"name\":\"resourceProfileId\",\"type\":\"int\"},{\"name\":\"isExcluded\",\"type\":\"boolean\"},{\"name\":\"excludedInStages\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null}]},\"null\"]}],\"default\":null},{\"name\":\"environment\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"environment\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record\",\"fields\":[{\"name\":\"runtime\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"runtime\",\"namespace\":\"hoodie.spark_application_hudi.spa
rk_application_hudi_record.environment\",\"fields\":[{\"name\":\"javaVersion\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"javaHome\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"scalaVersion\",\"type\":[\"null\",\"string\"],\"default\":null}]}],\"default\":null},{\"name\":\"sparkProperties\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"array\",\"items\":[\"string\",\"null\"]},\"null\"]}],\"default\":null},{\"name\":\"hadoopProperties\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"array\",\"items\":[\"string\",\"null\"]},\"null\"]}],\"default\":null},{\"name\":\"systemProperties\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"array\",\"items\":[\"string\",\"null\"]},\"null\"]}],\"default\":null},{\"name\":\"classpathEntries\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"array\",\"items\":[\"string\",\"null\"]},\"null\"]}],\"default\":null},{\"name\":\"resourceProfiles\",\"type\":[\"nu
ll\",{\"type\":\"array\",\"items\":[{\"type\":\"record\",\"name\":\"resourceProfiles\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.environment\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"},{\"name\":\"executorResources\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[{\"type\":\"record\",\"name\":\"executorResources\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.environment.resourceProfiles\",\"fields\":[{\"name\":\"resourceName\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"amount\",\"type\":\"long\"},{\"name\":\"discoveryScript\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"vendor\",\"type\":[\"null\",\"string\"],\"default\":null}]},\"null\"]}],\"default\":null},{\"name\":\"taskResources\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[{\"type\":\"record\",\"name\":\"taskResources\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.environment.resourcePro
files\",\"fields\":[{\"name\":\"resourceName\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"amount\",\"type\":\"double\"}]},\"null\"]}],\"default\":null}]},\"null\"]}],\"default\":null}]}],\"default\":null},{\"name\":\"sqlInfo\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"record\",\"name\":\"sqlInfo\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"status\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"description\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"planDescription\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"submissionTime\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"duration\",\"type\":\"long\"},{\"name\":\"runningJobIds\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null},{\"name\":\"successJobIds\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}
],\"default\":null},{\"name\":\"failedJobIds\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null},{\"name\":\"nodes\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"record\",\"name\":\"nodes\",\"namespace\":\"hoodie.spark_application_hudi.spark_application_hudi_record.sqlInfo\",\"fields\":[{\"name\":\"nodeId\",\"type\":\"int\"},{\"name\":\"nodeName\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"wholeStageCodegenId\",\"type\":\"int\"},{\"name\":\"metrics\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"map\",\"values\":[\"string\",\"null\"]},\"null\"]}],\"default\":null}]},\"null\"]}],\"default\":null},{\"name\":\"edges\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[{\"type\":\"map\",\"values\":\"int\"},\"null\"]}],\"default\":null}]},\"null\"]}],\"default\":null}]}"
}
```
For clustering, now we can enable
`hoodie.datasource.write.row.writer.enable=true` to avoid this issue.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
