steveloughran commented on PR #39185:
URL: https://github.com/apache/spark/pull/39185#issuecomment-1419461503
latest pr will save to a report dir aggregate IOStats collected from the
task attempts, in addition any _SUCCESS reports created by the inner committer.
This is to align later with collection of context IOStats, where all stream
read/write stats are collected.
For now, a json summary of the stats collected/reported by the commmitters
are collected, for example
```
{
"counters" : {
"committer_commit_job" : 1,
"op_msync" : 1,
"op_msync.failures" : 0,
"job_stage_optional_validate_output" : 0,
"job_stage_create_target_dirs" : 1,
"op_load_manifest.failures" : 0,
"op_rename.failures" : 0,
"op_load_all_manifests" : 1,
"job_stage_load_manifests" : 1,
"committer_commit_job.failures" : 0,
"task_stage_save_task_manifest" : 3,
"job_stage_rename_files" : 1,
"committer_bytes_committed" : 9,
"op_create_one_directory.failures" : 0,
"op_mkdirs.failures" : 0,
"committer_files_committed" : 1,
"store_io_rate_limited.failures" : 0,
"op_load_all_manifests.failures" : 0,
"committer_task_manifest_file_size" : 0,
"task_stage_save_manifest" : 2,
"task_stage_scan_directory" : 3,
"task_stage_setup.failures" : 0,
"op_is_file.failures" : 0,
"op_create_directories.failures" : 0,
"op_create_one_directory" : 0,
"committer_task_file_count" : 3,
"task_stage_save_manifest.failures" : 0,
"committer_task_directory_depth" : 6,
"job_stage_setup.failures" : 0,
"job_stage_load_manifests.failures" : 0,
"op_directory_scan" : 0,
"op_rename" : 3,
"job_stage_abort" : 0,
"op_get_file_status" : 9,
"commit_file_rename_recovered" : 0,
"store_io_rate_limited" : 0,
"op_delete_file_under_destination.failures" : 0,
"op_prepare_dir_ancestors" : 0,
"job_stage_abort.failures" : 0,
"op_is_directory.failures" : 0,
"job_stage_optional_validate_output.failures" : 0,
"task_stage_setup" : 3,
"commit_file_rename.failures" : 0,
"op_delete" : 6,
"op_delete_file_under_destination" : 0,
"job_stage_save_success_marker" : 1,
"commit_file_rename" : 1,
"op_mkdir_returned_false" : 0,
"op_directory_scan.failures" : 0,
"object_list_request.failures" : 0,
"object_continue_list_request.failures" : 0,
"op_load_manifest" : 1,
"committer_tasks_completed" : 2,
"job_stage_rename_files.failures" : 0,
"op_get_file_status.failures" : 8,
"task_stage_scan_directory.failures" : 0,
"op_delete.failures" : 0,
"object_list_request" : 0,
"op_list_status" : 8,
"task_stage_abort_task.failures" : 0,
"job_stage_setup" : 1,
"task_stage_commit" : 3,
"committer_tasks_failed" : 0,
"task_stage_commit.failures" : 0,
"task_stage_abort_task" : 0,
"committer_task_file_size" : 27,
"job_stage_create_target_dirs.failures" : 0,
"committer_task_directory_count" : 3,
"object_continue_list_request" : 0,
"job_stage_save_success_marker.failures" : 0,
"op_create_directories" : 1,
"op_mkdir_returned_false.failures" : 0,
"task_stage_save_task_manifest.failures" : 0,
"job_stage_cleanup.failures" : 0,
"job_stage_cleanup" : 1,
"op_mkdirs" : 6,
"op_is_directory" : 0,
"op_prepare_dir_ancestors.failures" : 0,
"op_is_file" : 0,
"op_list_status.failures" : 0
},
"gauges" : { },
"minimums" : {
"task_stage_commit.min" : 1,
"task_stage_save_task_manifest.min" : 21,
"op_create_directories.failures.min" : -1,
"op_is_directory.failures.min" : -1,
"job_stage_abort.min" : -1,
"object_list_request.min" : -1,
"task_stage_save_task_manifest.failures.min" : -1,
"object_continue_list_request.failures.min" : -1,
"op_prepare_dir_ancestors.failures.min" : -1,
"object_list_request.failures.min" : -1,
"op_directory_scan.min" : -1,
"job_stage_cleanup.min" : 14,
"op_list_status.min" : 0,
"op_mkdirs.failures.min" : -1,
"task_stage_setup.failures.min" : -1,
"job_stage_abort.failures.min" : -1,
"committer_bytes_committed" : -1,
"job_stage_rename_files.failures.min" : -1,
"op_is_file.failures.min" : -1,
"committer_files_committed" : -1,
"job_stage_rename_files.min" : 13,
"op_load_all_manifests.min" : 12,
"op_rename.failures.min" : -1,
"op_delete_file_under_destination.failures.min" : -1,
"task_stage_setup.min" : 29,
"job_stage_create_target_dirs.min" : 14,
"op_msync.failures.min" : -1,
"commit_file_rename.failures.min" : -1,
"job_stage_cleanup.failures.min" : -1,
"job_stage_save_success_marker.failures.min" : -1,
"store_io_rate_limited.min" : -1,
"job_stage_save_success_marker.min" : 28,
"committer_task_manifest_file_size" : -1,
"job_stage_create_target_dirs.failures.min" : -1,
"op_load_manifest.failures.min" : -1,
"committer_commit_job.min" : 83,
"committer_commit_job.failures.min" : -1,
"op_delete.min" : 0,
"op_prepare_dir_ancestors.min" : -1,
"committer_task_file_count" : 1,
"job_stage_setup.min" : 74,
"committer_task_directory_depth" : 2,
"op_get_file_status.min" : 0,
"job_stage_load_manifests.min" : 13,
"op_load_all_manifests.failures.min" : -1,
"job_stage_optional_validate_output.failures.min" : -1,
"op_get_file_status.failures.min" : 0,
"op_create_one_directory.failures.min" : -1,
"op_load_manifest.min" : 2,
"op_create_one_directory.min" : -1,
"commit_file_rename_recovered" : -1,
"op_mkdirs.min" : 13,
"task_stage_scan_directory.failures.min" : -1,
"op_msync.min" : 0,
"op_mkdir_returned_false.min" : -1,
"op_is_file.min" : -1,
"store_io_rate_limited.failures.min" : -1,
"task_stage_save_manifest.failures.min" : -1,
"op_list_status.failures.min" : -1,
"task_stage_abort_task.min" : -1,
"op_directory_scan.failures.min" : -1,
"committer_tasks_completed" : -1,
"commit_file_rename.min" : 0,
"op_delete_file_under_destination.min" : -1,
"object_continue_list_request.min" : -1,
"job_stage_load_manifests.failures.min" : -1,
"committer_tasks_failed" : -1,
"committer_task_file_size" : 9,
"job_stage_optional_validate_output.min" : -1,
"job_stage_setup.failures.min" : -1,
"committer_task_directory_count" : 1,
"op_delete.failures.min" : -1,
"task_stage_commit.failures.min" : -1,
"op_create_directories.min" : 13,
"task_stage_abort_task.failures.min" : -1,
"op_mkdir_returned_false.failures.min" : -1,
"op_is_directory.min" : -1,
"task_stage_scan_directory.min" : 1,
"task_stage_save_manifest.min" : 21,
"op_rename.min" : 0
},
"maximums" : {
"job_stage_cleanup.max" : 14,
"job_stage_load_manifests.max" : 13,
"op_prepare_dir_ancestors.max" : -1,
"op_load_all_manifests.failures.max" : -1,
"op_mkdir_returned_false.failures.max" : -1,
"job_stage_rename_files.failures.max" : -1,
"op_create_one_directory.max" : -1,
"op_is_file.failures.max" : -1,
"op_msync.failures.max" : -1,
"op_is_directory.failures.max" : -1,
"op_msync.max" : 0,
"job_stage_save_success_marker.max" : 28,
"object_list_request.failures.max" : -1,
"committer_bytes_committed" : -1,
"op_prepare_dir_ancestors.failures.max" : -1,
"committer_files_committed" : -1,
"task_stage_save_task_manifest.max" : 28,
"job_stage_optional_validate_output.max" : -1,
"job_stage_setup.max" : 74,
"op_directory_scan.max" : -1,
"job_stage_optional_validate_output.failures.max" : -1,
"op_mkdirs.max" : 51,
"task_stage_abort_task.max" : -1,
"committer_task_manifest_file_size" : -1,
"op_get_file_status.failures.max" : 0,
"op_create_one_directory.failures.max" : -1,
"commit_file_rename.max" : 0,
"op_delete_file_under_destination.max" : -1,
"op_create_directories.failures.max" : -1,
"committer_task_file_count" : 1,
"op_rename.failures.max" : -1,
"committer_task_directory_depth" : 2,
"op_delete_file_under_destination.failures.max" : -1,
"job_stage_abort.max" : -1,
"commit_file_rename.failures.max" : -1,
"job_stage_create_target_dirs.failures.max" : -1,
"op_is_file.max" : -1,
"commit_file_rename_recovered" : -1,
"job_stage_save_success_marker.failures.max" : -1,
"op_directory_scan.failures.max" : -1,
"op_delete.max" : 2,
"object_list_request.max" : -1,
"task_stage_setup.max" : 29,
"op_get_file_status.max" : 0,
"committer_commit_job.max" : 83,
"op_delete.failures.max" : -1,
"object_continue_list_request.max" : -1,
"op_is_directory.max" : -1,
"task_stage_save_manifest.max" : 21,
"task_stage_scan_directory.failures.max" : -1,
"store_io_rate_limited.max" : -1,
"op_list_status.failures.max" : -1,
"job_stage_create_target_dirs.max" : 14,
"job_stage_cleanup.failures.max" : -1,
"op_mkdir_returned_false.max" : -1,
"op_load_manifest.max" : 2,
"committer_tasks_completed" : -1,
"op_load_manifest.failures.max" : -1,
"store_io_rate_limited.failures.max" : -1,
"committer_commit_job.failures.max" : -1,
"task_stage_save_manifest.failures.max" : -1,
"op_create_directories.max" : 13,
"task_stage_commit.failures.max" : -1,
"op_mkdirs.failures.max" : -1,
"task_stage_setup.failures.max" : -1,
"task_stage_abort_task.failures.max" : -1,
"committer_tasks_failed" : -1,
"op_list_status.max" : 1,
"committer_task_file_size" : 9,
"task_stage_scan_directory.max" : 1,
"job_stage_rename_files.max" : 13,
"op_rename.max" : 0,
"committer_task_directory_count" : 1,
"op_load_all_manifests.max" : 12,
"job_stage_load_manifests.failures.max" : -1,
"job_stage_abort.failures.max" : -1,
"job_stage_setup.failures.max" : -1,
"object_continue_list_request.failures.max" : -1,
"task_stage_save_task_manifest.failures.max" : -1,
"task_stage_commit.max" : 22
},
"meanstatistics" : {
"commit_file_rename.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"task_stage_save_manifest.mean" : {
"samples" : 2,
"sum" : 42
},
"job_stage_optional_validate_output.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"job_stage_save_success_marker.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_rename.mean" : {
"samples" : 3,
"sum" : 0
},
"op_load_manifest.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_is_directory.mean" : {
"samples" : 0,
"sum" : 0
},
"op_create_one_directory.mean" : {
"samples" : 0,
"sum" : 0
},
"commit_file_rename.mean" : {
"samples" : 1,
"sum" : 0
},
"op_rename.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_mkdir_returned_false.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"job_stage_rename_files.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"job_stage_load_manifests.mean" : {
"samples" : 1,
"sum" : 13
},
"committer_bytes_committed" : {
"samples" : 0,
"sum" : 0
},
"op_delete.mean" : {
"samples" : 6,
"sum" : 2
},
"job_stage_cleanup.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"committer_files_committed" : {
"samples" : 0,
"sum" : 0
},
"task_stage_setup.mean" : {
"samples" : 3,
"sum" : 87
},
"task_stage_save_task_manifest.mean" : {
"samples" : 3,
"sum" : 70
},
"op_load_manifest.mean" : {
"samples" : 1,
"sum" : 2
},
"task_stage_abort_task.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"store_io_rate_limited.mean" : {
"samples" : 0,
"sum" : 0
},
"op_mkdir_returned_false.mean" : {
"samples" : 0,
"sum" : 0
},
"committer_task_manifest_file_size" : {
"samples" : 1,
"sum" : 18835
},
"op_is_directory.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_mkdirs.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"task_stage_scan_directory.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"task_stage_setup.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"job_stage_abort.mean" : {
"samples" : 0,
"sum" : 0
},
"committer_task_file_count" : {
"samples" : 3,
"sum" : 3
},
"object_list_request.mean" : {
"samples" : 0,
"sum" : 0
},
"committer_task_directory_depth" : {
"samples" : 3,
"sum" : 6
},
"task_stage_save_task_manifest.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_is_file.mean" : {
"samples" : 0,
"sum" : 0
},
"op_prepare_dir_ancestors.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"commit_file_rename_recovered" : {
"samples" : 0,
"sum" : 0
},
"task_stage_save_manifest.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"job_stage_setup.mean" : {
"samples" : 1,
"sum" : 74
},
"object_list_request.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_get_file_status.failures.mean" : {
"samples" : 8,
"sum" : 0
},
"task_stage_commit.mean" : {
"samples" : 5,
"sum" : 47
},
"object_continue_list_request.mean" : {
"samples" : 0,
"sum" : 0
},
"job_stage_cleanup.mean" : {
"samples" : 1,
"sum" : 14
},
"job_stage_create_target_dirs.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_delete_file_under_destination.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_directory_scan.mean" : {
"samples" : 0,
"sum" : 0
},
"op_get_file_status.mean" : {
"samples" : 1,
"sum" : 0
},
"job_stage_create_target_dirs.mean" : {
"samples" : 1,
"sum" : 14
},
"committer_tasks_completed" : {
"samples" : 0,
"sum" : 0
},
"job_stage_abort.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_load_all_manifests.mean" : {
"samples" : 1,
"sum" : 12
},
"op_create_directories.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_list_status.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"job_stage_save_success_marker.mean" : {
"samples" : 1,
"sum" : 28
},
"committer_commit_job.mean" : {
"samples" : 1,
"sum" : 83
},
"op_is_file.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_create_one_directory.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_load_all_manifests.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_directory_scan.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_mkdirs.mean" : {
"samples" : 6,
"sum" : 174
},
"store_io_rate_limited.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"job_stage_load_manifests.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_list_status.mean" : {
"samples" : 8,
"sum" : 4
},
"op_prepare_dir_ancestors.mean" : {
"samples" : 0,
"sum" : 0
},
"committer_commit_job.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"committer_tasks_failed" : {
"samples" : 0,
"sum" : 0
},
"committer_task_file_size" : {
"samples" : 3,
"sum" : 27
},
"job_stage_optional_validate_output.mean" : {
"samples" : 0,
"sum" : 0
},
"committer_task_directory_count" : {
"samples" : 3,
"sum" : 3
},
"op_create_directories.mean" : {
"samples" : 1,
"sum" : 13
},
"task_stage_scan_directory.mean" : {
"samples" : 3,
"sum" : 3
},
"op_delete_file_under_destination.mean" : {
"samples" : 0,
"sum" : 0
},
"object_continue_list_request.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"task_stage_commit.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"task_stage_abort_task.mean" : {
"samples" : 0,
"sum" : 0
},
"job_stage_rename_files.mean" : {
"samples" : 1,
"sum" : 13
},
"op_delete.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"op_msync.mean" : {
"samples" : 1,
"sum" : 0
},
"op_msync.failures.mean" : {
"samples" : 0,
"sum" : 0
},
"job_stage_setup.failures.mean" : {
"samples" : 0,
"sum" : 0
}
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
