SourabhBadhya commented on code in PR #5076:
URL: https://github.com/apache/hive/pull/5076#discussion_r1505606712
##########
ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java:
##########
@@ -254,18 +286,26 @@ private void generateActualTasks(HiveConf conf,
List<Task<?>> resTsks,
long trgtSize, long avgConditionSize, Task<?> mvTask,
Task<?> mrTask, Task<?> mrAndMvTask, Path dirPath,
FileSystem inpFs, ConditionalResolverMergeFilesCtx ctx, MapWork work,
int dpLbLevel,
- boolean manifestFilePresent)
- throws IOException {
+ boolean manifestFilePresent, HiveStorageHandler storageHandler)
+ throws IOException, ClassNotFoundException {
DynamicPartitionCtx dpCtx = ctx.getDPCtx();
List<FileStatus> statusList;
- Map<FileStatus, List<FileStatus>> manifestDirToFile = new HashMap<>();
+ Map<FileStatus, List<FileStatus>> parentDirToFile = new HashMap<>();
+ boolean useCustomStorageHandler = storageHandler != null &&
storageHandler.supportsMergeFiles();
+ StorageHandlerMergeProperties mergeProperties = useCustomStorageHandler ?
+
storageHandler.getStorageHandlerMergeProperties(ctx.getCustomStorageHandlerProps())
: null;
if (manifestFilePresent) {
// Get the list of files from manifest file.
List<FileStatus> fileStatuses = getManifestFilePaths(conf, dirPath);
// Setup the work to include all the files present in the manifest.
setupWorkWhenUsingManifestFile(work, fileStatuses, dirPath, false);
- manifestDirToFile = getManifestDirs(inpFs, fileStatuses);
- statusList = new ArrayList<>(manifestDirToFile.keySet());
+ parentDirToFile = getParentDirToFileMap(inpFs, fileStatuses);
+ statusList = new ArrayList<>(parentDirToFile.keySet());
+ } else if (useCustomStorageHandler) {
+ List<FileStatus> fileStatuses =
storageHandler.getMergeInputFiles(ctx.getCustomStorageHandlerProps());
+ setupWorkWhenUsingCustomHandler(work, dirPath, mergeProperties);
+ parentDirToFile = getParentDirToFileMap(inpFs, fileStatuses);
+ statusList = new ArrayList<>(parentDirToFile.keySet());
Review Comment:
Done.
##########
ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java:
##########
@@ -254,18 +286,26 @@ private void generateActualTasks(HiveConf conf,
List<Task<?>> resTsks,
long trgtSize, long avgConditionSize, Task<?> mvTask,
Task<?> mrTask, Task<?> mrAndMvTask, Path dirPath,
FileSystem inpFs, ConditionalResolverMergeFilesCtx ctx, MapWork work,
int dpLbLevel,
- boolean manifestFilePresent)
- throws IOException {
+ boolean manifestFilePresent, HiveStorageHandler storageHandler)
+ throws IOException, ClassNotFoundException {
DynamicPartitionCtx dpCtx = ctx.getDPCtx();
List<FileStatus> statusList;
- Map<FileStatus, List<FileStatus>> manifestDirToFile = new HashMap<>();
+ Map<FileStatus, List<FileStatus>> parentDirToFile = new HashMap<>();
+ boolean useCustomStorageHandler = storageHandler != null &&
storageHandler.supportsMergeFiles();
+ StorageHandlerMergeProperties mergeProperties = useCustomStorageHandler ?
+
storageHandler.getStorageHandlerMergeProperties(ctx.getCustomStorageHandlerProps())
: null;
if (manifestFilePresent) {
// Get the list of files from manifest file.
List<FileStatus> fileStatuses = getManifestFilePaths(conf, dirPath);
// Setup the work to include all the files present in the manifest.
setupWorkWhenUsingManifestFile(work, fileStatuses, dirPath, false);
- manifestDirToFile = getManifestDirs(inpFs, fileStatuses);
- statusList = new ArrayList<>(manifestDirToFile.keySet());
+ parentDirToFile = getParentDirToFileMap(inpFs, fileStatuses);
+ statusList = new ArrayList<>(parentDirToFile.keySet());
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
