[
https://issues.apache.org/jira/browse/HIVE-24109?focusedWorklogId=503697&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-503697
]
ASF GitHub Bot logged work on HIVE-24109:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 22/Oct/20 12:16
Start Date: 22/Oct/20 12:16
Worklog Time Spent: 10m
Work Description: aasha commented on a change in pull request #1529:
URL: https://github.com/apache/hive/pull/1529#discussion_r510111928
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
##########
@@ -207,119 +220,88 @@ private void addConsolidatedPartitionDesc() throws
Exception {
tableDesc.getTableName(), true, partitions);
//don't need to add ckpt task separately. Added as part of add partition
task
- addPartition((toPartitionCount < totalPartitionCount),
consolidatedPartitionDesc, null);
- if (partitions.size() > 0) {
- LOG.info("Added {} partitions", partitions.size());
+ addPartition((toPartitionCount < totalPartitionCount),
consolidatedPartitionDesc);
+ if (!tracker.canAddMoreTasks()) {
+ //No need to do processing as no more tasks can be added. Will be
processed in next run. State is already
+ //updated in add partition task
+ return;
}
currentPartitionCount = toPartitionCount;
}
}
private TaskTracker forNewTable() throws Exception {
- if (isMetaDataOp() ||
TableType.EXTERNAL_TABLE.equals(table.getTableType())) {
- // Place all partitions in single task to reduce load on HMS.
- addConsolidatedPartitionDesc();
- return tracker;
- }
-
- Iterator<AlterTableAddPartitionDesc> iterator =
event.partitionDescriptions(tableDesc).iterator();
- while (iterator.hasNext() && tracker.canAddMoreTasks()) {
- AlterTableAddPartitionDesc currentPartitionDesc = iterator.next();
- /*
- the currentPartitionDesc cannot be inlined as we need the hasNext() to
be evaluated post the
- current retrieved lastReplicatedPartition
- */
- addPartition(iterator.hasNext(), currentPartitionDesc, null);
- }
+ // Place all partitions in single task to reduce load on HMS.
+ addConsolidatedPartitionDesc(null);
return tracker;
}
- private void addPartition(boolean hasMorePartitions,
AlterTableAddPartitionDesc addPartitionDesc, Task<?> ptnRootTask)
+ private void addPartition(boolean hasMorePartitions,
AlterTableAddPartitionDesc addPartitionDesc)
throws Exception {
- tracker.addTask(tasksForAddPartition(table, addPartitionDesc,
ptnRootTask));
- if (hasMorePartitions && !tracker.canAddMoreTasks()) {
+ boolean processingComplete = addTasksForPartition(table, addPartitionDesc,
null,
+ PartitionState.Stage.PARTITION);
+ //If processing is not complete, means replication state is already
updated with copy or move tasks which need
Review comment:
Removed
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 503697)
Time Spent: 1h 10m (was: 1h)
> Load partitions in parallel for managed tables in the bootstrap phase
> ---------------------------------------------------------------------
>
> Key: HIVE-24109
> URL: https://issues.apache.org/jira/browse/HIVE-24109
> Project: Hive
> Issue Type: Task
> Reporter: Aasha Medhi
> Assignee: Aasha Medhi
> Priority: Major
> Labels: pull-request-available
> Attachments: HIVE-24109.01.patch, HIVE-24109.02.patch,
> HIVE-24109.03.patch
>
> Time Spent: 1h 10m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
