[
https://issues.apache.org/jira/browse/HDDS-1207?focusedWorklogId=222010&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-222010
]
ASF GitHub Bot logged work on HDDS-1207:
----------------------------------------
Author: ASF GitHub Bot
Created on: 02/Apr/19 21:26
Start Date: 02/Apr/19 21:26
Worklog Time Spent: 10m
Work Description: arp7 commented on pull request #662: HDDS-1207.
Refactor Container Report Processing logic and plugin new Replication Manager.
URL: https://github.com/apache/hadoop/pull/662#discussion_r271502348
##########
File path:
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
##########
@@ -146,68 +111,89 @@ public void onMessage(final ContainerReportFromDatanode
reportFromDatanode,
}
+ /**
+ * Processes the ContainerReport.
+ *
+ * @param datanodeDetails Datanode from which this report was received
+ * @param replicas list of ContainerReplicaProto
+ */
private void processContainerReplicas(final DatanodeDetails datanodeDetails,
- final List<ContainerReplicaProto> replicas,
- final EventPublisher publisher) {
- final PendingDeleteStatusList pendingDeleteStatusList =
- new PendingDeleteStatusList(datanodeDetails);
+ final List<ContainerReplicaProto> replicas) {
for (ContainerReplicaProto replicaProto : replicas) {
try {
- final ContainerID containerID = ContainerID.valueof(
- replicaProto.getContainerID());
-
- ReportHandlerHelper.processContainerReplica(containerManager,
- containerID, replicaProto, datanodeDetails, publisher, LOG);
-
- final ContainerInfo containerInfo = containerManager
- .getContainer(containerID);
-
- if (containerInfo.getDeleteTransactionId() >
- replicaProto.getDeleteTransactionId()) {
- pendingDeleteStatusList
- .addPendingDeleteStatus(replicaProto.getDeleteTransactionId(),
- containerInfo.getDeleteTransactionId(),
- containerInfo.getContainerID());
- }
+ processContainerReplica(datanodeDetails, replicaProto);
} catch (ContainerNotFoundException e) {
- LOG.error("Received container report for an unknown container {} from"
- + " datanode {} {}", replicaProto.getContainerID(),
+ LOG.error("Received container report for an unknown container" +
+ " {} from datanode {}.", replicaProto.getContainerID(),
datanodeDetails, e);
} catch (IOException e) {
- LOG.error("Exception while processing container report for container"
- + " {} from datanode {} {}", replicaProto.getContainerID(),
+ LOG.error("Exception while processing container report for container" +
+ " {} from datanode {}.", replicaProto.getContainerID(),
datanodeDetails, e);
}
}
- if (pendingDeleteStatusList.getNumPendingDeletes() > 0) {
- publisher.fireEvent(SCMEvents.PENDING_DELETE_STATUS,
- pendingDeleteStatusList);
- }
}
- private void checkReplicationState(ContainerID containerID,
- EventPublisher publisher) {
- try {
- ContainerInfo container = containerManager.getContainer(containerID);
- replicateIfNeeded(container, publisher);
- } catch (ContainerNotFoundException ex) {
- LOG.warn("Container is missing from containerStateManager. Can't request
"
- + "replication. {} {}", containerID, ex);
+ /**
+ * Process the missing replica on the given datanode.
+ *
+ * @param datanodeDetails DatanodeDetails
+ * @param missingReplicas ContainerID which are missing on the given datanode
+ */
+ private void processMissingReplicas(final DatanodeDetails datanodeDetails,
+ final Set<ContainerID> missingReplicas) {
+ for (ContainerID id : missingReplicas) {
+ try {
+ containerManager.getContainerReplicas(id).stream()
+ .filter(replica -> replica.getDatanodeDetails()
+ .equals(datanodeDetails)).findFirst()
+ .ifPresent(replica -> {
+ try {
+ containerManager.removeContainerReplica(id, replica);
+ } catch (ContainerNotFoundException |
+ ContainerReplicaNotFoundException ignored) {
+ // This should not happen, but even if it happens, not an issue
+ }
+ });
+ } catch (ContainerNotFoundException e) {
+ LOG.warn("Cannot remove container replica, container {} not found.",
+ id, e);
+ }
}
-
}
- private void replicateIfNeeded(ContainerInfo container,
- EventPublisher publisher) throws ContainerNotFoundException {
- if (!container.isOpen() && replicationStatus.isReplicationEnabled()) {
- final int existingReplicas = containerManager
- .getContainerReplicas(container.containerID()).size();
- final int expectedReplicas =
container.getReplicationFactor().getNumber();
- if (existingReplicas != expectedReplicas) {
- publisher.fireEvent(SCMEvents.REPLICATE_CONTAINER,
- new ReplicationRequest(container.getContainerID(),
- existingReplicas, expectedReplicas));
+ /**
+ * Updates the Delete Transaction Id for the given datanode.
+ *
+ * @param datanodeDetails DatanodeDetails
+ * @param replicas List of ContainerReplicaProto
+ * @param publisher EventPublisher reference
+ */
+ private void updateDeleteTransaction(final DatanodeDetails datanodeDetails,
Review comment:
Had a few questions on this function. I am not familiar with this part.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 222010)
Time Spent: 1h (was: 50m)
> Refactor Container Report Processing logic and plugin new Replication Manager
> -----------------------------------------------------------------------------
>
> Key: HDDS-1207
> URL: https://issues.apache.org/jira/browse/HDDS-1207
> Project: Hadoop Distributed Data Store
> Issue Type: Improvement
> Components: SCM
> Reporter: Nanda kumar
> Assignee: Nanda kumar
> Priority: Major
> Labels: pull-request-available
> Time Spent: 1h
> Remaining Estimate: 0h
>
> HDDS-1205 brings in new ReplicationManager, this Jira is to refactor
> ContainerReportProcessing logic in SCM so that it complements
> ReplicationManager and plugin the new ReplicationManager code.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
