[
https://issues.apache.org/jira/browse/HDFS-16331?focusedWorklogId=687767&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-687767
]
ASF GitHub Bot logged work on HDFS-16331:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 30/Nov/21 05:22
Start Date: 30/Nov/21 05:22
Worklog Time Spent: 10m
Work Description: tasanuma commented on a change in pull request #3676:
URL: https://github.com/apache/hadoop/pull/3676#discussion_r758939114
##########
File path:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
##########
@@ -533,78 +536,119 @@ protected Configuration getNewConf() {
public String reconfigurePropertyImpl(String property, String newVal)
throws ReconfigurationException {
switch (property) {
- case DFS_DATANODE_DATA_DIR_KEY: {
- IOException rootException = null;
+ case DFS_DATANODE_DATA_DIR_KEY: {
+ IOException rootException = null;
+ try {
+ LOG.info("Reconfiguring {} to {}", property, newVal);
+ this.refreshVolumes(newVal);
+ return getConf().get(DFS_DATANODE_DATA_DIR_KEY);
+ } catch (IOException e) {
+ rootException = e;
+ } finally {
+ // Send a full block report to let NN acknowledge the volume changes.
try {
- LOG.info("Reconfiguring {} to {}", property, newVal);
- this.refreshVolumes(newVal);
- return getConf().get(DFS_DATANODE_DATA_DIR_KEY);
+ triggerBlockReport(
+ new BlockReportOptions.Factory().setIncremental(false).build());
} catch (IOException e) {
- rootException = e;
+ LOG.warn("Exception while sending the block report after refreshing"
+ + " volumes {} to {}", property, newVal, e);
+ if (rootException == null) {
+ rootException = e;
+ }
} finally {
- // Send a full block report to let NN acknowledge the volume changes.
- try {
- triggerBlockReport(
- new
BlockReportOptions.Factory().setIncremental(false).build());
- } catch (IOException e) {
- LOG.warn("Exception while sending the block report after
refreshing"
- + " volumes {} to {}", property, newVal, e);
- if (rootException == null) {
- rootException = e;
- }
- } finally {
- if (rootException != null) {
- throw new ReconfigurationException(property, newVal,
- getConf().get(property), rootException);
- }
+ if (rootException != null) {
+ throw new ReconfigurationException(property, newVal,
+ getConf().get(property), rootException);
}
}
- break;
}
- case DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY: {
- ReconfigurationException rootException = null;
- try {
- LOG.info("Reconfiguring {} to {}", property, newVal);
- int movers;
- if (newVal == null) {
- // set to default
- movers = DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT;
- } else {
- movers = Integer.parseInt(newVal);
- if (movers <= 0) {
- rootException = new ReconfigurationException(
- property,
- newVal,
- getConf().get(property),
- new IllegalArgumentException(
- "balancer max concurrent movers must be larger than 0"));
- }
- }
- boolean success = xserver.updateBalancerMaxConcurrentMovers(movers);
- if (!success) {
+ break;
+ }
+ case DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY: {
+ ReconfigurationException rootException = null;
+ try {
+ LOG.info("Reconfiguring {} to {}", property, newVal);
+ int movers;
+ if (newVal == null) {
+ // set to default
+ movers = DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT;
+ } else {
+ movers = Integer.parseInt(newVal);
+ if (movers <= 0) {
rootException = new ReconfigurationException(
property,
newVal,
getConf().get(property),
new IllegalArgumentException(
- "Could not modify concurrent moves thread count"));
+ "balancer max concurrent movers must be larger than 0"));
}
- return Integer.toString(movers);
- } catch (NumberFormatException nfe) {
+ }
+ boolean success = xserver.updateBalancerMaxConcurrentMovers(movers);
+ if (!success) {
rootException = new ReconfigurationException(
- property, newVal, getConf().get(property), nfe);
- } finally {
- if (rootException != null) {
- LOG.warn(String.format(
- "Exception in updating balancer max concurrent movers %s to
%s",
- property, newVal), rootException);
- throw rootException;
+ property,
+ newVal,
+ getConf().get(property),
+ new IllegalArgumentException(
+ "Could not modify concurrent moves thread count"));
+ }
+ return Integer.toString(movers);
+ } catch (NumberFormatException nfe) {
+ rootException = new ReconfigurationException(
+ property, newVal, getConf().get(property), nfe);
+ } finally {
+ if (rootException != null) {
+ LOG.warn(String.format(
+ "Exception in updating balancer max concurrent movers %s to %s",
+ property, newVal), rootException);
+ throw rootException;
+ }
+ }
+ break;
+ }
+ case DFS_BLOCKREPORT_INTERVAL_MSEC_KEY: {
+ ReconfigurationException rootException = null;
+ try {
+ LOG.info("Reconfiguring {} to {}", property, newVal);
+ long intervalMs;
+ if (newVal == null) {
+ // Set to default.
+ intervalMs = DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT;
+ } else {
+ intervalMs = Long.parseLong(newVal);
+ if (intervalMs < 0) {
+ rootException = new ReconfigurationException(
+ property,
+ newVal,
+ getConf().get(property),
+ new IllegalArgumentException(
+ "block report interval must be larger than or equal to
0"));
+ }
+ }
+ dnConf.setBlockReportInterval(intervalMs);
+ for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) {
+ if (bpos != null) {
+ for (BPServiceActor actor : bpos.getBPServiceActors()) {
+ actor.getScheduler().setBlockReportIntervalMs(intervalMs);
+ }
Review comment:
`intervalMs` still can be negative here. If `intervalMs` is negative, it
should throw an exception before setting the value, shouldn't it?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 687767)
Time Spent: 3h 50m (was: 3h 40m)
> Make dfs.blockreport.intervalMsec reconfigurable
> ------------------------------------------------
>
> Key: HDFS-16331
> URL: https://issues.apache.org/jira/browse/HDFS-16331
> Project: Hadoop HDFS
> Issue Type: New Feature
> Reporter: tomscut
> Assignee: tomscut
> Priority: Major
> Labels: pull-request-available
> Attachments: image-2021-11-18-09-33-24-236.png,
> image-2021-11-18-09-35-35-400.png
>
> Time Spent: 3h 50m
> Remaining Estimate: 0h
>
> We have a cold data cluster, which stores as EC policy. There are 24 fast
> disks on each node and each disk is 7 TB.
> Recently, many nodes have more than 10 million blocks, and the interval of
> FBR is 6h as default. Frequent FBR caused great pressure on NN.
> !image-2021-11-18-09-35-35-400.png|width=334,height=229!
> !image-2021-11-18-09-33-24-236.png|width=566,height=159!
> We want to increase the interval of FBR, but have to rolling restart the DNs,
> this operation is very heavy. In this scenario, it is necessary to make
> _dfs.blockreport.intervalMsec_ reconfigurable.
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
