Repository: flume Updated Branches: refs/heads/trunk 0602cb032 -> 19fc3f41e
FLUME-3243 hdfs.callTimeout deafault increased and deprecated The default hdfs.callTimeout used by the HDFS sink was too low only 10 seconds that can cause problems on a busy system. The new default is 30 sec. I think this parameter should be deprecated and some new more error tolerant solution should be used. To enable the future change I indicated this in the code and in the Users Guide. Tested only with the unit tests. This closes #243 Reviewers: Ferenc Szabo, Peret Turcsanyi (Endre Major via Ferenc Szabo) Project: http://git-wip-us.apache.org/repos/asf/flume/repo Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/19fc3f41 Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/19fc3f41 Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/19fc3f41 Branch: refs/heads/trunk Commit: 19fc3f41ea7ff6b7e12c69c75becb4080fbe613f Parents: 0602cb0 Author: Endre Major <[email protected]> Authored: Fri Nov 23 11:05:17 2018 +0100 Committer: Ferenc Szabo <[email protected]> Committed: Fri Nov 23 11:05:17 2018 +0100 ---------------------------------------------------------------------- flume-ng-doc/sphinx/FlumeUserGuide.rst | 10 ++++++++-- .../org/apache/flume/sink/hdfs/AbstractHDFSWriter.java | 3 ++- .../java/org/apache/flume/sink/hdfs/HDFSEventSink.java | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flume/blob/19fc3f41/flume-ng-doc/sphinx/FlumeUserGuide.rst ---------------------------------------------------------------------- diff --git a/flume-ng-doc/sphinx/FlumeUserGuide.rst b/flume-ng-doc/sphinx/FlumeUserGuide.rst index 01eb81d..58164a5 100644 --- a/flume-ng-doc/sphinx/FlumeUserGuide.rst +++ b/flume-ng-doc/sphinx/FlumeUserGuide.rst @@ -2396,8 +2396,6 @@ hdfs.fileType SequenceFile File format: currently ``SequenceFile``, ` hdfs.maxOpenFiles 5000 Allow only this number of open files. If this number is exceeded, the oldest file is closed. hdfs.minBlockReplicas -- Specify minimum number of replicas per HDFS block. If not specified, it comes from the default Hadoop config in the classpath. hdfs.writeFormat Writable Format for sequence file records. One of ``Text`` or ``Writable``. Set to ``Text`` before creating data files with Flume, otherwise those files cannot be read by either Apache Impala (incubating) or Apache Hive. -hdfs.callTimeout 10000 Number of milliseconds allowed for HDFS operations, such as open, write, flush, close. - This number should be increased if many HDFS timeout operations are occurring. hdfs.threadsPoolSize 10 Number of threads per HDFS sink for HDFS IO ops (open, write, etc.) hdfs.rollTimerPoolSize 1 Number of threads per HDFS sink for scheduling timed file rolling hdfs.kerberosPrincipal -- Kerberos user principal for accessing secure HDFS @@ -2421,6 +2419,14 @@ serializer ``TEXT`` Other possible options include ``avro_even serializer.* ====================== ============ ====================================================================== +Deprecated Properties + +Name Default Description +====================== ============ ====================================================================== +hdfs.callTimeout 30000 Number of milliseconds allowed for HDFS operations, such as open, write, flush, close. + This number should be increased if many HDFS timeout operations are occurring. +====================== ============ ====================================================================== + Example for agent named a1: .. code-block:: properties http://git-wip-us.apache.org/repos/asf/flume/blob/19fc3f41/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java ---------------------------------------------------------------------- diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java index 2fe309f..d2a07fa 100644 --- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java +++ b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/AbstractHDFSWriter.java @@ -63,7 +63,8 @@ public abstract class AbstractHDFSWriter implements HDFSWriter { if (numberOfCloseRetries > 1) { try { - timeBetweenCloseRetries = context.getLong("hdfs.callTimeout", 10000L); + //hdfs.callTimeout is deprecated from 1.9 + timeBetweenCloseRetries = context.getLong("hdfs.callTimeout", 30000L); } catch (NumberFormatException e) { logger.warn("hdfs.callTimeout can not be parsed to a long: " + context.getLong("hdfs.callTimeout")); http://git-wip-us.apache.org/repos/asf/flume/blob/19fc3f41/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java ---------------------------------------------------------------------- diff --git a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java index bb2390e..4393b6d 100644 --- a/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java +++ b/flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSEventSink.java @@ -90,7 +90,7 @@ public class HDFSEventSink extends AbstractSink implements Configurable, BatchSi * Default length of time we wait for blocking BucketWriter calls * before timing out the operation. Intended to prevent server hangs. */ - private static final long defaultCallTimeout = 10000; + private static final long defaultCallTimeout = 30000; /** * Default number of threads available for tasks * such as append/open/close/flush with hdfs.
