Also added in GitHub at Flume config file | | | | | |
| | | | | Flume config file Flume config file | | | | From: Dheeraj Rokade <dheerajrok...@yahoo.com.INVALID> To: Balazs Donat Bessenyei <bes...@cloudera.com>; "dev@flume.apache.org" <dev@flume.apache.org> Cc: Attila Simon <s...@cloudera.com> Sent: Monday, 17 October 2016 7:59 PM Subject: Re: Flume channel error : Unable to put batch on required channel: FileChannel file-channel-1 Please find inline the configuration file # list the sources, sinks and channels in the agentagent_foo.sources = hdfs-Cluster1-sourceagent_foo.sinks = hdfs-Cluster1-sink1 hdfs-Cluster1-sink2 hdfs-Cluster1-sink3 hdfs-Cluster1-sink4agent_foo.channels = file-channel-1 file-channel-2 file-channel-3 # set channels for sourceagent_foo.sources.hdfs-Cluster1-source.channels = file-channel-1 file-channel-2 file-channel-3 # Describe/configure the sourceagent_foo.sources.hdfs-Cluster1-source.type = org.apache.flume.source.kafka.KafkaSource agent_foo.sources.hdfs-Cluster1-source.zookeeperConnect = 10.32.22.67:2181agent_foo.sources.hdfs-Cluster1-source.topic = test3agent_foo.sources.hdfs-Cluster1-source.groupId = flumeagent_foo.sources.hdfs-Cluster1-source.kafka.auto.offset.reset = smallest agent_foo.sources.hdfs-Cluster1-source.batchSize = 5 agent_foo.sources.hdfs-Cluster1-source.kafka.consumer.timeout.ms = 100 agent_foo.channels.file-channel-1.type = fileagent_foo.channels.file-channel-1.checkpointDir = /home/dheeraj.rokade/flume/.flume/file-channel1/checkpoint agent_foo.channels.file-channel-1.backupCheckpointDir = /home/dheeraj.rokade/flume/.flume/file-channel1/backupagent_foo.channels.file-channel-1.dataDirs = /home/dheeraj.rokade/flume/.flume/file-channel1/dataagent_foo.channels.file-channel-1.transactionCapacity = 205agent_foo.channels.file-channel-1.checkpointInterval = 3000agent_foo.channels.file-channel-1.capacity = 10000agent_foo.channels.file-channel-1.keep-alive = 5agent_foo.channels.file-channel-1.use-fast-replay=false agent_foo.channels.file-channel-2.type = fileagent_foo.channels.file-channel-2.checkpointDir = /home/dheeraj.rokade/flume/.flume/file-channel2/checkpoint agent_foo.channels.file-channel-2.backupCheckpointDir = /home/dheeraj.rokade/flume/.flume/file-channel2/backup agent_foo.channels.file-channel-2.dataDirs = /home/dheeraj.rokade/flume/.flume/file-channel2/data agent_foo.channels.file-channel-2.transactionCapacity = 25agent_foo.channels.file-channel-2.checkpointInterval = 300agent_foo.channels.file-channel-2.capacity = 100agent_foo.channels.file-channel-2.keep-alive = 5agent_foo.channels.file-channel-2.use-fast-replay=false agent_foo.channels.file-channel-3.type = fileagent_foo.channels.file-channel-3.checkpointDir = /home/dheeraj.rokade/flume/.flume/file-channel3/checkpoint agent_foo.channels.file-channel-3.backupCheckpointDir = /home/dheeraj.rokade/flume/.flume/file-channel3/backup agent_foo.channels.file-channel-3.dataDirs = /home/dheeraj.rokade/flume/.flume/file-channel3/dataagent_foo.channels.file-channel-3.transactionCapacity = 25agent_foo.channels.file-channel-3.checkpointInterval = 300agent_foo.channels.file-channel-3.capacity = 100agent_foo.channels.file-channel-3.keep-alive = 5agent_foo.channels.file-channel-3.use-fast-replay=false # set channel for sinksagent_foo.sinks.hdfs-Cluster1-sink1.channel = file-channel-1agent_foo.sinks.hdfs-Cluster1-sink2.channel = file-channel-2agent_foo.sinks.hdfs-Cluster1-sink3.channel = file-channel-3agent_foo.sinks.hdfs-Cluster1-sink4.channel = file-channel-1 agent_foo.sources.hdfs-Cluster1-source.interceptors = i1agent_foo.sources.hdfs-Cluster1-source.interceptors.i1.type = regex_extractoragent_foo.sources.hdfs-Cluster1-source.interceptors.i1.regex =(\\w+)|(\\w+)|#agent_foo.sources.hdfs-Cluster1-source.interceptors.i1.serializers.delimiter = :#agent_foo.sources.hdfs-Cluster1-source.interceptors.i1.regex = (\\d).*agent_foo.sources.hdfs-Cluster1-source.interceptors.i1.serializers = s1 s2agent_foo.sources.hdfs-Cluster1-source.interceptors.i1.serializers.s1.type = org.apache.flume.interceptor.RegexExtractorInterceptorPassThroughSerializer agent_foo.sources.hdfs-Cluster1-source.interceptors.i1.serializers.s2.type = org.apache.flume.interceptor.RegexExtractorInterceptorPassThroughSerializer agent_foo.sources.hdfs-Cluster1-source.interceptors.i1.serializers.s1.name = val1agent_foo.sources.hdfs-Cluster1-source.interceptors.i1.serializers.s2.name = col2 # channel selector configurationagent_foo.sources.hdfs-Cluster1-source.selector.type = multiplexingagent_foo.sources.hdfs-Cluster1-source.selector.header = val1 agent_foo.sources.hdfs-Cluster1-source.selector.mapping.I = file-channel-1agent_foo.sources.hdfs-Cluster1-source.selector.mapping.s_case = file-channel-2agent_foo.sources.hdfs-Cluster1-source.selector.mapping.s_user = file-channel-3agent_foo.sources.hdfs-Cluster1-source.selector.default = file-channel-3 agent_foo.sinks.hdfs-Cluster1-sink1.type = hdfsagent_foo.sinks.hdfs-Cluster1-sink1.hdfs.path = hdfs://10.32.22.65:8020/user/dheeraj.rokade/flume_test1agent_foo.sinks.hdfs-Cluster1-sink1.rollInterval = 10agent_foo.sinks.hdfs-Cluster1-sink1.hdfs.writeFormat=Textagent_foo.sinks.hdfs-Cluster1-sink1.hdfs.fileType=DataStreamagent_foo.sinks.hdfs-Cluster1-sink1.hdfs.batchSize = 5 agent_foo.sinks.hdfs-Cluster1-sink2.type = hdfsagent_foo.sinks.hdfs-Cluster1-sink2.hdfs.path = hdfs://10.32.22.65:8020/user/dheeraj.rokade/flume_test2 agent_foo.sinks.hdfs-Cluster1-sink2.rollInterval = 10agent_foo.sinks.hdfs-Cluster1-sink2.hdfs.writeFormat=Textagent_foo.sinks.hdfs-Cluster1-sink2.hdfs.fileType=DataStreamagent_foo.sinks.hdfs-Cluster1-sink2.hdfs.batchSize = 5 agent_foo.sinks.hdfs-Cluster1-sink3.type = hdfsagent_foo.sinks.hdfs-Cluster1-sink3.hdfs.path = hdfs://10.32.22.65:8020/user/dheeraj.rokade/flume_test3 agent_foo.sinks.hdfs-Cluster1-sink3.rollInterval = 10agent_foo.sinks.hdfs-Cluster1-sink3.hdfs.writeFormat=Textagent_foo.sinks.hdfs-Cluster1-sink3.hdfs.fileType=DataStreamagent_foo.sinks.hdfs-Cluster1-sink3.hdfs.batchSize = 5 agent_foo.sinks.hdfs-Cluster1-sink4.type = hdfsagent_foo.sinks.hdfs-Cluster1-sink4.hdfs.path = hdfs://10.32.22.65:8020/user/dheeraj.rokade/flume_test4agent_foo.sinks.hdfs-Cluster1-sink4.rollInterval = 10agent_foo.sinks.hdfs-Cluster1-sink4.hdfs.writeFormat=Textagent_foo.sinks.hdfs-Cluster1-sink4.hdfs.fileType=DataStreamagent_foo.sinks.hdfs-Cluster1-sink4.hdfs.batchSize = 5 From: Balazs Donat Bessenyei <bes...@cloudera.com> To: dev@flume.apache.org; "dheerajrok...@yahoo.com" <dheerajrok...@yahoo.com> Cc: Attila Simon <s...@cloudera.com> Sent: Monday, 17 October 2016 6:58 PM Subject: Re: Flume channel error : Unable to put batch on required channel: FileChannel file-channel-1 As far as I know, apache.org mailing lists "ignore" attachments. You should either upload them to some service like https://gist.github.com/ or inline the relevant parts. Btw, when do you get these errors? When starting Flume or after some time? Thank you, Donat On Mon, Oct 17, 2016 at 7:51 PM, Dheeraj Rokade <dheerajrok...@yahoo.com.invalid> wrote: > Please find the configuration file attached > > Thanks > Dheeraj > > Sent from Yahoo Mail on Android > > On Mon, 17 Oct, 2016 at 18:23, Attila Simon > <s...@cloudera.com> wrote: > Hi, > It might've been my mobile client for gmail but couldn't find the flume > conf attached. Could you please double check? > Cheers, > Attila > > On Monday, 17 October 2016, Dheeraj Rokade <dheerajrok...@yahoo.com.invalid> > wrote: > >> Hello Team, >> >> My development activity is stuck due to below error in the Apache agent. >> >> *org.apache.flume.ChannelException: Unable to put batch on required >> channel: FileChannel file-channel-1 * >> >> I have the following configuration >> One Source - *Kafka* >> Three channels - *File channel* >> Four Sinks - *HDFS Sinks* >> >> Attached is the flume agent config file and below is the error message >> >> Things tried >> 1) Lowering or increasing the batch size for Kafka source and Hive HDFS >> sink >> 2) Lowering or increasing the TransactionCapacity and Capacity for File >> channel >> 3) With some help from the net, tried keeping File channel >> TransactionCapacity to be 5x of Kafka and HDFS Batch size but in vain. >> >> Please help advise >> >> Error message in detail : >> 16/10/17 18:06:08 ERROR kafka.KafkaSource: KafkaSource EXCEPTION, {} >> org.apache.flume.ChannelException: Unable to put batch on required >> channel: FileChannel file-channel-1 { dataDirs: >> [/home/dheeraj.rokade/flume/.flume/file-channel1/data] } >> at org.apache.flume.channel.ChannelProcessor.processEventBatch( >> ChannelProcessor.java:200) >> at org.apache.flume.source.kafka.KafkaSource.process( >> KafkaSource.java:123) >> at org.apache.flume.source.PollableSourceRunner$PollingRunner.run( >> PollableSourceRunner.java:139) >> at java.lang.Thread.run(Thread.java:745) >> Caused by: java.lang.NullPointerException >> at org.apache.flume.channel.file.proto.ProtosFactory$ >> FlumeEventHeader$Builder.setValue(ProtosFactory.java:7415) >> at org.apache.flume.channel.file.Put.writeProtos(Put.java:85) >> at org.apache.flume.channel.file.TransactionEventRecord. >> toByteBuffer(TransactionEventRecord.java:174) >> at org.apache.flume.channel.file.Log.put(Log.java:642) >> at org.apache.flume.channel.file.FileChannel$ >> FileBackedTransaction.doPut(FileChannel.java:468) >> at org.apache.flume.channel.BasicTransactionSemantics.put( >> BasicTransactionSemantics.java:93) >> at org.apache.flume.channel.BasicChannelSemantics.put( >> BasicChannelSemantics.java:80) >> at org.apache.flume.channel.ChannelProcessor.processEventBatch( >> ChannelProcessor.java:189) >> ... 3 more >> 16/10/17 18:06:13 ERROR kafka.KafkaSource: KafkaSource EXCEPTION, {} >> org.apache.flume.ChannelException: Unable to put batch on required >> channel: FileChannel file-channel-1 { dataDirs: >> [/home/dheeraj.rokade/flume/.flume/file-channel1/data] } >> at org.apache.flume.channel.ChannelProcessor.processEventBatch( >> ChannelProcessor.java:200) >> at org.apache.flume.source.kafka.KafkaSource.process( >> KafkaSource.java:123) >> at org.apache.flume.source.PollableSourceRunner$PollingRunner.run( >> PollableSourceRunner.java:139) >> at java.lang.Thread.run(Thread.java:745) >> Caused by: java.lang.NullPointerException >> at org.apache.flume.channel.file.proto.ProtosFactory$ >> FlumeEventHeader$Builder.setValue(ProtosFactory.java:7415) >> at org.apache.flume.channel.file.Put.writeProtos(Put.java:85) >> at org.apache.flume.channel.file.TransactionEventRecord. >> toByteBuffer(TransactionEventRecord.java:174) >> at org.apache.flume.channel.file.Log.put(Log.java:642) >> at org.apache.flume.channel.file.FileChannel$ >> FileBackedTransaction.doPut(FileChannel.java:468) >> at org.apache.flume.channel.BasicTransactionSemantics.put( >> BasicTransactionSemantics.java:93) >> at org.apache.flume.channel.BasicChannelSemantics.put( >> BasicChannelSemantics.java:80) >> at org.apache.flume.channel.ChannelProcessor.processEventBatch( >> ChannelProcessor.java:189) >> ... 3 more > >> >> > > -- > > *Attila Simon* > Software Engineer > Email: s...@cloudera.com > > [image: Cloudera Inc.] >