##### global defination ############################################################## lc_srv03.sources = src_avro lc_srv03.channels = ch_null ch_hdfs ch_kafka ch_bypass lc_srv03.sinks = sink_hdfs sink_kafka sink_bypass ##### source defination ############################################################## lc_srv03.sources.src_avro.channels = ch_null ch_hdfs ch_kafka ch_bypass lc_srv03.sources.src_avro.type = avro lc_srv03.sources.src_avro.bind = 0.0.0.0 lc_srv03.sources.src_avro.port = 4400 lc_srv03.sources.src_avro.threads = 32 lc_srv03.sources.src_avro.selector.type = multiplexing lc_srv03.sources.src_avro.selector.header = category lc_srv03.sources.src_avro.selector.mapping.fe_perf_benchmark = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mtcos = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_nginx = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.dealrank = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_push_service = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_data_location = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_groupapi_search = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.b = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.b_business = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.passport = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.sphinx_deal = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.deal_query = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.filestorage = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobileapp = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.cdn = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_group_recommend = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.basync = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_movie_statistics = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.eventlog = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.nginx = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.applog = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.couponverify = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.couponverifylog = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.magiccard = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_groupapi_stid = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.paylog = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.payrequest = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.ecomlogin = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mailsubscribe = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.cos_mis = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.bizappapi = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.appbind = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_group_push = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_report_loc = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_mars_locate = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mtsg = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_xmserver_netstat = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mtcos_data_productshow = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mtcos_data_auditcommit = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_order_nomsg = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.cos_errorlog = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.captcha = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_group_push = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.mobile_mars_locate = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.cos_sso = ch_hdfs lc_srv03.sources.src_avro.selector.mapping.test = ch_hdfs lc_srv03.sources.src_avro.selector.default = ch_null lc_srv03.sources.src_avro.selector.optional.mobile_push_service = ch_bypass lc_srv03.sources.src_avro.selector.optional.applog = ch_bypass lc_srv03.sources.src_avro.selector.optional.couponverify = ch_bypass lc_srv03.sources.src_avro.selector.optional.couponverifylog = ch_bypass lc_srv03.sources.src_avro.selector.optional.b = ch_bypass lc_srv03.sources.src_avro.selector.optional.eventlog = ch_bypass lc_srv03.sources.src_avro.selector.optional.sphinx_deal = ch_bypass lc_srv03.sources.src_avro.selector.optional.deal_query = ch_bypass lc_srv03.sources.src_avro.selector.optional.mobile_xmserver_netstat = ch_bypass lc_srv03.sources.src_avro.selector.optional.stormlog = ch_bypass lc_srv03.sources.src_avro.selector.optional.cos_errorlog = ch_kafka lc_srv03.sources.src_avro.selector.optional.mobile_nginx = ch_kafka lc_srv03.sources.src_avro.selector.optional.mobile_groupapi_search = ch_kafka lc_srv03.sources.src_avro.selector.optional.nginx = ch_kafka ##### channel defination ############################################################## lc_srv03.channels.ch_null.type = null lc_srv03.channels.ch_hdfs.type = dual lc_srv03.channels.ch_hdfs.memory.capacity = 1000000 lc_srv03.channels.ch_hdfs.memory.transactionCapacity = 100000 lc_srv03.channels.ch_hdfs.file.checkpointDir = /opt/meituan/flume_collector/data/ch_hdfs.checkpoint lc_srv03.channels.ch_hdfs.file.dataDirs = /opt/meituan/flume_collector/data/ch_hdfs.data lc_srv03.channels.ch_hdfs.file.capacity = 200000000 lc_srv03.channels.ch_hdfs.file.transactionCapacity = 100000 lc_srv03.channels.ch_hdfs.file.checkpointInterval = 300000 lc_srv03.channels.ch_hdfs.file.maxFileSize = 2146435071 lc_srv03.channels.ch_hdfs.file.keep-alive = 30 lc_srv03.channels.ch_hdfs.file.write-timeout = 30 lc_srv03.channels.ch_kafka.type = dual lc_srv03.channels.ch_kafka.memory.capacity = 1000000 lc_srv03.channels.ch_kafka.memory.transactionCapacity = 100000 lc_srv03.channels.ch_kafka.file.checkpointDir = /opt/meituan/flume_collector/data/ch_kafka.checkpoint lc_srv03.channels.ch_kafka.file.dataDirs = /opt/meituan/flume_collector/data/ch_kafka.data lc_srv03.channels.ch_kafka.file.capacity = 100000000 lc_srv03.channels.ch_kafka.file.transactionCapacity = 100000 lc_srv03.channels.ch_kafka.file.checkpointInterval = 300000 lc_srv03.channels.ch_kafka.file.maxFileSize = 2146435071 lc_srv03.channels.ch_kafka.file.keep-alive = 30 lc_srv03.channels.ch_kafka.file.write-timeout = 30 lc_srv03.channels.ch_bypass.type = dual lc_srv03.channels.ch_bypass.memory.capacity = 1000000 lc_srv03.channels.ch_bypass.memory.transactionCapacity = 100000 lc_srv03.channels.ch_bypass.file.checkpointDir = /opt/meituan/flume_collector/data/ch_bypass.checkpoint lc_srv03.channels.ch_bypass.file.dataDirs = /opt/meituan/flume_collector/data/ch_bypass.data lc_srv03.channels.ch_bypass.file.capacity = 100000000 lc_srv03.channels.ch_bypass.file.transactionCapacity = 100000 lc_srv03.channels.ch_bypass.file.checkpointInterval = 300000 lc_srv03.channels.ch_bypass.file.maxFileSize = 2146435071 lc_srv03.channels.ch_bypass.file.keep-alive = 30 lc_srv03.channels.ch_bypass.file.write-timeout = 30 ##### sink defination ############################################################## lc_srv03.sinks.sink_hdfs.channel = ch_hdfs lc_srv03.sinks.sink_hdfs.type = hdfs lc_srv03.sinks.sink_hdfs.switchon = true lc_srv03.sinks.sink_hdfs.hdfs.path = /user/hive/warehouse/originallog.db/%{category}org/dt=%Y%m%d/hour=%H lc_srv03.sinks.sink_hdfs.hdfs.filePrefix = lc_srv03 lc_srv03.sinks.sink_hdfs.hdfs.rollInterval = 1200 lc_srv03.sinks.sink_hdfs.hdfs.rollSize = 3072000000 lc_srv03.sinks.sink_hdfs.hdfs.rollCount = 0 lc_srv03.sinks.sink_hdfs.hdfs.batchSize = 30000 lc_srv03.sinks.sink_hdfs.hdfs.callTimeout = 60000 lc_srv03.sinks.sink_hdfs.hdfs.threadsPoolSize = 30 lc_srv03.sinks.sink_hdfs.hdfs.codeC = LzopCodec #lc_srv03.sinks.sink_hdfs.hdfs.codeC = bzip2 lc_srv03.sinks.sink_hdfs.hdfs.fileType = CompressedStream #lc_srv03.sinks.sink_hdfs.hdfs.fileType = SequenceFile lc_srv03.sinks.sink_hdfs.hdfs.writeFormat = Text lc_srv03.sinks.sink_hdfs.hdfs.useLocalTimeStamp = True lc_srv03.sinks.sink_hdfs.serializer = TEXT lc_srv03.sinks.sink_hdfs.serializer.appendNewline = false lc_srv03.sinks.sink_kafka.channel = ch_kafka lc_srv03.sinks.sink_kafka.type = kafka lc_srv03.sinks.sink_kafka.zkConnect = *******/kafka lc_srv03.sinks.sink_kafka.zkTimeout = 30000 lc_srv03.sinks.sink_kafka.topicPrefix = org lc_srv03.sinks.sink_kafka.batchSize = 5000 lc_srv03.sinks.sink_bypass.channel = ch_bypass lc_srv03.sinks.sink_bypass.type = avro lc_srv03.sinks.sink_bypass.hostname = ****** lc_srv03.sinks.sink_bypass.port = 4490 lc_srv03.sinks.sink_bypass.batch-size = 1000 lc_srv03.sinks.sink_bypass.connect-timeout = 30000 lc_srv03.sinks.sink_bypass.request-timeout = 60000 ##### sinkgroups defination #########################################################