[
https://issues.apache.org/jira/browse/FLUME-3021?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15932394#comment-15932394
]
tycho_yang edited comment on FLUME-3021 at 3/20/17 9:50 AM:
------------------------------------------------------------
agent1.sources = source1
agent1.sinks = sink1
agent1.channels = channel1
agent1.sources.source1.type = exec
agent1.sources.source1.command = tail -F /data/lb-www.access.log
agent1.sources.source1.interceptors = i1
#agent1.sources.source1.interceptors.i1.type = host
#agent1.sources.source1.interceptors.i1.hostHeader = hostname
agent1.sources.source1.interceptors.i0.regex = ([^
]*)\\s+-\\s+(.+?|-)\\s+\\[(.*)\\]\\s+\"([^ ]*)\\s+([^ ]*)\\s+([^
|\"]*)\"\\s+(-|[0-9]*)\\s+(-|[0-9]*)\\s+\"(.+?|-)\"\\s+\"(.+?|-)\"\\s+\"(.+?|-)\"\\s+\"(.+?|-)\"\\s+(.+?|-)\\s+(.+?|-)\\s+(.+?|-)\\s+\"(.+?|-)\"\\s+(.+?|-)\\s+(.*)
agent1.sources.source1.interceptors=i0 i3
agent1.sources.source1.interceptors.i0.type=regex_extractor
agent1.sources.source1.interceptors.i0.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9
s10 s11 s12 s13 s14 s15 s16 s17 s18
agent1.sources.source1.interceptors.i0.serializers.s1.name = remote_addr
agent1.sources.source1.interceptors.i0.serializers.s2.name = user
agent1.sources.source1.interceptors.i0.serializers.s3.type =
org.apache.flume.interceptor.RegexExtractorInterceptorMillisSerializer
agent1.sources.source1.interceptors.i0.serializers.s3.name = timestamp
agent1.sources.source1.interceptors.i0.serializers.s3.pattern=
yyyy-MM-dd'T'HH:mm:ssZ
agent1.sources.source1.interceptors.i0.serializers.s4.name = request_methond
agent1.sources.source1.interceptors.i0.serializers.s5.name = request_url
agent1.sources.source1.interceptors.i0.serializers.s6.name = request_http
agent1.sources.source1.interceptors.i0.serializers.s7.name = http_status
agent1.sources.source1.interceptors.i0.serializers.s8.name = body_bytes_sent
agent1.sources.source1.interceptors.i0.serializers.s9.name = referer
agent1.sources.source1.interceptors.i0.serializers.s10.name = user_Agent
agent1.sources.source1.interceptors.i0.serializers.s11.name =
http_x_forwarded_for
agent1.sources.source1.interceptors.i0.serializers.s12.name = gzip_ratio
agent1.sources.source1.interceptors.i0.serializers.s13.name = request_time
agent1.sources.source1.interceptors.i0.serializers.s14.name = bytes_sent
agent1.sources.source1.interceptors.i0.serializers.s15.name = request_length
agent1.sources.source1.interceptors.i0.serializers.s16.name = upstream_addr
agent1.sources.source1.interceptors.i0.serializers.s17.name = upstream_status
agent1.sources.source1.interceptors.i0.serializers.s18.name =
upstream_response_time
agent1.sources.source1.interceptors.i3.type=org.apache.flume.interceptor.HostInterceptor$Builder
agent1.sources.source1.interceptors.i3.hostHeader = host
agent1.sinks.sink1.type = elasticsearch
agent1.sinks.sink1.hostNames=192.168.3.63:9300
agent1.sinks.sink1.indexName = flume
agent1.sinks.sink1.indexType = nginx-access
agent1.sinks.sink1.clusterName = es5
agent1.sinks.sink1.batchSize = 100
agent1.sinks.sink1.ttl = 10
agent1.sinks.sink1.serializer.outputCharset = UTF-8
agent1.sinks.sink1.serializer =
org.apache.flume.sink.elasticsearch.ElasticSearchLogStashEventSerializer
#agent1.sinks.sink1.serializer =
org.apache.flume.sink.elasticsearch.ElasticSearchDynamicSerializer
agent1.channels.channel1.type = memory
agent1.channels.channel1.capacity = 10000
agent1.channels.channel1.transactionCapactiy = 3000
agent1.sources.source1.channels = channel1
agent1.sinks.sink1.channel = channel1
was (Author: ytc301):
agent1.sources = source1
agent1.sinks = sink1
agent1.channels = channel1
agent1.sources.source1.type = exec
agent1.sources.source1.command = tail -F /data/lb-www.access.log
agent1.sources.source1.interceptors = i1
#agent1.sources.source1.interceptors.i1.type = host
#agent1.sources.source1.interceptors.i1.hostHeader = hostname
agent1.sources.source1.interceptors.i0.regex = ([^
]*)\\s+-\\s+(.+?|-)\\s+\\[(.*)\\]\\s+\"([^ ]*)\\s+([^ ]*)\\s+([^
|\"]*)\"\\s+(-|[0-9]*)\\s+(-|[0-9]*)\\s+\"(.+?|-)\"\\s+\"(.+?|-)\"\\s+\"(.+?|-)\"\\s+\"(.+?|-)\"\\s+(.+?|-)\\s+(.+?|-)\\s+(.+?|-)\\s+\"(.+?|-)\"\\s+(.+?|-)\\s+(.*)
agent1.sources.source1.interceptors=i0 i3
agent1.sources.source1.interceptors.i0.type=regex_extractor
agent1.sources.source1.interceptors.i0.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9
s10 s11 s12 s13 s14 s15 s16 s17 s18
agent1.sources.source1.interceptors.i0.serializers.s1.name = remote_addr
agent1.sources.source1.interceptors.i0.serializers.s2.name = user
agent1.sources.source1.interceptors.i0.serializers.s3.type =
org.apache.flume.interceptor.RegexExtractorInterceptorMillisSerializer
agent1.sources.source1.interceptors.i0.serializers.s3.name = timestamp
agent1.sources.source1.interceptors.i0.serializers.s3.pattern=
yyyy-MM-dd'T'HH:mm:ssZ
agent1.sources.source1.interceptors.i0.serializers.s4.name = request_methond
agent1.sources.source1.interceptors.i0.serializers.s5.name = request_url
agent1.sources.source1.interceptors.i0.serializers.s6.name = request_http
agent1.sources.source1.interceptors.i0.serializers.s7.name = http_status
agent1.sources.source1.interceptors.i0.serializers.s8.name = body_bytes_sent
agent1.sources.source1.interceptors.i0.serializers.s9.name = referer
agent1.sources.source1.interceptors.i0.serializers.s10.name = user_Agent
agent1.sources.source1.interceptors.i0.serializers.s11.name =
http_x_forwarded_for
agent1.sources.source1.interceptors.i0.serializers.s12.name = gzip_ratio
agent1.sources.source1.interceptors.i0.serializers.s13.name = request_time
agent1.sources.source1.interceptors.i0.serializers.s14.name = bytes_sent
agent1.sources.source1.interceptors.i0.serializers.s15.name = request_length
agent1.sources.source1.interceptors.i0.serializers.s16.name = upstream_addr
agent1.sources.source1.interceptors.i0.serializers.s17.name = upstream_status
agent1.sources.source1.interceptors.i0.serializers.s18.name =
upstream_response_time
agent1.sources.source1.interceptors.i3.type=org.apache.flume.interceptor.HostInterceptor$Builder
agent1.sources.source1.interceptors.i3.hostHeader = host
# source1ink Elasticsearch
# Elasource1ticsource1earch lib ---> flume/lib
agent1.sinks.sink1.type = elasticsearch
#agent1.sinks.sink1.type = logger
agent1.sinks.sink1.hostNames=192.168.3.63:9300
agent1.sinks.sink1.indexName = flume
agent1.sinks.sink1.indexType = nginx-access
agent1.sinks.sink1.clusterName = es5
agent1.sinks.sink1.batchSize = 100
agent1.sinks.sink1.ttl = 10
agent1.sinks.sink1.serializer.outputCharset = UTF-8
agent1.sinks.sink1.serializer =
org.apache.flume.sink.elasticsearch.ElasticSearchLogStashEventSerializer
#agent1.sinks.sink1.serializer =
org.apache.flume.sink.elasticsearch.ElasticSearchDynamicSerializer
# Channel Memory
agent1.channels.channel1.type = memory
agent1.channels.channel1.capacity = 10000
agent1.channels.channel1.transactionCapactiy = 3000
agent1.sources.source1.channels = channel1
agent1.sinks.sink1.channel = channel1
> flume Elasticsearch 5.0 not support
> -----------------------------------
>
> Key: FLUME-3021
> URL: https://issues.apache.org/jira/browse/FLUME-3021
> Project: Flume
> Issue Type: Bug
> Components: Sinks+Sources
> Affects Versions: 1.7.0
> Reporter: tycho_yang
> Assignee: Yonghao Zou
> Fix For: 1.8.0
>
> Attachments: FLUME-3021-0.patch
>
>
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
