This is an automated email from the ASF dual-hosted git repository. ppkarwasz pushed a commit to branch docs/move-flume-site in repository https://gitbox.apache.org/repos/asf/logging-flume.git
commit 965b04836e2fdc4ec07778a079e0e2eefc911f42 Author: Ralph Goers <[email protected]> AuthorDate: Thu Jun 2 15:00:37 2022 -0700 Update site for 1.10.0 --- source/sphinx/FlumeDeveloperGuide.rst | 92 ++++++++++- source/sphinx/FlumeUserGuide.rst | 296 +++++++++++++++++++--------------- source/sphinx/download.rst | 8 +- source/sphinx/index.rst | 28 ++++ source/sphinx/releases/1.10.0.rst | 69 ++++++++ source/sphinx/releases/index.rst | 5 +- source/sphinx/team.rst | 70 ++++---- 7 files changed, 401 insertions(+), 167 deletions(-) diff --git a/source/sphinx/FlumeDeveloperGuide.rst b/source/sphinx/FlumeDeveloperGuide.rst index 5e98bd3e..f383cda7 100644 --- a/source/sphinx/FlumeDeveloperGuide.rst +++ b/source/sphinx/FlumeDeveloperGuide.rst @@ -15,7 +15,7 @@ ====================================== -Flume 1.9.0 Developer Guide +Flume 1.10.0 Developer Guide ====================================== Introduction @@ -866,3 +866,93 @@ Channel ~~~~~~~ TBD + +Initializable +~~~~~~~~~~~~~ + +As of Flume 1.10.0 Sources, Sinks, and Channels may implement the Intitializable interface. Doing so +allows the component to have access the materialized configuration before any of the components have been +started. + +This example shows a Sink being configured with the name of a Source. While initializing it will +retrieve the Source from the configuration and save it. During event processing a new event will be +sent to the Source, presumably after the event has be modified in some way. + +.. code-block:: java + + public class NullInitSink extends NullSink implements Initializable { + + private static final Logger logger = LoggerFactory.getLogger(NullInitSink.class); + private String sourceName = null; + private EventProcessor eventProcessor = null; + private long total = 0; + + public NullInitSink() { + super(); + } + + @Override + public void configure(Context context) { + sourceName = context.getString("targetSource"); + super.configure(context); + + } + + @Override + public void initialize(MaterializedConfiguration configuration) { + logger.debug("Locating source for event publishing"); + for (Map.Entry<String, SourceRunner> entry : configuration.getSourceRunners().entrySet()) { + if (entry.getKey().equals(sourceName)) { + Source source = entry.getValue().getSource(); + if (source instanceof EventProcessor) { + eventProcessor = (EventProcessor) source; + logger.debug("Found event processor {}", source.getName()); + return; + } + } + } + logger.warn("No Source named {} found for republishing events.", sourceName); + } + + @Override + public Status process() throws EventDeliveryException { + Status status = Status.READY; + + Channel channel = getChannel(); + Transaction transaction = channel.getTransaction(); + Event event = null; + CounterGroup counterGroup = getCounterGroup(); + long batchSize = getBatchSize(); + long eventCounter = counterGroup.get("events.success"); + + try { + transaction.begin(); + int i = 0; + for (i = 0; i < batchSize; i++) { + event = channel.take(); + if (event != null) { + long id = Long.parseLong(new String(event.getBody())); + total += id; + event.getHeaders().put("Total", Long.toString(total)); + eventProcessor.processEvent(event); + logger.info("Null sink {} successful processed event {}", getName(), id); + } else { + status = Status.BACKOFF; + break; + } + } + transaction.commit(); + counterGroup.addAndGet("events.success", (long) Math.min(batchSize, i)); + counterGroup.incrementAndGet("transaction.success"); + } catch (Exception ex) { + transaction.rollback(); + counterGroup.incrementAndGet("transaction.failed"); + logger.error("Failed to deliver event. Exception follows.", ex); + throw new EventDeliveryException("Failed to deliver event: " + event, ex); + } finally { + transaction.close(); + } + + return status; + } + } \ No newline at end of file diff --git a/source/sphinx/FlumeUserGuide.rst b/source/sphinx/FlumeUserGuide.rst index b740507f..7d7b3fde 100644 --- a/source/sphinx/FlumeUserGuide.rst +++ b/source/sphinx/FlumeUserGuide.rst @@ -14,9 +14,9 @@ limitations under the License. -=============================== -Flume 1.9.0 User Guide -=============================== +================================ +Flume 1.10.0 User Guide +================================ Introduction ============ @@ -109,18 +109,20 @@ There's also a memory channel which simply stores the events in an in-memory queue, which is faster but any events still left in the memory channel when an agent process dies can't be recovered. +Flume's `KafkaChannel` uses Apache Kafka to stage events. Using a replicated +Kafka topic as a channel helps avoiding event loss in case of a disk failure. + Setup ===== Setting up an agent ------------------- -Flume agent configuration is stored in a local configuration file. This is a -text file that follows the Java properties file format. -Configurations for one or more agents can be specified in the same -configuration file. The configuration file includes properties of each source, -sink and channel in an agent and how they are wired together to form data -flows. +Flume agent configuration is stored in one or more configuration files that +follow the Java properties file format. Configurations for one or more agents +can be specified in these configuration files. The configuration includes +properties of each source, sink and channel in an agent and how they are wired +together to form data flows. Configuring individual components ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -222,30 +224,110 @@ The original Flume terminal will output the event in a log message. Congratulations - you've successfully configured and deployed a Flume agent! Subsequent sections cover agent configuration in much more detail. -Using environment variables in configuration files -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Configuration from URIs +~~~~~~~~~~~~~~~~~~~~~~~ +As of version 1.10.0 Flume supports being configured using URIs instead of just from local files. Direct support +for HTTP(S), file, and classpath URIs is included. The HTTP support includes support for authentication using +basic authorization but other authorization mechanisms may be supported by specifying the fully qualified name +of the class that implements the AuthorizationProvider interface using the --auth-provider option. HTTP also +supports reloading of configuration files using polling if the target server properly responds to the If-Modified-Since +header. + +To specify credentials for HTTP authentication add:: + + --conf-user userid --conf-password password + +to the startup command. + +Multiple Configuration Files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +As of version 1.10.0 Flume supports being configured from multiple configuration files instead of just one. +This more easily allows values to be overridden or added based on specific environments. Each file should +be configured using its own --conf-file or --conf-uri option. However, all files should either be provided +with --conf-file or with --conf-uri. If --conf-file and --conf-uri appear together as options all --conf-uri +configurations will be processed before any of the --conf-file configurations are merged. + +For example, a configuration of:: + + $ bin/flume-ng agent --conf conf --conf-file example.conf --conf-uri http://localhost:80/flume.conf --conf-uri http://localhost:80/override.conf --name a1 -Dflume.root.logger=INFO,console + +will cause flume.conf to be read first, override.conf to be merged with it and finally example.conf would be +merged last. If it is desirec to have example.conf be the base configuration it should be specified using the +--conf-uri option either as:: + + --conf-uri classpath://example.conf + or + --conf-uri file:///example.conf + +depending on how it should be accessed. + +Using environment variables, system properies, or other properties configuration files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Flume has the ability to substitute environment variables in the configuration. For example:: a1.sources = r1 a1.sources.r1.type = netcat a1.sources.r1.bind = 0.0.0.0 - a1.sources.r1.port = ${NC_PORT} + a1.sources.r1.port = ${env:NC_PORT} a1.sources.r1.channels = c1 NB: it currently works for values only, not for keys. (Ie. only on the "right side" of the `=` mark of the config lines.) -This can be enabled via Java system properties on agent invocation by setting `propertiesImplementation = org.apache.flume.node.EnvVarResolverProperties`. +As of version 1.10.0 Flume resolves configuration values using Apache Commons Text's StringSubstitutor +class using the default set of Lookups along with a lookup that uses the configuration files as a +source for replacement values. For example:: - $ NC_PORT=44444 bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger=INFO,console -DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties + $ NC_PORT=44444 bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger=INFO,console Note the above is just an example, environment variables can be configured in other ways, including being set in `conf/flume-env.sh`. +As noted, system properties are also supported, so the configuration:: + + a1.sources = r1 + a1.sources.r1.type = netcat + a1.sources.r1.bind = 0.0.0.0 + a1.sources.r1.port = ${sys:NC_PORT} + a1.sources.r1.channels = c1 + +could be used and the startup command could be:: + + $ bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger=INFO,console -DNC_PORT=44444 + +Furthermore, because multiple configuration files are allowed the first file could contain:: + + a1.sources = r1 + a1.sources.r1.type = netcat + a1.sources.r1.bind = 0.0.0.0 + a1.sources.r1.port = ${NC_PORT} + a1.sources.r1.channels = c1 + +and the override file could contain:: + + NC_PORT = 44444 + +In this case the startup command could be:: + + $ bin/flume-ng agent --conf conf --conf-file example.conf --conf-file override.conf --name a1 -Dflume.root.logger=INFO,console + +Note that the method for specifying environment variables as was done in prior versions will stil work +but has been deprecated in favor of using ${env:varName}. + +Using a command options file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Instead of specifying all the command options on the command line as of version 1.10.0 command +options may be placed in either /etc/flume/flume.opts or flume.opts on the classpath. An example +might be:: + + conf-file = example.conf + conf-file = override.conf + name = a1 + Logging raw data ~~~~~~~~~~~~~~~~ -Logging the raw stream of data flowing through the ingest pipeline is not desired behaviour in +Logging the raw stream of data flowing through the ingest pipeline is not desired behavior in many production environments because this may result in leaking sensitive data or security related configurations, such as secret keys, to Flume log files. By default, Flume will not log such information. On the other hand, if the data pipeline is broken, @@ -487,10 +569,10 @@ component: <Agent>.sources.<Source>.<someProperty> = <someValue> # properties for channels - <Agent>.channel.<Channel>.<someProperty> = <someValue> + <Agent>.channels.<Channel>.<someProperty> = <someValue> # properties for sinks - <Agent>.sources.<Sink>.<someProperty> = <someValue> + <Agent>.sinks.<Sink>.<someProperty> = <someValue> The property "type" needs to be set for each component for Flume to understand what kind of object it needs to be. Each source, sink and channel type has its @@ -539,7 +621,7 @@ linked to form multiple flows: <Agent>.channels = <Channel1> <Channel2> Then you can link the sources and sinks to their corresponding channels (for -sources) of channel (for sinks) to setup two different flows. For example, if +sources) or channel (for sinks) to setup two different flows. For example, if you need to setup two flows in an agent, one going from an external avro client to external HDFS and another from output of a tail to avro sink, then here's a config to do that: @@ -1248,7 +1330,7 @@ recursiveDirectorySearch false Whether to monitor sub directories for maxBackoff 4000 The maximum time (in millis) to wait between consecutive attempts to write to the channel(s) if the channel is full. The source will start at a low backoff and increase it exponentially each time the channel throws a - ChannelException, upto the value specified by this parameter. + ChannelException, up to the value specified by this parameter. batchSize 100 Granularity at which to batch transfer to the channel inputCharset UTF-8 Character set used by deserializers that treat the input file as text. decodeErrorPolicy ``FAIL`` What to do when we see a non-decodable character in the input file. @@ -1260,7 +1342,7 @@ deserializer ``LINE`` Specify the deserializer used to parse Defaults to parsing each line as an event. The class specified must implement ``EventDeserializer.Builder``. deserializer.* Varies per event deserializer. -bufferMaxLines -- (Obselete) This option is now ignored. +bufferMaxLines -- (Obsolete) This option is now ignored. bufferMaxLineLength 5000 (Deprecated) Maximum length of a line in the commit buffer. Use deserializer.maxLineLength instead. selector.type replicating replicating or multiplexing selector.* Depends on the selector.type value @@ -1412,7 +1494,7 @@ Twitter 1% firehose Source (experimental) Use at your own risk. Experimental source that connects via Streaming API to the 1% sample twitter -firehose, continously downloads tweets, converts them to Avro format and +firehose, continuously downloads tweets, converts them to Avro format and sends Avro events to a downstream Flume sink. Requires the consumer and access tokens and secrets of a Twitter developer account. Required properties are in **bold**. @@ -1460,7 +1542,7 @@ Property Name Default Description **kafka.bootstrap.servers** -- List of brokers in the Kafka cluster used by the source kafka.consumer.group.id flume Unique identified of consumer group. Setting the same id in multiple sources or agents indicates that they are part of the same consumer group -**kafka.topics** -- Comma-separated list of topics the kafka consumer will read messages from. +**kafka.topics** -- Comma-separated list of topics the Kafka consumer will read messages from. **kafka.topics.regex** -- Regex that defines set of topics the source is subscribed on. This property has higher priority than ``kafka.topics`` and overrides ``kafka.topics`` if exists. batchSize 1000 Maximum number of messages written to Channel in one batch @@ -1505,8 +1587,8 @@ Property Name Default Description =============================== =================== ================================================================================================ topic -- Use kafka.topics groupId flume Use kafka.consumer.group.id -zookeeperConnect -- Is no longer supported by kafka consumer client since 0.9.x. Use kafka.bootstrap.servers - to establish connection with kafka cluster +zookeeperConnect -- Is no longer supported by Kafka consumer client since 0.9.x. Use kafka.bootstrap.servers + to establish connection with Kafka cluster migrateZookeeperOffsets true When no Kafka stored offset is found, look up the offsets in Zookeeper and commit them to Kafka. This should be true to support seamless Kafka client migration from older versions of Flume. Once migrated this can be set to false, though that should generally not be required. @@ -1579,7 +1661,7 @@ Example configuration with server side authentication and data encryption. a1.sources.source1.kafka.consumer.ssl.truststore.location=/path/to/truststore.jks a1.sources.source1.kafka.consumer.ssl.truststore.password=<password to access the truststore> -Specyfing the truststore is optional here, the global truststore can be used instead. +Specifying the truststore is optional here, the global truststore can be used instead. For more details about the global SSL setup, see the `SSL/TLS support`_ section. Note: By default the property ``ssl.endpoint.identification.algorithm`` @@ -2416,10 +2498,12 @@ serializer.* Deprecated Properties +====================== ============ ====================================================================================== Name Default Description -====================== ============ ====================================================================== -hdfs.callTimeout 30000 Number of milliseconds allowed for HDFS operations, such as open, write, flush, close. This number should be increased if many HDFS timeout operations are occurring. -====================== ============ ====================================================================== +====================== ============ ====================================================================================== +hdfs.callTimeout 30000 Number of milliseconds allowed for HDFS operations, such as open, write, flush, close. + This number should be increased if many HDFS timeout operations are occurring. +====================== ============ ====================================================================================== Example for agent named a1: @@ -2429,7 +2513,7 @@ Example for agent named a1: a1.sinks = k1 a1.sinks.k1.type = hdfs a1.sinks.k1.channel = c1 - a1.sinks.k1.hdfs.path = /flume/events/%y-%m-%d/%H%M/%S + a1.sinks.k1.hdfs.path = /flume/events/%Y-%m-%d/%H%M/%S a1.sinks.k1.hdfs.filePrefix = events- a1.sinks.k1.hdfs.round = true a1.sinks.k1.hdfs.roundValue = 10 @@ -2564,7 +2648,7 @@ Example for agent named a1: a1.sinks.k1.hive.metastore = thrift://127.0.0.1:9083 a1.sinks.k1.hive.database = logsdb a1.sinks.k1.hive.table = weblogs - a1.sinks.k1.hive.partition = asia,%{country},%y-%m-%d-%H-%M + a1.sinks.k1.hive.partition = asia,%{country},%Y-%m-%d-%H-%M a1.sinks.k1.useLocalTimeStamp = false a1.sinks.k1.round = true a1.sinks.k1.roundValue = 10 @@ -2997,74 +3081,6 @@ Example for agent named a1: # a1.sinks.k1.batchSize = 1000 # a1.sinks.k1.batchDurationMillis = 1000 -ElasticSearchSink -~~~~~~~~~~~~~~~~~ - -This sink writes data to an elasticsearch cluster. By default, events will be written so that the `Kibana <http://kibana.org>`_ graphical interface -can display them - just as if `logstash <https://logstash.net>`_ wrote them. - -The elasticsearch and lucene-core jars required for your environment must be placed in the lib directory of the Apache Flume installation. -Elasticsearch requires that the major version of the client JAR match that of the server and that both are running the same minor version -of the JVM. SerializationExceptions will appear if this is incorrect. To -select the required version first determine the version of elasticsearch and the JVM version the target cluster is running. Then select an elasticsearch client -library which matches the major version. A 0.19.x client can talk to a 0.19.x cluster; 0.20.x can talk to 0.20.x and 0.90.x can talk to 0.90.x. Once the -elasticsearch version has been determined then read the pom.xml file to determine the correct lucene-core JAR version to use. The Flume agent -which is running the ElasticSearchSink should also match the JVM the target cluster is running down to the minor version. - -Events will be written to a new index every day. The name will be <indexName>-yyyy-MM-dd where <indexName> is the indexName parameter. The sink -will start writing to a new index at midnight UTC. - -Events are serialized for elasticsearch by the ElasticSearchLogStashEventSerializer by default. This behaviour can be -overridden with the serializer parameter. This parameter accepts implementations of org.apache.flume.sink.elasticsearch.ElasticSearchEventSerializer -or org.apache.flume.sink.elasticsearch.ElasticSearchIndexRequestBuilderFactory. Implementing ElasticSearchEventSerializer is deprecated in favour of -the more powerful ElasticSearchIndexRequestBuilderFactory. - -The type is the FQCN: org.apache.flume.sink.elasticsearch.ElasticSearchSink - -Required properties are in **bold**. - -================ ======================================================================== ======================================================================================================= -Property Name Default Description -================ ======================================================================== ======================================================================================================= -**channel** -- -**type** -- The component type name, needs to be ``org.apache.flume.sink.elasticsearch.ElasticSearchSink`` -**hostNames** -- Comma separated list of hostname:port, if the port is not present the default port '9300' will be used -indexName flume The name of the index which the date will be appended to. Example 'flume' -> 'flume-yyyy-MM-dd' - Arbitrary header substitution is supported, eg. %{header} replaces with value of named event header -indexType logs The type to index the document to, defaults to 'log' - Arbitrary header substitution is supported, eg. %{header} replaces with value of named event header -clusterName elasticsearch Name of the ElasticSearch cluster to connect to -batchSize 100 Number of events to be written per txn. -ttl -- TTL in days, when set will cause the expired documents to be deleted automatically, - if not set documents will never be automatically deleted. TTL is accepted both in the earlier form of - integer only e.g. a1.sinks.k1.ttl = 5 and also with a qualifier ms (millisecond), s (second), m (minute), - h (hour), d (day) and w (week). Example a1.sinks.k1.ttl = 5d will set TTL to 5 days. Follow - http://www.elasticsearch.org/guide/reference/mapping/ttl-field/ for more information. -serializer org.apache.flume.sink.elasticsearch.ElasticSearchLogStashEventSerializer The ElasticSearchIndexRequestBuilderFactory or ElasticSearchEventSerializer to use. Implementations of - either class are accepted but ElasticSearchIndexRequestBuilderFactory is preferred. -serializer.* -- Properties to be passed to the serializer. -================ ======================================================================== ======================================================================================================= - -.. note:: Header substitution is a handy to use the value of an event header to dynamically decide the indexName and indexType to use when storing the event. - Caution should be used in using this feature as the event submitter now has control of the indexName and indexType. - Furthermore, if the elasticsearch REST client is used then the event submitter has control of the URL path used. - -Example for agent named a1: - -.. code-block:: properties - - a1.channels = c1 - a1.sinks = k1 - a1.sinks.k1.type = elasticsearch - a1.sinks.k1.hostNames = 127.0.0.1:9200,127.0.0.2:9300 - a1.sinks.k1.indexName = foo_index - a1.sinks.k1.indexType = bar_type - a1.sinks.k1.clusterName = foobar_cluster - a1.sinks.k1.batchSize = 500 - a1.sinks.k1.ttl = 5d - a1.sinks.k1.serializer = org.apache.flume.sink.elasticsearch.ElasticSearchDynamicSerializer - a1.sinks.k1.channel = c1 - Kite Dataset Sink ~~~~~~~~~~~~~~~~~ @@ -4037,6 +4053,29 @@ In the above configuration, c3 is an optional channel. Failure to write to c3 is simply ignored. Since c1 and c2 are not marked optional, failure to write to those channels will cause the transaction to fail. +Load Balancing Channel Selector +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Load balancing channel selector provides the ability to load-balance flow over multiple channels. This +effectively allows the incoming data to be processed on multiple threads. It maintains an indexed list of active channels on which the load must be distributed. Implementation supports distributing load using either via round_robin or random selection mechanisms. The choice of selection mechanism defaults to round_robin type, but can be overridden via configuration. + +Required properties are in **bold**. + +================== ===================== ================================================= +Property Name Default Description +================== ===================== ================================================= +selector.type replicating The component type name, needs to be ``load_balancing`` +selector.policy ``round_robin`` Selection mechanism. Must be either ``round_robin`` or ``random``. +================== ===================== ================================================= + +Example for agent named a1 and it's source called r1: + +.. code-block:: properties + + a1.sources = r1 + a1.channels = c1 c2 c3 c4 + a1.sources.r1.channels = c1 c2 c3 c4 + a1.sources.r1.selector.type = load_balancing + a1.sources.r1.selector.policy = round_robin Multiplexing Channel Selector ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -4279,7 +4318,7 @@ Example for agent named a1: a1.sinks.k1.type = hdfs a1.sinks.k1.channel = c1 - a1.sinks.k1.hdfs.path = /flume/events/%y-%m-%d/%H%M/%S + a1.sinks.k1.hdfs.path = /flume/events/%Y-%m-%d/%H%M/%S a1.sinks.k1.serializer = avro_event a1.sinks.k1.serializer.compressionCodec = snappy @@ -4805,7 +4844,7 @@ Log4J Appender Appends Log4j events to a flume agent's avro source. A client using this appender must have the flume-ng-sdk in the classpath (eg, -flume-ng-sdk-1.9.0.jar). +flume-ng-sdk-1.10.0.jar). Required properties are in **bold**. ===================== ======= ================================================================================== @@ -4869,7 +4908,7 @@ Load Balancing Log4J Appender Appends Log4j events to a list of flume agent's avro source. A client using this appender must have the flume-ng-sdk in the classpath (eg, -flume-ng-sdk-1.9.0.jar). This appender supports a round-robin and random +flume-ng-sdk-1.10.0.jar). This appender supports a round-robin and random scheme for performing the load balancing. It also supports a configurable backoff timeout so that down agents are removed temporarily from the set of hosts Required properties are in **bold**. @@ -5028,33 +5067,33 @@ Sources 2 Sinks 1 ~~~~~~~ -+------------------------+-------------+------------+---------------+-------+--------+ -| | Avro/Thrift | AsyncHBase | ElasticSearch | HBase | HBase2 | -+------------------------+-------------+------------+---------------+-------+--------+ -| BatchCompleteCount | x | x | x | x | x | -+------------------------+-------------+------------+---------------+-------+--------+ -| BatchEmptyCount | x | x | x | x | x | -+------------------------+-------------+------------+---------------+-------+--------+ -| BatchUnderflowCount | x | x | x | x | x | -+------------------------+-------------+------------+---------------+-------+--------+ -| ChannelReadFail | x | | | | x | -+------------------------+-------------+------------+---------------+-------+--------+ -| ConnectionClosedCount | x | x | x | x | x | -+------------------------+-------------+------------+---------------+-------+--------+ -| ConnectionCreatedCount | x | x | x | x | x | -+------------------------+-------------+------------+---------------+-------+--------+ -| ConnectionFailedCount | x | x | x | x | x | -+------------------------+-------------+------------+---------------+-------+--------+ -| EventDrainAttemptCount | x | x | x | x | x | -+------------------------+-------------+------------+---------------+-------+--------+ -| EventDrainSuccessCount | x | x | x | x | x | -+------------------------+-------------+------------+---------------+-------+--------+ -| EventWriteFail | x | | | | x | -+------------------------+-------------+------------+---------------+-------+--------+ -| KafkaEventSendTimer | | | | | | -+------------------------+-------------+------------+---------------+-------+--------+ -| RollbackCount | | | | | | -+------------------------+-------------+------------+---------------+-------+--------+ ++------------------------+-------------+------------+-------+--------+ +| | Avro/Thrift | AsyncHBase | HBase | HBase2 | ++------------------------+-------------+------------+-------+--------+- +| BatchCompleteCount | x | x | x | x | ++------------------------+-------------+------------+-------+--------+ +| BatchEmptyCount | x | x | x | x | ++------------------------+-------------+------------+-------+--------+ +| BatchUnderflowCount | x | x | x | x | ++------------------------+-------------+------------+-------+--------+ +| ChannelReadFail | x | | | x | ++------------------------+-------------+------------+-------+--------+ +| ConnectionClosedCount | x | x | x | x | ++------------------------+-------------+------------+-------+--------+ +| ConnectionCreatedCount | x | x | x | x | ++------------------------+-------------+------------+-------+--------+ +| ConnectionFailedCount | x | x | x | x | ++------------------------+-------------+------------+-------+--------+ +| EventDrainAttemptCount | x | x | x | x | ++------------------------+-------------+------------+-------+--------+ +| EventDrainSuccessCount | x | x | x | x | ++------------------------+-------------+------------+-------+--------+ +| EventWriteFail | x | | | x | ++------------------------+-------------+------------+-------+--------+ +| KafkaEventSendTimer | | | | | ++------------------------+-------------+------------+-------+--------+ +| RollbackCount | | | | | ++------------------------+-------------+------------+-------+--------+ Sinks 2 ~~~~~~~ @@ -5540,7 +5579,6 @@ org.apache.flume.Sink hdfs org.apache.flume.Sink hbase org.apache.flume.sink.hbase.HBaseSink org.apache.flume.Sink hbase2 org.apache.flume.sink.hbase2.HBase2Sink org.apache.flume.Sink asynchbase org.apache.flume.sink.hbase.AsyncHBaseSink -org.apache.flume.Sink elasticsearch org.apache.flume.sink.elasticsearch.ElasticSearchSink org.apache.flume.Sink file_roll org.apache.flume.sink.RollingFileSink org.apache.flume.Sink irc org.apache.flume.sink.irc.IRCSink org.apache.flume.Sink thrift org.apache.flume.sink.ThriftSink diff --git a/source/sphinx/download.rst b/source/sphinx/download.rst index 2cffa8d0..b72626a7 100644 --- a/source/sphinx/download.rst +++ b/source/sphinx/download.rst @@ -10,8 +10,8 @@ originals on the main distribution server. .. csv-table:: - "Apache Flume binary (tar.gz)", `apache-flume-1.9.0-bin.tar.gz <http://www.apache.org/dyn/closer.lua/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz>`_, `apache-flume-1.9.0-bin.tar.gz.sha512 <http://www.apache.org/dist/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz.sha512>`_, `apache-flume-1.9.0-bin.tar.gz.asc <http://www.apache.org/dist/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz.asc>`_ - "Apache Flume source (tar.gz)", `apache-flume-1.9.0-src.tar.gz <http://www.apache.org/dyn/closer.lua/flume/1.9.0/apache-flume-1.9.0-src.tar.gz>`_, `apache-flume-1.9.0-src.tar.gz.sha512 <http://www.apache.org/dist/flume/1.9.0/apache-flume-1.9.0-src.tar.gz.sha512>`_, `apache-flume-1.9.0-src.tar.gz.asc <http://www.apache.org/dist/flume/1.9.0/apache-flume-1.9.0-src.tar.gz.asc>`_ + "Apache Flume binary (tar.gz)", `apache-flume-1.10.0-bin.tar.gz <http://www.apache.org/dyn/closer.lua/flume/1.10.0/apache-flume-1.10.0-bin.tar.gz>`_, `apache-flume-1.10.0-bin.tar.gz.sha512 <http://www.apache.org/dist/flume/1.10.0/apache-flume-1.10.0-bin.tar.gz.sha512>`_, `apache-flume-1.10.0-bin.tar.gz.asc <http://www.apache.org/dist/flume/1.10.0/apache-flume-1.10.0-bin.tar.gz.asc>`_ + "Apache Flume source (tar.gz)", `apache-flume-1.10.0-src.tar.gz <http://www.apache.org/dyn/closer.lua/flume/1.10.0/apache-flume-1.10.0-src.tar.gz>`_, `apache-flume-1.10.0-src.tar.gz.sha512 <http://www.apache.org/dist/flume/1.10.0/apache-flume-1.10.0-src.tar.gz.sha512>`_, `apache-flume-1.10.0-src.tar.gz.asc <http://www.apache.org/dist/flume/1.10.0/apache-flume-1.10.0-src.tar.gz.asc>`_ It is essential that you verify the integrity of the downloaded files using the PGP or MD5 signatures. Please read `Verifying Apache HTTP Server Releases <http://httpd.apache.org/dev/verification.html>`_ for more information on @@ -23,9 +23,9 @@ as well as the asc signature file for the relevant distribution. Make sure you g Then verify the signatures using:: % gpg --import KEYS - % gpg --verify apache-flume-1.9.0-src.tar.gz.asc + % gpg --verify apache-flume-1.10.0-src.tar.gz.asc -Apache Flume 1.9.0 is signed by Ferenc Szabo 79E8E648 +Apache Flume 1.10.0 is signed by Ralph Goers B3D8E1BA Alternatively, you can verify the MD5 or SHA1 signatures of the files. A program called md5, md5sum, or shasum is included in many Unix distributions for this purpose. diff --git a/source/sphinx/index.rst b/source/sphinx/index.rst index b5744119..a7507343 100644 --- a/source/sphinx/index.rst +++ b/source/sphinx/index.rst @@ -31,6 +31,34 @@ application. .. rubric:: News +.. raw:: html + + <h3>June 5, 2022 - Apache Flume 1.10.0 Released</h3> + +The Apache Flume team is pleased to announce the release of Flume 1.10.0. + +Flume is a distributed, reliable, and available service for efficiently +collecting, aggregating, and moving large amounts of streaming event data. + +Flume 1.10.0 is stable, production-ready software, and is backwards-compatible with +previous versions of the Flume 1.x codeline. + +This version of Flume upgrades many dependencies, resolving the CVEs associated with them. +Enhancements included in this release include the addition of a LoadBalancingChannelSelector, +the ability to retrieve the Flume configuration from a remote source such as a Spring +Cloud Config Server, and support for composite configurations. + +Flume has been updated to use Log4j 2.x instead of Log4j 1.x. + +The full change log and documentation are available on the +`Flume 1.10.0 release page <releases/1.10.0.html>`__. + +This release can be downloaded from the Flume `Download <download.html>`__ page. + +Your contributions, feedback, help and support make Flume better! +For more information on how to report problems or contribute, +please visit our `Get Involved <getinvolved.html>`__ page. + .. raw:: html <h3>January 8, 2019 - Apache Flume 1.9.0 Released</h3> diff --git a/source/sphinx/releases/1.10.0.rst b/source/sphinx/releases/1.10.0.rst new file mode 100644 index 00000000..681b5b16 --- /dev/null +++ b/source/sphinx/releases/1.10.0.rst @@ -0,0 +1,69 @@ +=============== +Version 1.10.0 +=============== + +.. rubric:: Status of this release + +Apache Flume 1.10.0 is the twelfth release of Flume as an Apache top-level project +(TLP). Apache Flume 1.10.0 is production-ready software. + +.. rubric:: Release Documentation + +* `Flume 1.10.0 User Guide <content/1.10.0/FlumeUserGuide.html>`__ (also in `pdf <content/1.10.0/FlumeUserGuide.pdf>`__) +* `Flume 1.10.0 Developer Guide <content/1.10.0/FlumeDeveloperGuide.html>`__ (also in `pdf <content/1.10.0/FlumeDeveloperGuide.pdf>`__) +* `Flume 1.10.0 API Documentation <content/1.10.0/apidocs/index.html>`__ + +.. rubric:: Changes + +Release Notes - Flume - Version v1.10.0 + +** Bug + * [`FLUME-3151 <https://issues.apache.org/jira/browse/FLUME-3151>`__] - Upgrade Hadoop to 2.10.1 + * [`FLUME-3311 <https://issues.apache.org/jira/browse/FLUME-3311>`__] - Update Wrong Use In HDFS Sink + * [`FLUME-3316 <https://issues.apache.org/jira/browse/FLUME-3316>`__] - Syslog Rfc3164Date test fails when the test date falls on a leap day + * [`FLUME-3328 <https://issues.apache.org/jira/browse/FLUME-3328>`__] - Fix Deprecated Properties table of HDFS Sink + * [`FLUME-3356 <https://issues.apache.org/jira/browse/FLUME-3356>`__] - Probable security issue in Flume + * [`FLUME-3360 <https://issues.apache.org/jira/browse/FLUME-3360>`__] - Maven assemble failed on macOS + * [`FLUME-3395 <https://issues.apache.org/jira/browse/FLUME-3395>`__] - Fix for CVE-2021-44228 + * [`FLUME-3407 <https://issues.apache.org/jira/browse/FLUME-3407>`__] - workaround for jackson-mapper-asl-1.9.13.jar @ flume-ng + * [`FLUME-3409 <https://issues.apache.org/jira/browse/FLUME-3409>`__] - upgrade httpclient due to cve + * [`FLUME-3416 <https://issues.apache.org/jira/browse/FLUME-3416>`__] - Improve input validation + * [`FLUME-3421 <https://issues.apache.org/jira/browse/FLUME-3421>`__] - Default log4j settings do not log to console after FLUME-2050 + * [`FLUME-3426 <https://issues.apache.org/jira/browse/FLUME-3426>`__] - Unresolved Security Issues + +** New Feature + * [`FLUME-3412 <https://issues.apache.org/jira/browse/FLUME-3412>`__] - Add LoadBalancingChannelSelector + +** Improvement + * [`FLUME-199 <https://issues.apache.org/jira/browse/FLUME-199>`__] - Unit tests should hunt for available ports if defaults are in use + * [`FLUME-2050 <https://issues.apache.org/jira/browse/FLUME-2050>`__] - Upgrade to log4j2 (when GA) + * [`FLUME-3045 <https://issues.apache.org/jira/browse/FLUME-3045>`__] - Document GitHub Pull Requests in How to Contribute Guide + * [`FLUME-3335 <https://issues.apache.org/jira/browse/FLUME-3335>`__] - Support configuration and reconfiguration via HTTP(S) + * [`FLUME-3338 <https://issues.apache.org/jira/browse/FLUME-3338>`__] - Doc Flume Recoverability with Kafka + * [`FLUME-3363 <https://issues.apache.org/jira/browse/FLUME-3363>`__] - CVE-2019-20445 + * [`FLUME-3368 <https://issues.apache.org/jira/browse/FLUME-3368>`__] - Update Jackson to 2.9.10 + * [`FLUME-3389 <https://issues.apache.org/jira/browse/FLUME-3389>`__] - Build and test Apache Flume on ARM64 CPU architecture + * [`FLUME-3397 <https://issues.apache.org/jira/browse/FLUME-3397>`__] - Upgrade Log4 to 2.17.1 and SLF4J to 1.7.32 + * [`FLUME-3398 <https://issues.apache.org/jira/browse/FLUME-3398>`__] - Upgrade Kafka to a supported version. + * [`FLUME-3399 <https://issues.apache.org/jira/browse/FLUME-3399>`__] - Update Jackson to 2.13.1 + * [`FLUME-3403 <https://issues.apache.org/jira/browse/FLUME-3403>`__] - The parquet-avro version used by flume is 1.4.1, which is vulnerable. + * [`FLUME-3405 <https://issues.apache.org/jira/browse/FLUME-3405>`__] - Reopened - The parquet-avro version used by flume is 1.4.1, which is vulnerable. + * [`FLUME-3413 <https://issues.apache.org/jira/browse/FLUME-3413>`__] - Add "initialization" phase to components. + +** Wish + * [`FLUME-3400 <https://issues.apache.org/jira/browse/FLUME-3400>`__] - Upgrade commons-io to 2.11.0 + +** Task + * [`FLUME-3401 <https://issues.apache.org/jira/browse/FLUME-3401>`__] - Remove Kite Dataset Sink + * [`FLUME-3402 <https://issues.apache.org/jira/browse/FLUME-3402>`__] - remove org.codehaus.jackson dependencies + * [`FLUME-3404 <https://issues.apache.org/jira/browse/FLUME-3404>`__] - Update Commons CLI to 1.5.0, Commons Codec to 1.15, Commons Compress to 1.21 and Commons Lang to 2.6 + * [`FLUME-3410 <https://issues.apache.org/jira/browse/FLUME-3410>`__] - upgrade hbase version + * [`FLUME-3411 <https://issues.apache.org/jira/browse/FLUME-3411>`__] - upgrade hive sink to 1.2.2 + * [`FLUME-3417 <https://issues.apache.org/jira/browse/FLUME-3417>`__] - Remove Elasticsearch sink that requires Elasticsearch 0.90.1 + * [`FLUME-3419 <https://issues.apache.org/jira/browse/FLUME-3419>`__] - Review project LICENSE and NOTICE + * [`FLUME-3424 <https://issues.apache.org/jira/browse/FLUME-3424>`__] - Upgrade Twitter4j to version 4.0.7+ + +** Dependency upgrade + * [`FLUME-3339 <https://issues.apache.org/jira/browse/FLUME-3339>`__] - Remove Xerces and Xalan dependencies + * [`FLUME-3385 <https://issues.apache.org/jira/browse/FLUME-3385>`__] - flume-ng-sdk uses Avro-IPC version with vulnerable version of Jetty + * [`FLUME-3386 <https://issues.apache.org/jira/browse/FLUME-3386>`__] - flume-ng-sdk uses vulnerable version of netty diff --git a/source/sphinx/releases/index.rst b/source/sphinx/releases/index.rst index e9c353ed..79065685 100644 --- a/source/sphinx/releases/index.rst +++ b/source/sphinx/releases/index.rst @@ -3,13 +3,13 @@ Releases .. rubric:: Current Release -The current stable release is `Apache Flume Version 1.9.0 <1.9.0.html>`__. +The current stable release is `Apache Flume Version 1.10.0 <1.10.0.html>`__. .. toctree:: :maxdepth: 1 :hidden: - 1.9.0 + 1.10.0 .. rubric:: Previous Releases @@ -17,6 +17,7 @@ The current stable release is `Apache Flume Version 1.9.0 <1.9.0.html>`__. :maxdepth: 1 :glob: + 1.9.0 1.8.0 1.7.0 1.6.0 diff --git a/source/sphinx/team.rst b/source/sphinx/team.rst index 6e5405ba..b1a4c638 100644 --- a/source/sphinx/team.rst +++ b/source/sphinx/team.rst @@ -10,39 +10,47 @@ Team to the Members. The number of Contributors to the project is unbounded. Get involved today. All contributions to the project are greatly appreciated. - The following individuals are recognized as PMC Members or Project Committers. + The following individuals are recognized as currently active PMC Members or Project Committers. .. csv-table:: :header: "Name", "Email", "Id", "Organization", "Role" :widths: 30, 25, 15, 15, 15 - "Aaron Kimball", "[email protected]", "kimballa", "Zymergen", "PMC Member" - "Ashish Paliwal", "[email protected]", "apaliwal", "Apple", "Committer" - "Andrew Bayer", "[email protected]", "abayer", "CloudBees", "PMC Member" - "Ahmed Radwan", "[email protected]", "ahmed", "Apple", "PMC Member" - "Arvind Prabhakar", "[email protected]", "arvind", "StreamSets", "PMC Member" - "Balázs Donát Bessenyei", "[email protected]", "bessbd", "Ericsson", "PMC Member" - "Brock Noland", "[email protected]", "brock", "phData", "PMC Member" - "Bruce Mitchener", "[email protected]", "brucem", "Data Fueled", "PMC Member" - "Derek Deeter", "[email protected]", "ddeeter", "Vanderbilt University", "PMC Member" - "Denes Arvay", "[email protected]", "denes", "Cloudera", "PMC Member" - "Eric Sammer", "[email protected]", "esammer", "Splunk", "PMC Member" - "Hari Shreedharan", "[email protected]", "hshreedharan", "StreamSets", "PMC Member" - "Henry Robinson", "[email protected]", "henry", "Cloudera", "PMC Member" - "Jaroslav Cecho", "[email protected]", "jarcec", "StreamSets", "PMC Member" - "Johny Rufus", "[email protected]", "johnyrufus", "Microsoft", "Committer" - "Jonathan Hsieh", "[email protected]", "jmhsieh", "Cloudera", "PMC Member" - "Juhani Connolly", "[email protected]", "juhanic", "CyberAgent", "PMC Member" - "Mike Percy", "[email protected]", "mpercy", "Cloudera", "PMC Member" - "Mingjie Lai", "[email protected]", "mlai", "Apple", "PMC Member" - "Mubarak Seyed", "[email protected]","mubarak", "Apple", "Committer" - "Nick Verbeck", "[email protected]", "nerdynick", "", "PMC Member" - "Patrick Hunt", "[email protected]", "phunt", "Cloudera", "PMC Member" - "Patrick Wendell", "[email protected]", "pwendell", "Databricks", "Committer" - "Prasad Mujumdar", "[email protected]", "prasadm", "BlueTalon", "PMC Member" - "Ralph Goers", "[email protected]", "rgoers", "Nextiva", "PMC Member" - "Roshan Naik", "[email protected]", "roshannaik", "Hortonworks", "PMC Member" - "Attila Simon", "[email protected]", "sati", "RapidMiner", "Committer" - "Ferenc Szabo", "[email protected]", "szaboferee", "Cloudera", "Committer" - "Wolfgang Hoschek", "[email protected]", "whoschek", "Cloudera", "Committer" - "Will McQueen", "[email protected]", "will", "", "PMC Member" + "Arvind Prabhakar", "arvind at apache.org", "arvind", "StreamSets", "PMC Member" + "Balázs Donát Bessenyei", "bessbd at apache.org", "bessbd", "Ericsson", "PMC Chair" + "Denes Arvay", "denes at apache.org", "denes", "Cloudera", "PMC Member" + "Jaroslav Cecho", "jarcec at apache.org", "jarcec", "StreamSets", "PMC Member" + "Jonathan Hsieh", "jmhsieh at apache.org", "jmhsieh", "Cloudera", "PMC Member" + "Juhani Connolly", "juhanic at apache.org", "juhanic", "CyberAgent", "PMC Member" + "Mike Percy", "mpercy at apache.org", "mpercy", "Cloudera", "PMC Member" + "Ahmed Radwan", "ahmed at apache.org", "ahmed", "Apple", "PMC Member" + "Ralph Goers", "rgoers at apache.org", "rgoers", "Nextiva", "PMC Member" + "Tristan Stevens", "tristan at apache.org", "tristan", "Cloudera", "PMC Member" + +The following individuals are recognized as former PMC Members or Project Committers + +.. csv-table:: + :header: "Name", "Email", "Id", "Organization", "Role", "Status" + :widths: 25, 25, 10, 10, 15, 15 + + "Aaron Kimball", "kimballa at apache.org", "kimballa", "Zymergen", "PMC Member", "Last active 2011" + "Ashish Paliwal", "apaliwal at apache.org", "apaliwal", "Apple", "Committer", "Last active 2017" + "Andrew Bayer", "abayer at apache.org", "abayer", "CloudBees", "PMC Member", "Last active 2015" + "Brock Noland", "brock at apache.org", "brock", "phData", "PMC Member", "Last active 2019" + "Bruce Mitchener", "brucem at apache.org", "brucem", "Data Fueled", "PMC Member", "Last active - project creation" + "Derek Deeter", "ddeeter at apache.org", "ddeeter", "Vanderbilt University", "PMC Member", "Last active - project creation" + "Eric Sammer", "esammer at apache.org", "esammer", "Splunk", "PMC Member", "Last active 2017" + "Hari Shreedharan", "hshreedharan at apache.org", "hshreedharan", "StreamSets", "PMC Member", "Emeritus 2022" + "Henry Robinson", "henry at apache.org", "henry", "Cloudera", "PMC Member", "Last active - project creation" + "Johny Rufus", "johnyrufus at apache.org", "johnyrufus", "Microsoft", "Committer", "Last active 2017" + "Mingjie Lai", "mlai at apache.org", "mlai", "Apple", "PMC Member", "Last active 2012" + "Mubarak Seyed", "mubarak at apache.org","mubarak", "Apple", "Committer", "Last active 2017" + "Nick Verbeck", "nerdynick at apache.org", "nerdynick", "", "PMC Member", "Last active 2011" + "Patrick Hunt", "phunt at apache.org", "phunt", "Cloudera", "PMC Member", "Last active 2012" + "Patrick Wendell", "pwendell at apache.org", "pwendell", "Databricks", "Committer", "Last active 2015" + "Prasad Mujumdar", "prasadm at apache.org", "prasadm", "BlueTalon", "PMC Member", "Last active 2015" + "Roshan Naik", "roshannaik at apache.org", "roshannaik", "Hortonworks", "PMC Member", "Last active 2017" + "Attila Simon", "sati at apache.org", "sati", "RapidMiner", "Committer", "Last active 2017" + "Ferenc Szabo", "szaboferee at apache.org", "szaboferee", "Cloudera", "Committer", "Last active 2019" + "Wolfgang Hoschek", "whoschek at apache.org", "whoschek", "Cloudera", "Committer", "Last active 2016" + "Will McQueen", "will at apache.org", "will", "", "PMC Member", "Last active 2017"
