[nifi] branch master updated: NIFI-6687: - Updating exclusions and overrides accordingly for Ranger 2.0.
This is an automated email from the ASF dual-hosted git repository.
bbende pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new c2da6e7 NIFI-6687: - Updating exclusions and overrides accordingly
for Ranger 2.0.
c2da6e7 is described below
commit c2da6e7aa2bb0a0e11a0b61ebb411c75d7bea86d
Author: Matt Gilman
AuthorDate: Wed Oct 23 14:35:42 2019 -0400
NIFI-6687:
- Updating exclusions and overrides accordingly for Ranger 2.0.
This closes #3835.
Signed-off-by: Bryan Bende
---
.../nifi-ranger-bundle/nifi-ranger-plugin/pom.xml | 11 +-
nifi-nar-bundles/nifi-ranger-bundle/pom.xml| 25 --
2 files changed, 1 insertion(+), 35 deletions(-)
diff --git a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml
b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml
index 35775cc..45871c5 100644
--- a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml
+++ b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml
@@ -25,7 +25,7 @@
nifi-ranger-plugin
jar
-3.0.0
+3.1.1
@@ -37,11 +37,6 @@
nifi-properties
-com.sun.jersey
-jersey-bundle
-1.19
-
-
org.apache.ranger
ranger-plugins-common
${ranger.version}
@@ -54,10 +49,6 @@
com.google.code.findbugs
jsr305
-
-com.sun.jersey
-jersey-bundle
-
diff --git a/nifi-nar-bundles/nifi-ranger-bundle/pom.xml
b/nifi-nar-bundles/nifi-ranger-bundle/pom.xml
index c2180a9..6ae02e7 100644
--- a/nifi-nar-bundles/nifi-ranger-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-ranger-bundle/pom.xml
@@ -27,35 +27,10 @@
1.10.0-SNAPSHOT
pom
-
-
4.4.1
-
4.4.1
-
4.4.1
-
-
nifi-ranger-plugin
nifi-ranger-nar
nifi-ranger-resources
-
-
-
-org.apache.httpcomponents
-httpclient
-${httpcomponents.httpclient.version}
-
-
-org.apache.httpcomponents
-httpcore
-${httpcomponents.httpcore.version}
-
-
-org.apache.httpcomponents
-httpmime
-${httpcomponents.httpmime.version}
-
-
-
[nifi] branch master updated: NIFI-6797: Add support for specifying Partition via EL or RecordPath for PublishKafka(Record)_1_0 and PublishKafka(Record)_2_0
This is an automated email from the ASF dual-hosted git repository.
bbende pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new 3543b9c NIFI-6797: Add support for specifying Partition via EL or
RecordPath for PublishKafka(Record)_1_0 and PublishKafka(Record)_2_0
3543b9c is described below
commit 3543b9c42cb00b8365059a990aa4f77f23c4892b
Author: Mark Payne
AuthorDate: Tue Oct 22 14:30:52 2019 -0400
NIFI-6797: Add support for specifying Partition via EL or RecordPath for
PublishKafka(Record)_1_0 and PublishKafka(Record)_2_0
This closes #3834.
Signed-off-by: Bryan Bende
---
.../nifi-kafka-1-0-processors/pom.xml | 8 +-
.../nifi/processors/kafka/pubsub/Partitioners.java | 40 +-
.../kafka/pubsub/PublishKafkaRecord_1_0.java | 88 +++-
.../processors/kafka/pubsub/PublishKafka_1_0.java | 82 +---
.../processors/kafka/pubsub/PublisherLease.java| 49 +++
.../processors/kafka/pubsub/PublisherPool.java | 11 +-
.../processors/kafka/pubsub/TestPublishKafka.java | 43 +++---
.../kafka/pubsub/TestPublishKafkaRecord_1_0.java | 61 +
.../kafka/pubsub/TestPublisherLease.java | 16 +--
.../nifi-kafka-2-0-processors/pom.xml | 6 +
.../nifi/processors/kafka/pubsub/Partitioners.java | 40 +-
.../kafka/pubsub/PublishKafkaRecord_2_0.java | 149 -
.../processors/kafka/pubsub/PublishKafka_2_0.java | 45 ++-
.../processors/kafka/pubsub/PublisherLease.java| 49 +++
.../processors/kafka/pubsub/PublisherPool.java | 9 +-
.../kafka/pubsub/TestPublishKafkaRecord_2_0.java | 82 ++--
.../kafka/pubsub/TestPublishKafka_2_0.java | 13 +-
.../kafka/pubsub/TestPublisherLease.java | 16 +--
18 files changed, 605 insertions(+), 202 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/pom.xml
b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/pom.xml
index 9b9b047..597cc80 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/pom.xml
@@ -52,7 +52,7 @@
org.apache.nifi
nifi-kerberos-credentials-service-api
-
+
org.apache.kafka
kafka-clients
@@ -92,6 +92,12 @@
2.6
test
+
+org.apache.nifi
+nifi-record-path
+1.10.0-SNAPSHOT
+compile
+
diff --git
a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/Partitioners.java
b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/Partitioners.java
index 64ab4ce..a7b20f2 100644
---
a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/Partitioners.java
+++
b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/Partitioners.java
@@ -16,11 +16,11 @@
*/
package org.apache.nifi.processors.kafka.pubsub;
-import java.util.Map;
-
import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;
+import java.util.Map;
+
/**
* Collection of implementation of common Kafka {@link Partitioner}s.
*/
@@ -59,4 +59,40 @@ final public class Partitioners {
return index++;
}
}
+
+public static class RecordPathPartitioner implements Partitioner {
+@Override
+public int partition(final String topic, final Object key, final
byte[] keyBytes, final Object value, final byte[] valueBytes, final Cluster
cluster) {
+// When this partitioner is used, it is always overridden by
creating the ProducerRecord with the Partition directly specified. However, we
must have a unique value
+// to set in the Producer's config, so this class exists
+return 0;
+}
+
+@Override
+public void close() {
+}
+
+@Override
+public void configure(final Map configs) {
+}
+}
+
+
+public static class ExpressionLanguagePartitioner implements Partitioner {
+@Override
+public int partition(final String topic, final Object key, final
byte[] keyBytes, final Object value, final byte[] valueBytes, final Cluster
cluster) {
+// When this partitioner is used, it is always overridden by
creating the ProducerRecord with the Partition directly specified. However, we
must have a unique value
+// to set in the Producer's config, so this class exists
+return 0;
+}
+
+@Override
+public void close() {
svn commit: r1868808 - /nifi/site/trunk/mailing_lists.html
Author: aldrin Date: Wed Oct 23 15:20:19 2019 New Revision: 1868808 URL: http://svn.apache.org/viewvc?rev=1868808=rev Log: Update Slack invite link for community managed channel Modified: nifi/site/trunk/mailing_lists.html Modified: nifi/site/trunk/mailing_lists.html URL: http://svn.apache.org/viewvc/nifi/site/trunk/mailing_lists.html?rev=1868808=1868807=1868808=diff == --- nifi/site/trunk/mailing_lists.html (original) +++ nifi/site/trunk/mailing_lists.html Wed Oct 23 15:20:19 2019 @@ -225,7 +225,7 @@ Apache NiFi Slack Workspace: https://apachenifi.slack.com/;>https://apachenifi.slack.com/ -New users can join the workspace using the following https://join.slack.com/t/apachenifi/shared_invite/enQtNDI2NDMyMTY3MTA5LWJmZDI3MmM1ZmYyODQwZDYwM2MyMDY5ZjkyMDkxY2JmOGMyNmEzYTE0MTRkZTYwYzZlYTJkY2JhZTYyMzcyZGI;>invite link. +New users can join the workspace using the following https://join.slack.com/t/apachenifi/shared_invite/enQtNDI2NDMyMTY3MTA5LTQ0YmQwMjljZDQ4NjkzODI0ZWU1YmU0YmU0YjJiNjVjMjhmMWQwZjY0Njk0NDY1OTlkMjM5NmNiODg4MGFmYWU;>invite link. IRC: #nifi on http://webchat.freenode.net/?channels=#nifi;>irc.freenode.net
[nifi-site] branch master updated: Refresh invite link for community managed Slack channel.
This is an automated email from the ASF dual-hosted git repository. aldrin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nifi-site.git The following commit(s) were added to refs/heads/master by this push: new 052763d Refresh invite link for community managed Slack channel. 052763d is described below commit 052763de49f2b0f1b317c38270a4817ec9c4629f Author: Aldrin Piri AuthorDate: Wed Oct 23 11:00:33 2019 -0400 Refresh invite link for community managed Slack channel. --- src/pages/html/mailing_lists.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/html/mailing_lists.hbs b/src/pages/html/mailing_lists.hbs index 7ebf36c..4232757 100644 --- a/src/pages/html/mailing_lists.hbs +++ b/src/pages/html/mailing_lists.hbs @@ -121,7 +121,7 @@ title: Apache NiFi Mailing Lists Chat Apache NiFi Slack Workspace: https://apachenifi.slack.com/;>https://apachenifi.slack.com/ -New users can join the workspace using the following https://join.slack.com/t/apachenifi/shared_invite/enQtNDI2NDMyMTY3MTA5LWJmZDI3MmM1ZmYyODQwZDYwM2MyMDY5ZjkyMDkxY2JmOGMyNmEzYTE0MTRkZTYwYzZlYTJkY2JhZTYyMzcyZGI;>invite link. +New users can join the workspace using the following https://join.slack.com/t/apachenifi/shared_invite/enQtNDI2NDMyMTY3MTA5LTQ0YmQwMjljZDQ4NjkzODI0ZWU1YmU0YmU0YjJiNjVjMjhmMWQwZjY0Njk0NDY1OTlkMjM5NmNiODg4MGFmYWU;>invite link. IRC: #nifi on http://webchat.freenode.net/?channels=#nifi;>irc.freenode.net
[nifi-minifi-cpp] branch master updated: MINIFICPP-1048 - Add PublishKafka docker tests
This is an automated email from the ASF dual-hosted git repository.
aboda pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
The following commit(s) were added to refs/heads/master by this push:
new c0efe8c MINIFICPP-1048 - Add PublishKafka docker tests
c0efe8c is described below
commit c0efe8cf1ee74a53800b307588f4c18fa8741647
Author: Nghia Le
AuthorDate: Wed Sep 18 15:39:39 2019 +0200
MINIFICPP-1048 - Add PublishKafka docker tests
Signed-off-by: Arpad Boda
This closes #657
---
docker/Dockerfile | 2 +-
docker/test/integration/minifi/__init__.py | 60 +++-
docker/test/integration/minifi/test/__init__.py | 120 +---
docker/test/integration/test_rdkafka.py | 76 +++
4 files changed, 237 insertions(+), 21 deletions(-)
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 38bcaf7..7d88477 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -76,7 +76,7 @@ ENV MINIFI_HOME
$MINIFI_BASE_DIR/nifi-minifi-cpp-${MINIFI_VERSION}
RUN cd ${MINIFI_BASE_DIR} \
&& mkdir build \
&& cd build \
- && cmake -DDISABLE_JEMALLOC=ON -DSTATIC_BUILD= -DSKIP_TESTS=true
-DENABLE_JNI=ON .. \
+ && cmake -DDISABLE_JEMALLOC=ON -DSTATIC_BUILD= -DSKIP_TESTS=true
-DENABLE_JNI=ON -DENABLE_LIBRDKAFKA=ON .. \
&& make -j8 package \
&& tar -xzvf
${MINIFI_BASE_DIR}/build/nifi-minifi-cpp-${MINIFI_VERSION}-bin.tar.gz -C
${MINIFI_BASE_DIR}
diff --git a/docker/test/integration/minifi/__init__.py
b/docker/test/integration/minifi/__init__.py
index 48f25b3..cfb2eb5 100644
--- a/docker/test/integration/minifi/__init__.py
+++ b/docker/test/integration/minifi/__init__.py
@@ -27,6 +27,9 @@ import os
import yaml
from copy import copy
+import time
+from collections import OrderedDict
+
class Cluster(object):
"""
@@ -64,7 +67,7 @@ class SingleNodeDockerCluster(Cluster):
self.minifi_root = '/opt/minifi/nifi-minifi-cpp-' + self.minifi_version
self.nifi_root = '/opt/nifi/nifi-' + self.nifi_version
self.network = None
-self.containers = []
+self.containers = OrderedDict()
self.images = []
self.tmp_files = []
@@ -93,12 +96,22 @@ class SingleNodeDockerCluster(Cluster):
if self.network is None:
net_name = 'nifi-' + str(uuid.uuid4())
logging.info('Creating network: %s', net_name)
-self.network = self.client.networks.create(net_name)
+# Set IP
+ipam_pool = docker.types.IPAMPool(
+subnet='192.168.42.0/24',
+gateway='192.168.42.1'
+)
+ipam_config = docker.types.IPAMConfig(
+pool_configs=[ipam_pool]
+)
+self.network = self.client.networks.create(net_name,
ipam=ipam_config)
if engine == 'nifi':
self.deploy_nifi_flow(flow, name, vols)
elif engine == 'minifi-cpp':
self.deploy_minifi_cpp_flow(flow, name, vols)
+elif engine == 'kafka-broker':
+self.deploy_kafka_broker(name)
else:
raise Exception('invalid flow engine: \'%s\'' % engine)
@@ -148,7 +161,7 @@ class SingleNodeDockerCluster(Cluster):
logging.info('Started container \'%s\'', container.name)
-self.containers.append(container)
+self.containers[container.name] = container
def deploy_nifi_flow(self, flow, name, vols):
dockerfile = dedent("""FROM {base_image}
@@ -198,7 +211,35 @@ class SingleNodeDockerCluster(Cluster):
logging.info('Started container \'%s\'', container.name)
-self.containers.append(container)
+self.containers[container.name] = container
+
+def deploy_kafka_broker(self, name):
+dockerfile = dedent("""FROM {base_image}
+USER root
+CMD $KAFKA_HOME/bin/kafka-console-consumer.sh
--bootstrap-server host.docker.internal:9092 --topic test > heaven_signal.txt
+""".format(base_image='spotify/kafka:latest'))
+
+logging.info('Creating and running docker container for kafka
broker...')
+
+broker = self.client.containers.run(
+self.client.images.pull("spotify/kafka:latest"),
+detach=True,
+name='kafka-broker',
+ports={'2181/tcp': 2181, '9092/tcp': 9092},
+environment=["ADVERTISED_HOST=192.168.42.4",
"ADVERTISED_PORT=9092"]
+)
+self.network.connect(broker, ipv4_address='192.168.42.4')
+
+configured_image = self.build_image(dockerfile, [])
+consumer = self.client.containers.run(
+configured_image[0],
+detach=True,
+name='kafka-consumer',
+network=self.network.name
+)
+
+
