This is an automated email from the ASF dual-hosted git repository.
fanningpj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pekko-connectors.git
The following commit(s) were added to refs/heads/main by this push:
new a8f15734f Using apache kudu docker images and update kudu-client
version (#626)
a8f15734f is described below
commit a8f15734f74d770d535ad8c10dcef383d88a321c
Author: Laglangyue <[email protected]>
AuthorDate: Tue Apr 23 21:06:06 2024 +0800
Using apache kudu docker images and update kudu-client version (#626)
* update kudu connector to new version
* revert not need changes
* parse encryptionPolicy from conf file
* fix ci fmt
* revert a change
---
.github/workflows/check-build-test.yml | 2 +-
docker-compose.yml | 46 ++++++++++++----------
kudu/src/main/resources/reference.conf | 3 ++
.../stream/connectors/kudu/KuduClientExt.scala | 6 ++-
project/Dependencies.scala | 10 ++---
5 files changed, 37 insertions(+), 30 deletions(-)
diff --git a/.github/workflows/check-build-test.yml
b/.github/workflows/check-build-test.yml
index 5135bc59d..d22b46de4 100644
--- a/.github/workflows/check-build-test.yml
+++ b/.github/workflows/check-build-test.yml
@@ -105,7 +105,7 @@ jobs:
- { connector: jms, pre_cmd:
'docker-compose up -d ibmmq' }
- { connector: json-streaming }
- { connector: kinesis }
- - { connector: kudu, pre_cmd:
'docker-compose up -d kudu-master-data kudu-tserver-data kudu-master
kudu-tserver' }
+ - { connector: kudu, pre_cmd:
'docker-compose up -d kudu-master kudu-tserver' }
- { connector: mongodb, pre_cmd:
'docker-compose up -d mongo' }
- { connector: mqtt, pre_cmd:
'docker-compose up -d mqtt' }
- { connector: mqtt-streaming, pre_cmd:
'docker-compose up -d mqtt' }
diff --git a/docker-compose.yml b/docker-compose.yml
index 7456ba7d0..3bfc6873e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -178,32 +178,36 @@ services:
- "AUTH_HOST=http://ironauth:8090";
ports:
- "8080:8080"
- kudu-master-data:
- image: kunickiaj/kudu
- volumes:
- - /var/lib/kudu/master
- kudu-tserver-data:
- image: kunickiaj/kudu
- volumes:
- - kudu-tserver-data:/var/lib/kudu/tserver
kudu-master:
- image: kunickiaj/kudu
+ image: apache/kudu:1.17.0
ports:
- - 7051:7051
- volumes:
- - kudu-tserver-data:/var/lib/kudu/tserver
- command: master
- kudu-tserver:
- image: kunickiaj/kudu
+ - "7051:7051"
+ command: [ "master" ]
environment:
- - KUDU_MASTER=kudu-master
- ports:
- - 7050:7050
- volumes:
- - kudu-tserver-data:/var/lib/kudu/tserver
- command: tserver
+ - KUDU_MASTERS=kudu-master:7051
+ - >
+ MASTER_ARGS=--fs_wal_dir=/var/lib/kudu/master
+ --rpc_bind_addresses=0.0.0.0:7051
+ --stderrthreshold=0
+ --use_hybrid_clock=false
+ --unlock_unsafe_flags=true
+ kudu-tserver:
+ image: apache/kudu:1.17.0
links:
- kudu-master
+ ports:
+ - "7050:7050"
+ command: [ "tserver" ]
+ environment:
+ - KUDU_MASTERS=kudu-master:7051
+ - >
+ TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver
+ --rpc_bind_addresses=0.0.0.0:7050
+ --stderrthreshold=0
+ --use_hybrid_clock=false
+ --unlock_unsafe_flags=true
+ deploy:
+ replicas: 1
mongo:
image: mongo
ports:
diff --git a/kudu/src/main/resources/reference.conf
b/kudu/src/main/resources/reference.conf
index bc22e6beb..988b3cfa6 100644
--- a/kudu/src/main/resources/reference.conf
+++ b/kudu/src/main/resources/reference.conf
@@ -4,5 +4,8 @@ pekko.connectors.kudu {
# the hostname and port to locate the master
master-address = ""
+ # options: OPTIONAL,REQUIRED_REMOTE,REQUIRED
+ # reference to org.apache.kudu.client.AsyncKuduClient.EncryptionPolicy
+ encryptionPolicy = "OPTIONAL"
}
diff --git
a/kudu/src/main/scala/org/apache/pekko/stream/connectors/kudu/KuduClientExt.scala
b/kudu/src/main/scala/org/apache/pekko/stream/connectors/kudu/KuduClientExt.scala
index b1abf6b39..92ac1bb70 100644
---
a/kudu/src/main/scala/org/apache/pekko/stream/connectors/kudu/KuduClientExt.scala
+++
b/kudu/src/main/scala/org/apache/pekko/stream/connectors/kudu/KuduClientExt.scala
@@ -16,6 +16,7 @@ package org.apache.pekko.stream.connectors.kudu
import org.apache.pekko
import pekko.actor.{ ClassicActorSystemProvider, ExtendedActorSystem,
Extension, ExtensionId, ExtensionIdProvider }
import org.apache.kudu.client.KuduClient
+import org.apache.kudu.client.AsyncKuduClient.EncryptionPolicy
/**
* Manages one [[org.apache.kudu.client.KuduClient]] per `ActorSystem`.
@@ -23,9 +24,10 @@ import org.apache.kudu.client.KuduClient
final class KuduClientExt private (sys: ExtendedActorSystem) extends Extension
{
val client = {
val masterAddress =
sys.settings.config.getString("pekko.connectors.kudu.master-address")
- new KuduClient.KuduClientBuilder(masterAddress).build
+ new KuduClient.KuduClientBuilder(masterAddress)
+ .encryptionPolicy(EncryptionPolicy.valueOf(
+
sys.settings.config.getString("pekko.connectors.kudu.encryptionPolicy"))).build()
}
-
sys.registerOnTermination(client.shutdown())
}
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index bfb1dac21..48d19da39 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -64,6 +64,7 @@ object Dependencies {
val GoogleAuthVersion = "1.23.0"
val JwtScalaVersion = "10.0.0"
+ val Log4jVersion = "2.23.1"
// Releases https://github.com/FasterXML/jackson-databind/releases
// CVE issues
https://github.com/FasterXML/jackson-databind/issues?utf8=%E2%9C%93&q=+label%3ACVE
@@ -364,14 +365,11 @@ object Dependencies {
"org.slf4j" % "slf4j-api" % Slf4jVersion % Test,
"ch.qos.logback" % "logback-classic" % LogbackVersion % Test) ++ Mockito)
- val KuduVersion = "1.10.1"
+ val KuduVersion = "1.17.0"
val Kudu = Seq(
libraryDependencies ++= Seq(
- "org.apache.kudu" % "kudu-client-tools" % KuduVersion,
- "org.apache.kudu" % "kudu-client" % KuduVersion % Test),
- dependencyOverrides ++= Seq(
- "org.slf4j" % "slf4j-api" % Slf4jLegacyVersion,
- "ch.qos.logback" % "logback-classic" % LogbackLegacyVersion))
+ "org.apache.kudu" % "kudu-client" % KuduVersion,
+ "org.apache.logging.log4j" % "log4j-to-slf4j" % Log4jVersion % Test))
val MongoDb = Seq(
crossScalaVersions -= Scala3,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
