This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1b94818311 Simplify kafka build and remove old kafka 0.9 files (#11638)
1b94818311 is described below
commit 1b94818311f706e97aa4eb235e5def9de2c09562
Author: Xiang Fu <[email protected]>
AuthorDate: Wed Sep 20 21:41:53 2023 -0700
Simplify kafka build and remove old kafka 0.9 files (#11638)
---
pinot-distribution/pinot-assembly.xml | 8 ++--
pinot-distribution/pom.xml | 2 +-
pinot-integration-test-base/pom.xml | 2 +-
pinot-integration-tests/pom.xml | 2 +-
pinot-perf/pom.xml | 2 +-
pinot-tools/pom.xml | 4 +-
.../airlineStats_realtime_table_config.json | 35 ----------------
.../meetupRsvp_realtime_table_config.json | 48 ----------------------
pom.xml | 6 ---
9 files changed, 10 insertions(+), 99 deletions(-)
diff --git a/pinot-distribution/pinot-assembly.xml
b/pinot-distribution/pinot-assembly.xml
index 73aefa1c11..eef6100d27 100644
--- a/pinot-distribution/pinot-assembly.xml
+++ b/pinot-distribution/pinot-assembly.xml
@@ -48,10 +48,10 @@
<!-- Start Include Pinot Stream Ingestion Plugins-->
<file>
<source>
-
${pinot.root}/pinot-plugins/pinot-stream-ingestion/pinot-kafka-${kafka.version}/target/pinot-kafka-${kafka.version}-${project.version}-shaded.jar
+
${pinot.root}/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/target/pinot-kafka-2.0-${project.version}-shaded.jar
</source>
<destName>
-
plugins/pinot-stream-ingestion/pinot-kafka-${kafka.version}/pinot-kafka-${kafka.version}-${project.version}-shaded.jar
+
plugins/pinot-stream-ingestion/pinot-kafka-2.0/pinot-kafka-2.0-${project.version}-shaded.jar
</destName>
</file>
<file>
@@ -245,13 +245,13 @@
</fileSet>
<fileSet>
<useDefaultExcludes>false</useDefaultExcludes>
-
<directory>${pinot.root}/pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_${kafka.version}
+
<directory>${pinot.root}/pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_2.0
</directory>
<outputDirectory>examples/stream/meetupRsvp/</outputDirectory>
</fileSet>
<fileSet>
<useDefaultExcludes>false</useDefaultExcludes>
-
<directory>${pinot.root}/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_${kafka.version}
+
<directory>${pinot.root}/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_2.0
</directory>
<outputDirectory>examples/stream/airlineStats/</outputDirectory>
</fileSet>
diff --git a/pinot-distribution/pom.xml b/pinot-distribution/pom.xml
index 51fff252cc..6b12eed8b6 100644
--- a/pinot-distribution/pom.xml
+++ b/pinot-distribution/pom.xml
@@ -90,7 +90,7 @@
</exclusion>
<exclusion>
<groupId>org.apache.pinot</groupId>
- <artifactId>pinot-kafka-${kafka.version}</artifactId>
+ <artifactId>pinot-kafka-2.0</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.pinot</groupId>
diff --git a/pinot-integration-test-base/pom.xml
b/pinot-integration-test-base/pom.xml
index 44d68dd379..9bd7661d4c 100644
--- a/pinot-integration-test-base/pom.xml
+++ b/pinot-integration-test-base/pom.xml
@@ -149,7 +149,7 @@
</dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
- <artifactId>pinot-kafka-${kafka.version}</artifactId>
+ <artifactId>pinot-kafka-2.0</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
diff --git a/pinot-integration-tests/pom.xml b/pinot-integration-tests/pom.xml
index a703728eaa..c47394a107 100644
--- a/pinot-integration-tests/pom.xml
+++ b/pinot-integration-tests/pom.xml
@@ -314,7 +314,7 @@
</dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
- <artifactId>pinot-kafka-${kafka.version}</artifactId>
+ <artifactId>pinot-kafka-2.0</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
diff --git a/pinot-perf/pom.xml b/pinot-perf/pom.xml
index 062c1c4ba5..adba15be78 100644
--- a/pinot-perf/pom.xml
+++ b/pinot-perf/pom.xml
@@ -53,7 +53,7 @@
</dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
- <artifactId>pinot-kafka-${kafka.version}</artifactId>
+ <artifactId>pinot-kafka-2.0</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml
index 7b5d5c8393..a362dc8b27 100644
--- a/pinot-tools/pom.xml
+++ b/pinot-tools/pom.xml
@@ -143,7 +143,7 @@
</dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
- <artifactId>pinot-kafka-${kafka.version}</artifactId>
+ <artifactId>pinot-kafka-2.0</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
@@ -588,7 +588,7 @@
<resources>
<resource>
<!-- source -->
-
<directory>src/main/resources/sample_data/kafka_${kafka.version}</directory>
+
<directory>src/main/resources/sample_data/kafka_2.0</directory>
</resource>
</resources>
</configuration>
diff --git
a/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_0.9/airlineStats_realtime_table_config.json
b/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_0.9/airlineStats_realtime_table_config.json
deleted file mode 100644
index 758e638745..0000000000
---
a/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_0.9/airlineStats_realtime_table_config.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "tableName": "airlineStats",
- "tableType": "REALTIME",
- "segmentsConfig": {
- "timeColumnName": "DaysSinceEpoch",
- "timeType": "DAYS",
- "retentionTimeUnit": "DAYS",
- "retentionTimeValue": "5",
- "segmentPushType": "APPEND",
- "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy",
- "schemaName": "airlineStats",
- "replication": "1",
- "replicasPerPartition": "1"
- },
- "tenants": {},
- "tableIndexConfig": {
- "loadMode": "MMAP",
- "streamConfigs": {
- "streamType": "kafka",
- "stream.kafka.consumer.type": "simple",
- "stream.kafka.topic.name": "flights-realtime",
- "stream.kafka.decoder.class.name":
"org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder",
- "stream.kafka.consumer.factory.class.name":
"org.apache.pinot.plugin.stream.kafka09.KafkaConsumerFactory",
- "stream.kafka.hlc.zk.connect.string": "localhost:2191/kafka",
- "stream.kafka.zk.broker.url": "localhost:2191/kafka",
- "stream.kafka.broker.list": "localhost:19092",
- "realtime.segment.flush.threshold.time": "3600000",
- "realtime.segment.flush.threshold.size": "50000",
- "stream.kafka.consumer.prop.auto.offset.reset": "smallest"
- }
- },
- "metadata": {
- "customConfigs": {}
- }
-}
diff --git
a/pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_0.9/meetupRsvp_realtime_table_config.json
b/pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_0.9/meetupRsvp_realtime_table_config.json
deleted file mode 100644
index e2420e12ab..0000000000
---
a/pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_0.9/meetupRsvp_realtime_table_config.json
+++ /dev/null
@@ -1,48 +0,0 @@
-{
- "tableName": "meetupRsvp",
- "tableType": "REALTIME",
- "segmentsConfig": {
- "timeColumnName": "mtime",
- "timeType": "MILLISECONDS",
- "segmentPushType": "APPEND",
- "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy",
- "schemaName": "meetupRsvp",
- "replication": "1",
- "replicasPerPartition": "1",
- "retentionTimeUnit": "DAYS",
- "retentionTimeValue": "1"
- },
- "tenants": {},
- "fieldConfigList": [
- {
- "name": "mtime",
- "encodingType": "DICTIONARY",
- "indexTypes": ["TIMESTAMP"],
- "timestampConfig": {
- "granularities": [
- "DAY",
- "WEEK",
- "MONTH"
- ]
- }
- }
- ],
- "tableIndexConfig": {
- "loadMode": "MMAP",
- "streamConfigs": {
- "streamType": "kafka",
- "stream.kafka.consumer.type": "lowLevel",
- "stream.kafka.topic.name": "meetupRSVPEvents",
- "stream.kafka.decoder.class.name":
"org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder",
- "stream.kafka.consumer.factory.class.name":
"org.apache.pinot.plugin.stream.kafka09.KafkaConsumerFactory",
- "stream.kafka.zk.broker.url": "localhost:2191/kafka",
- "stream.kafka.broker.list": "localhost:19092",
- "stream.kafka.consumer.prop.auto.offset.reset": "largest",
- "realtime.segment.flush.threshold.time": "12h",
- "realtime.segment.flush.threshold.size": "100M"
- }
- },
- "metadata": {
- "customConfigs": {}
- }
-}
diff --git a/pom.xml b/pom.xml
index 2f3ba865dc..db2d8962bc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -164,12 +164,6 @@
<joda-time.version>2.12.5</joda-time.version>
<!-- Sets the VM argument line used when unit tests are run. -->
<argLine>-Xms4g -Xmx4g</argLine>
-
- <!-- handle kafka imports
- kafka dependency is still explicitly defined in pinot-integration-tests,
pinot-tools and pinot-perf pom files.
- To change kafka connector dependency, we only need to update this version
number config.
- TODO: figure out a way to inject kafka dependency instead of explicitly
setting the kafka module dependency -->
- <kafka.version>2.0</kafka.version>
<protobuf.version>3.22.0</protobuf.version>
<grpc.version>1.53.0</grpc.version>
<confluent.version>5.5.3</confluent.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
