This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch move_sample_data_to_examples in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit f10757be8fbb150d3614d48504bb043f1d8da503 Author: Xiang Fu <[email protected]> AuthorDate: Thu Jan 9 18:11:49 2020 -0800 Move sample_data to examples --- docker/images/pinot/README.md | 4 +-- docs/in_production.rst | 4 +-- kubernetes/helm/pinot-realtime-quickstart.yml | 4 +-- .../skaffold/gke/pinot-realtime-quickstart.yml | 4 +-- pinot-distribution/pinot-assembly.xml | 20 +++++------ .../pinot-stream-ingestion/pinot-kafka-2.0/pom.xml | 4 +-- .../org/apache/pinot/tools/HybridQuickstart.java | 8 ++--- .../java/org/apache/pinot/tools/Quickstart.java | 6 ++-- .../org/apache/pinot/tools/RealtimeQuickStart.java | 4 +-- .../baseballStats_offline_table_config.json | 0 .../batch/baseballStats}/baseballStats_schema.json | 0 .../baseballStats/rawdata}/baseballStats_data.csv | 0 .../airlineStats_realtime_table_config.json | 0 .../stream/airlineStats}/airlineStats_schema.json | 0 .../docker/airlineStats_realtime_table_config.json | 0 .../airlineStats_realtime_table_config.json | 1 + .../airlineStats_realtime_table_config.json | 0 .../sample_data/airlineStats_data.avro | Bin .../sample_data/airlineStats_data.json | 0 .../sample_data/airlineStats_data.orc | Bin .../meetupRsvp_realtime_table_config.json | 3 +- .../meetupRsvp_realtime_table_config.json | 0 .../meetupRsvp_realtime_table_config.json | 0 .../stream/meetupRsvp}/meetupRsvp_schema.json | 0 .../airlineStats_offline_table_config.json | 21 ------------ .../docker/airlineStats_offline_table_config.json | 21 ------------ .../airlineStats_realtime_table_config.json | 37 --------------------- .../meetupRsvp_realtime_table_config.json | 28 ---------------- pom.xml | 14 +++++++- 29 files changed, 45 insertions(+), 138 deletions(-) diff --git a/docker/images/pinot/README.md b/docker/images/pinot/README.md index 6d97b1f..f18aa79 100644 --- a/docker/images/pinot/README.md +++ b/docker/images/pinot/README.md @@ -131,12 +131,12 @@ docker-compose -f docker-compose.yml up Below is the script to create airlineStats table ```SHELL -docker run --network=docker_default winedepot/pinot:0.3.0-SNAPSHOT AddTable -schemaFile sample_data/airlineStats_schema.json -tableConfigFile sample_data/docker/airlineStats_realtime_table_config.json -controllerHost pinot-controller -controllerPort 9000 -exec +docker run --network=docker_default winedepot/pinot:0.3.0-SNAPSHOT AddTable -schemaFile examples/stream/airlineStats/airlineStats_schema.json -tableConfigFile examples/stream/airlineStats/docker/airlineStats_realtime_table_config.json -controllerHost pinot-controller -controllerPort 9000 -exec ``` Below is the script to ingest airplane stats data to Kafka ```SHELL -docker run --network=docker_default winedepot/pinot:0.3.0-SNAPSHOT StreamAvroIntoKafka -avroFile sample_data/airlineStats_data.avro -kafkaTopic flights-realtime -kafkaBrokerList kafka:9092 -zkAddress zookeeper:2181 +docker run --network=docker_default winedepot/pinot:0.3.0-SNAPSHOT StreamAvroIntoKafka -avroFile examples/stream/airlineStats/sample_data/airlineStats_data.avro -kafkaTopic flights-realtime -kafkaBrokerList kafka:9092 -zkAddress zookeeper:2181 ``` In order to query pinot, try to open `localhost:9000/query` from your browser. diff --git a/docs/in_production.rst b/docs/in_production.rst index 08b0605..2b1e06e 100644 --- a/docs/in_production.rst +++ b/docs/in_production.rst @@ -76,8 +76,8 @@ Here is an example of invoking the command to create a pinot segment: .. code-block:: none - $ ./pinot-distribution/target/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/bin/pinot-admin.sh CreateSegment -dataDir /Users/host1/Desktop/test/ -format CSV -outDir /Users/host1/Desktop/test2/ -tableName baseballStats -segmentName baseballStats_data -overwrite -schemaFile ./pinot-distribution/target/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/sample_data/baseballStats_schema.json - Executing command: CreateSegment -generatorConfigFile null -dataDir /Users/host1/Desktop/test/ -format CSV -outDir /Users/host1/Desktop/test2/ -overwrite true -tableName baseballStats -segmentName baseballStats_data -timeColumnName null -schemaFile ./pinot-distribution/target/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/sample_data/baseballStats_schema.json -readerConfigFile null -enableStarTreeIndex false -starTreeIndexSpecFile null -hllSize 9 [...] + $ ./pinot-distribution/target/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/bin/pinot-admin.sh CreateSegment -dataDir /Users/host1/Desktop/test/ -format CSV -outDir /Users/host1/Desktop/test2/ -tableName baseballStats -segmentName baseballStats_data -overwrite -schemaFile ./pinot-distribution/target/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/examples/batch/baseballStats/baseballStats_schema.json + Executing command: CreateSegment -generatorConfigFile null -dataDir /Users/host1/Desktop/test/ -format CSV -outDir /Users/host1/Desktop/test2/ -overwrite true -tableName baseballStats -segmentName baseballStats_data -timeColumnName null -schemaFile ./pinot-distribution/target/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/apache-pinot-incubating-0.1.0-SNAPSHOT-bin/examples/batch/baseballStats/baseballStats_schema.json -readerConfigFile null -enableStarTreeIndex false -starTreeIndexSpecFil [...] Accepted files: [/Users/host1/Desktop/test/baseballStats_data.csv] Finished building StatsCollector! Collected stats for 97889 documents diff --git a/kubernetes/helm/pinot-realtime-quickstart.yml b/kubernetes/helm/pinot-realtime-quickstart.yml index 8e60658..e5a613a 100644 --- a/kubernetes/helm/pinot-realtime-quickstart.yml +++ b/kubernetes/helm/pinot-realtime-quickstart.yml @@ -497,10 +497,10 @@ spec: containers: - name: loading-json-data-to-kafka image: fx19880617/pinot:0.2.0-SNAPSHOT - args: [ "StreamAvroIntoKafka", "-avroFile", "sample_data/airlineStats_data.avro", "-kafkaTopic", "flights-realtime", "-kafkaBrokerList", "kafka:9092", "-zkAddress", "kafka-zookeeper:2181" ] + args: [ "StreamAvroIntoKafka", "-avroFile", "examples/stream/airlineStats/sample_data/airlineStats_data.avro", "-kafkaTopic", "flights-realtime", "-kafkaBrokerList", "kafka:9092", "-zkAddress", "kafka-zookeeper:2181" ] - name: loading-avro-data-to-kafka image: fx19880617/pinot:0.2.0-SNAPSHOT - args: [ "StreamAvroIntoKafka", "-avroFile", "sample_data/airlineStats_data.avro", "-kafkaTopic", "flights-realtime-avro", "-kafkaBrokerList", "kafka:9092", "-zkAddress", "kafka-zookeeper:2181", "-outputFormat", "avro" ] + args: [ "StreamAvroIntoKafka", "-avroFile", "examples/stream/airlineStats/sample_data/airlineStats_data.avro", "-kafkaTopic", "flights-realtime-avro", "-kafkaBrokerList", "kafka:9092", "-zkAddress", "kafka-zookeeper:2181", "-outputFormat", "avro" ] restartPolicy: OnFailure backoffLimit: 3 diff --git a/kubernetes/skaffold/gke/pinot-realtime-quickstart.yml b/kubernetes/skaffold/gke/pinot-realtime-quickstart.yml index 7de21bd..cd39c22 100644 --- a/kubernetes/skaffold/gke/pinot-realtime-quickstart.yml +++ b/kubernetes/skaffold/gke/pinot-realtime-quickstart.yml @@ -28,10 +28,10 @@ spec: containers: - name: loading-data-to-kafka image: winedepot/pinot:0.3.0-SNAPSHOT - args: [ "StreamAvroIntoKafka", "-avroFile", "sample_data/airlineStats_data.avro", "-kafkaTopic", "flights-realtime", "-kafkaBrokerList", "kafka:9092", "-zkAddress", "zookeeper:2181" ] + args: [ "StreamAvroIntoKafka", "-avroFile", "examples/stream/airlineStats/sample_data/airlineStats_data.avro", "-kafkaTopic", "flights-realtime", "-kafkaBrokerList", "kafka:9092", "-zkAddress", "zookeeper:2181" ] - name: pinot-add-example-realtime-table image: winedepot/pinot:0.3.0-SNAPSHOT - args: [ "AddTable", "-schemaFile", "sample_data/airlineStats_schema.json", "-tableConfigFile", "sample_data/docker/airlineStats_realtime_table_config.json", "-controllerHost", "pinot-controller", "-controllerPort", "9000", "-exec" ] + args: [ "AddTable", "-schemaFile", "examples/stream/airlineStats/airlineStats_schema.json", "-tableConfigFile", "examples/stream/airlineStats/docker/airlineStats_realtime_table_config.json", "-controllerHost", "pinot-controller", "-controllerPort", "9000", "-exec" ] restartPolicy: OnFailure nodeSelector: cloud.google.com/gke-nodepool: default-pool diff --git a/pinot-distribution/pinot-assembly.xml b/pinot-distribution/pinot-assembly.xml index a410657..e5e4eed 100644 --- a/pinot-distribution/pinot-assembly.xml +++ b/pinot-distribution/pinot-assembly.xml @@ -83,18 +83,13 @@ </fileSet> <fileSet> <useDefaultExcludes>false</useDefaultExcludes> - <directory>${pinot.root}/pinot-tools/src/main/resources/sample_data/kafka_${kafka.version}</directory> - <outputDirectory>sample_data/</outputDirectory> + <directory>${pinot.root}/pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_${kafka.version}</directory> + <outputDirectory>examples/stream/meetupRsvp/</outputDirectory> </fileSet> <fileSet> <useDefaultExcludes>false</useDefaultExcludes> - <directory>${pinot.root}/pinot-tools/src/main/resources/sample_data</directory> - <outputDirectory>sample_data/</outputDirectory> - </fileSet> - <fileSet> - <useDefaultExcludes>false</useDefaultExcludes> - <directory>${pinot.root}/pinot-tools/src/main/resources/sample_data_realtime</directory> - <outputDirectory>sample_data</outputDirectory> + <directory>${pinot.root}/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_${kafka.version}</directory> + <outputDirectory>examples/stream/airlineStats/</outputDirectory> </fileSet> <fileSet> <useDefaultExcludes>false</useDefaultExcludes> @@ -117,12 +112,17 @@ <exclude>**/pinot-file-system/pinot-file-system/**</exclude> <exclude>**/pinot-input-format/pinot-input-format/**</exclude> <exclude>**/pinot-stream-ingestion/pinot-stream-ingestion/**</exclude> - <exclude>**/pinot-stream-ingestion/pinot-kafka-base/**</exclude> + <exclude>**/pinot-stream-ingestion/pinot-kafka-*/**</exclude> <exclude>**/pinot-batch-ingestion/pinot-batch-ingestion/**</exclude> <exclude>**/pinot-batch-ingestion/pinot-batch-ingestion-base/**</exclude> <exclude>**/pinot-batch-ingestion/pinot-ingestion-common/**</exclude> <exclude>**/pinot-batch-ingestion/v0_deprecated/**</exclude> </excludes> </fileSet> + <fileSet> + <useDefaultExcludes>false</useDefaultExcludes> + <directory>${pinot.root}/pinot-plugins/target/plugins/pinot-stream-ingestion/pinot-kafka-${kafka.version}</directory> + <outputDirectory>plugins/pinot-stream-ingestion/pinot-kafka-${kafka.version}</outputDirectory> + </fileSet> </fileSets> </assembly> diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml index a484b3a..164c2cd 100644 --- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml +++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml @@ -65,11 +65,11 @@ <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> - <version>2.12.8</version> + <version>2.11.11</version> </dependency> <dependency> <groupId>org.apache.kafka</groupId> - <artifactId>kafka_2.12</artifactId> + <artifactId>kafka_2.11</artifactId> <version>${kafka.lib.version}</version> <exclusions> <exclusion> diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java index aa33073..4547493 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java @@ -70,13 +70,13 @@ public class HybridQuickstart { File tableConfigFile = new File(_offlineQuickStartDataDir, "airlineStats_offline_table_config.json"); ClassLoader classLoader = Quickstart.class.getClassLoader(); - URL resource = classLoader.getResource("sample_data/airlineStats_schema.json"); + URL resource = classLoader.getResource("examples/batch/airlineStats/airlineStats_schema.json"); Preconditions.checkNotNull(resource); FileUtils.copyURLToFile(resource, _schemaFile); - resource = classLoader.getResource("sample_data/airlineStats_data.avro"); + resource = classLoader.getResource("examples/stream/airlineStats/sample_data/airlineStats_data.avro"); Preconditions.checkNotNull(resource); FileUtils.copyURLToFile(resource, _dataFile); - resource = classLoader.getResource("sample_data/airlineStats_offline_table_config.json"); + resource = classLoader.getResource("examples/batch/airlineStats/airlineStats_offline_table_config.json"); Preconditions.checkNotNull(resource); FileUtils.copyURLToFile(resource, tableConfigFile); @@ -95,7 +95,7 @@ public class HybridQuickstart { File tableConfigFile = new File(_realtimeQuickStartDataDir, "airlineStats_realtime_table_config.json"); URL resource = Quickstart.class.getClassLoader().getResource( - "sample_data/airlineStats_realtime_table_config.json"); + "examples/stream/airlineStats/airlineStats_realtime_table_config.json"); Preconditions.checkNotNull(resource); FileUtils.copyURLToFile(resource, tableConfigFile); diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java index 9427b52..31b7f31 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java @@ -128,13 +128,13 @@ public class Quickstart { File tableConfigFile = new File(quickStartDataDir, "baseballStats_offline_table_config.json"); ClassLoader classLoader = Quickstart.class.getClassLoader(); - URL resource = classLoader.getResource("sample_data/baseballStats_schema.json"); + URL resource = classLoader.getResource("examples/batch/baseballStats/baseballStats_schema.json"); com.google.common.base.Preconditions.checkNotNull(resource); FileUtils.copyURLToFile(resource, schemaFile); - resource = classLoader.getResource("sample_data/baseballStats_data.csv"); + resource = classLoader.getResource("examples/batch/baseballStats/rawdata/baseballStats_data.csv"); com.google.common.base.Preconditions.checkNotNull(resource); FileUtils.copyURLToFile(resource, dataFile); - resource = classLoader.getResource("sample_data/baseballStats_offline_table_config.json"); + resource = classLoader.getResource("examples/batch/baseballStats/baseballStats_offline_table_config.json"); com.google.common.base.Preconditions.checkNotNull(resource); FileUtils.copyURLToFile(resource, tableConfigFile); diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java index 82f76e2..c5dc44c 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java @@ -60,10 +60,10 @@ public class RealtimeQuickStart { File tableConfigFile = new File(quickStartDataDir, "meetupRsvp_realtime_table_config.json"); ClassLoader classLoader = Quickstart.class.getClassLoader(); - URL resource = classLoader.getResource("sample_data/meetupRsvp_schema.json"); + URL resource = classLoader.getResource("examples/stream/meetupRsvp/meetupRsvp_schema.json"); com.google.common.base.Preconditions.checkNotNull(resource); FileUtils.copyURLToFile(resource, schemaFile); - resource = classLoader.getResource("sample_data/meetupRsvp_realtime_table_config.json"); + resource = classLoader.getResource("examples/stream/meetupRsvp/meetupRsvp_realtime_table_config.json"); com.google.common.base.Preconditions.checkNotNull(resource); FileUtils.copyURLToFile(resource, tableConfigFile); diff --git a/pinot-tools/src/main/resources/sample_data/baseballStats_offline_table_config.json b/pinot-tools/src/main/resources/examples/batch/baseballStats/baseballStats_offline_table_config.json similarity index 100% rename from pinot-tools/src/main/resources/sample_data/baseballStats_offline_table_config.json rename to pinot-tools/src/main/resources/examples/batch/baseballStats/baseballStats_offline_table_config.json diff --git a/pinot-tools/src/main/resources/sample_data/baseballStats_schema.json b/pinot-tools/src/main/resources/examples/batch/baseballStats/baseballStats_schema.json similarity index 100% rename from pinot-tools/src/main/resources/sample_data/baseballStats_schema.json rename to pinot-tools/src/main/resources/examples/batch/baseballStats/baseballStats_schema.json diff --git a/pinot-tools/src/main/resources/sample_data/baseballStats_data.csv b/pinot-tools/src/main/resources/examples/batch/baseballStats/rawdata/baseballStats_data.csv similarity index 100% rename from pinot-tools/src/main/resources/sample_data/baseballStats_data.csv rename to pinot-tools/src/main/resources/examples/batch/baseballStats/rawdata/baseballStats_data.csv diff --git a/pinot-tools/src/main/resources/sample_data/kafka_2.0/airlineStats_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_realtime_table_config.json similarity index 100% copy from pinot-tools/src/main/resources/sample_data/kafka_2.0/airlineStats_realtime_table_config.json copy to pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_realtime_table_config.json diff --git a/pinot-tools/src/main/resources/sample_data/airlineStats_schema.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_schema.json similarity index 100% rename from pinot-tools/src/main/resources/sample_data/airlineStats_schema.json rename to pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_schema.json diff --git a/pinot-tools/src/main/resources/sample_data/docker/airlineStats_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/docker/airlineStats_realtime_table_config.json similarity index 100% rename from pinot-tools/src/main/resources/sample_data/docker/airlineStats_realtime_table_config.json rename to pinot-tools/src/main/resources/examples/stream/airlineStats/docker/airlineStats_realtime_table_config.json diff --git a/pinot-tools/src/main/resources/sample_data/airlineStats_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_0.9/airlineStats_realtime_table_config.json similarity index 91% rename from pinot-tools/src/main/resources/sample_data/airlineStats_realtime_table_config.json rename to pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_0.9/airlineStats_realtime_table_config.json index 4ea745e..f5c4652 100644 --- a/pinot-tools/src/main/resources/sample_data/airlineStats_realtime_table_config.json +++ b/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_0.9/airlineStats_realtime_table_config.json @@ -23,6 +23,7 @@ "stream.kafka.consumer.type": "simple", "stream.kafka.topic.name": "flights-realtime", "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder", + "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka09.KafkaConsumerFactory", "stream.kafka.hlc.zk.connect.string": "localhost:2191/kafka", "stream.kafka.zk.broker.url": "localhost:2191/kafka", "stream.kafka.broker.list": "localhost:19092", diff --git a/pinot-tools/src/main/resources/sample_data/kafka_2.0/airlineStats_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_2.0/airlineStats_realtime_table_config.json similarity index 100% rename from pinot-tools/src/main/resources/sample_data/kafka_2.0/airlineStats_realtime_table_config.json rename to pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_2.0/airlineStats_realtime_table_config.json diff --git a/pinot-tools/src/main/resources/sample_data/airlineStats_data.avro b/pinot-tools/src/main/resources/examples/stream/airlineStats/sample_data/airlineStats_data.avro similarity index 100% rename from pinot-tools/src/main/resources/sample_data/airlineStats_data.avro rename to pinot-tools/src/main/resources/examples/stream/airlineStats/sample_data/airlineStats_data.avro diff --git a/pinot-tools/src/main/resources/sample_data/airlineStats_data.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/sample_data/airlineStats_data.json similarity index 100% rename from pinot-tools/src/main/resources/sample_data/airlineStats_data.json rename to pinot-tools/src/main/resources/examples/stream/airlineStats/sample_data/airlineStats_data.json diff --git a/pinot-tools/src/main/resources/sample_data/airlineStats_data.orc b/pinot-tools/src/main/resources/examples/stream/airlineStats/sample_data/airlineStats_data.orc similarity index 100% rename from pinot-tools/src/main/resources/sample_data/airlineStats_data.orc rename to pinot-tools/src/main/resources/examples/stream/airlineStats/sample_data/airlineStats_data.orc diff --git a/pinot-tools/src/main/resources/sample_data/kafka_0.9/meetupRsvp_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_0.9/meetupRsvp_realtime_table_config.json similarity index 87% rename from pinot-tools/src/main/resources/sample_data/kafka_0.9/meetupRsvp_realtime_table_config.json rename to pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_0.9/meetupRsvp_realtime_table_config.json index 9b145bc..8d00c37 100644 --- a/pinot-tools/src/main/resources/sample_data/kafka_0.9/meetupRsvp_realtime_table_config.json +++ b/pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_0.9/meetupRsvp_realtime_table_config.json @@ -19,7 +19,8 @@ "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder", "stream.kafka.hlc.zk.connect.string": "localhost:2191/kafka", "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka09.KafkaConsumerFactory", - "stream.kafka.zk.broker.url": "localhost:2191/kafka" + "stream.kafka.zk.broker.url": "localhost:2191/kafka", + "stream.kafka.hlc.bootstrap.server": "localhost:19092" } }, "metadata": { diff --git a/pinot-tools/src/main/resources/sample_data/kafka_2.0/meetupRsvp_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_2.0/meetupRsvp_realtime_table_config.json similarity index 100% copy from pinot-tools/src/main/resources/sample_data/kafka_2.0/meetupRsvp_realtime_table_config.json copy to pinot-tools/src/main/resources/examples/stream/meetupRsvp/kafka_2.0/meetupRsvp_realtime_table_config.json diff --git a/pinot-tools/src/main/resources/sample_data/kafka_2.0/meetupRsvp_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/meetupRsvp/meetupRsvp_realtime_table_config.json similarity index 100% rename from pinot-tools/src/main/resources/sample_data/kafka_2.0/meetupRsvp_realtime_table_config.json rename to pinot-tools/src/main/resources/examples/stream/meetupRsvp/meetupRsvp_realtime_table_config.json diff --git a/pinot-tools/src/main/resources/sample_data/meetupRsvp_schema.json b/pinot-tools/src/main/resources/examples/stream/meetupRsvp/meetupRsvp_schema.json similarity index 100% rename from pinot-tools/src/main/resources/sample_data/meetupRsvp_schema.json rename to pinot-tools/src/main/resources/examples/stream/meetupRsvp/meetupRsvp_schema.json diff --git a/pinot-tools/src/main/resources/sample_data/airlineStats_offline_table_config.json b/pinot-tools/src/main/resources/sample_data/airlineStats_offline_table_config.json deleted file mode 100644 index de73b4c..0000000 --- a/pinot-tools/src/main/resources/sample_data/airlineStats_offline_table_config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "tableName": "airlineStats", - "tableType": "OFFLINE", - "segmentsConfig": { - "timeColumnName": "DaysSinceEpoch", - "timeType": "DAYS", - "segmentPushType": "APPEND", - "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy", - "replication": "1" - }, - "tenants": { - "broker": "airline_broker", - "server": "airline" - }, - "tableIndexConfig": { - "loadMode": "MMAP" - }, - "metadata": { - "customConfigs": {} - } -} diff --git a/pinot-tools/src/main/resources/sample_data/docker/airlineStats_offline_table_config.json b/pinot-tools/src/main/resources/sample_data/docker/airlineStats_offline_table_config.json deleted file mode 100644 index de73b4c..0000000 --- a/pinot-tools/src/main/resources/sample_data/docker/airlineStats_offline_table_config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "tableName": "airlineStats", - "tableType": "OFFLINE", - "segmentsConfig": { - "timeColumnName": "DaysSinceEpoch", - "timeType": "DAYS", - "segmentPushType": "APPEND", - "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy", - "replication": "1" - }, - "tenants": { - "broker": "airline_broker", - "server": "airline" - }, - "tableIndexConfig": { - "loadMode": "MMAP" - }, - "metadata": { - "customConfigs": {} - } -} diff --git a/pinot-tools/src/main/resources/sample_data/kafka_0.9/airlineStats_realtime_table_config.json b/pinot-tools/src/main/resources/sample_data/kafka_0.9/airlineStats_realtime_table_config.json deleted file mode 100644 index 4ea745e..0000000 --- a/pinot-tools/src/main/resources/sample_data/kafka_0.9/airlineStats_realtime_table_config.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "tableName": "airlineStats", - "tableType": "REALTIME", - "segmentsConfig": { - "timeColumnName": "DaysSinceEpoch", - "timeType": "DAYS", - "retentionTimeUnit": "DAYS", - "retentionTimeValue": "5", - "segmentPushType": "APPEND", - "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy", - "schemaName": "airlineStats", - "replication": "1", - "replicasPerPartition": "1" - }, - "tenants": { - "broker": "airline_broker", - "server": "airline" - }, - "tableIndexConfig": { - "loadMode": "MMAP", - "streamConfigs": { - "streamType": "kafka", - "stream.kafka.consumer.type": "simple", - "stream.kafka.topic.name": "flights-realtime", - "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder", - "stream.kafka.hlc.zk.connect.string": "localhost:2191/kafka", - "stream.kafka.zk.broker.url": "localhost:2191/kafka", - "stream.kafka.broker.list": "localhost:19092", - "realtime.segment.flush.threshold.time": "3600000", - "realtime.segment.flush.threshold.size": "50000", - "stream.kafka.consumer.prop.auto.offset.reset": "smallest" - } - }, - "metadata": { - "customConfigs": {} - } -} diff --git a/pinot-tools/src/main/resources/sample_data/meetupRsvp_realtime_table_config.json b/pinot-tools/src/main/resources/sample_data/meetupRsvp_realtime_table_config.json deleted file mode 100644 index 9b145bc..0000000 --- a/pinot-tools/src/main/resources/sample_data/meetupRsvp_realtime_table_config.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "tableName": "meetupRsvp", - "tableType": "REALTIME", - "segmentsConfig": { - "timeColumnName": "mtime", - "timeType": "MILLISECONDS", - "segmentPushType": "APPEND", - "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy", - "schemaName": "meetupRsvp", - "replication": "1" - }, - "tenants": {}, - "tableIndexConfig": { - "loadMode": "MMAP", - "streamConfigs": { - "streamType": "kafka", - "stream.kafka.consumer.type": "highLevel", - "stream.kafka.topic.name": "meetupRSVPEvents", - "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder", - "stream.kafka.hlc.zk.connect.string": "localhost:2191/kafka", - "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka09.KafkaConsumerFactory", - "stream.kafka.zk.broker.url": "localhost:2191/kafka" - } - }, - "metadata": { - "customConfigs": {} - } -} diff --git a/pom.xml b/pom.xml index c152598..e9576c9 100644 --- a/pom.xml +++ b/pom.xml @@ -137,11 +137,23 @@ kafka dependency is still explicitly defined in pinot-integration-tests, pinot-tools and pinot-perf pom files. To change kafka connector dependency, we only need to update this version number config. TODO: figure out a way to inject kafka dependency instead of explicitly setting the kafka module dependency --> - <kafka.version>0.9</kafka.version> + <kafka.version>2.0</kafka.version> </properties> <profiles> <profile> + <id>kafka-0.9</id> + <activation> + <property> + <name>kafka.version</name> + <value>0.9</value> + </property> + </activation> + <properties> + <kafka.version>0.9</kafka.version> + </properties> + </profile> + <profile> <id>kafka-2.0</id> <activation> <property> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
