Hi, First of all I am new to both kafka and druid. I am trying to create a demo project for streaming data which is in mysql I used kafka-connect to extract data into kafka . which is in below format.
Kafka Topic name: test-sqlite-jdbc-manoj1 INPUT: bin/kafka-avro-console-consumer --bootstrap-server localhost:9092 --topic test-sqlite-jdbc-manoj1 --from-beginning OUTPUT: {"id1":123,"name1":"Sai_new_2","date1":1543143234000} {"id1":124,"name1":"Sai_new_3","date1":1543143238000} {"id1":125,"name1":"Sai_new_3","date1":1543165872000} {"id1":126,"name1":"Sai_new_4","date1":1543165875000} {"id1":127,"name1":"Sai_new_5","date1":1543165877000} Now I am trying to pipeline this data from kafka into druid. with below supervisor json, curl -XPOST -H'Content-Type: application/json' -d @examples/test7.json http://localhost:8090/druid/indexer/v1/supervisor {"id":"test-sqlite-jdbc-manoj1"} test7.json: { "type": "kafka", "dataSchema": { "dataSource": "test-sqlite-jdbc-manoj1", "parser": { "type": "avro_stream", "avroBytesDecoder" : { "type" : "schema_repo", "subjectAndIdConverter" : { "type" : "confluent", "subject" : "test-sqlite-jdbc-manoj1" }, "schemaRepository" : { "type" : "confluent_client", "url" : "http://localhost:8081", "identityMapCapacity" : 1000 } }, "parseSpec": { "format": "json", "timestampSpec": { "column": "date1", "format": "auto" }, "dimensionsSpec": { "dimensions": [ "id1", "name1" ] } } }, "metricsSpec" : [ { "type":"count", "name":"count" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "NONE", "rollup": false } }, "tuningConfig": { "type": "kafka", "reportParseExceptions": false }, "ioConfig": { "topic": "test-sqlite-jdbc-manoj1", "replicas": 1, "taskDuration": "PT10M", "completionTimeout": "PT20M", "consumerProperties": { "bootstrap.servers": "localhost:9092" } } } But no druid datasource is created, I also tried quering druid with below cmd and json, It hase no data in it INPUT:curl -X 'POST' -H 'Content-Type:application/json' -d @examples/test1.json http://localhost:8086/druid/v2?pretty OUTPUT:[ ] test1.json : { "queryType" : "topN", "dataSource" : "test-sqlite-jdbc-manoj1", "intervals" : ["2015-09-12/2019-09-13"], "granularity" : "all", "dimension" : "name1", "metric" : "count", "threshold" : 10, "aggregations" : [ { "type" : "count", "name" : "count" } ] }