They are hosted on AWS and I dont think there are any network issues because I tried testing other Queuing systems with no issues however I am using a node.js client with the following code. I am not sure if there are any errors or anything I didn't set in the following code?
//producer var kafka = require('kafka-node'); var Producer = kafka.Producer; var Client = kafka.Client; var client = new Client('172.31.21.175:2181'); var argv = require('optimist').argv; var topic = argv.topic || 'kafka_test'; var p = argv.p || 0; var a = argv.a || 0; var producer = new Producer(client, { requireAcks: 1}); var num = 300005; producer.on('ready', function () { var message = 'Hello World'; for (var i=0; i<num; i++) { producer.send([ { topic: topic, partition: p, messages: message, attributes: a } ], function (err, result) { console.log(err || result); //process.exit(); }); } }); producer.on('error', function (err) { console.log('error', err); process.exit(); }); //Consumer var kafka = require('kafka-node'); var Consumer = kafka.Consumer; var Offset = kafka.Offset; var Client = kafka.Client; var argv = require('optimist').argv; var topic = argv.topic || 'kafka_test'; var client = new Client('172.31.21.175:2181'); var topics = [ {topic: topic, partition: 0} ]; var options = { autoCommit: false, fetchMaxWaitMs: 1000 }; var consumer = new Consumer(client, topics, options); var offset = new Offset(client); var start; var received = 0; var target = 200000; var hash = 1000; consumer.on('message', function (message) { console.log(message); received += 1; if (received === 1) { start = new Date(); } if (received === target) { var stop = new Date(); console.log('\nDone test'); var mps = parseInt(target/((stop-start)/1000)); console.log('Received at ' + mps + ' msgs/sec'); process.exit(); } else if (received % hash === 0){ process.stdout.write(received + '\n'); } }); consumer.on('error', function (err) { console.log('error', err); });
Not using Mixmax yet? On Wed, Sep 14, 2016 11:58 PM, Ali Akhtar ali.rac...@gmail.com wrote: It sounds like a network issue. Where are the 3 servers located / hosted? On Thu, Sep 15, 2016 at 11:51 AM, kant kodali <kanth...@gmail.com> wrote:
Hi,
I have the following setup.
Single Kafka broker and Zookeeper on Machine 1single Kafka producer on
Machine 2
Single Kafka Consumer on Machine 3
When a producer client sends a message to the Kafka broker by pointing at
the
Zookeeper Server the consumer doesn't seem to get the message right away
instead
it gets after a minute or something (pretty late). I am not sure what
settings I
need to change. any ideas?
Thanks,kant