Using npm librdkafka i am producing large volume of data to the Kafka. I am using HighlevelProducer. For 50K messages this configuration is working fine. But for larger volume like 100K messages, there is data loss. Data loss is varying from 200-1K based on configurations. Tried various configurations but unable to find best configuration to achieve 100% accuracy. any suggestions are helpful. Below is my configuration.
var producerConfigOptions = {
'request.required.acks': requireAcknowledge,
'client.id': producerClientId,
'compression.codec': 'gzip',
'metadata.broker.list': msgSystemConnectionString,
'security.protocol': 'ssl',
'ssl.key.location': clintCertKeyFile,
'ssl.certificate.location': clientCertFile,
'ssl.ca.location': caFile,
'retry.backoff.ms': 200,
'message.send.max.retries': 10,
'socket.keepalive.enable': true,
'queue.buffering.max.messages': 50000,
'queue.buffering.max.ms': 100,
'batch.num.messages': 50000,
'dr_cb': true,
}
var ProducerInstance= new Kafka.HighLevelProducer(producerConfigOptions);
ProducerInstance.connect({}, (err) => {
if(err)
{
console.log(error)
}
else
{
ProducerInstance.produce(topicName, lastUsedPartition, Buffer.from(msg), msgKey, Date.now(), (err, offset) => {
//logic to store offset
}
}
}