Make sure BullMQ queue pushes data to single node in redis cluster

Question

I've integrated BullMQ into our Node.js application, leveraging Redis Cluster for production. According to the BullMQ and Redis Cluster documentation I understood that we can give some prefix in key while creating queue in bullMQ which internally takes CRC16 of the key modulo 16384 (https://docs.bullmq.io/bull/patterns/redis-cluster) but this doesn't give us control data to decide destination node which can result in hot shard if hash of 2 queues prefix falls in same node

Is there any way to ensure that each queue in BullMQ pushes data to predefined Redis nodes within the cluster?

Below is my demo code:

const Redis = require('ioredis');
const { Queue, Worker } = require('bullmq');

const clusterNodes = [
  { port: 6380, host: "127.0.0.1" },
  { port: 6381, host: "127.0.0.1" },
  { port: 6382, host: "127.0.0.1" }
];

const cluster = new Redis.Cluster(clusterNodes);

async function bullMqTest() {

  const queue1 = new Queue(`myQueue1`, {
    connection: cluster,
    prefix: "{queue_1}",
  });
  const queue2 = new Queue(`myQueue2`, {
    connection: cluster,
    prefix: "{queue_2}",
  });
  const queue3 = new Queue(`myQueue3`, {
    connection: cluster,
    prefix: "{queue_3}",
  });
  const queue4 = new Queue(`myQueue4`, {
    connection: cluster,
    prefix: "{queue_4}",
  });
  const queue5 = new Queue(`myQueue5`, {
    connection: cluster,
    prefix: "{queue_5}",
  });
  const queue6 = new Queue(`myQueue6`, {
    connection: cluster,
    prefix: "{queue_6}",
  });


  queue1.add('myJob', { foo: 'bar' });
  queue2.add('myJob', { foo: 'baz' });
  queue3.add('myJob', { foo: 'qux' });
  queue4.add('myJob', { foo: 'xyz' });
  queue5.add('myJob', { foo: 'lmn' });
  queue6.add('myJob', { foo: 'qpr' });
}

module.exports = { pushDataToRedis, fetchDataFromRedis, bullMqTest };

Answer 1

This will meet your requirement using multiple hosts.

async function bullMqTest() {

  const redis1 = new Redis(host1);
  const queue1 = new Queue(`myQueue1`, {
    connection: redis1 ,
    prefix: "{queue_1}",
  });

  const redis2 = new Redis(host2);
  const queue2 = new Queue(`myQueue2`, {
    connection: redis2 ,
    prefix: "{queue_2}",
  });

  const redis3 = new Redis(host3);
  const queue3 = new Queue(`myQueue3`, {
    connection: redis3 ,
    prefix: "{queue_3}",
  });
  .....