Conference keeps participant in call if they reject and have a 'leave after tone message'

Question

I am implementing a call transfer functionality in the frontend. In my case, whenever a transfer is needed, I add another participant and leave myself. However, I noticed that if the other party rejects the call, the call is still considered to be in progress. I need the call to be marked as completed when the other party rejects it. Is there any better way to implement call transfer than using the conference feature?

router.post("/testEndpointTwilio", async (req, res) => {
  try {
    const isTransfer = req.body.isTransfer;
    if (isTransfer) {
      const number = req.body.To;
      const confName = req.body.confName;

      console.log("\n\nisTransfer status", isTransfer, "\n\n");
      const response = new VoiceResponse();
      const dial = response.dial();
      dial.conference(confName, {
        startConferenceOnEnter: true,
        endConferenceOnExit: false,
      });
      add_to_conf(
        number,
        confName,
        "sehej_customer" + new Date().getTime().toString(),
      );
      res.set("Content-Type", "text/xml");
      res.send(response.toString());
    } else {
      // yet to write
    }
  } catch (err) {
    console.log(err);
    res.status(403).json({ data: "Internal Server Error", success: false });
  }
});

let confMap = new Map();

function add_to_conf(contact, conf_name, label) {
  twilioClient
    .conferences(conf_name)
    .participants.create({
      label: label,
      from: "+61483947285",
      to: contact,
      startConferenceOnEnter: false,
      endConferenceOnExit: true,
      statusCallback: "{my_live_api_path}/handleStatusCallBack",
      statusCallbackMethod: "POST",
      statusCallbackEvent: ["initiated", "ringing", "answered", "completed"],
    })
    .then((participant) => {
      confMap.set(participant.callSid, conf_name);
      console.log(participant.callSid);
    })
    .done();
}

router.post("/handleStatusCallBack", function (req, res) {
  const status = req.body.CallStatus;
  const callSid = req.body.CallSid;
  let confName = null;

  if (confMap.has(callSid)) {
    try {
      confName = confMap.get(callSid);
    } catch (err) {
      console.log("\n\nerror getting confName, ", err, "\n\n");
    }
  } else {
    console.error("Key does not exist in Map");
  }

  console.log("\n\n\nhandleStatus data", status, callSid, confName);

  if (status === "initiated" || status === "ringing") {
    console.log("\n\nCall Initiated or Ringing...\n\n");
  } else if (status === "answered") {
    console.log("\n\nCall Answered.\n\n");
  } else if (
    status === "completed" ||
    status === "busy" ||
    status === "no-answer" ||
    status === "failed" ||
    status === "canceled"
  ) {
    console.log("\n\nCall Ended.\n\n");

    twilioClient
      .conferences(confName)
      .participants(callSid)
      .remove()
      .then((participant) =>
        console.log("\n\nParticipant removed from conference\n\n"),
      )
      .catch((err) =>
        console.log("error in removing participant\n\n", err, "\n\n"),
      );
  }

  res.set("Content-Type", "text/xml");
  res.sendStatus(200);
});

I haven't really been able to solve this one.