Mongoose aggregation not returning latest document

Question

I have an aggregation that returns a document, but I need it to return a document that is the latest one in the collection messageTrackingId. The one that has the most recent value in the creationDate field I have a sort to accomplish this, but I keep getting returned documents that aren't the latest and I'm not sure why. The purpose is to show the user the time stamp of the last message sent between that user and another user regardless of who sent it. appreciate any help.

app.get("/api/messages", (req, res, next) => {
  query = {};
  inbox = false;
  messageId = false;
  console.log(req.query.recipientId);
  if (req.query.recipientId) {
    query = { recipientId: req.query.recipientId };
    inbox = true;

    Messages.aggregate([
      {
        $match: {
          $or: [
            { recipientId: req.query.recipientId },
            { creator: mongoose.Types.ObjectId(req.query.recipientId) }
          ]
        }
      },
      {
        $addFields: {
          conversant: {
            $cond: [
              { $ne: ["$recipientId", req.query.recipientId] },
              "$recipientId",
              "$creator"
            ]
          }
        }
      },
      {
        $sort: { creationDate: 1 }
      },
      {
        $group: {
          _id: "$conversant",
          message: { $last: "$message" },
          recipientId: { $last: "$recipientId" },
          creator: { $last: "$creator" },
          messageTrackingId: { $last: "$messageTrackingId" },
          recipient: { $last: "$recipient" },
          creationDate: { $last: "$creationDate" }
        }
      },
      {
        $lookup: {
          from: "users",
          let: { creator: "$creator" },
          pipeline: [
            { $match: { $expr: { $eq: ["$_id", "$$creator"] } } },
            { $project: { instagramName: 1 } }
          ],
          as: "creatorName"
        }
      }
    ]).then(documents => {


      if (inbox === true) {
        res.status(200).json({
          message: "User's Inbox Retrieved!",
          posts: documents
        });
      }

    });

I cleared out the message table and sent 3 messsages back and forth to demonstrate. When I reload the page, I only receive 2 of the docs in the JSON response instead of the latest one. Of the 2 received, none of them seem to be the latest either. I'm not sure why it's not returning one. Please see output on page load below.

Messages :

/* 1 */
{
    "_id" : "5e16463be5fcba1d2c56ebb7",
    "creator" : "5e0a1d41f86e1e4234fc1a77",
    "recipient" : "andy",
    "recipientId" : "5df0014e25ee451beccf588a",
    "message" : "First message",
    "messageTrackingId" : "17c4fc99-d92f-46ef-9bae-1408f415287a",
    "creatorName" : "buyer1",
    "creationDate" : ISODate("2020-01-08T21:14:35.104Z"),
    "__v" : 0.0
}

/* 2 */
{
    "_id" : "5e16464de5fcba1d2c56ebb9",
    "creator" : "5df0014e25ee451beccf588a",
    "recipient" : "buyer1",
    "recipientId" : "5e0a1d41f86e1e4234fc1a77",
    "message" : "Second Message",
    "messageTrackingId" : "17c4fc99-d92f-46ef-9bae-1408f415287a",
    "creatorName" : "andy",
    "creationDate" : ISODate("2020-01-08T21:14:53.618Z"),
    "__v" : 0.0
}

/* 3 */
{
    "_id" : "5e16465ce5fcba1d2c56ebbb",
    "creator" : "5e0a1d41f86e1e4234fc1a77",
    "recipient" : "andy",
    "recipientId" : "5df0014e25ee451beccf588a",
    "message" : "Third Message",
    "messageTrackingId" : "17c4fc99-d92f-46ef-9bae-1408f415287a",
    "creatorName" : "buyer1",
    "creationDate" : ISODate("2020-01-08T21:15:08.500Z"),
    "__v" : 0.0
}

users :

/* 1 */
{
    "_id" : "5df00d08c713f722909c99c1",
    "email" : "[email protected]",
    "password" : "$2b$10$5eWhwL4RNT8TyQRCC191J.myYBwFcJ8hCARqGfofYWUmRaq6jJFQm",
    "instagramName" : "joe",
    "over21" : "Yes",
    "role" : "Artist",
    "fullName" : "testdsfdfsfs",
    "address1" : "adfsdf",
    "address2" : "dssdf",
    "city" : "sdfsdfsdf",
    "state" : "dsffsdsd",
    "zip" : "dsdfsdf",
    "passwordCreated" : ISODate("2020-01-07T23:46:57.861Z"),
    "__v" : 0
}

/* 2 */
{
    "_id" : "5df0014e25ee451beccf588a",
    "email" : "[email protected]",
    "password" : "$2b$10$fNHmIoDb4mX6x.YeMLXHbu2yIaeW6HVQvNpZR8Nt/a4xVFkhxM1Ey",
    "instagramName" : "andy",
    "over21" : "Yes",
    "role" : "Artist",
    "fullName" : "dsfdsfdfdsf",
    "address1" : "111",
    "address2" : "111",
    "city" : "atlanta",
    "state" : "ga",
    "zip" : "33222",
    "passwordCreated" : ISODate("2019-12-19T15:29:46.528Z"),
    "__v" : 0
}

Required output (Return last message from each user) :

/* 1 */
{
    "_id" : {
        "conversant" : "5df0014e25ee451beccf588a",
        "recipient" : "andy"
    },
    "message" : "Third Message",
    "recipientId" : "5df0014e25ee451beccf588a",
    "creator" : "5e0a1d41f86e1e4234fc1a77",
    "messageTrackingId" : "17c4fc99-d92f-46ef-9bae-1408f415287a",
    "recipient" : "andy",
    "creationDate" : ISODate("2020-01-08T21:15:08.500Z"),
    "creatorName" : [ 
        {
            "_id" : "5df0014e25ee451beccf588a",
            "instagramName" : "andy"
        }
    ]
}

/* 2 */
{
    "_id" : {
        "conversant" : "5df0014e25ee451beccf588a",
        "recipient" : "buyer1"
    },
    "message" : "Second Message",
    "recipientId" : "5e0a1d41f86e1e4234fc1a77",
    "creator" : "5df0014e25ee451beccf588a",
    "messageTrackingId" : "17c4fc99-d92f-46ef-9bae-1408f415287a",
    "recipient" : "buyer1",
    "creationDate" : ISODate("2020-01-08T21:14:53.618Z"),
    "creatorName" : [ 
        {
            "_id" : "5df0014e25ee451beccf588a",
            "instagramName" : "andy"
        }
    ]
}

Answer 1

Changes has to be

1) If I understand correctly at this stage you wanted to group by recipientId on a field(since you can't group on two fields for a single value) So you're creating a new field called conversant, So it has to be $eq or you need to swap recipientId & creator & also it has to be $toString:

{
      $addFields: {
        conversant: {
          $cond: [
            { $eq: ["$recipientId", req.query.recipientId] },
            "$recipientId",
            {$toString : "$creator"} 
          ]
        }
      }
    }

2) You don't need this $sort stage, as all latest additions to collection will be added at the end (Expecting creationDate is once created but never updated) :

// Not useful
{
     $sort: { creationDate: 1 }
}

Also you need to group by _id: { conversant: "$conversant", recipient: '$recipient' } for group on last messages based on user. In $lookup change to this let: { conversant: "$_id.conversant" }.

Try this :

 Messages.aggregate([
    {
        $match: {
            $or: [
                { recipientId: req.query.recipientId },
                { creator: mongoose.Types.ObjectId(req.query.recipientId) }
            ]
        }
    },
    {
        $addFields: {
            conversant: {
                $cond: [
                    { $eq: ["$recipientId", req.query.recipientId] },
                    "$recipientId",
                    { $toString: "$creator" }
                ]
            }
        }
    },
    {
        $group: {
            _id: { conversant: "$conversant", recipient: '$recipient' },
            message: { $last: "$message" },
            recipientId: { $last: "$recipientId" },
            creator: { $last: "$creator" },
            messageTrackingId: { $last: "$messageTrackingId" },
            recipient: { $last: "$recipient" },
            creationDate: { $last: "$creationDate" }
        }
    },
    {
        $lookup: {
            from: "users",
            let: { conversant: "$_id.conversant" },
            pipeline: [
                { $match: { $expr: { $eq: ["$_id", "$$conversant"] } } },
                { $project: { instagramName: 1 } }
            ],
            as: "creatorName"
        }
    },
     // this is optional only if you wanted to return latest document irrespective of user.
     { $sort: { creationDate: -1 } }
])