How do I make an interactive video where an object's data displays on the screen when a user clicks the object?

Question

I'm trying to display object data on a video when a user clicks on the object in the video but having some difficulty. I'm using a MERN stack in addition to Google Vision API and have the data in my MongoDB and can GET the data via Postman but when I click on the object in the browser a fetch request isn't made, nothing happens. I was thinking that I'm going to have to get the position of the object in each frame and make the necessary correspondences but I'm not too sure. Can't seem to figure it out so if anyone has suggestions I'd be happy to hear them!

Thanks!

Client

import React, { useEffect, useState } from 'react';

function VideoPlayer() {
  const [objects, setObjects] = useState([]);
  const [selectedObject, setSelectedObject] = useState(null);
  const [videoUrl, setVideoUrl] = useState('');
  const [error, setError] = useState(null);
  const [isLoading, setIsLoading] = useState(true);

  // Fetch object data when an object is clicked
  function fetchObjectData(objectId) {
    // Set loading state while fetching
    setIsLoading(true);

    // Fetch detected object data for the specific object
    fetch(`http://localhost:3001/annotation/${objectId}`)
      .then(response => {
        if (!response.ok) {
          throw new Error(`HTTP error! Status: ${response.status}`);
        }
        return response.json(); // Assuming the response contains JSON data
      })
      .then(data => {
        setSelectedObject(data); // Update selectedObject with the fetched data
        setIsLoading(false); // Clear loading state
      })
      .catch(error => {
        console.error('Error fetching object data:', error);
        setIsLoading(false); // Clear loading state on error
      });
  }

  useEffect(() => {
    // Fetch video URL from the backend
    fetch('http://localhost:3001/videourl')
      .then(response => {
        if (!response.ok) {
          throw new Error(`HTTP error! Status: ${response.status}`);
        }
        const contentType = response.headers.get('content-type');
        if (contentType && contentType.startsWith('video')) {
          return response.blob();
        } else {
          console.error('Unexpected response type:', contentType);
          setError(new Error('Unexpected response type'));
        }
      })
      .then(videoBlob => {
        setVideoUrl(URL.createObjectURL(videoBlob));
      })
      .catch(error => {
        console.error('Error fetching video URL:', error);
        setError(error);
      });
      
  }, []);

  function handleObjectClick(objectId) {
    console.log('Clicked object ID:', objectId);
    // Clear selectedObject when a new object is clicked
    setSelectedObject(null);
    // Fetch object data when an object is clicked
    fetchObjectData(objectId);
  }

  return (
    <div>
      {error ? (
        <div>Error: {error.message}</div>
      ) : (
        <>
          <video
            src={videoUrl}
            muted
            autoPlay
            playsInline
            loop
            style={{ width: '100%', height: '300pt' }}
          />
          <div className="annotationsContainer">
            {isLoading ? (
              <div>Loading data...</div>
            ) : (
              objects.map((object, index) => (
                <div
                  key={index}
                  className="annotation"
                  onClick={() => handleObjectClick(object.id)}
                >
                  <strong>{object.entity.description}</strong>
                  <br />
                  Confidence: {object.confidence}
                </div>
              ))
            )}
          </div>
          {selectedObject && (
            <div className="selectedObject">
              <strong>Selected Object:</strong> {selectedObject.entity.description}
              <br />
              Confidence: {selectedObject.confidence}
            </div>
          )}
        </>
      )}
    </div>
  );
}

export default VideoPlayer;

Server and Route

const express = require('express');
const router = express.Router();
const {ObjectModel} = require('../../model/index');


router.get('/:objectId', async (req, res) => {
  const objectId = req.params.objectId;

  try {
    // Use the objectId to retrieve the specific object from the database
    const object = await ObjectModel.findById(objectId);
    if (!object) {
      return res.status(404).json({ error: 'Object not found' });
    }

    res.json(object);
  } catch (error) {
    console.error('Error fetching object:', error);
    res.status(500).json({ error: 'Internal Server Error' });
  }
});

module.exports = router;

const router = require('express').Router();
const annotationRoutes = require('./api/annotationroute');
const videoURLRoutes = require('./api/videourlroute');

router.use('/annotation', annotationRoutes);
router.use('/videourl', videoURLRoutes);

router.use((req, res) => {
  return res.send('Wrong route!');
});

module.exports = router;

Model

const { Schema, model } = require('mongoose');

const objectSchema = new Schema({

    objectDescription: {
        type: String,  
    },
    objectEntityId: {
        type: String,
    },
    objectStartTime: {
        type: String,
    },
    objectEndTime: {
        type: String,
    },
    objectConfidence: {
        type: Number,
    },
   
});

const ObjectModel = model('Object', objectSchema);

module.exports = ObjectModel;

I've tried making various console.logs but I'm not getting any errors and the Network tab isn't telling me anything either- fetch request aren't being made. I've tried searching for Google vision API demos for making clickable videos on Youtube as well as searched documentation but nothing's turned up.