Filter and Hooks caused to re-render issue

Question

I have a file system that I am building with tus. So for each file upload, I will have an array with ReactElement which will kickstart the uploading process. Adding an element into the array has no issue, but what I faced is the issue of deleting the upload if say I uploaded a file and wish to remove it from the array. Using filter() will create a new array which cause my ReactElement to re-render = re-upload the folder. I have tried using splice method which doesn't work as well. Please advise.

const [files, addFile] = useState<JSX.Element>([]);
const startUpload = (e: React.ChangeEvent<HTMLInputElement>): void => {
    // some code here
    addFile([...files, <File/>]
    // <File delete={deleteUpload}/> contains the tus uploading process
}

const deleteUpload = (name: string): void => {
    // Using filter() method
    addFile(prevState => prevState.filter(file => file.name !== name))
    
    // Using splice
    let temp = [...files];
    for(let i=0; i<temp.length; i++){
        if(temp[i].name === name){
             temp.splice(i,1)
        }
    }
    addFile(temp)

    // Both filter() and splice() does not work as filter() cause my <File/> to
    // re-render within the array and splice() will delete everything below
}

// Further edit based on the comments The <File/> looks similar to

const File = (props) => {
    useEffect(()=>{

    let uploadOptions: any = {
          endpoint: 'http://localhost:5000/files',
          chunkSize: 8 * 1024 * 1024,
          resume: true,
          retryDelays: [0, 3000, 5000, 10000, 20000],
          headers: {
            filename: file.name,
            filetype: file.type,
          },
          metadata: {
            filename: file.name,
            filetype: file.type,
          },
          onError(error: any) {
            console.log(`Failed because: ${error}`);
          },
          onProgress(bytesUploaded: number, bytesTotal: number) {
            const percentage = ((bytesUploaded / bytesTotal) * 100).toFixed(2);
            console.log(bytesUploaded, bytesTotal, percentage + "%")
      },
      onSuccess() {
        console.log("Download %s from %s", upload.file.name, upload.url)
      },
    };

        // Create a new tus upload
        upload = new tus.Upload(file, uploadOptions);
        upload.start() // starting the upload process the moment I drop a file in
    },[])

    return (<> some progress bars and stop buttons to stop pause upload </>)
}