How do I send my Desktop Screen over the net through a python script and display it in a unity app?

45 views Asked by At

I am trying to create a python script that sends the feed of my desktop through the net to my unity app and trying to display it on a plane but I get this error.

Error: ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine

import socket
import cv2
import numpy as np
from PIL import ImageGrab

# Set up the server
server_ip = 'Insert IP Here'  
server_port = Insert Port Here  

# Create a socket object
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

# Bind the socket to the address
server_socket.bind((server_ip, server_port))

# Listen for incoming connections
server_socket.listen(1)

print("Waiting for Unity application to connect...")

# Accept incoming connection
connection, address = server_socket.accept()
print("Connected to Unity application at", address)

# Function to capture desktop frames and send them over the network
def send_frames():
    try:
        while True:
            # Capture desktop frame
            frame = capture_screen()

            # Send frame size
            connection.sendall(len(frame).to_bytes(4, byteorder='big'))

            # Send frame data
            connection.sendall(frame)

    except KeyboardInterrupt:
        print("Stopped sending frames.")
        connection.close()

def capture_screen():
    # Capture desktop screen using OpenCV
    screen = cv2.cvtColor(np.array(ImageGrab.grab()), cv2.COLOR_RGB2BGR)
    
    # Convert frame to bytes
    _, frame_encoded = cv2.imencode('.jpg', screen)
    return frame_encoded.tobytes()

if __name__ == "__main__":
    send_frames()
using UnityEngine;
using System;
using TMPro;
using System.Net;
using System.Net.Sockets;
using System.Threading.Tasks;

public class ScreenMirror : MonoBehaviour
{
    public Material planeMaterial;
    public string serverIp = "Insert IP Here";
    public TMP_Text networkStatusText;

    private TcpClient client;
    private NetworkStream stream;
    private bool receiving = false;
    private const int bufferSize = 1024 * 1024; // 1 MB buffer size

    private async void Start()
    {
        try
        {
            client = new TcpClient();
            await client.ConnectAsync(serverIp, Insert Port Here);
            stream = client.GetStream();
            receiving = true;
            networkStatusText.text = "Connected";

            // Start receiving frames asynchronously
            await ReceiveFramesAsync();
        }
        catch (Exception e)
        {
            Debug.LogError("Error connecting to server: " + e.Message);
            networkStatusText.text = "Error: " + e.Message;
        }
    }

    private async Task ReceiveFramesAsync()
    {
        try
        {
            byte[] buffer = new byte[bufferSize];

            while (receiving)
            {
                // Read frame size
                int bytesRead = await stream.ReadAsync(buffer, 0, 4);
                if (bytesRead != 4)
                {
                    Debug.LogWarning("Incomplete frame size received.");
                    break;
                }

                int frameSize = BitConverter.ToInt32(buffer, 0);

                // Read frame data
                bytesRead = 0;
                while (bytesRead < frameSize)
                {
                    bytesRead += await stream.ReadAsync(buffer, bytesRead, frameSize - bytesRead);
                }

                // Convert frame data to texture
                Texture2D texture = new Texture2D(1, 1);
                texture.LoadImage(buffer);

                // Apply texture to plane material
                planeMaterial.mainTexture = texture;
            }
        }
        catch (Exception e)
        {
            Debug.LogError("Error receiving frames: " + e.Message);
            networkStatusText.text = "Error: " + e.Message;
        }
        finally
        {
            if (stream != null)
                stream.Close();
            if (client != null)
                client.Close();
            networkStatusText.text = "Disconnected";
        }
    }
}

I tried creating inbound and outbound rules for my port but it did not affect the error message.

0

There are 0 answers