Slow serial communication with Arduino and Python

Question

In a project of mine, I have to take a picture from a camera connected to a Sony Spresense Arduino board which is linked to my computer via an USB port. I am not very experienced in Arduino and serial communication, not at all in fact, so I am asking for some help.

I want to read the data from the camera which is sent in hexadecimal from the Arduino to my computer via a Serial.begin at a given baud rate, processed by a Python program in order to collect the hex code, correct it (there have been some print errors which are now solved), and convert it to a JPEG image. I am able to do it, but the serial communication part of my program where Python collects the data from the Arduino is significantly slow, it takes 34 seconds to obtain an image that weighs "only" 100 ko (I don't know if it's a lot to handle for an Arduino), at a baud rate of 115200. If I try to increase this number, the hex code collected shows some errors and the image can not be converted. I can also change the pixel resolution of the image, but I'd like to be able to work with HD pictures.

In detail, here is the code from the Arduino, I have found no other way than this to get the data from the camera (there is no designed function in the Spresense library for serial communication). The relevant part is at the end :

#include <SDHCI.h>
#include <stdio.h>  /* for sprintf */

#include <Camera.h>

#define BAUDRATE                (115200)

/**
 * Print error message
 */

void printError(enum CamErr err)
{
  Serial.print("Error: ");
  switch (err)
    {
      case CAM_ERR_NO_DEVICE:
        Serial.println("No Device");
        break;
      case CAM_ERR_ILLEGAL_DEVERR:
        Serial.println("Illegal device error");
        break;
      case CAM_ERR_ALREADY_INITIALIZED:
        Serial.println("Already initialized");
        break;
      case CAM_ERR_NOT_INITIALIZED:
        Serial.println("Not initialized");
        break;
      case CAM_ERR_NOT_STILL_INITIALIZED:
        Serial.println("Still picture not initialized");
        break;
      case CAM_ERR_CANT_CREATE_THREAD:
        Serial.println("Failed to create thread");
        break;
      case CAM_ERR_INVALID_PARAM:
        Serial.println("Invalid parameter");
        break;
      case CAM_ERR_NO_MEMORY:
        Serial.println("No memory");
        break;
      case CAM_ERR_USR_INUSED:
        Serial.println("Buffer already in use");
        break;
      case CAM_ERR_NOT_PERMITTED:
        Serial.println("Operation not permitted");
        break;
      default:
        break;
    }
}

void CamCB(CamImage img)
{

  /* Check the img instance is available or not. */

  if (img.isAvailable())
    {

      /* If you want RGB565 data, convert image data format to RGB565 */

      img.convertPixFormat(CAM_IMAGE_PIX_FMT_RGB565);

    }
  else
    {
      Serial.print("Failed to get video stream image\n");
    }
}

/**
 * @brief Initialize camera
 */
void setup()
{
  CamErr err;

  /* Open serial communications and wait for port to open */

  Serial.begin(BAUDRATE);
  while (!Serial)
    {
      ; /* wait for serial port to connect. Needed for native USB port only */
    }


  /* begin() without parameters means that
   * number of buffers = 1, 30FPS, QVGA, YUV 4:2:2 format */

  Serial.println("Prepare camera");
  err = theCamera.begin();
  if (err != CAM_ERR_SUCCESS)
    {
      printError(err);
    }

  /* Start video stream.
   * If received video stream data from camera device,
   *  camera library call CamCB.
   */

  Serial.println("Start streaming");
  err = theCamera.startStreaming(true, CamCB);
  if (err != CAM_ERR_SUCCESS)
    {
      printError(err);
    }

  /* Auto white balance configuration */

 // Serial.println("Set Auto white balance parameter");
  err = theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_DAYLIGHT);
  if (err != CAM_ERR_SUCCESS)
    {
      printError(err);
    }

  /* Set parameters about still picture.
   * In the following case, QUADVGA and JPEG.
   */

  Serial.println("Set still picture format");
//  err = theCamera.setStillPictureImageFormat(
//     CAM_IMGSIZE_QUADVGA_H,
//     CAM_IMGSIZE_QUADVGA_V,
//     CAM_IMAGE_PIX_FMT_JPG);
    //err = theCamera.setStillPictureImageFormat(320, 240, CAM_IMAGE_PIX_FMT_JPG);
    err = theCamera.setStillPictureImageFormat(CAM_IMGSIZE_QUADVGA_H, CAM_IMGSIZE_QUADVGA_V, CAM_IMAGE_PIX_FMT_JPG);
  if (err != CAM_ERR_SUCCESS)
    {
      printError(err);
    }


/**
 * @brief Take picture with format JPEG per second
 */
/******** Affichage serie ********/
/* une ligne de vide et l'image */
  Serial.println(" ");
  CamImage img = theCamera.takePicture();

  /* Check availability of the img instance. */
  /* If any error was occured, the img is not available. */

  if (img.isAvailable())
    {
        /*Indicateur de debut img : FFD8FF (Magic number of the jpeg format) */      

       for(int i=0;i<img.getImgSize();i++)
       {
        Serial.print(*(img.getImgBuff()+i),HEX); //img.getImgBuff() gets the data address of the picture, so the * before.
        Serial.print(";");
       }


        /*End indicator : FFD9FF (Magic number of jpeg format) */

    }

}

void loop() 
{

  // put your main code here, to run repeatedly:
  sleep(1);
}

And here is the Python code :

## Serial collecting the data of the picture taken by the camera

import serial
from serial import Serial

import binascii
import string

from PIL import Image

import time
start_time = time.time()



ser = serial.Serial('COM3', baudrate=115200, timeout=1)

# writing the data in a text file
data = open("data.txt", "w")
data.write(str(ser.readlines()))

## Correcting the data

data = open("data.txt", "r")

string = str(data.readlines())


# spliting the string into a list
# I chose to use the ";" to split the different hex couples in the Arduino program
tab=string.split(";")

tab[0] = 'FF'
tab.pop(-1)

N = len(tab)

# correcting the arguments that are not couples :
# Indeed, when Arduino encounter a couple starting with a 0,
# it omits it, so I have to add it back manually
for i in range(N):
    if len(tab[i]) == 1:
        tab[i] = '0' + tab[i]


newdata = open("newdata.txt", "w")

# writing the new data in a text file
for s in tab:
    newdata.write(s)

newdata.close()
data.close()

## Converting the hex data into a JPEG file

file = open("newdata.txt", "r")
data= file.read()

# conversion
data = binascii.a2b_hex(data)

# creation of the JPEG file
with open('imagenew.jpg', 'wb') as image_file:
    image_file.write(data)


file.close()


img = Image.open('imagenew.jpg')
img.show()

print("--- %s seconds ---" % (time.time() - start_time))

If you have any idea or any advice in order to speed up this process I am taking it. I heard that there are some flow control and buffer stories here and there, but I am quite lost when I try to find something relevant to my situation. Thanks in advance.