MBEDTLS_ERR_NET_INVALID_CONTEXT Error in mbedtls. Epoll gives EPOLLIN event notification when no data is available to read with TLS

Question

I made a epoll HTTPS server with mbedtls. It listens on two ports: one for HTTP and the other is for HTTPS. The HTTP one works perfectly well. But for HTTPS I seem to get EPOLLIN notification when no data is available to read with mbedtls_ssl_read().

It could be that the data is available, but the problem is with the SSL context, since mbedtls_ssl_read() returns MBEDTLS_ERR_NET_INVALID_CONTEXT. But I can't see where the problem could originate from. I have correctly initialized the context and even made a TLS handshake with it.

Here are the important parts of the code: Note that I understand this code will fail in a lot of cases. I just made an example to reproduce the error. This code just describes everything that goes on when I accept I single client and then, after receiving EPOLLIN, try to read data from it.

#define REQUEST_SIZE 8192

struct HTTP_server
{
    mbedtls_net_context https_context;
    mbedtls_entropy_context entropy;
    mbedtls_ctr_drbg_context ctr_drbg;
    mbedtls_ssl_config conf;
    mbedtls_x509_crt srvcert;
    mbedtls_pk_context pkey;
    const char *pers;
};

typedef struct client_data 
{
    char *request;
    proto_t protocol;
    union
    {
        int client_fd;
        mbedtls_net_context https_context;
    };
    mbedtls_ssl_context ssl;

} client_data_t;

struct HTTP_server server;

server->pers = "ssl_server";

mbedtls_net_init(&server->https_context);
mbedtls_ssl_config_init(&server->conf);

mbedtls_x509_crt_init(&server->srvcert);
mbedtls_pk_init(&server->pkey);
mbedtls_entropy_init(&server->entropy);
mbedtls_ctr_drbg_init(&server->ctr_drbg);

if (mbedtls_ctr_drbg_seed(&server->ctr_drbg, mbedtls_entropy_func, &server->entropy,
        (const unsigned char *) server->pers, strlen(server->pers)) != 0) {
    return 0;
}

if (mbedtls_x509_crt_parse_file(&server->srvcert, "server.crt") != 0)
{
    return 0;
}

if (mbedtls_pk_parse_keyfile(&server->pkey, "server.key", "pass_phrase", mbedtls_ctr_drbg_random, &server->ctr_drbg) != 0)
{
    return 0;
}

if (mbedtls_net_bind(&server->https_context, NULL, "8000", MBEDTLS_NET_PROTO_TCP) != 0)
{
    return 0;
}

mbedtls_net_set_nonblock(&server->https_context);

if (mbedtls_ssl_config_defaults(&server->conf, MBEDTLS_SSL_IS_SERVER, MBEDTLS_SSL_TRANSPORT_STREAM, 
    MBEDTLS_SSL_PRESET_DEFAULT) != 0)
{
    return 0;
}

mbedtls_ssl_conf_rng(&server->conf, mbedtls_ctr_drbg_random, &server->ctr_drbg);

if (mbedtls_ssl_conf_own_cert(&server->conf, &server->srvcert, &server->pkey) != 0)
{
    return 0;
}

int epoll_fd = epoll_create1(0);
    
if (epoll_fd == -1) 
{
    return;
}

event.data.fd = server->https_context.fd;
event.events = EPOLLIN;

if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, server->https_context.fd, &event) == -1) 
{
    goto epoll_close;
}

while (1)
{
    int nevents = epoll_wait(epoll_fd, events, 128, -1);
    if (nevents == -1) 
    {
        goto epoll_close;
    }
    else if (errno == EINTR)
    {
        goto epoll_close;
    }
    for (int i = 0; i < nevents; ++i) 
    {
         if ((events[i].events & (EPOLLERR | EPOLLHUP))) 
         {
             continue;
         } 
         else if (events[i].data.fd == server->https_context.fd)
         {
              client_data_t client_status = malloc(sizeof(client_data_t));

              mbedtls_net_context client_socket;
              mbedtls_net_accept(&server->https_context, &client_socket, NULL, 0, NULL);

              client_status->https_context = client_socket;
              mbedtls_ssl_init(&client_status->ssl);

              if (mbedtls_ssl_setup(&client_status->ssl, &server->conf) != 0)
              {
                  return 0;
              }

              mbedtls_ssl_set_bio(&client_status->ssl, &client_socket, mbedtls_net_send, mbedtls_net_recv, NULL);

              int ret;
              while (!program_interrupted && ((ret = mbedtls_ssl_handshake(&client_status->ssl)) != 0)) 
              {
                   if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) 
                   {
                        mbedtls_printf(" failed\n  ! mbedtls_ssl_handshake returned %d\n\n", ret);
                        return 0;
                    }
                }

                client_status->request = malloc(REQUEST_SIZE + 1);

                event.data.fd = client_socket.fd;
                event.data.ptr = client_status;
                event.events = EPOLLIN | EPOLLET;
                if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_socket.fd, &event) == -1) 
                {
                     continue;
                }
                   
          }
          else 
          {
                client_data_t *client_status = events[i].data.ptr;
                        
                // Every function call until this was successful, I checked, here nbytes == -69 or nbytes == -76
                // mbedtls_ssl_get_bytes_avail(&client_status->ssl) gives 0
                ssize_t nbytes = mbedtls_ssl_read(&client_status->ssl, (unsigned char *)(client_status->request), REQUEST_SIZE);  
                // other stuff
               
          }
    }
}

epoll_close:

    close(epoll_fd);

The mbedtls debug error messages look like this:

ssl_msg.c:5662: => read
ssl_msg.c:4110: => read record
ssl_msg.c:2155: => fetch input
ssl_msg.c:2295: in_left: 0, nb_want: 5
ssl_msg.c:2315: in_left: 0, nb_want: 5
ssl_msg.c:2318: ssl->f_recv(_timeout)() returned -69 (-0x0045)
ssl_msg.c:4782: mbedtls_ssl_fetch_input() returned -69 (-0x0045)
ssl_msg.c:4141: ssl_get_next_record() returned -69 (-0x0045)
ssl_msg.c:5722: mbedtls_ssl_read_record() returned -69 (-0x0045)

Edit: note that I get EPOLLIN after sending a request from browser and I use epoll() in edge-triggered mode, so I use mbedtls_ssl_read() until the socket would block each time, that is, until I get MBEDTLS_ERR_SSL_WANT_READ.

This is how I read the message:

while (1)
{
    nbytes = mbedtls_ssl_read(&client_status->ssl, (unsigned char *)(request + client_status->request_bytes_read), server_properties->max_request - client_status->request_bytes_read);
            
            if (nbytes < 0) 
            { 
           
                if (errno == EAGAIN || errno == EWOULDBLOCK || nbytes == MBEDTLS_ERR_SSL_WANT_READ) 
                {
                    // no application data to read, wait for another epoll_wait() notification
                    break;
                } 
                else 
                {
                    // print error
                    break;
                }

                // I also tried to replace the code above with this, but mbedtls_ssl_read() returns -69, which doesn't compare equal to MBEDTLS_ERR_SSL_WANT_READ and so there must be some sort of error
                // In case there is control data pending, but no application data, so that epoll_wait() wouldn't hang

                /* if (nbytes == MBEDTLS_ERR_SSL_WANT_READ)
                {
                    continue;
                }
                else if (errno == EAGAIN || errno == EWOULDBLOCK) 
                {
                    *blocked = 1;
                    break;
                } 
                else 
                {
                    ERROR_LOG("read() failed");
                    break;
                } */

            } 
            else if (nbytes == 0) 
            {
                break;
            }
            else 
            {
                client_status->request_bytes_read += nbytes;
                client_status->request[client_status->request_bytes_read] = '\0';
                char *CRLF = strstr(client_status->request + client_status->request_bytes_read - nbytes, "\r\n\r\n");
                if (CRLF)
                {
                    // parse the headers, get Content-Length and move to reading body
                    // body is read in a similar manner
                    break;
                }
            }
}

Answer 1

TLS adds a layer on top of TCP which has its own record framing and which also has records with no application payload (handshake, session tickets). mbedtls_ssl_read will return only data after a full record containing application data was received, because only then the data can be decrypted.

EPOLL works only at the level of the TCP socket though. This means it will signal activity whenever something at the socket can be read, no matter if this is a partial TLS record or a full TLS record, no matter if these are application data or just control information.

This means you cannot be sure that you can mbedtls_ssl_read whenever EPOLL returns the socket to be readable. Even more, you might be able to mbedtls_ssl_read even if EPOLL does not say that the socket is readable because the last socket read got more than one TLS record with application data. Not sure about Mbed-TLS, but with OpenSSL the SSL_read only contains data from a single TLS record and one need to check with SSL_pending to find out if there are more data available. Mbed-TLS has a similar function mbedtls_ssl_check_pending.

Answer 2

The problem is that client_socket in mbedtls_ssl_set_bio(&client_status->ssl, &client_socket, mbedtls_net_send, mbedtls_net_recv, NULL); goes out of scope, and subsequent BIO callbacks in read/write function cannot use that shared context after that happens.

Should've been mbedtls_ssl_set_bio(&client_status->ssl, &client_status->https_context, mbedtls_net_send, mbedtls_net_recv, NULL);