Why my tcp bandwidth is only 1/3 compareing with iperf3?

Question

#include <netinet/in.h>
#include <pthread.h>
#include <sys/types.h>         
#include <sys/socket.h>
#include <assert.h>

static int create_local_sock(void)
{
    struct sockaddr_in un;
    int sockfd = socket(AF_INET, SOCK_STREAM, 0);
    memset(&un, 0, sizeof(un));
    un.sin_family = AF_INET;   
    un.sin_port = htons(8888);
    un.sin_addr.s_addr = 0;
    if(bind(sockfd, (struct sockaddr *)&un, sizeof(un))<0){
        assert(0);
    }
    listen(sockfd, 10);

    return sockfd; 
}
ssize_t prev_recv, curr_recv;
void *print_fun(void *arg)
{
    while(1){
        sleep(1);
        ssize_t curr_recv1 = curr_recv;
        printf("Recv %d MB/s\n", (curr_recv1 - prev_recv)/1024/1024);
        prev_recv = curr_recv1;
    }
}
int s_main(int argc, char **argv)
{
    char buf[2048];
    int fd = create_local_sock();
    pthread_t thr;
    pthread_create(&thr, NULL, print_fun, NULL);
    while(1){
        int client = accept(fd, NULL, NULL);
        while(1){
            ssize_t len = recv(client, buf, sizeof(buf), 0);
        if(len > 0){
            curr_recv += len;
        }else{
            break;
        }
        }

    }
}

ssize_t prev_send, curr_send;
void *print_fun_c(void *arg)
{
    while(1){
        sleep(1);
        ssize_t curr_send1 = curr_send;
        printf("Send %d MB/s\n", (curr_send1 - prev_send)/1024/1024);
        prev_send = curr_send1;
    }
}

int c_main(int argc, char **argv)
{
    char buf[2048];
    struct sockaddr_in un;
    int fd = socket(AF_INET, SOCK_STREAM, 0);
    memset(&un, 0, sizeof(un));
    un.sin_family = AF_INET;   
    un.sin_port = htons(8888);
    un.sin_addr.s_addr = inet_addr(argv[1]);
    connect(fd, (struct sockaddr*)&un, sizeof(un));
    pthread_t thr;
    pthread_create(&thr, NULL, print_fun_c, NULL);
    while(1){
        ssize_t len = send(fd, buf, 1400, 0);
        if(len > 0){
            curr_send += len;
        }else{
            break;
        }
    }
}

int main(int argc, char **argv){
    if(strcmp(argv[1], "s")==0) return s_main(argc,argv);
    return c_main(argc,argv);
}

I have a simple tcp receiving server and client, they both receive/send packets in a loop as quick as possible. I use the payload 1400 seems the mtu is 1500. In real testing, the throughput is only 1/3 comparing with iperf3. I cannot understand why it has so much difference. I have checked the cpu usage. Very similar. What hurts my performance?

Answer 1

Information for those who may read this (relatively old) question. The default TCP message size iperf3 is using is 128KB. Therefore, sending only 1400 bytes per message has much higher overhead. This might be the root of the issue.