Linux sockets

References

• Socket options SO_REUSEADDR and SO_REUSEPORT, how do they differ? Do they mean the same across all major operating systems?

A very detailed answer about the differences between SO_REUSEADDR and SO_REUSEPORT.

• Programmed Instruction Course

A course on internet technologies, including Domain Naming (DNS...), the IP protocol, the TCP protocol, ...

• Manpages:

• ip (7)
• socket (7)

API

C

read(2) - Read from a file descriptor

#include <unistd.h>
ssize_t read(int fd, void *buf, size_t count);

• Mostly equivalent to recv except there is no specification for flags (e.g. to change blocking/non-blocking behaviour).

recv(2), recvfrom(2), recmmsg(2) - Receive a message from a socket

#include <sys/types.h>
#include <sys/socket.h>

ssize_t recv(int sockfd, void *buf, size_t len, int flags);
ssize_t recvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen);
ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);

write(2) - Write to a file descriptor

#include <unistd.h>

ssize_t write(int fd, const void *buf, size_t count);

• The main difference with send is that when wrote returns EPIPE, the writing process will also receive a SIGPIPE signal, terminating it unless it catches it or ignore it.

send(2), sendto(2), sendmsg(2) - Send a message on a socket

#include <sys/types.h>
#include <sys/socket.h>

ssize_t send(int sockfd, const void *buf, size_t len, int flags);
ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);
ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags);

• The main difference with write is the presence of flags, which among other can configured to prevent the generation of SIGPIPE signal.

Examples and tutorials

Some examples or tutorials on the internet:

• Socket programming in C on Linux – tutorial.

A thoough step-by-step example of client and socket code.

• C Socket Programming for Linux with a Server and Client Example Code

Simple example of client and server code using Linux sockets.

• Sockets Tutorial

Step-by-step detailed examples of server and client code.

C server

The following example comes from https://www.binarytides.com/socket-programming-c-linux-tutorial/.

To use this code:

gcc server.c -lpthread -o server
./server
# In another window:
telnet localhost 8888

Changes

• Set socket option SO_REUSEADDR to allow reconnect to same port while previous server instance is closing the connection (socket in TIME_WAIT state).
• Use send instead of write to avoid SIGPIPE signal if clients closed the socket.
• Don't use strlen on received message because there is no guarantee of terminating zero. Instead use read_size.
• Test error code of send / write.
• Close connection fd on exit (to notify client).
• Close listener socket on exit.

// Server - Example code from https://www.binarytides.com/socket-programming-c-linux-tutorial/
//
// Example of use:
//
// * In one window:
// |
// |   ./server
// |
// * In another window:
// |
// |   telnet localhost 8888
// |
// | Then type anything.

#include <stdio.h>
#include <string.h>                                                  // strlen
#include <stdlib.h>                                                  // strlen
#include <sys/socket.h>
#include <arpa/inet.h>                                               // inet_addr
#include <unistd.h>                                                  // write
#include <errno.h>

#include <pthread.h>                                                 // for threading , link with lpthread

void * connection_handler(void *);

int main(int argc, char *argv[])
{
    int                 sockfd, new_socket, c, *new_sock;
    struct sockaddr_in  server, client;
    char *              message;

    // Create socket
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if ( sockfd == -1 ) {
        printf("Could not create socket");
        return 1;
    }

    // Allow address/port reuse in a case server is killed (socket in TIME_WAIT state)
    if ( setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &(int){1 }, sizeof(int)) < 0 ) {
        printf("setsockopt(SO_REUSEADDR) failed");
        return 1;
    }

    // Prepare the sockaddr_in structure
    server.sin_family      = AF_INET;
    server.sin_addr.s_addr = INADDR_ANY;
    server.sin_port        = htons(8888);

    // Bind
    if ( bind(sockfd, (struct sockaddr *)&server, sizeof(server)) < 0 ) {
        printf("bind failed: %s (errno %d)", strerror(errno), errno);
        return 1;
    }
    puts("bind done");

    // Listen
    listen(sockfd, 3);

    // Accept and incoming connection
    puts("Waiting for incoming connections...");
    c = sizeof(struct sockaddr_in);
    while ((new_socket = accept(sockfd, (struct sockaddr *)&client, (socklen_t *)&c))) {
        puts("Connection accepted");

        // Reply to the client
        message   = "Hello Client , I have received your connection. And now I will assign a handler for you\n";
        if ( send(new_socket, message, strlen(message), MSG_NOSIGNAL) < 0 ) {
            perror("send failed");
        }

        pthread_t  sniffer_thread;
        new_sock  = malloc(4);
        *new_sock = new_socket;

        if ( pthread_create(&sniffer_thread, NULL, connection_handler, (void *)new_sock) < 0 ) {
            perror("could not create thread");
            return 1;
        }

        puts("Handler assigned");
    }

    // TODO: Join the running threads, so that we don't terminate before them
    // for( int i=0; ...; ... ) pthread_join( sniffer_thread[i] , NULL);

    if ( new_socket < 0 ) {
        perror("accept failed");
        return 1;
    }

    return 0;
}

/*
 * This will handle connection for each client
 * */
void * connection_handler(void *sockfd)
{
    // Get the socket descriptor
    int   sock = *(int *)sockfd;
    int   read_size;
    char *message, client_message[2048];

    // Send some messages to the client
    message = "Greetings! I am your connection handler\n";
    if ( send(sock, message, strlen(message), MSG_NOSIGNAL) < 0 ) {
        perror("send failed");
    }

    message = "Now type something and i shall repeat what you type \n";
    if ( send(sock, message, strlen(message), MSG_NOSIGNAL) < 0 ) {
        perror("send failed");
    }

    // Receive a message from client
    while ((read_size = recv(sock, client_message, sizeof(client_message), 0)) > 0 ) {
        // Output received message (assuming zero-terminated)
        write(STDOUT_FILENO, client_message, read_size);

        // Send the message back to client
        send(sock, client_message, read_size, MSG_NOSIGNAL);
    }

    if ( read_size == 0 ) {
        puts("Client disconnected");
        fflush(stdout);
    } else if ( read_size == -1 ) {
        perror("recv failed");
    }

    // Free the socket pointer
    free(sockfd);

    return 0;
}

C client

The following example comes from https://www.binarytides.com/socket-programming-c-linux-tutorial/.

To use this code:

gcc client.c -o server
./client

Changes

• Close connection fd on exit (to notify server).

// client - Sample code from https://www.binarytides.com/socket-programming-c-linux-tutorial/

#include <stdio.h>
#include <string.h>                                                  // strlen
#include <sys/socket.h>
#include <arpa/inet.h>                                               // inet_addr
#include <unistd.h>                                                  // close

int main(int argc, char *argv[])
{
    int                 sockfd;
    struct sockaddr_in  server;
    char *              message, server_reply[2000];

    // Create socket
    sockfd                 = socket(AF_INET, SOCK_STREAM, 0);
    if ( sockfd == -1 ) {
        printf("Could not create socket");
    }

    server.sin_addr.s_addr = inet_addr("216.58.206.4");
    server.sin_family      = AF_INET;
    server.sin_port        = htons(80);

    // Connect to remote server
    if ( connect(sockfd, (struct sockaddr *)&server, sizeof(server)) < 0 ) {
        puts("connect error");
        return 1;
    }

    puts("Connected\n");

    // Send some data
    message = "GET / HTTP/1.1\r\n\r\n";
    if ( send(sockfd, message, strlen(message), 0) < 0 ) {
        puts("Send failed");
        return 1;
    }
    puts("Data Send\n");

    // Receive a reply from the server
    if ( recv(sockfd, server_reply, 2000, 0) < 0 ) {
        puts("recv failed");
    }
    puts("Reply received\n");
    puts(server_reply);

    close(sockfd);

    return 0;
}

How-to

Manage timeout

By default sockets are in blocking mode. One can change the timeout by setting the socket options SO_RCVTIMEO and SO_SNDTIMEO.

if ( setsockopt(sock,
                SOL_SOCKET,
                SO_RCVTIMEO,
                &(struct timeval){.tv_sec = 30, .tv_usec = 0 },
                sizeof(struct timeval)) < 0 ) {
    err(1, "setsockopt(SO_RCVTIMEO) failed");
}
if ( setsockopt(sock,
                SOL_SOCKET,
                SO_SNDTIMEO,
                &(struct timeval){.tv_sec = 30, .tv_usec = 0 },
                sizeof(struct timeval)) < 0 ) {
    err(1, "setsockopt(SO_SNDTIMEO) failed");
}

Note that even it the timeout is set, calls to recv and send may return before the timeout elapsed:

• recv returns 0 if the socket was properly shutdown (see recv(2)).
• recv returns a negative value and set errno in case of error (see recv(2)).
• recv may return fewer bytes than requested. The call must be repeated until all bytes are received:

int      nread = 0, nrecv;
uint8_t *o = buf;

while (nread < n) {
    nrecv = recv(fd, o, n - nread, 0);
    if ( nrecv < 0 ) {
        err("recv failed");
        set_error(error, ERROR_SOCKET);
        return;
    }
    if ( nrecv == 0 ) {
        warnx("recv returned 0, assuming shutdown.\n");
        set_error(error, ERROR_SHUTDOWN);
        return;
    }
    if ( nrecv != (n - nread)) {
        warnx("%s: Requested n %d but got nrecv %d\n", __FUNCTION__, n - nread, nrecv);
    }
    nread += nrecv;
    o     += nread;
}

Troubleshooting

Use errno / perror / strerror to get error

if( bind(socket_desc,(struct sockaddr *)&server , sizeof(server)) < 0)
{
    printf("bind failed: %s (errno %d)", strerror(errno), errno);
    return 1;
}

Or there is also the perror function:

if( bind(socket_desc,(struct sockaddr *)&server , sizeof(server)) < 0)
{
    perror("bind failed");
    return 1;
}

Use netstat to get socket status

./server
# bind done
# Waiting for incoming connections...
# Connection accepted
# Handler assigned
# ^C

./server
# bind failed: Address already in use (errno 98)

netstat -a | grep 8888
# tcp        0      0 zavcxl0005:58888        165.225.76.32:http      ESTABLISHED
# tcp        0      0 zavcxl0005:48888        10.75.126.1:https       ESTABLISHED
# tcp        0      0 localhost.localdom:8888 localhost.localdo:50310 TIME_WAIT  

# ... So port is still in use

bind failed: Address already in use (errno 98)

Call to bind fails with error

bind failed: Address already in use (errno 98)

On a server, this is probably due to an old instance of the server that tries to bind to the same port and address [1].

As a fix, try to bind with option SO_REUSEADDR and/or SO_REUSEPORT. From SO:

if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) < 0)
    error("setsockopt(SO_REUSEADDR) failed");

For more information, see socket(7) and ip(7) manpages.

Program crashes on write with exit code -13

Exit code -13 means the program died because of SIGPIPE signal, meaning it tried to write to a closed socket.

Solutions:

• Ignore SIGPIPE in the program [2]:

signal (SIGPIPE, SIG_IGN);

• Use send instead of write. Send returns -1 and set errno to EPIPE rather than generating a fatal SIGPIPE like write does [3]:

n = send(newsockfd, data.c_str(), data.length()+1, MSG_NOSIGNAL);

Detect connection shut down

When connection is properly shut down, recv returns 0.

Unlimited loop with recv (recv returns 0 / empty response)

A common mistake is to expect that recv will always return a non-empty response or fail with an error/exception otherwise. This is not true, and might be the cause of unlimited loop in the code. recv may return an empty response in the case the peer decided it won't send anything anymore but might still accept incoming data. [4]

For instance, in Python:

s = socket.create_connection((host,port), 0.1)
while True:
    try:
        block = s.recv(1024)               # WRONG - block might be empty
        buf = buf + block
        if block.find(b'\n') >= 0:
            break
    except socket.timeout:
        return None

The code above is wrong because it might never exit the while loop if recv returns an empty message.

Correct implementation:

s = socket.create_connection((host,port), 0.1)
while True:
    try:
        block = s.recv(1024)
        if len(block) == 0:
            return None
        buf = buf + block
        if block.find(b'\n') >= 0:
            break
    except socket.timeout:
        return None

Likewise, in C:

do {
    nrecv = recv(fd, o, n - nread, 0);
    if (nrecv < 0) {
        warn("recv failed");                  // Socket is broken
        return;
    }
    if (nrecv == 0) {
        warnx("socket shut down by peer");    // peer won't send anything anymore
        return;
    }
    nread += nrecv;
    o     += nread;
} while (nread < n);

Fix delay / high latency / bad performance in communication

The most probable root cause is Nagle's algorithm that may add a 500ms latency to every packet sent.

Below are some examples on how to disable that algorithm. Note that if both ends do send packets, the Naggle's algorithm must be disabled at each sender. Also, this will improve the communication responsiveness but will not increase the communication speed.

in C

#include <netinet/tcp.h>

// ...
fd = accept(sockfd, (struct sockaddr *)&client, (void *)&c)
if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &(int){1}, sizeof(int)) < 0) {
    errx(1, "setsockopt(TCP_NODELAY) failed\n");
}

in java

s = new Socket(hostName, port);
s.setTcpNoDelay(true);

in Python

import socket

# s = socket.create_connection((host, port), timeout)
# s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# ... setting TCP_NODELAY w/ using create_connection does not work...

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
s.connect((host, port))