可能很多人都像我一样,很早就听说过TCP_NODELAY这个选项,明白它跟nagle算法相关,但是就只是停留在表面上了。
最近,实验室在做一套低时延可靠传输协议。作为对比,我希望搞清楚TCP在实际场景中丢包重传到底要消耗多少时间。为此,通过netem在loopback上模拟丢包和延时,我只需要在send以及recv这两个时刻打时间戳就可以知道每个包大概的时延。
以上是背景。然后问题来了。
在MacOS 10.11上,直接进行测试,时延基本为0,符合预计,毕竟没有人为加丢包和延时;
在Linux 4.13上,进行测试,发现时延经常出现30ms、40ms这样的异常值,但是,当把单个数据包的长度降低到500B以下时,恢复正常;这个现象很有趣,也扰乱了思路。
当然,最后才想起可能是TCP_NODELAY,所以纸上得来终觉浅,设置TCP_NODELAY之后,Linux上恢复正常。
#include <stdio.h> #include <string.h> #include <sys/socket.h> #include <netinet/in.h> #include <arpa/inet.h> #include <sys/time.h> #include <fcntl.h> #include <unistd.h> #include <assert.h> #include <netinet/tcp.h> #include <errno.h> #define N (20000) #define USETCP (1) #define LEN (1024) long getts() { struct timeval tv; gettimeofday(&tv, NULL); long ts = tv.tv_sec * 1000 + tv.tv_usec / 1000; return ts; } ssize_t nrecv(int sock, void *buf, size_t len, int flags) { #ifdef USETCP size_t nr; size_t nleft = len; void *ptr = buf; while (nleft > 0) { if ((nr = recv(sock, ptr, nleft, flags)) < 0) { if (errno == EINTR) nr = 0; else return -1; } else if (nr == 0) break; nleft -= nr; ptr += nr; } return (len - nleft); #else return recv(sock, buf, len, flags); #endif } typedef struct { long ts; char buf[LEN - sizeof(long)]; } DataWrapper; int main(int argc, char *argv[]) { if (argc != 2) { printf("Not Enough Parameter\n"); return -1; } else if (memcmp(argv[1], "c", 1) == 0) { #ifdef USETCP // Init Socket Begins int sock = socket(PF_INET, SOCK_STREAM, 0); struct sockaddr_in addr; memset(&addr, 0, sizeof(addr)); inet_pton(PF_INET, "127.0.0.1", &addr.sin_addr); addr.sin_family = AF_INET; addr.sin_port = htons(7777); connect(sock, (struct sockaddr *) &addr, sizeof(addr)); int on = 1; setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)); // Init Socket Ends #else int sock = socket(PF_INET, SOCK_DGRAM, 0); struct sockaddr_in addr; memset(&addr, 0, sizeof(addr)); inet_pton(PF_INET, "127.0.0.1", &addr.sin_addr); addr.sin_family = AF_INET; addr.sin_port = htons(7777); connect(sock, (struct sockaddr *) &addr, sizeof(addr)); #endif DataWrapper dw; for (int i = 0; i < N; i++) { dw.ts = getts(); send(sock, &dw, sizeof(dw), 0); // 1000KBps => 1KB/ms => 1 packet / 1000us usleep(1000); // roughly } } else if (memcmp(argv[1], "s", 1) == 0) { #ifdef USETCP // Init Socket Begins int listensock = socket(PF_INET, SOCK_STREAM, 0); int opt = 1; setsockopt(listensock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); struct sockaddr_in addr; memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; addr.sin_addr.s_addr = htons(INADDR_ANY); addr.sin_port = htons(7777); if (bind(listensock, (struct sockaddr *)&addr, sizeof(addr)) < 0) { printf("tcpsock Bind Failed\n"); return -1; } listen(listensock, 128); int sock = accept(listensock, NULL, NULL); int on = 1; setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)); // Init Socket Ends #else int sock = socket(PF_INET, SOCK_DGRAM, 0); struct sockaddr_in addr; memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; addr.sin_addr.s_addr = htons(INADDR_ANY); addr.sin_port = htons(7777); if (bind(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) { printf("tcpsock Bind Failed\n"); return -1; } struct timeval timeout = {2, 0}; setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (void *)&timeout, sizeof(struct timeval)); #endif DataWrapper dw; for (int i = 0; i < N; i++) { int nrcv = nrecv(sock, &dw, sizeof(dw), 0); if (nrcv != sizeof(dw)) { fprintf(stderr, "nrcv: %d\n", nrcv); return -1; } printf("%ld\n", getts() - dw.ts); } } else { printf("Error Parameter\n"); return -1; } }