LINUX socket與VRF

時間  2019-11-21
標籤 linux socket vrf 欄目 Linux 简体版
原文   原文鏈接

實驗環境以下圖所示：
圖片.png
配置以下：shell

#!/bin/bash
sudo ip netns add ns1 
sudo ip link add ns1veth1 type veth peer name eth0 netns ns1
sudo ip netns add ns2
sudo ip link add ns2veth1 type veth peer name eth0 netns ns2
sudo ip link set ns1veth1 master vrftest
sudo ip link set ns2veth1 master vrftest
sudo ip link set ns2veth1 up
sudo ip link set ns1veth1 up
sudo ip addr add 1.1.1.254/24 dev ns1veth1 
sudo ip addr add 2.2.2.254/24 dev ns2veth1 
sudo ip netns exec ns2 ip addr add 2.2.2.1/24 dev eth0 
sudo ip netns exec ns1 ip addr add 1.1.1.1/24 dev eth0 
sudo ip netns exec ns1 ip link set eth0 up
sudo ip netns exec ns1 ip link set lo up
sudo ip netns exec ns1 ip route add default via 1.1.1.254 dev eth0
sudo ip netns exec ns2 ip link set eth0 up
sudo ip netns exec ns2 ip link set lo up
sudo ip netns exec ns2 ip route add default via 2.2.2.254 dev eth0

實驗使用c語言寫了兩個套接字交互程序：ubuntu

  • 服務器:vrfs
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<errno.h>
#include<sys/types.h>
#include<sys/socket.h>
#include<netinet/in.h>
#include <unistd.h>

#define MAXLINE 4096

int main(int argc, char** argv)
{
    int    listenfd, connfd;
    struct sockaddr_in     servaddr;
    char    buff[4096];
    int     n;
    int     on = 1;



    if( (listenfd = socket(AF_INET, SOCK_STREAM, 0)) == -1 ){
        printf("create socket error: %s(errno: %d)\n",strerror(errno),errno);
        exit(0);
    }
    setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, (void *)&on,
                         sizeof(on));
    setsockopt(listenfd, SOL_SOCKET, SO_REUSEPORT, (void *)&on,
                         sizeof(on));

    memset(&servaddr, 0, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(6666);

    if(argc == 2){
        printf("vrf device name: %s\r\n", argv[1]);
        if(0 > setsockopt(listenfd, SOL_SOCKET, SO_BINDTODEVICE, argv[1], strlen(argv[1])+1)){
             printf("bind socket master dev error: %s(errno: %d)\n",strerror(errno),errno);
             exit(0);
        }
    }

    if( bind(listenfd, (struct sockaddr*)&servaddr, sizeof(servaddr)) == -1){
        printf("bind socket error: %s(errno: %d)\n",strerror(errno),errno);
        exit(0);
    }

    if( listen(listenfd, 10) == -1){
        printf("listen socket error: %s(errno: %d)\n",strerror(errno),errno);
        exit(0);
    }

    printf("======waiting for client's request======\n");
    while(1){
        if((connfd = accept(listenfd, (struct sockaddr*)NULL, NULL)) == -1){
                printf("accept socket error: %s(errno: %d)",strerror(errno),errno);
                continue;
        }
        n = recv(connfd, buff, MAXLINE, 0);
        buff[n] = '\0';
        printf("recv msg from client: %s\n", buff);
        close(connfd);
    }

    close(listenfd);
}
  • 客戶端程序：vrfc
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<errno.h>
#include<sys/types.h>
#include<sys/socket.h>
#include<netinet/in.h>
#include<unistd.h>
#define MAXLINE 4096
#include <arpa/inet.h>

int main(int argc, char** argv)
{
    int    sockfd, n;
    char   *sendline = "hello vrf";
    struct sockaddr_in    servaddr;

    if( argc != 2){
    printf("usage: ./client <ipaddress> [master device]\n");
    exit(0);
    }

    if( (sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0){
    printf("create socket error: %s(errno: %d)\n", strerror(errno),errno);
    exit(0);
    }

    memset(&servaddr, 0, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_port = htons(6666);
    if( inet_pton(AF_INET, argv[1], &servaddr.sin_addr) <= 0){
    printf("inet_pton error for %s\n",argv[1]);
    exit(0);
    }

    if(argc == 3){
        printf("vrf device name: %s\r\n", argv[2]);
        if(0 > setsockopt(sockfd, SOL_SOCKET, SO_BINDTODEVICE, argv[2], strlen(argv[2])+1)){
             printf("bind socket master dev error: %s(errno: %d)\n",strerror(errno),errno);
             exit(0);
        }
    }
    if( connect(sockfd, (struct sockaddr*)&servaddr, sizeof(servaddr)) < 0){
    printf("connect error: %s(errno: %d)\n",strerror(errno),errno);
    exit(0);
    }

    printf("send msg to server: hello vrf\n");
    
    if( send(sockfd, sendline, strlen(sendline), 0) < 0)
    {
    printf("send msg error: %s(errno: %d)\n", strerror(errno), errno);
    exit(0);
    }

    close(sockfd);
    exit(0);
}

實驗一：驚羣效應

在默認VRF環境下，啓動兩個進程，監聽相同的端口和地址：程序中套接口使用了SO_REUSEADDR和SO_REUSEPORT。查看內核如何處理驚羣效應。bash

console1:服務器

admin@ubuntu:~/vrfsocket$ for i in {0..9}; do ./vrfc 127.0.0.1; done  
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$

console2:負載均衡

admin@ubuntu:~/vrfsocket$ ./vrfs 
======waiting for client's request======
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf

console3:socket

admin@ubuntu:~/vrfsocket$ ./vrfs 
======waiting for client's request======
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf

結論

新內核彷佛已經可以處理驚羣效應了，收到請求時再也不通知全部監聽該端口的服務器程序，而是會進行必定的負載均衡調度處理。tcp

實驗二：啓動兩個服務器，一個綁定VRF，一個不綁定，客戶端不綁定VRF

console1:

admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 127.0.0.1; done
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$

console2:spa

root@ubuntu:/home/jd/vrfsocket# ./vrfs
======waiting for client's request======
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf

console3:code

root@ubuntu:/home/jd/vrfsocket# ./vrfs vrftest
vrf device name: vrftest
======waiting for client's request======

結論：服務器監聽套接字綁定VRF後，再也不處理默認VRF中的請求server

實驗三：啓動兩個服務器，一個綁定VRF，一個不綁定，客戶端綁定VRF

console1：

admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 1.1.1.254 vrftest; done
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$

console2：在root用戶下運行

root@ubuntu:/home/jd/vrfsocket# ./vrfs
======waiting for client's request======

console3：在root用戶下運行。

root@ubuntu:/home/jd/vrfsocket# ./vrfs vrftest
vrf device name: vrftest
======waiting for client's request======
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf

結論：服務器監聽套接字不綁定VRF，不能處理非默認VRF中的請求

實驗四：設置sudo sysctl -w net.ipv4.tcp_l3mdev_accept=1

啓動兩個服務器，一個綁定VRF，一個不綁定，客戶端不綁定VRF

console1：

admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 127.0.0.1; done
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$

console2:

root@ubuntu:/home/jd/vrfsocket# ./vrfs
======waiting for client's request======
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf

console3:

root@ubuntu:/home/jd/vrfsocket# ./vrfs vrftest
vrf device name: vrftest
======waiting for client's request======

啓動一個服務器，綁定VRF，客戶端不綁定VRF

console1：

admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 127.0.0.1; done
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
admin@ubuntu:~/vrfsocket$

console3:

root@ubuntu:/home/jd/vrfsocket# ./vrfs vrftest
vrf device name: vrftest
======waiting for client's request======

啓動一個服務器，綁定VRF，客戶端綁定VRF

console1：

admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 1.1.1.254 vrftest; done
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$

console3:

root@ubuntu:/home/jd/vrfsocket# ./vrfs vrftest
vrf device name: vrftest
======waiting for client's request======
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf

啓動兩個服務器，一個綁定VRF，一個不綁定，客戶端綁定VRF

console1：

admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 1.1.1.254 vrftest; done
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$

console2:

root@ubuntu:/home/jd/vrfsocket# ./vrfs
======waiting for client's request======
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf
recv msg from client: hello vrf

console3:

root@ubuntu:/home/jd/vrfsocket# ./vrfs vrftest
vrf device name: vrftest
======waiting for client's request======

在打開sudo sysctl -w net.ipv4.tcp_l3mdev_accept=1後，默認VRF中的監聽套接字可以處理全部VRF中的請求，且優先級高於其它的VRF的監聽套接字。

總結

序號 結論
1 多個服務器器監聽同一地址和端口，內核會進行負載均衡，選擇喚醒其中一個進程處理請求。
2 默認VRF中的服務器進程不能處理非默認VRF中的請求，非默認VRF中的服務器進程不能處理其它VRF中的請求
3 開啓net.ipv4.tcp_l3mdev_accept=1後，默認VRF中的服務器進程能夠處理任意VRF中的請求，且優先級最高
4 開啓net.ipv4.tcp_l3mdev_accept=1後，非默認VRF中的服務器進程不能處理其它VRF中的請求，在處理本VRF中的流量時，優先級低於默認VRF中的進程。
相關文章
相關標籤/搜索
每日一句
    每一个你不满意的现在，都有一个你没有努力的曾经。
本站公眾號
   歡迎關注本站公眾號,獲取更多信息
聯繫我們 最近搜索 最新文章 沪ICP备13005482号-10 MyBatis教程 SQL 教程 MySQL教程 Java 教程 Thymeleaf 教程 Hibernate教程 Spring教程 Redis教程