1. 当前环境
[root@vm1 ~]# ovs-vsctl show
c152c245-2f6c-478c-9c07-2e4a3c7a2403
[root@vm1 ~]# cat /proc/sys/net/ipv4/ip_forward
0
[root@vm1 ~]# iptables -t nat -F
[root@vm1 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.19.0.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.19.0.0 0.0.0.0 255.255.240.0 U 0 0 0 eth0
安装ovs可以参考 在 Centos 上安装 ovs
2. 用ovs模拟docker
添加一个ovs 网桥br0 并且配置ip为192.168.1.250/24
[root@vm1 ~]# ovs-vsctl add-br br0
[root@vm1 ~]# ovs-vsctl show
c152c245-2f6c-478c-9c07-2e4a3c7a2403
Bridge "br0"
Port "br0"
Interface "br0"
type: internal
ovs_version: "2.5.1"
[root@vm1 ~]# ifconfig br0 192.168.1.250/24
[root@vm1 ~]# ifconfig br0
br0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.1.250 netmask 255.255.255.0 broadcast 192.168.1.255
inet6 fe80::c0b6:92ff:fe4d:7649 prefixlen 64 scopeid 0x20<link>
ether c2:b6:92:4d:76:49 txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 8 overruns 0 frame 0
TX packets 6 bytes 508 (508.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
创建一个network namespace ns1, 创建一对veth为veth0和veth1, 将veth0加入到br0中, 将veth1加入到ns1中, 并且给ns1配置ip为192.168.1.1/24.
// 创建一个network namespace ns1
[root@vm1 ~]# ip netns add ns1
// 创建一对veth pair (veth0 和 veth1)
[root@vm1 ~]# ip link add veth0 type veth peer name veth1
// 将veth0加入到br0中
[root@vm1 ~]# ip link set veth0 up
[root@vm1 ~]# ovs-vsctl add-port br0 veth0
[root@vm1 ~]# ovs-vsctl show
c152c245-2f6c-478c-9c07-2e4a3c7a2403
Bridge "br0"
Port "veth0"
Interface "veth0"
Port "br0"
Interface "br0"
type: internal
ovs_version: "2.5.1"
[root@vm1 ~]#
// 将veth1加入到ns1中
[root@vm1 ~]# ip link set veth1 netns ns1
// 设置veth1 ip 与br0是同一个网络
[root@vm1 ~]# ip netns exec ns1 ip addr add 192.168.1.1/24 dev veth1
[root@vm1 ~]# ip netns exec ns1 ip link set veth1 up
[root@vm1 ~]# ip netns exec ns1 ip link set lo up
// 在ns1中ping br0成功
[root@vm1 ~]# ip netns exec ns1 ping -c 1 192.168.1.250
PING 192.168.1.250 (192.168.1.250) 56(84) bytes of data.
64 bytes from 192.168.1.250: icmp_seq=1 ttl=64 time=0.392 ms
--- 192.168.1.250 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.392/0.392/0.392/0.000 ms
给ns1增加路由
[root@vm1 ~]# ip netns exec ns1 route add default gw 192.168.1.250
[root@vm1 ~]# ip netns exec ns1 ping -c 1 172.19.0.12
PING 172.19.0.12 (172.19.0.12) 56(84) bytes of data.
64 bytes from 172.19.0.12: icmp_seq=1 ttl=64 time=0.224 ms
--- 172.19.0.12 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.224/0.224/0.224/0.000 ms
打开ip_forward功能和加iptables规则
[root@vm1 ~]# echo 1 > /proc/sys/net/ipv4/ip_forward
[root@vm1 ~]# iptables -t nat -A POSTROUTING -s 192.168.1.0/24 -o eth0 -j MASQUERADE
[root@vm1 ~]# ip netns exec ns1 ping -c 1 www.baidu.com
PING www.wshifen.com (103.235.46.39) 56(84) bytes of data.
64 bytes from 103.235.46.39 (103.235.46.39): icmp_seq=1 ttl=55 time=1.77 ms
--- www.wshifen.com ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.772/1.772/1.772/0.000 ms
[root@vm1 ~]#
3. ovs-docker
3.1 配置
[root@vm1 ~]# docker version
Client:
Version: 18.09.6
[root@vm1 ~]# docker run -d --name con1 --net=none busybox top
[root@vm1 ~]# docker exec -it con1 ifconfig
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)
// 此时利用ovs-docker 设置此容器ip地址为192.168.1.2/24 网关为192.168.1.250
[root@vm1 ~]# ovs-docker add-port br0 eth0 con1 --ipaddress=192.168.1.2/24 --gateway=192.168.1.250
[root@vm1 ~]# docker exec -it con1 ifconfig
eth0 Link encap:Ethernet HWaddr 56:39:36:6A:B0:61
inet addr:192.168.1.2 Bcast:0.0.0.0 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:7 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:578 (578.0 B) TX bytes:0 (0.0 B)
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)
[root@vm1 ~]# docker exec -it con1 route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.1.250 0.0.0.0 UG 0 0 0 eth0
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0
[root@vm1 ~]# ovs-vsctl show
c152c245-2f6c-478c-9c07-2e4a3c7a2403
Bridge "br0"
Port "veth0"
Interface "veth0"
Port "7506959a37594_l"
Interface "7506959a37594_l"
Port "br0"
Interface "br0"
type: internal
ovs_version: "2.5.1"
[root@vm1 ~]#
从上面结果可以知道ovs-docker所做的操作的就是2. 用ovs模拟docker中所做的内容,说白了就是一些
shell命令的集合.
3.2 测试
// 访问br0
[root@vm1 ~]# docker exec -it con1 ping -c 1 192.168.1.250
PING 192.168.1.250 (192.168.1.250): 56 data bytes
64 bytes from 192.168.1.250: seq=0 ttl=64 time=3.972 ms
--- 192.168.1.250 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 3.972/3.972/3.972 ms
// 访问ns1
[root@vm1 ~]# docker exec -it con1 ping -c 1 192.168.1.1
PING 192.168.1.1 (192.168.1.1): 56 data bytes
64 bytes from 192.168.1.1: seq=0 ttl=64 time=3.751 ms
--- 192.168.1.1 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 3.751/3.751/3.751 ms
// 访问本机ip
[root@vm1 ~]# docker exec -it con1 ping -c 1 172.19.0.12
PING 172.19.0.12 (172.19.0.12): 56 data bytes
64 bytes from 172.19.0.12: seq=0 ttl=64 time=4.743 ms
--- 172.19.0.12 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 4.743/4.743/4.743 ms
// 访问另外一台机器vm2
[root@vm1 ~]# docker exec -it con1 ping -c 1 172.19.0.8
PING 172.19.0.8 (172.19.0.8): 56 data bytes
64 bytes from 172.19.0.8: seq=0 ttl=63 time=3.829 ms
--- 172.19.0.8 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 3.829/3.829/3.829 ms
// 访问互联网
[root@vm1 ~]# docker exec -it con1 ping -c 1 www.baidu.com
PING www.baidu.com (119.63.197.151): 56 data bytes
64 bytes from 119.63.197.151: seq=0 ttl=49 time=50.983 ms
--- www.baidu.com ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 50.983/50.983/50.983 ms
从ns1中访问con1
[root@vm1 ~]# ip netns exec ns1 ping -c 1 192.168.1.2
PING 192.168.1.2 (192.168.1.2) 56(84) bytes of data.
64 bytes from 192.168.1.2: icmp_seq=1 ttl=64 time=0.230 ms
--- 192.168.1.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.230/0.230/0.230/0.000 ms
[root@vm1 ~]#
如果ovs-docker 配置不成功, 可以在创建容器的时候加上
--privileged=true.
3.3 ovs-docker 原理
源码访问 http://github.com/openvswitch/ovs/raw/master/utilities/ovs-docker
...
add_port () {
BRIDGE="$1"
INTERFACE="$2"
CONTAINER="$3"
if [ -z "$BRIDGE" ] || [ -z "$INTERFACE" ] || [ -z "$CONTAINER" ]; then
echo >&2 "$UTIL add-port: not enough arguments (use --help for help)"
exit 1
fi
shift 3
while [ $# -ne 0 ]; do
case $1 in
--ipaddress=*)
ADDRESS=`expr X"$1" : 'X[^=]*=\(.*\)'`
shift
;;
--macaddress=*)
MACADDRESS=`expr X"$1" : 'X[^=]*=\(.*\)'`
shift
;;
--gateway=*)
GATEWAY=`expr X"$1" : 'X[^=]*=\(.*\)'`
shift
;;
--mtu=*)
MTU=`expr X"$1" : 'X[^=]*=\(.*\)'`
shift
;;
*)
echo >&2 "$UTIL add-port: unknown option \"$1\""
exit 1
;;
esac
done
# Check if a port is already attached for the given container and interface
PORT=`get_port_for_container_interface "$CONTAINER" "$INTERFACE" \
2>/dev/null`
if [ -n "$PORT" ]; then
echo >&2 "$UTIL: Port already attached" \
"for CONTAINER=$CONTAINER and INTERFACE=$INTERFACE"
exit 1
fi
if ovs_vsctl br-exists "$BRIDGE" || \
ovs_vsctl add-br "$BRIDGE"; then :; else
echo >&2 "$UTIL: Failed to create bridge $BRIDGE"
exit 1
fi
if PID=`docker inspect -f '{{.State.Pid}}' "$CONTAINER"`; then :; else
echo >&2 "$UTIL: Failed to get the PID of the container"
exit 1
fi
create_netns_link
# Create a veth pair.
ID=`uuidgen | sed 's/-//g'`
PORTNAME="${ID:0:13}"
ip link add "${PORTNAME}_l" type veth peer name "${PORTNAME}_c"
# Add one end of veth to OVS bridge.
if ovs_vsctl --may-exist add-port "$BRIDGE" "${PORTNAME}_l" \
-- set interface "${PORTNAME}_l" \
external_ids:container_id="$CONTAINER" \
external_ids:container_iface="$INTERFACE"; then :; else
echo >&2 "$UTIL: Failed to add "${PORTNAME}_l" port to bridge $BRIDGE"
ip link delete "${PORTNAME}_l"
exit 1
fi
ip link set "${PORTNAME}_l" up
# Move "${PORTNAME}_c" inside the container and changes its name.
ip link set "${PORTNAME}_c" netns "$PID"
ip netns exec "$PID" ip link set dev "${PORTNAME}_c" name "$INTERFACE"
ip netns exec "$PID" ip link set "$INTERFACE" up
if [ -n "$MTU" ]; then
ip netns exec "$PID" ip link set dev "$INTERFACE" mtu "$MTU"
fi
if [ -n "$ADDRESS" ]; then
ip netns exec "$PID" ip addr add "$ADDRESS" dev "$INTERFACE"
fi
if [ -n "$MACADDRESS" ]; then
ip netns exec "$PID" ip link set dev "$INTERFACE" address "$MACADDRESS"
fi
if [ -n "$GATEWAY" ]; then
ip netns exec "$PID" ip route add default via "$GATEWAY"
fi
}
...
很简单就看到这个方法就是ovs-docker刚才的操作过程, 基本上也就是2. 用ovs模拟docker的基础上加入了一些判断.
4. 参考
1. https://blog.csdn.net/silvester123/article/details/80867168
2. https://blog.csdn.net/yeya24/article/details/79829240