为了满足应用就近访问的需求,Kubernetes 提供了 Service Topology Aware Hints(即拓扑感知) 能力,本文介绍该特性的基本用法(基于 v1.21 版本)。
集群准备
# cat << EOF > cluster.yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
image: kindest/node:v1.21.10@sha256:84709f09756ba4f863769bdcabe5edafc2ada72d3c8c44d6515fc581b66b029c
- role: worker
image: kindest/node:v1.21.10@sha256:84709f09756ba4f863769bdcabe5edafc2ada72d3c8c44d6515fc581b66b029c
- role: worker
image: kindest/node:v1.21.10@sha256:84709f09756ba4f863769bdcabe5edafc2ada72d3c8c44d6515fc581b66b029c
- role: worker
image: kindest/node:v1.21.10@sha256:84709f09756ba4f863769bdcabe5edafc2ada72d3c8c44d6515fc581b66b029c
EOF
# kind create cluster --config cluster.yaml
开启 Feature Gate (TopologyAwareHints)
# docker exec -it kind-control-plane bash
root@kind-control-plane:/# apt update
root@kind-control-plane:/# apt install vim -y
root@kind-control-plane:/# vim /etc/kubernetes/manifests/kube-apiserver.yaml
root@kind-control-plane:/# ## 添加 --feature-gates=TopologyAwareHints=true 启动参数后保存
root@kind-control-plane:/# vim /etc/kubernetes/manifests/kube-controller-manager.yaml
root@kind-control-plane:/# ## 添加 --feature-gates=TopologyAwareHints=true 启动参数后保存
root@kind-control-plane:/# exit
# kubectl edit cm -n kube-system kube-proxy
# 按下图配置 kube-proxy feature gates
# kubectl delete pod -l k8s-app=kube-proxy -n kube-system
为 Node 设置拓扑
# kubectl label node kind-control-plane topology.kubernetes.io/zone=a
# kubectl label node kind-worker topology.kubernetes.io/zone=a
# kubectl label node kind-worker2 topology.kubernetes.io/zone=b
# kubectl label node kind-worker3 topology.kubernetes.io/zone=b
为 Service 开启拓扑感知
# kubectl edit svc -n kube-system kube-dns
# 添加 service.kubernetes.io/topology-aware-hints: auto 注解
# kubectl get endpoints -n kube-system kube-dns
# kubectl get endpointslice -n kube-system -l k8s-app=kube-dns -oyaml
apiVersion: v1
items:
- addressType: IPv4
apiVersion: discovery.k8s.io/v1
endpoints:
- addresses:
- 10.244.0.5
conditions:
ready: true
hints:
forZones:
- name: a
nodeName: kind-control-plane
targetRef:
kind: Pod
name: coredns-558bd4d5db-pxphb
namespace: kube-system
resourceVersion: "7327"
uid: 539be8dd-747e-463c-8204-01e160f3a083
zone: a
- addresses:
- 10.244.2.2
conditions:
ready: true
hints:
forZones:
- name: b
nodeName: kind-worker3
targetRef:
kind: Pod
name: coredns-558bd4d5db-rhk7r
namespace: kube-system
resourceVersion: "6936"
uid: 5a221a61-d978-41a7-8506-9128d69865fa
zone: b
- addresses:
- 10.244.3.2
conditions:
ready: true
hints:
forZones:
- name: a
nodeName: kind-worker
targetRef:
kind: Pod
name: coredns-558bd4d5db-hfbmj
namespace: kube-system
resourceVersion: "5565"
uid: f2a6a303-6b17-4016-8ed7-7878d858921f
zone: a
- addresses:
- 10.244.1.2
conditions:
ready: true
hints:
forZones:
- name: b
nodeName: kind-worker2
targetRef:
kind: Pod
name: coredns-558bd4d5db-hswkq
namespace: kube-system
resourceVersion: "5558"
uid: 63ca17b6-bf03-4f91-8598-959d0cc762a4
zone: b
kind: EndpointSlice
metadata:
creationTimestamp: "2022-04-19T15:52:15Z"
generateName: kube-dns-
generation: 1
labels:
endpointslice.kubernetes.io/managed-by: endpointslice-controller.k8s.io
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: CoreDNS
kubernetes.io/service-name: kube-dns
Manager: Kube-Controller-Manager
operation: Update
time: "2022-04-19T15:52:15Z"
name: kube-dns-wt2f9
namespace: kube-system
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Service
name: kube-dns
uid: b1c3f974-cdd7-4949-b78e-888d1f292153
resourceVersion: "9907"
uid: 31267f84-37ea-4b09-a2a7-7b4d9db680ec
ports:
- name: metrics
port: 9153
protocol: TCP
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
kind: List
metadata:
resourceVersion: ""
selfLink: ""
验证 IPVS 规则
# kubectl get pod -n kube-system -l k8s-app=kube-dns -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
coredns-558bd4d5db-hfbmj 1/1 Running 0 45m 10.244.3.2 kind-worker <none> <none>
coredns-558bd4d5db-hswkq 1/1 Running 0 45m 10.244.1.2 kind-worker2 <none> <none>
coredns-558bd4d5db-pxphb 1/1 Running 0 31m 10.244.0.5 kind-control-plane <none> <none>
coredns-558bd4d5db-rhk7r 1/1 Running 0 34m 10.244.2.2 kind-worker3 <none> <none>
实验中,每个节点都运行了一个 coredns 副本
登陆到 kind-worker2 上,查看 IPVS 规则如下:
root@kind-worker2:/# ipvsadm -ln
TCP 10.96.0.10:53 rr
-> 10.244.1.2:53 Masq 1 0 0
-> 10.244.2.2:53 Masq 1 0 0
可以看出,coredns 地址被解析到了 10.244.1.2 和 10.244.2.2,目标 Pod 均在 topology.kubernetes.io/zone=b 节点上,满足就近访问原则。
同理,zone a 节点上的 IPVS 规则指向的服务 Pod IP 也是 zone a 的 coredns Pod 地址:
root@kind-control-plane:/# ipvsadm -ln
TCP 10.96.0.10:53 rr
-> 10.244.0.5:53 Masq 1 0 0
-> 10.244.3.2:53 Masq 1 0 0
缩小 coredns 规模(每可用区一个)
# kubectl get pod -n kube-system -l k8s-app=kube-dns -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
coredns-558bd4d5db-hfbmj 1/1 Running 0 60m 10.244.3.2 kind-worker <none> <none>
coredns-558bd4d5db-hswkq 1/1 Running 0 60m 10.244.1.2 kind-worker2 <none> <none>
如此,每个可用区只运行一个 coredns,检查新的 IPVS 规则
登陆到 kind-worker 上,查看 IPVS 规则如下:
# ipvsadm -ln
TCP 10.96.0.10:53 rr
-> 10.244.3.2:53 Masq 1 0 0
登陆到 kind-control-plane 上,查看 IPVS 规则如下:
# ipvsadm -ln
TCP 10.96.0.10:53 rr
-> 10.244.3.2:53 Masq 1 0 0
登陆到 kind-worker2 上,查看 IPVS 规则如下:
# ipvsadm -ln
TCP 10.96.0.10:53 rr
-> 10.244.1.2:53 Masq 1 0 0
可以看出,均服务同可用区就近访问原则
缩小 coredns 规模(单可用区可用)
# kubectl get pod -n kube-system -l k8s-app=kube-dns -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
coredns-558bd4d5db-hfbmj 1/1 Running 0 66m 10.244.3.2 kind-worker <none> <none>
# kubectl get endpointslice -n kube-system -l k8s-app=kube-dns -oyaml
apiVersion: v1
items:
- addressType: IPv4
apiVersion: discovery.k8s.io/v1
endpoints:
- addresses:
- 10.244.3.2
conditions:
ready: true
nodeName: kind-worker
targetRef:
kind: Pod
name: coredns-558bd4d5db-hfbmj
namespace: kube-system
resourceVersion: "5565"
uid: f2a6a303-6b17-4016-8ed7-7878d858921f
zone: a
kind: EndpointSlice
metadata:
creationTimestamp: "2022-04-19T15:52:15Z"
generateName: kube-dns-
generation: 4
labels:
endpointslice.kubernetes.io/managed-by: endpointslice-controller.k8s.io
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: CoreDNS
kubernetes.io/service-name: kube-dns
manager: kube-controller-manager
operation: Update
time: "2022-04-19T16:15:47Z"
name: kube-dns-wt2f9
namespace: kube-system
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Service
name: kube-dns
uid: b1c3f974-cdd7-4949-b78e-888d1f292153
resourceVersion: "13630"
uid: 31267f84-37ea-4b09-a2a7-7b4d9db680ec
ports:
- name: metrics
port: 9153
protocol: TCP
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
kind: List
metadata:
resourceVersion: ""
selfLink: ""
登陆到 kind-worker 上,查看 IPVS 规则如下:
# ipvsadm -ln
TCP 10.96.0.10:53 rr
-> 10.244.3.2:53 Masq 1 0 0
登陆到 kind-worker2 上,查看 IPVS 规则如下:
# ipvsadm -ln
TCP 10.96.0.10:53 rr
-> 10.244.3.2:53 Masq 1 0 0
可以看到,当本地可用区没有 coredns 服务时,跨可用区访问依然可行。