Description
在 kubernetes 中有 4 种 DNS 策略,分别是 ClusterFirstWithHostNet 、ClusterFirst 、Default、和 None,这些策略可以通过 dnsPolicy 来定义,如果 Pod、Deployment 或者 RC 等资源时没有定义 dnsPolicy,默认使用 ClusterFirst 策略
a. clusterFirstWithHostNet
当 pod 以 host 模式(和宿主机共享网络)启动时,pod 中的所有容器都会使用宿主机的 /etc/resolv.conf 配置DNS。如果在 Pod 中仍然想使用 k8s 集群 的 DNS 服务时,需要将 dnsPolicy 设置为 ClusterFirstWithHostNet
apiVersion: apps/v1
kind: Deployment
metadata:
name: perf
labels:
app: perf
spec:
replicas: 1
selector:
matchLabels:
app: perf
template:
metadata:
labels:
app: perf
spec:
hostNetwork: true
containers:
- name: perf
image: qperf:latest
imagePullPolicy: IfNotPresent
dnsPolicy: ClusterFirstWithHostNet
进入容器查看 /etc/resolv.conf
$ kubectl exec -it perf-859b6d5d46-4j5ms cat /etc/resolv.conf
nameserver 10.200.254.254
search default.svc.cluster.local. svc.cluster.local. cluster.local.
options ndots:5
b. ClusterFirst
pod 内的 DNS 优先会使用 k8s 集群内的 DNS 服务,会使用 kubedns 或者 coredns 进行域名解析。如果解析不成功才使用宿主机的 DNS 配置进行解析。
c. Default
kubelet 的默认方式使用宿主机的 /etc/resolv.conf 来进行解析。通过设置 kubelet 的启动参数, --resolv-conf=/etc/resolv.conf 来决定该 DNS 服务使用的解析文件的地址
注意,部署集群 DNS 需要将 dnsPolicy 设置成 Default,而非使用默认值 ClusterFirst,否则该 DNS 服务的上游解析地址会变成它自身的 Service 的 ClusterIP(我解析我自己),导致域名无法解析,比如 coredns的配置
spec:
containers:
- name: coredns
image: zhangzhonglin/coredns:test
imagePullPolicy: IfNotPresent
resources:
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
args: [ "-conf", "/etc/coredns/Corefile" ]
dnsPolicy: Default
d. None
不使用集群和宿主机的 DNS 策略。和 dnsConfig 配合一起使用,来自定义 DNS 配置
dnsConfig
包括下面属性:
nameservers
: DNS Server 的列表,最多 3 个searches
: search 域名列表,也就是/etc/resolv.conf
中的search
字段的配置,最多配置6个options
: u选项列表,也就是/etc/resolv.conf
中的option
字段的配置
注意,如果设置 namespaces 超过三个,则报错 The Deployment "perf" is invalid: spec.template.spec.dnsConfig.nameservers: Invalid value: []string{"1.2.3.4", "8.8.8.8", "9.9.8.8", "7.7.8.8"}: must not have more than 3 nameservers
apiVersion: apps/v1
kind: Deployment
metadata:
name: perf
labels:
app: perf
spec:
replicas: 1
selector:
matchLabels:
app: perf
template:
metadata:
labels:
app: perf
spec:
containers:
- name: perf
image: qperf:latest
imagePullPolicy: IfNotPresent
dnsPolicy: None
dnsConfig:
nameservers:
- 1.2.3.4
- 8.8.8.8
searches:
- aa.bb.cc
- bb.cc.dd
options:
- name: ndots
value: "5"
查看 pod 中容器的 /etc/resolv.conf
$ kubectl exec -it perf-54844bb5cf-5bqcc cat /etc/resolv.conf
nameserver 1.2.3.4
nameserver 8.8.8.8
search aa.bb.cc bb.cc.dd
options ndots:5
开始关于 dns 部分的源码分析
定义 dnsConfigurer,路径 pkg/kubelet/kubelet.go
// Kubelet is the main kubelet implementation. type Kubelet struct { kubeletConfiguration kubeletconfiginternal.KubeletConfiguration // dnsConfigurer is used for setting up DNS resolver configuration when launching pods. dnsConfigurer *dns.Configurer
// Configurer is used for setting up DNS resolver configuration when launching pods.
type Configurer struct {
recorder record.EventRecorder
nodeRef *v1.ObjectReference
nodeIP net.IP
// If non-nil, use this for container DNS server.
clusterDNS []net.IP
// If non-empty, use this for container DNS search.
ClusterDomain string
// The path to the DNS resolver configuration file used as the base to generate
// the container's DNS resolver configuration file. This can be used in
// conjunction with clusterDomain and clusterDNS.
ResolverConfig string
}
1. dns 配置初始化
NewMainKubelet 函数中实例化 kubelet,初始化 dnsConfigurer
// NewConfigurer returns a DNS configurer for launching pods.
func NewConfigurer(recorder record.EventRecorder, nodeRef *v1.ObjectReference, nodeIP net.IP, clusterDNS []net.IP, clusterDomain, resolverConfig string) *Configurer {
return &Configurer{
recorder: recorder,
nodeRef: nodeRef,
nodeIP: nodeIP,
clusterDNS: clusterDNS,
ClusterDomain: clusterDomain,
ResolverConfig: resolverConfig,
}
}
2. 创建 pod 检查 resolv.conf
主要作用是限制 resolv.conf,调用 parseResolvConf 函数解析文件读取,nameservers,searches,options
// HandlePodAdditions is the callback in SyncHandler for pods being added from
// a config source.
func (kl *Kubelet) HandlePodAdditions(pods []*v1.Pod) {
start := kl.clock.Now()
sort.Sort(sliceutils.PodsByCreationTime(pods))
for _, pod := range pods {
// Responsible for checking limits in resolv.conf
if kl.dnsConfigurer != nil && kl.dnsConfigurer.ResolverConfig != "" {
kl.dnsConfigurer.CheckLimitsForResolvConf()
}
existingPods := kl.podManager.GetPods()
2.1 CheckLimitsForResolvConf
主要作用是限制 resolv.conf,调用 parseResolvConf 函数解析文件读取,nameservers,searches,options,validation.MaxDNSSearchPaths 限制 searches 最多 6 个,
createPodSandbox
--> generatePodSandboxConfig
--> m.runtimeHelper.GetPodDNS (第 3 章节讲解)
--> m.runtimeService.RunPodSandbox
--> rewriteResolvFile (第 4 章节讲解)
3. 获取 dns 配置
// GetPodDNS returns DNS settings for the pod.
// This function is defined in kubecontainer.RuntimeHelper interface so we
// have to implement it.
func (kl *Kubelet) GetPodDNS(pod *v1.Pod) (*runtimeapi.DNSConfig, error) {
return kl.dnsConfigurer.GetPodDNS(pod)
}
3.1 GetPodDNS 函数
// GetPodDNS returns DNS settings for the pod.
func (c *Configurer) GetPodDNS(pod *v1.Pod) (*runtimeapi.DNSConfig, error) {
dnsConfig, err := c.getHostDNSConfig()
if err != nil {
return nil, err
}
3.1.1 getHostDNSConfig
如果指定 host resolv.conf 配置,则读取,获得参数
func (c *Configurer) getHostDNSConfig() (*runtimeapi.DNSConfig, error) {
var hostDNS, hostSearch, hostOptions []string
// Get host DNS settings
if c.ResolverConfig != "" {
f, err := os.Open(c.ResolverConfig)
if err != nil {
return nil, err
}
defer f.Close()
hostDNS, hostSearch, hostOptions, err = parseResolvConf(f)
if err != nil {
return nil, err
}
}
return &runtimeapi.DNSConfig{
Servers: hostDNS,
Searches: hostSearch,
Options: hostOptions,
}, nil
}
3.1.2 getPodDNSType 函数获取四种 DNS 策略中的类型
- clusterFirstWithHostNet:podDNSCluster
- None:podDNSNone
- clusterFirst:podDNSCluster
- default:podDNSHost
func getPodDNSType(pod *v1.Pod) (podDNSType, error) {
dnsPolicy := pod.Spec.DNSPolicy
switch dnsPolicy {
case v1.DNSNone:
return podDNSNone, nil
case v1.DNSClusterFirstWithHostNet:
return podDNSCluster, nil
case v1.DNSClusterFirst:
if !kubecontainer.IsHostNetworkPod(pod) {
return podDNSCluster, nil
}
// Fallback to DNSDefault for pod on hostnetowrk.
fallthrough
case v1.DNSDefault:
return podDNSHost, nil
}
// This should not happen as kube-apiserver should have rejected
// invalid dnsPolicy.
return podDNSCluster, fmt.Errorf(fmt.Sprintf("invalid DNSPolicy=%v", dnsPolicy))
}
3.1.3 DNS None 策略
switch dnsType {
case podDNSNone:
// DNSNone should use empty DNS settings as the base.
dnsConfig = &runtimeapi.DNSConfig{}
3.1.4 clusterFirstWithHostNet 和 ClusterFirst
如果 clusterDNS 未设置,则 fallthrough 使用 host,generateSearchesForDNSClusterFirst 生成三个 search
case podDNSCluster:
if len(c.clusterDNS) != 0 {
// For a pod with DNSClusterFirst policy, the cluster DNS server is
// the only nameserver configured for the pod. The cluster DNS server
// itself will forward queries to other nameservers that is configured
// to use, in case the cluster DNS server cannot resolve the DNS query
// itself.
dnsConfig.Servers = []string{}
for _, ip := range c.clusterDNS {
dnsConfig.Servers = append(dnsConfig.Servers, ip.String())
}
dnsConfig.Searches = c.generateSearchesForDNSClusterFirst(dnsConfig.Searches, pod)
dnsConfig.Options = defaultDNSOptions
break
}
// clusterDNS is not known. Pod with ClusterDNSFirst Policy cannot be created.
nodeErrorMsg := fmt.Sprintf("kubelet does not have ClusterDNS IP configured and cannot create Pod using %q policy. Falling back to %q policy.", v1.DNSClusterFirst, v1.DNSDefault)
c.recorder.Eventf(c.nodeRef, v1.EventTypeWarning, "MissingClusterDNS", nodeErrorMsg)
c.recorder.Eventf(pod, v1.EventTypeWarning, "MissingClusterDNS", "pod: %q. %s", format.Pod(pod), nodeErrorMsg)
// Fallback to DNSDefault.
fallthrough
3.1.4.1 看这里知道为啥 search 里的设置
search default.svc.cluster.local. svc.cluster.local. cluster.local.
func (c *Configurer) generateSearchesForDNSClusterFirst(hostSearch []string, pod *v1.Pod) []string {
if c.ClusterDomain == "" {
return hostSearch
}
nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, c.ClusterDomain)
svcDomain := fmt.Sprintf("svc.%s", c.ClusterDomain)
clusterSearch := []string{nsSvcDomain, svcDomain, c.ClusterDomain}
return omitDuplicates(append(clusterSearch, hostSearch...))
}
3.1.5 None 模式设置 dnsConfig 加入进来
if pod.Spec.DNSConfig != nil {
dnsConfig = appendDNSConfig(dnsConfig, pod.Spec.DNSConfig)
}
3.1.6 限制 nameservers 最多 3 个
func (c *Configurer) formDNSNameserversFitsLimits(nameservers []string, pod *v1.Pod) []string {
if len(nameservers) > validation.MaxDNSNameservers {
nameservers = nameservers[0:validation.MaxDNSNameservers]
log := fmt.Sprintf("Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: %s", strings.Join(nameservers, " "))
c.recorder.Event(pod, v1.EventTypeWarning, "DNSConfigForming", log)
klog.Error(log)
}
return nameservers
formDNSSearchFitsLimits 最多限制 6 个 search
4. rewriteResolvFile
rewriteResolvFile 写入文件路径如上图最后一行所示,写入 nameserver search options
// rewriteResolvFile rewrites resolv.conf file generated by docker.
func rewriteResolvFile(resolvFilePath string, dns []string, dnsSearch []string, dnsOptions []string) error {
if len(resolvFilePath) == 0 {
klog.Errorf("ResolvConfPath is empty.")
return nil
}
if _, err := os.Stat(resolvFilePath); os.IsNotExist(err) {
return fmt.Errorf("ResolvConfPath %q does not exist", resolvFilePath)
}
var resolvFileContent []string
for _, srv := range dns {
resolvFileContent = append(resolvFileContent, "nameserver "+srv)
}
if len(dnsSearch) > 0 {
resolvFileContent = append(resolvFileContent, "search "+strings.Join(dnsSearch, " "))
}
if len(dnsOptions) > 0 {
resolvFileContent = append(resolvFileContent, "options "+strings.Join(dnsOptions, " "))
}
if len(resolvFileContent) > 0 {
resolvFileContentStr := strings.Join(resolvFileContent, "\n")
resolvFileContentStr += "\n"
klog.V(4).Infof("Will attempt to re-write config file %s with: \n%s", resolvFilePath, resolvFileContent)
if err := rewriteFile(resolvFilePath, resolvFileContentStr); err != nil {
klog.Errorf("resolv.conf could not be updated: %v", err)
return err
}
}
return nil
}