keepalived实现haproxy负载均衡高可用
一、什么是keepalived
keepalived是集群管理中保证集群高可用的一个服务软件,其功能类似于heartbeat,用来防止单点故障。
二、keepalived如何实现故障切换
keepalived 服务工作时,主master节点会不断地向备用节点发送心跳信息,告诉backup节点自己还活着。当主节点发生故障时,就无法发送心跳了,于是会调用自身的接管程序,接管主节点的ip资源和服务。
三、keepalived重要功能
keepalived 有三个重要的功能,分别是:
- 管理LVS负载均衡软件
- 实现LVS集群节点的健康检查
- 作为系统网络服务的高可用性(failover)
四、keepalived实现haproxy负载均衡高可用部署
环境说明:
主机名 | IP地址 | 安装的服务 | 系统 |
---|---|---|---|
master | 192.168.183.135 | haproxy、keepalived | centos8 |
backup | 192.168.183.136 | haproxy、keepalived | centos8 |
RS1 | 192.168.183.137 | httpd | centos8 |
RS2 | 192.168.183.138 | nginx | centos8 |
1、部署web界面
配置RS1
//修改主机名关闭防火墙和selinux
[root@localhost ~]# hostnamectl set-hostname RS1
[root@localhost ~]# bash
[root@RS1 ~]# setenforce 0
[root@RS1 ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@RS1 ~]# systemctl disable --now firewalld.service
Removed /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
//配置yum源
[root@RS1 ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 2495 100 2495 0 0 9415 0 --:--:-- --:--:-- --:--:-- 9379
[root@RS1 ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
//安装httpd
[root@RS1 ~]# dnf -y install httpd
[root@RS1 ~]# echo 'httpd' > /var/www/html/index.html
[root@RS1 ~]# systemctl enable --now httpd
Created symlink /etc/systemd/system/multi-user.target.wants/httpd.service → /usr/lib/systemd/system/httpd.service.
[root@RS1 ~]# curl 192.168.183.137
httpd
[root@RS1 ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 *:80 *:*
LISTEN 0 128 [::]:22 [::]:*
配置RS2
//修改主机名关闭防火墙和selinux
[root@localhost ~]# hostnamectl set-hostname RS2
[root@localhost ~]# bash
[root@RS2 ~]# setenforce 0
[root@RS2 ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@RS2 ~]# systemctl disable --now firewalld
Removed /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
//配置yum源
[root@RS2 ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 2495 100 2495 0 0 12729 0 --:--:-- --:--:-- --:--:-- 12729
[root@RS2 ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
//安装nginx
[root@RS2 ~]# dnf install -y nginx
[root@RS2 ~]# echo 'nginx' > /usr/share/nginx/html/index.html
[root@RS2 ~]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@RS2 ~]# curl 192.168.183.138
nginx
[root@RS2 ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 128 [::]:80 [::]:*
2、部署haproxy负载均衡
配置master
//修改主机名关闭防火墙和selinux
[root@localhost ~]# hostnamectl set-hostname master
[root@localhost ~]# bash
[root@master ~]# systemctl disable --now firewalld
Removed /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@master ~]# setenforce 0
[root@master ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
//下载haproxy软件包及所需依赖包
[root@master ~]# ls
anaconda-ks.cfg haproxy-2.1.3.tar.gz
[root@master ~]# dnf -y install make gcc pcre-devel bzip2-devel openssl-devel systemd-devel wget vim
//创建用户
[root@master ~]# useradd -rMs /sbin/nologin haproxy
//解压软件包进行安装
[root@master ~]# tar -xf haproxy-2.1.3.tar.gz
[root@master ~]# cd haproxy-2.1.3
[root@master haproxy-2.1.3]# make -j $(grep 'processor' /proc/cpuinfo |wc -l) \
> TARGET=linux-glibc \
> USE_OPENSSL=1 \
> USE_ZLIB=1 \
> USE_PCRE=1 \
> USE_SYSTEMD=1
[root@master haproxy-2.1.3]# make install PREFIX=/usr/local/haproxy
//复制命令到/usr/sbin目录下
[root@master haproxy-2.1.3]# cp haproxy /usr/sbin/
//修改内核参数
[root@master haproxy-2.1.3]# cd
[root@master ~]# vim /etc/sysctl.conf
[root@master ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
//修改配置文件
[root@master ~]# cat /etc/haproxy/haproxy.cfg
global
daemon
maxconn 256
defaults
mode http
timeout connect 5000ms
timeout client 50000ms
timeout server 50000ms
frontend http-in
bind *:80
default_backend servers
backend servers
server web01 192.168.183.137:80
server web02 192.168.183.138:80
//写service文件启动服务
[root@master ~]# cat >> /usr/lib/systemd/system/haproxy.service <<EOF
> [Unit]
> Description=HAProxy Load Balancer
> After=syslog.target network.target
>
> [Service]
> ExecStartPre=/usr/local/haproxy/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
> ExecStart=/usr/local/haproxy/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid
> ExecReload=/bin/kill -USR2 $MAINPID
>
> [Install]
> WantedBy=multi-user.target
> EOF
[root@master ~]# systemctl daemon-reload
[root@master ~]# systemctl start haproxy
//查看效果
[root@master ~]# curl 192.168.183.135
httpd
[root@master ~]# curl 192.168.183.135
nginx
[root@master ~]# curl 192.168.183.135
httpd
[root@master ~]# curl 192.168.183.135
nginx
配置backup
//修改主机名关闭防火墙和selinux
[root@localhost ~]# hostnamectl set-hostname backup
[root@localhost ~]# bash
[root@backup ~]# systemctl disable --now firewalld
Removed /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@backup ~]# setenforce 0
[root@backup ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
//下载haproxy软件包及所需依赖包
[root@backup ~]# ls
anaconda-ks.cfg haproxy-2.1.3.tar.gz
[root@backup ~]# dnf -y install make gcc pcre-devel bzip2-devel openssl-devel systemd-devel wget vim
//创建用户
[root@backup ~]# useradd -rMs /sbin/nologin haproxy
//解压软件包进行安装
[root@backup ~]# tar -xf haproxy-2.1.3.tar.gz
[root@backup ~]# cd haproxy-2.1.3
[root@backup haproxy-2.1.3]# make -j $(grep 'processor' /proc/cpuinfo |wc -l) \
> TARGET=linux-glibc \
> USE_OPENSSL=1 \
> USE_ZLIB=1 \
> USE_PCRE=1 \
> USE_SYSTEMD=1
[root@backup haproxy-2.1.3]# make install PREFIX=/usr/local/haproxy
//复制命令到/usr/sbin/目录下
[root@backup haproxy-2.1.3]# cp haproxy /usr/sbin/
//修改内核参数
[root@backup haproxy-2.1.3]# cd
[root@backup ~]# vim /etc/sysctl.conf
[root@backup ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
//修改配置文件
[root@backup ~]# mkdir /etc/haproxy
[root@backup ~]# cat /etc/haproxy/haproxy.cfg
global
daemon
maxconn 256
defaults
mode http
timeout connect 5000ms
timeout client 50000ms
timeout server 50000ms
frontend http-in
bind *:80
default_backend servers
backend servers
server web01 192.168.183.137:80
server web02 192.168.183.138:80
//写service文件启动服务
[root@backup ~]# cat >> /usr/lib/systemd/system/haproxy.service <<EOF
> [Unit]
> Description=HAProxy Load Balancer
> After=syslog.target network.target
>
> [Service]
> ExecStartPre=/usr/local/haproxy/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
> ExecStart=/usr/local/haproxy/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid
> ExecReload=/bin/kill -USR2 $MAINPID
>
> [Install]
> WantedBy=multi-user.target
> EOF
[root@backup ~]# systemctl daemon-reload
[root@backup ~]# systemctl start haproxy
//查看效果
[root@backup ~]# curl 192.168.183.136
httpd
[root@backup ~]# curl 192.168.183.136
nginx
[root@backup ~]# curl 192.168.183.136
httpd
[root@backup ~]# curl 192.168.183.136
nginx
//停掉backup端的haproxy
[root@backup ~]# systemctl stop haproxy
3、部署keepalived高可用
配置master
//安装keepalived
[root@master ~]# dnf -y install keepalived
//编辑配置文件启动服务
[root@master ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lb01
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.183.250
}
}
virtual_server 192.168.183.250 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 50
protocol TCP
real_server 192.168.183.135 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.183.136 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
[root@master ~]# systemctl enable --now keepalived
Created symlink /etc/systemd/system/multi-user.target.wants/keepalived.service → /usr/lib/systemd/system/keepalived.service.
//查看vip
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:7f:37:b0 brd ff:ff:ff:ff:ff:ff
inet 192.168.183.135/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
valid_lft 1034sec preferred_lft 1034sec
inet 192.168.183.250/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe7f:37b0/64 scope link noprefixroute
valid_lft forever preferred_lft forever
//vip访问web界面
[root@master ~]# curl 192.168.183.250
httpd
[root@master ~]# curl 192.168.183.250
nginx
[root@master ~]# curl 192.168.183.250
httpd
[root@master ~]# curl 192.168.183.250
nginx
配置backup
//安装keepalived
[root@backup ~]# dnf -y install keepalived
//编辑配置文件启动服务
[root@backup ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lb02
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
virtual_router_id 51
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.183.250
}
}
virtual_server 192.168.183.250 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 50
protocol TCP
real_server 192.168.183.135 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.183.136 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
[root@backup ~]# systemctl start keepalived
4、编写脚本
配置master
[root@master ~]# mkdir /scripts
[root@master ~]# cd /scripts/
[root@master scripts]# cat check_hp.sh
#!/bin/bash
haproxy_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bhaproxy\b'|wc -l)
if [ $haproxy_status -lt 1 ];then
systemctl stop keepalived
fi
[root@master scripts]# cat notify.sh
#!/bin/bash
VIP=$2
case "$1" in
master)
haproxy_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bhaproxy\b'|wc -l)
if [ $haproxy_status -lt 1 ];then
systemctl start haproxy
fi
;;
backup)
haproxy_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bhaproxy\b'|wc -l)
if [ $haproxy_status -gt 0 ];then
systemctl stop haproxy
fi
;;
*)
echo "Usage:$0 master|backup VIP"
;;
esac
[root@master scripts]# chmod +x check_hp.sh notify.sh
[root@master scripts]# ll
total 8
-rwxr-xr-x. 1 root root 148 Oct 9 20:57 check_hp.sh
-rwxr-xr-x. 1 root root 443 Oct 9 21:00 notify.sh
配置backup
[root@backup ~]# mkdir /scripts
[root@backup ~]# cd /scripts/
[root@backup scripts]# scp [email protected]:/scripts/notify.sh .
The authenticity of host '192.168.183.135 (192.168.183.135)' can't be established.
ECDSA key fingerprint is SHA256:c/bKicNnB6SvIpxi/x93PuBCTI8v7FuwiL4pI+1R16w.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '192.168.183.135' (ECDSA) to the list of known hosts.
[email protected]'s password:
notify.sh 100% 443 528.7KB/s 00:00
[root@backup scripts]# ll
total 4
-rwxr-xr-x. 1 root root 443 Oct 9 21:03 notify.sh
5、配置keepalived加入监控脚本
配置master
[root@master ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lb01
}
vrrp_script haproxy_check {
//添加这部分
script "/scripts/check_hp.sh"
interval 1
weight -20
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.183.250
}
track_script {
//及这部分
haproxy_check
}
notify_master "/scripts/notify.sh master 192.168.183.250"
}
virtual_server 192.168.183.250 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 50
protocol TCP
real_server 192.168.183.135 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.183.136 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
[root@master ~]# systemctl restart keepalived
配置backup
backup无需检测nginx是否正常,当升级为MASTER时启动nginx,当降级为BACKUP时关闭
[root@backup ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lb02
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
virtual_router_id 51
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.183.250
}
notify_master "/scripts/notify.sh master 192.168.183.250"
notify_backup "/scripts/notify.sh backup 192.168.183.250" //增加着两条
}
virtual_server 192.168.183.250 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 50
protocol TCP
real_server 192.168.183.135 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.183.136 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
[root@backup ~]# systemctl restart keepalived
6、检验
master端
[root@master ~]# curl 192.168.183.250
httpd
[root@master ~]# curl 192.168.183.250
nginx
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:7f:37:b0 brd ff:ff:ff:ff:ff:ff
inet 192.168.183.135/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
valid_lft 1609sec preferred_lft 1609sec
inet 192.168.183.250/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe7f:37b0/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@master ~]# systemctl stop haproxy
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:7f:37:b0 brd ff:ff:ff:ff:ff:ff
inet 192.168.183.135/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
valid_lft 1593sec preferred_lft 1593sec
inet6 fe80::20c:29ff:fe7f:37b0/64 scope link noprefixroute
valid_lft forever preferred_lft forever
backup端
[root@backup ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:07:de:9b brd ff:ff:ff:ff:ff:ff
inet 192.168.183.136/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
valid_lft 1541sec preferred_lft 1541sec
inet 192.168.183.250/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe07:de9b/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@backup ~]# curl 192.168.183.250
httpd
[root@backup ~]# curl 192.168.183.250
nginx
[root@backup ~]# curl 192.168.183.250
httpd
[root@backup ~]# curl 192.168.183.250
nginx