首页 > 代码库 > Nginx+Keepalived实现Nginx负载均衡及高可用WEB服务器集群
Nginx+Keepalived实现Nginx负载均衡及高可用WEB服务器集群
环境：
操作系统Centos 6.5 X86_64(final)
Nginx-Master:192.168.2.32
Nginx-Backup:192.168.3.31
VIP:192.168.2.33
Web3:192.168.2.29
Web4:192.168.2.30
1、分别在Nginx-Master和Nginx-Backup上安装nginx
[root@Nginx-Master ~]# rpm -i http://nginx.org/packages/centos/6/noarch/RPMS/nginx-release-centos-6-0.el6.ngx.noarch.rpm
[root@Nginx-Backup ~]# rpm -i http://nginx.org/packages/centos/6/noarch/RPMS/nginx-release-centos-6-0.el6.ngx.noarch.rpm
[root@Nginx-Master ~]# yum install nginx
[root@Nginx-Backup ~]# yum install nginx
2、先配置Nginx-Master上的nginx，建立配置一个proxy.conf文件。并复制一份到Nginx-Backup上，删除两台机器上默认的default.conf,启动nginx服务；并将服务添加到开机启动。
[root@Nginx-Master ~]# vi /etc/nginx/conf.d/proxy.conf
upstream web {
＃ip_hash;
server 192.168.2.29:80; #默认为rr轮询，如需解决session的问题采有哈希（ip_hash)模块。
server 192.168.2.30:80;
}
server {
listen 80;
index index.php index.html index.htm;
location / {
proxy_pass http://web;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
[root@Nginx-Master ~]# scp /etc/nginx/conf.d/proxy.conf root@192.168.2.31:/etc/nginx/conf.d/
[root@Nginx-Master ~]# service nginx start
Starting nginx: [ OK ]
[root@Nginx-Backup ~]# service nginx start
Starting nginx: [ OK ]
[root@Nginx-Master ~]# chkconfig nginx on
[root@Nginx-Backup ~]# chkconfig nginx on
3、分别在Nginx-Master和Nginx-Backup上安装keepalived并配置。
[root@Nginx-Master ~]# yum install keepalived
[root@Nginx-Backup ~]# yum install keepalived
[root@Nginx-Master ~]# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_script chk_nginx { #监控nginx服务进程脚本
script "/root/nginx.sh"
interval 2
weight 2
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
mcast_src_ip 192.168.2.32
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
track_script { #调用监控脚本
chk_nginx
}
virtual_ipaddress {
192.168.2.33
}
}
[root@Nginx-Backup ~]# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_script chk_nginx { #监控nginx服务进程脚本
script "/root/nginx.sh"
interval 2
weight 2
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 51
priority 90
mcast_src_ip 192.168.2.31
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
track_script { #调用监控脚本
chk_nginx
virtual_ipaddress {
192.168.2.33
}
}
[root@Nginx-Master ~]# service keepalived start
Starting keepalived: [ OK ]
[root@Nginx-Backup ~]# service keepalived start
Starting keepalived: [ OK ]
3.1.对keepalived的不足写的一个脚本，用来检测本机的nginx是否正常的运行，如果nginx挂掉试着重新启动，如果启动后又挂掉，那么就直接停止keepalived进程，keepalived将转移到另一台备用上，实现故障转移
[root@Nginx-Master ~]# vi /root/nginx.sh
#!/bin/bash
A=`ps -C nginx --no-header |wc -l` # 查看是否有 nginx进程 把值赋给变量A
if [ $A -eq 0 ];then #如果没有进程值为零
/usr/sbin/nginx
sleep 1
if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then
/etc/init.d/keepalived stop # 则停止keepalived 进程
fi
fi
[root@Nginx-Master ~]# chmod 755 /root/nginx.sh
[root@Nginx-Master ~]# scp /root/nginx.sh root@192.168.2.31:/root/nginx.sh
3.2.重启keepalived服务，查看日志脚本是否正常
Nginx-Master日志
[root@Nginx-Master ~]# tail -f /var/log/messages
Sep 5 15:20:48 Nginx-Master Keepalived_vrrp[5136]: VRRP sockpool: [ifindex(2), proto(112), fd(10,11)]
Sep 5 15:20:48 Nginx-Master Keepalived_vrrp[5136]: VRRP_Script(chk_nginx) succeeded ＃检查脚本成功，表示正常
Sep 5 15:20:48 Nginx-Master Keepalived_vrrp[5136]: VRRP_Instance(VI_1) Transition to MASTER STATE
Sep 5 15:20:49 Nginx-Master Keepalived_vrrp[5136]: VRRP_Instance(VI_1) Entering MASTER STATE
Sep 5 15:20:49 Nginx-Master Keepalived_vrrp[5136]: VRRP_Instance(VI_1) setting protocol VIPs.
Nginx-Backup日志
[root@Nginx-Backup ~]# tail -f /var/log/messages
Sep 5 15:20:41 Nginx-Backup Keepalived_vrrp[7670]: Using LinkWatch kernel netlink reflector...
Sep 5 15:20:41 Nginx-Backup Keepalived_vrrp[7670]: VRRP_Instance(VI_1) Entering BACKUP STATE
Sep 5 15:20:41 Nginx-Backup Keepalived_vrrp[7670]: VRRP sockpool: [ifindex(2), proto(112), fd(10,11)]
Sep 5 15:20:41 Nginx-Backup Keepalived_vrrp[7670]: VRRP_Script(chk_nginx) succeeded #检查脚本成功，表示正常
[root@Nginx-Backup ~]# tail -f /var/log/messages
Sep 5 15:10:39 Nginx-Backup Keepalived_vrrp[5265]: Process [5913] didn‘t respond to SIGTERM
Sep 5 15:10:41 Nginx-Backup Keepalived_vrrp[5265]: Process [5923] didn‘t respond to SIGTERM #如出现这种，看下脚本是否正确。
Sep 5 15:10:43 Nginx-Backup Keepalived_vrrp[5265]: Process [5933] didn‘t respond to SIGTERM
3.3.另外一种查看脚本是否生效方法,停止nginx服务，马上会自动启动
[root@Nginx-Master ~]# service nginx status
nginx (pid 1114) is running...
[root@Nginx-Master ~]# service nginx stop
Stopping nginx: [ OK ]
[root@Nginx-Master ~]# service nginx status
nginx (pid 28751) is running...
4、测试VIP，检查是否能主备切换
4.1.当两台主机同时启动时，只有Nginx-Master服务器拥有VIP地址，备服务器没有。
[root@Nginx-Master ~]# ip addr
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:50:56:a6:00:13 brd ff:ff:ff:ff:ff:ff
inet 192.168.2.32/24 brd 192.168.2.255 scope global eth0
inet 192.168.2.33/32 scope global eth0 #可以看到Nginx-Master服务器上拥有192.168.2.33这个VIP地址
inet6 fe80::250:56ff:fea6:13/64 scope link
valid_lft forever preferred_lft forever
[root@Nginx-Backup ~]# ip addr
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
link/ether 08:00:27:77:d3:82 brd ff:ff:ff:ff:ff:ff
inet 192.168.2.31/24 brd 192.168.2.255 scope global eth0
inet6 fe80::a00:27ff:fe77:d382/64 scope link #Nginx-Backup服务器上没有
valid_lft forever preferred_lft forever
4.2.当停止Nginx-Master服务器的keepalived服务，再查看下两台主机的VIP地址，发现VIP地址已从主服务器转移到了备服务器
[root@Nginx-Master ~]# service keepalived stop
Stopping keepalived: [ OK ]
[root@Nginx-Master ~]# ip addr
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:50:56:a6:00:13 brd ff:ff:ff:ff:ff:ff
inet 192.168.2.32/24 brd 192.168.2.255 scope global eth0
inet6 fe80::250:56ff:fea6:13/64 scope link #Nginx-Master服务器VIP地址已移除
valid_lft forever preferred_lft forever
[root@Nginx-Backup ~]# ip addr
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
link/ether 08:00:27:77:d3:82 brd ff:ff:ff:ff:ff:ff
inet 192.168.2.31/24 brd 192.168.2.255 scope global eth0
inet 192.168.2.33/32 scope global eth0 #VIP地地已自动转移到Nginx-Backup服务器上
inet6 fe80::a00:27ff:fe77:d382/64 scope link
valid_lft forever preferred_lft forever
5、日志查看主备切换过程
5.1.当停止Nginx-Master上的keepalived服务时
root@Nginx-Master ~]# service keepalived stop
Nginx-Master日志
[root@Nginx-Master ~]# tail -f /var/log/messages
Sep 4 18:04:06 Nginx-Master Keepalived[3278]: Stopping Keepalived v1.2.7 (02/21,2013) ＃主服务器已停掉
Sep 4 18:04:06 Nginx-Master Keepalived_vrrp[3281]: VRRP_Instance(VI_1) sending 0 priority
Sep 4 18:04:06 Nginx-Master Keepalived_vrrp[3281]: VRRP_Instance(VI_1) removing protocol VIPs.
Nginx-Backup日志
[root@Nginx-Backup ~]# tail -f /var/log/messages
Sep 4 18:04:07 Nginx-Backup Keepalived_vrrp[1428]: VRRP_Instance(VI_1) Transition to MASTER STATE
Sep 4 18:04:08 Nginx-Backup Keepalived_vrrp[1428]: VRRP_Instance(VI_1) Entering MASTER STATE #Nginx-Backup转为MASTER STATE
Sep 4 18:04:08 Nginx-Backup Keepalived_vrrp[1428]: VRRP_Instance(VI_1) setting protocol VIPs.
Sep 4 18:04:08 Nginx-Backup Keepalived_healthcheckers[1427]: Netlink reflector reports IP 192.168.2.33 added
Sep 4 18:04:08 Nginx-Backup Keepalived_vrrp[1428]: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 192.168.2.33
Sep 4 18:04:13 Nginx-Backup Keepalived_vrrp[1428]: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 192.168.2.33
5.2.当Nginx-Master的keepalived服务再次启动时
[root@Nginx-Master ~]# service keepalived restart
Stopping keepalived: [ OK ]
Starting keepalived: [ OK ]
Nginx-Master日志
[root@Nginx-Master ~]# tail -f /var/log/messages
Sep 4 18:06:47 Nginx-Master Keepalived_vrrp[3316]: VRRP_Instance(VI_1) Transition to MASTER STATE
Sep 4 18:06:48 Nginx-Master Keepalived_vrrp[3316]: VRRP_Instance(VI_1) Entering MASTER STATE #Nginx-Master转回MASTER STATE
Sep 4 18:06:48 Nginx-Master Keepalived_vrrp[3316]: VRRP_Instance(VI_1) setting protocol VIPs.
Sep 4 18:06:48 Nginx-Master Keepalived_healthcheckers[3315]: Netlink reflector reports IP 192.168.2.33 added
Sep 4 18:06:48 Nginx-Master Keepalived_vrrp[3316]: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 192.168.2.33
Sep 4 18:06:53 Nginx-Master Keepalived_vrrp[3316]: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 192.168.2.33
Nginx-Backup日志
[root@Nginx-Backup ~]# tail -f /var/log/messages
Sep 4 18:06:47 Nginx-Backup Keepalived_vrrp[1428]: VRRP_Instance(VI_1) Received higher prio advert
Sep 4 18:06:47 Nginx-Backup Keepalived_vrrp[1428]: VRRP_Instance(VI_1) Entering BACKUP STATE #Nginx-Backup转回BACKUP STATE
Sep 4 18:06:47 Nginx-Backup Keepalived_vrrp[1428]: VRRP_Instance(VI_1) removing protocol VIPs.
Sep 4 18:06:47 Nginx-Backup Keepalived_healthcheckers[1427]: Netlink reflector reports IP 192.168.2.33 removed
6、安装后端两台httpd服务器，并添加内容测试
[root@web3 ~]# yum install httpd
[root@web3 ~]# vi /var/www/html/index.html
<h1>Welcome to web3(192.168.2.29)</h1>
[root@web3 ~]# service httpd start
[root@web4 ~]# yum install httpd
[root@web4 ~]# vi /var/www/html/index.html
<h1>Welcome to web4(192.168.2.30)</h1>
[root@web4 ~]# service httpd start
7.查看后端web服务器日志
7.1.后端web服务器日志，访问IP还是代理服务器的，不是真实客户端IP
[root@web3 ~]# tail -f /var/log/httpd/access_log
192.168.2.31 - - [05/Sep/2014:15:47:46 +0800] "GET / HTTP/1.0" 200 39 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.3.1.2000 Chrome/30.0.1599.101 Safari/537.36"
192.168.2.32 - - [05/Sep/2014:15:58:02 +0800] "GET / HTTP/1.0" 200 39 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.3.1.2000 Chrome/30.0.1599.101 Safari/537.36"
[root@web4 ~]# tail -f /var/log/httpd/access_log
192.168.2.31 - - [05/Sep/2014:15:48:47 +0800] "GET / HTTP/1.0" 200 39 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.3.1.2000 Chrome/30.0.1599.101 Safari/537.36"
192.168.2.32 - - [05/Sep/2014:15:59:55 +0800] "GET / HTTP/1.0" 200 39 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.3.1.2000 Chrome/30.0.1599.101 Safari/537.36"
7.2.虽然在配置proxy.conf时已配置了proxy_set_header X-Real-IP $remote_addr；但还需要在web端修改配围起文件显示
[root@web3 ~]# vi /etc/httpd/conf/httpd.conf
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
改为:
LogFormat "%{X-Real-IP}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
web4也同样修改以上数据后重启
[root@web3 ~]# service httpd restart
[root@web4 ~]# service httpd restart
7.3.最后再查看后端web服务器日志为真实客户端IP
[root@web3 ~]# tail -f /var/log/httpd/access_log
192.168.2.200 - - [05/Sep/2014:16:58:36 +0800] "GET / HTTP/1.0" 200 39 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.3.1.2000 Chrome/30.0.1599.101 Safari/537.36"
[root@web4 ~]# tail -f /var/log/httpd/access_log
192.168.2.200 - - [05/Sep/2014:16:58:42 +0800] "GET / HTTP/1.0" 200 39 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.3.1.2000 Chrome/30.0.1599.101 Safari/537.36"
最后测试主keepalived挂掉，会转移到备用keepalived上，nginx负载均衡服务，keepalived会一直调用脚本检查机制，如果nginx服务挂掉或不能从新启动，都会停止掉keepalived并立即转移到备用上继续工作，后端web服务器出现问题，nginx负载均衡能自动切换后端有故障的web服务器。
参考：http://network.51cto.com/art/201007/209823.htm
http://friendlinux.blog.51cto.com/6249249/1433295
Nginx+Keepalived实现Nginx负载均衡及高可用WEB服务器集群