Nginx双主集群搭建

技术分享 Nginx
技术分享
发布日期:   2024-01-11
更新日期:   2024-04-07
文章字数:   1.5k
阅读时长:   7 分

安装编译Nginx 1.22版本

# 安装依赖和工具
yum install -y vim wget gcc gcc-c++ make automake autoconf libtool pcre pcre-devel zlib zlib-devel openssl openssl-devel net-tools psmisc

# 关闭SELinux
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config

#下载nginx
wget https://nginx.org/download/nginx-1.22.0.tar.gz
#解压文件
tar -zxvf nginx-1.22.0.tar.gz && cd nginx-1.22.0

#编译

./configure --prefix=/usr/local/nginx --http-log-path=/var/log/nginx/access.log --error-log-path=/var/log/nginx/error.log --conf-path=/usr/local/nginx/conf/nginx.conf --lock-path=/var/lock/nginx.lock --pid-path=/run/nginx.pid --with-pcre-jit --with-http_ssl_module --with-http_v2_module --with-http_sub_module --with-stream --with-stream_ssl_module --with-http_gzip_static_module --with-http_stub_status_module --with-pcre --with-file-aio --with-http_realip_module --with-http_stub_status_module 

make 
#安装
make install

#验证版本：

/usr/local/nginx/sbin/nginx -V

# 创建快捷方式：

ln -s /usr/local/nginx/sbin/nginx /usr/bin/nginx



创建nginx.service文件，并编辑
vim /lib/systemd/system/nginx.service

#内容如下：

[Unit]
Description=nginx service
After=network.target
[Service]
Type=forking
ExecStart=/usr/bin/nginx
ExecReload=/usr/bin/nginx -s reload
ExecStop=/usr/bin/nginx -s stop
PrivateTmp=true
[Install]
WantedBy=multi-user.target


# 开放nginx的80端口
firewall-cmd --zone=public --add-port=80/tcp --permanent
firewall-cmd --reload

# 安装keepalived
wget https://keepalived.org/software/keepalived-2.2.8.tar.gz --no-check-certificate

tar -zxvf keepalived-2.2.8.tar.gz && cd keepalived-2.2.8
./configure --prefix=/usr/local/keepalived --disable-fwmark 
#选项--disable-fwmark 可用于禁用iptables规则,可防止VIP无法访问,无此选项默认会启用iptables规则
make -j 2 && make install

# 安装查看是否版本正确
/usr/local/keepalived/sbin/keepalived -v

ln -s /usr/local/keepalived/sbin/keepalived  /usr/bin/keepalived


# 修改nginx相关首页标识
vim /usr/local/nginx/html/index.html


节点一:172.16.2.191 keepalived 配置

vim /usr/local/keepalived/etc/keepalived/keepalived.conf

global_defs
{
    notification_email
    {
       75013085@qq.com
    }
    notification_email_from 75013085@qq.com
    smtp_server 127.0.0.1
    stmp_connect_timeout 30
    router_id LVS_DEVEL
}

vrrp_script Monitor_Nginx {
    script "/usr/local/keepalived/scripts/monitor_nginx.sh"
    interval 2
    weight -50
}

# 虚拟IP1, 本机作为Master
vrrp_instance VI_1 {
    state MASTER
    interface ens192
    virtual_router_id 51
    mcast_src_ip 172.16.2.191
    priority 120
    advert_int 1
    track_interface {
        ens192
    }

    authentication {
        auth_type PASS
        auth_pass 202311
    }
    virtual_ipaddress {
        172.16.2.193
    }i
    track_script {
        Monitor_Nginx
    }

}

# 虚拟IP2, 本机作为Backup
vrrp_instance VI_2 {
    state BACKUP
    interface ens192
    virtual_router_id 52
    mcast_src_ip 172.16.2.191
    priority 100
    advert_int 1
    track_interface {
        ens192
    }

    authentication {
        auth_type PASS
        auth_pass 202311
    }
    virtual_ipaddress {
        172.16.2.194
    }
    track_script {
        Monitor_Nginx
    }

}

节点二:172.16.2.191keepalived 配置

vim /usr/local/keepalived/etc/keepalived/keepalived.conf
global_defs
{
    notification_email
    {
       75013085@qq.com
    }
    notification_email_from 75013085@qq.com
    smtp_server 127.0.0.1
    stmp_connect_timeout 30
    router_id LVS_DEVEL
}

vrrp_script Monitor_Nginx {
    script "/usr/local/keepalived/scripts/monitor_nginx.sh"
    interval 2
    weight 2
}

# 虚拟IP1, 本机作为BACKUP
vrrp_instance VI_1 {
    state BACKUP
    interface ens192
    virtual_router_id 51
    mcast_src_ip 172.16.2.192
    priority 100
    advert_int 1
    track_interface {
        ens192
    }

    authentication {
        auth_type PASS
        auth_pass 202311
    }
    virtual_ipaddress {
        172.16.2.193
    }
    track_script {
        Monitor_Nginx
    }
}

# 虚拟IP2, 本机作为Master
vrrp_instance VI_2 {
    state MASTER
    interface ens192
    virtual_router_id 52
    mcast_src_ip 172.16.2.192
    priority 120
    advert_int 1
    track_interface {
        ens192
    }

    authentication {
        auth_type PASS
        auth_pass 202311
    }
    virtual_ipaddress {
        172.16.2.194
    }
    track_script {
        Monitor_Nginx
    }

}

允许vrrp

firewall-cmd --permanent --direct --add-rule ipv4 filter INPUT 0 -p vrrp -j ACCEPT
firewall-cmd --permanent --direct --add-rule ipv4 filter OUTPUT 0 -p vrrp -j ACCEPT
firewall-cmd --reload

编写nginx状态监测脚本

#如果 nginx 停止运行，尝试启动，如果无法启动则杀死本机的 keepalived 进程， keepalied将虚拟 ip 绑定到 BACKUP 机器上。
mkdir /usr/local/keepalived/scripts
vim /usr/local/keepalived/scripts/monitor_nginx.sh


# 监控nginx进程，若nginx主进程不存在则启动nginx
# 若5s后nginx进程还是不存在的话kill掉keepalived进程,防止nginx没运行该主机的keepalived还接管虚拟IP




#!/bin/bash
if [ "$(ps -ef | grep "nginx: master process"| grep -v grep )" == "" ]
        then
        /usr/local/nginx/sbin/nginx
        sleep 5
        if [ "$(ps -ef | grep "nginx: master process"| grep -v grep )" == "" ]
        then
                  killall keepalived
        fi
fi

chmod +x /usr/local/keepalived/scripts/monitor_nginx.sh
# 开机自动启动
systemctl enable keepalived
# 启动keepadlived
systemctl start keepalived
# 查看ip漂移情况
ip add

# 关闭查看是否漂移
nginx -s stop


如果出现编译失败，需要更新cmake 3.26.6

#删除cmake
yum remove cmake
wget https://github.com/Kitware/CMake/archive/refs/tags/v3.26.6.tar.gz
tar -zxvf v3.26.6.tar.gz && cd CMake-3.26.6



./bootstrap --prefix=/usr/local/cmake && make &&  make install
# 创建快捷方式
ln -s  /usr/local/cmake/bin/cmake /usr/bin/cmake

配置文件同步 rsync+lsyncd

两边同时部署后就可以双向同步


# 快速切换源
bash <(curl -sSL https://gitee.com/SuperManito/LinuxMirrors/raw/main/ChangeMirrors.sh)
vim /etc/yum.repos.d/CentOS-Base.repo
gpgcheck=0
yum update


yum install -y gcc gcc-c++ lua lua-devel  libxml2 libxml2-devel rsync




# 源码编译
wget https://github.com/lsyncd/lsyncd/archive/refs/tags/v2.3.1.tar.gz
tar xvf v2.3.1.tar.gz && cd lsyncd-2.3.1
cmake -DCMAKE_INSTALL_PREFIX=/usr/local/lsyncd
make && make install
ln -s /usr/local/lsyncd/bin/lsyncd /usr/bin/lsyncd


# 配置两边免密码登录
ssh-keygen
ssh-copy-id root@172.16.2.191
ssh-copy-id root@172.16.2.192

vim /etc/lsyncd.conf

settings {
    logfile = "/var/log/lsyncd.log",          --日志路径
    statusFile = "/var/log/lsyncd.status",    --状态文件
    pidfile = "/var/run/lsyncd.pid",          --pid文件路径
    statusInterval = 1,                       --状态文件写入最短时间
    nodaemon = false,                         --daemon运行
    maxProcesses = 1,                         --最大进程
    maxDelays = 1,                            --最大延迟
}
sync {
    default.rsyncssh,      --默认rsync+ssh,rsync版本需要升级3以上版本
    source = "/usr/local/nginx/conf/",        --源目录
    delete = true,                            --保持完全同步        
    host = "root@172.16.2.192",         
    targetdir = "/usr/local/nginx/conf/", --目标目录
    exclude={                 
             ".txt"            --需排除的文件
    },
rsync = {
    binary = "/usr/bin/rsync", --需先安装好rsync
    archive = true,            --归档
    compress = false,          --压缩
    owner = true,              --属主
    perms = true,              --权限
    whole_file = false
    },
ssh = {
    port = 22
    }
}


# 创建自动启动
$ vim /etc/sysconfig/lsyncd

添加如下内容：

LSYNCD_OPTIONS="/etc/lsyncd.conf"

# 创建启动文件：

$ vim /usr/lib/systemd/system/lsyncd.service

# 添加如下内容：

[Unit]
Description=Live Syncing (Mirror) Daemon
After=network.target

[Service]
Type=simple
EnvironmentFile=-/etc/sysconfig/lsyncd
ExecStart=/usr/local/lsyncd/bin/lsyncd -nodaemon $LSYNCD_OPTIONS

[Install]
WantedBy=multi-user.target

$ systemctl start lsyncd
$ systemctl enable lsyncd


# 查看日志
cat /var/log/lsyncd.log

总结

经过测试可以实现将172.16.2.191和172.16.2.193（VIP）绑定在节点一，172.16.2.192和172.16.2.194（VIP）绑定在节点二。业务上可以分别使用172.16.2.193承载一部分业务，172.16.2.194承载另外一半的业务，当其中一台挂掉以后，业务都会自动漂移到另外一台上，确保业务可以正常。

未来待解决问题

需要实现两台服务器可以配置完成后自动重启nginx的服务，但是目前测试下来还不行，要进行手工的重启。

文章作者: JasonWen
文章链接: https://www.v2ex.com.cn/2024/01/11/nginx-shuang-zhu-ji-qun-da-jian/
版权声明: 本博客所有文章除特別声明外，均采用 CC BY 4.0 许可协议。转载请注明来源 JasonWen !
技术分享 Nginx
赏
  目录