【实验环境】
主机名 IP地址 部署的服务
nginx1 20.0.0.19 nginx、keepalived
nginx2 20.0.0.20 nginx、keepalived
master 20.0.0.15 apiserver、scheduler、controller-manager、etcd
master2 20.0.0.18 apiserver、scheduler、controller-manager
node1 20.0.0.16 kubelet、kube-proxy、docker、flannel、etcd
node2 20.0.0.17 kubelet、kube-proxy、docker、flannel、etcd
VIP 20.0.0.200

一、master2节点操作

1.1 关闭防火墙，关闭核心防护，关闭网络管理功能（生成环境中一定要关闭它）

[root@localhost ~]# hostnamectl set-hostname master2
[root@localhost ~]# su
[root@master2 ~]# systemctl stop firewalld && systemctl disable firewalld
[root@master2 ~]#  setenforce 0 && sed -i "s/SELINUX=enforcing/SELNIUX=disabled/g" /etc/selinux/config
[root@master2 ~]# systemctl stop NetworkManager && systemctl disable NetworkManager
'//关闭网络管理功能'

1.2 master节点操作，将master节点的kubernetes配置文件和启动脚本复制到master2节点

[root@master ~]# scp -r /opt/kubernetes/ root@20.0.0.18:/opt/
[root@master ~]# scp /usr/lib/systemd/system/{kube-apiserver,kube-controller-manager,kube-scheduler}.service root@20.0.0.18:/usr/lib/systemd/system/

1.3 master2上修改apiserver配置文件中的IP地址

[root@master2 ~]# cd /opt/kubernetes/cfg/
[root@master2 cfg]# ls
kube-apiserver  kube-controller-manager  kube-scheduler  token.csv
[root@master2 cfg]# vim kube-apiserverKUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--etcd-servers=https://20.0.0.15:2379,https://20.0.0.16:2379,https://20.0.0.17:2379 \
--bind-address=20.0.0.18 \
'//修改此处的绑定IP地址'
--secure-port=6443 \
--advertise-address=20.0.0.18 \
'//修改此处的IP地址'
...省略

1.4 将master节点的etcd证书复制到master2节点（master2上一定要有etcd证书，用来与etcd通信）

[root@master ~]# scp -r /opt/etcd/ root@20.0.0.18:/opt

1.5 master2节点查看etcd证书，并启动三个服务

[root@master2 ~]# yum -y install tree
[root@master2 ~]# tree /opt/etcd
/opt/etcd
├── bin
│   ├── etcd
│   └── etcdctl
├── cfg
│   └── etcd
└── ssl├── ca-key.pem├── ca.pem├── server-key.pem└── server.pem[root@master2 ~]# systemctl start kube-apiserver && systemctl status kube-apiserver && systemctl enable kube-apiserver[root@master2 ~]# systemctl start kube-controller-manager && systemctl status kube-controller-manager && systemctl enable kube-controller-manager[root@master2 ~]# systemctl enable kube-scheduler && systemctl start kube-scheduler && systemctl status kube-scheduler

1.6 添加环境变量并查看状态

[root@master2 ~]# echo export PATH=$PATH:/opt/kubernetes/bin >> /etc/profile
[root@master2 ~]# source /etc/profile
[root@master2 ~]# kubectl get node
NAME        STATUS   ROLES    AGE    VERSION
20.0.0.16   Ready    <none>   168m   v1.12.3
20.0.0.17   Ready    <none>   167m   v1.12.3

二、nginx负载均衡集群部署

2.1 两个nginx主机开局优化（仅展示nginx1的操作）：关闭防火墙和核心防护，编辑nginx yum源

[root@localhost ~]# hostnamectl set-hostname nginx1
[root@localhost ~]# su
[root@nginx1 ~]# systemctl stop firewalld && systemctl disable firewalld
[root@nginx1 ~]# setenforce 0 && sed -i "s/SELINUX=enforcing/SELNIUX=disabled/g" /etc/selinux/config	
[root@nginx1 ~]# vi /etc/yum.repos.d/nginx.repo
[nginx]
name=nginx.repo
baseurl=http://nginx.org/packages/centos/7/$basearch/
enabled=1
gpgcheck=0[root@nginx1 ~]# rm -rf /var/run/yum.pid
[root@nginx1 ~]# yum clean all
[root@nginx1 ~]# yum makecache

2.2 两台nginx主机安装nginx并开启四层转发（仅展示nginx1的操作）

[root@nginx1 ~]# yum -y install nginx
[root@nginx1 ~]# vi /etc/nginx/nginx.conf
'//在events和http之间添加stream'
stream {log_format  main    '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';access_log /var/log/nginx/k8s-access.log main;upstream k8s-apiserver {server 20.0.0.15:6443;server 20.0.0.18:6443;}server {listen 6443;proxy_pass k8s-apiserver;}
}

2.3 启动nginx服务

[root@nginx1 ~]# nginx -t
[root@nginx1 ~]# systemctl start nginx && systemctl status nginx && netstat -ntap |grep nginx
'//nginx为运行状态'

2.4 两台nginx主机部署keepalived服务（仅展示nginx1的操作）

[root@nginx1 ~]# yum -y install keepalived
[root@nginx1 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalivedglobal_defs {notification_email {acassen@firewall.locfailover@firewall.locsysadmin@firewall.loc}notification_email_from Alexandre.Cassen@firewall.locsmtp_server 127.0.0.1smtp_connect_timeout 30router_id NGINX_MASTER
}vrrp_script check_nginx {script "/usr/local/nginx/sbin/check_nginx.sh"
}vrrp_instance VI_1 {state MASTERinterface ens33virtual_router_id 51priority 100advert_int 1authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {20.0.0.200/24}track_script {check_nginx}
}'//nginx2除了下面不一样，其余与nginx1一样'
[root@nginx2 ~]# vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {state BACKUPinterface ens33virtual_router_id 52priority 90'解释'
! Configuration File for keepalivedglobal_defs {# 接收邮件地址 notification_email {acassen@firewall.locfailover@firewall.locsysadmin@firewall.loc}# 邮件发送地址notification_email_from Alexandre.Cassen@firewall.locsmtp_server 127.0.0.1smtp_connect_timeout 30router_id NGINX_MASTER
}vrrp_script check_nginx {script "/usr/local/nginx/sbin/check_nginx.sh"	'//keepalived服务检查脚本的位置'
}vrrp_instance VI_1 {state MASTER 	'//nginx2设置为BACKUP'interface ens33virtual_router_id 51 '//nginx2可设置为52'priority 100    '//优先级，nginx2设置 90'advert_int 1    '//指定VRRP 心跳包通告间隔时间，默认1秒 'authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {20.0.0.200/24 	'//VIP地址'}track_script {check_nginx}
}

2.5 创建监控脚本，启动keepalived服务，查看VIP地址（只展示nginx1的操作）

[root@nginx1 ~]# mkdir -p /usr/local/nginx/sbin/
'//创建监控脚本目录'
[root@nginx1 ~]# cd /usr/local/nginx/sbin/
[root@nginx1 sbin]# vim check_nginx.sh
'//编写监控脚本配置文件'
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")if [ "$count" -eq 0 ];thensystemctl stop keepalived
fi[root@nginx1 sbin]# chmod +x check_nginx.sh 
[root@nginx1 sbin]# systemctl start keepalived && systemctl status keepalived
'//keepalived运行状态'
[root@nginx1 sbin]# ip a
'//两个nginx服务器查看IP地址，VIP在nginx1上'[root@nginx2 sbin]# ip a
'//nginx2查看不到VIP''问题'
先启动nginx1的nginx服务和keepalived服务。再启动nginx2的nginx服务和keepalived服务，VIP地址只在nginx1上查看到；关闭nginx1，VIP在nginx2上；再开启nginx1，VIP在nginx1和nginx2上同时存在。查看keepalived配置文件中的优先级，nginx1（100）优先级大于nginx2（90）。

2.6 验证漂移地址

[root@nginx1 ~]# pkill nginx
'//关闭nginx服务'
[root@nginx1 ~]# systemctl status keepalived
'//发现keepalived服务关闭了'
[root@nginx2 ~]# ip a
'//现在发现VIP地址跑到nginx2上了'

2.7 恢复漂移地址的操作

[root@nginx1 ~]# systemctl start nginx
[root@nginx1 ~]# systemctl start keepalived
'//先开启nginx，在启动keepalived服务'
[root@nginx1 ~]# ip a
'//再次查看，发现VIP回到了nginx1节点上'

2.8 修改两个node节点配置文件（bootstrap.kubeconfig 、kubelet.kubeconfig、kube-proxy.kubeconfig），统一VIP地址，仅展示node1节点的操作

[root@node1 ~]# vi /opt/kubernetes/cfg/bootstrap.kubeconfigserver: https://20.0.0.200:6443[root@node1 ~]# vi /opt/kubernetes/cfg/kubelet.kubeconfigserver: https://20.0.0.200:6443[root@node1 ~]# vi /opt/kubernetes/cfg/kube-proxy.kubeconfigserver: https://20.0.0.200:6443

2.9 重启两个node节点的服务

[root@node1 ~]# systemctl restart kubelet
[root@node1 ~]# systemctl restart kube-proxy
[root@node1 ~]# cd /opt/kubernetes/cfg/
[root@node1 cfg]# grep 200 *
'//VIP修改成功'
bootstrap.kubeconfig:    server: https://20.0.0.200:6443
kubelet.kubeconfig:    server: https://20.0.0.200:6443
kube-proxy.kubeconfig:    server: https://20.0.0.200:6443

2.10 在nginx1上查看k8s日志

[root@nginx1 ~]# tail /var/log/nginx/k8s-access.log
'//下面的日志是重启nginx和keepalived服务后产生的'
20.0.0.17 20.0.0.18:6443 - [20/Nov/2020:09:08:34 -0500] 200 1114
20.0.0.17 20.0.0.18:6443 - [20/Nov/2020:09:08:34 -0500] 200 1115
20.0.0.16 20.0.0.15:6443 - [20/Nov/2020:09:08:34 -0500] 200 1115
20.0.0.17 20.0.0.15:6443 - [20/Nov/2020:09:08:34 -0500] 200 1566
20.0.0.16 20.0.0.18:6443 - [20/Nov/2020:09:08:34 -0500] 200 1114
20.0.0.17 20.0.0.18:6443 - [20/Nov/2020:09:08:34 -0500] 200 1115
20.0.0.16 20.0.0.18:6443 - [20/Nov/2020:09:08:34 -0500] 200 1116
20.0.0.16 20.0.0.15:6443 - [20/Nov/2020:09:08:34 -0500] 200 1566

2.11 master节点测试创建pod

[root@master ~]# kubectl run nginx --image=nginx
'//创建一个nginx测试pod'
kubectl run --generator=deployment/apps.v1beta1 is DEPRECATED and will be removed in a future version. Use kubectl create instead.
deployment.apps/nginx created
[root@master ~]# kubectl get pods
'//查看状态，是正在创建'
NAME                    READY   STATUS              RESTARTS   AGE
nginx-dbddb74b8-84xgw   0/1     ContainerCreating   0          30s[root@master ~]# kubectl get pods
'//稍等一下再次查看，发现pod已经创建完成，在master2节点也可以查看'
NAME                    READY   STATUS    RESTARTS   AGE
nginx-dbddb74b8-84xgw   1/1     Running   0          2m23s

2.12 查看pod日志

[root@master ~]# kubectl logs nginx-dbddb74b8-84xgw
'//查看pod日志发现报错原因是权限问题'
Error from server (Forbidden): Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy) ( pods/log nginx-dbddb74b8-84xgw)[root@master ~]# kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=system:anonymous
'//指定集群中的匿名用户有管理员权限，指定后master2也可以查看'[root@master ~]# kubectl logs nginx-dbddb74b8-5s6h7
'//此时可以访问，有日志产生'
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Configuration complete; ready for start up

2.13 访问node节点的pod资源产生日志，并在两个master节点查看

[root@master ~]# kubectl get pods -o wide
'//查看podIP网络信息'
NAME                    READY   STATUS    RESTARTS   AGE     IP            NODE        NOMINATED NODE
nginx-dbddb74b8-84xgw   1/1     Running   0          6m18s   172.17.58.2   20.0.0.16   <none>[root@node1 ~]# curl 172.17.58.2
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>'//在对应的节点访问pod'
[root@master ~]# kubectl logs nginx-dbddb74b8-84xgw
'//再次在master节点访问日志情况，master2节点同样可以访问'
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
172.17.58.1 - - [20/Nov/2020:14:17:41 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"
172.17.58.1 - - [20/Nov/2020:14:17:48 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"