Kubernetes：在线部署集群（13700）

发表于 分类于 本文字数： 1k 阅读时长 ≈ 4 分钟
1. 环境准备 1.1 服务器资源清单 | IP地址 | 内存 | CPU核心 | 磁盘 | 密码/备注 | | | | | | | | 172.20.3.12 |...

1. 环境准备

1.1 服务器资源清单

IP地址 内存 CPU核心 磁盘 密码/备注
172.20.3.12 125G 32 40T ES/Kibana
172.20.3.28 125G 32 40T
172.20.3.29 125G 32 40T 场景化宽带运营平台前端
172.20.3.30 125G 32 40T 安全宽带报表数据对接
172.20.3.31 125G 32 40T 运营后台数据库
172.20.3.32 125G 32 40T YunWang@2022 (Master节点)
172.20.3.34 125G 32 40T %Yj3rRstg2#8 (Node1节点)
172.20.3.46 125G 32 40T %Yj3rRstg2#8 (Node2节点)

1.2 基础环境检查

1
2
3
4
5
6
7
8
9
10
11
# 查看CPU核心数
cat /proc/cpuinfo | grep processor | wc -l

# 查看内存
free -g

# 查看磁盘
df -lh

# 查看IP
ip addr | grep 172.20.3

2. 系统初始化配置

2.1 配置Yum源

1
2
3
4
5
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum clean all
yum makecache
yum update -y

2.2 关闭防火墙和SELinux

1
2
3
4
5
6
7
# 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld

# 关闭SELinux
sed -i 's/enforcing/disabled/' /etc/selinux/config
setenforce 0

2.3 关闭Swap分区

1
2
3
4
5
# 临时关闭
swapoff -a

# 永久关闭
sed -ri 's/.*swap.*/#&/' /etc/fstab

2.4 配置主机名和Hosts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# Master节点
hostnamectl set-hostname k8s-master

# Node1节点
hostnamectl set-hostname k8s-node1

# Node2节点
hostnamectl set-hostname k8s-node2

# 所有节点添加hosts
cat >> /etc/hosts <<EOF
172.20.3.32 k8s-master
172.20.3.34 k8s-node1
172.20.3.46 k8s-node2
EOF

2.5 内核参数配置

1
2
3
4
5
6
7
8
cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

modprobe br_netfilter
sysctl --system

2.6 时间同步

1
2
3
yum install -y ntpdate
ntpdate time.windows.com
ntpdate -u ntp1.aliyun.com

2.7 开启IPVS

1
2
3
4
5
6
7
8
9
10
11
12
13
14
yum -y install ipset ipvsadm

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && \
bash /etc/sysconfig/modules/ipvs.modules && \
lsmod | grep -e ip_vs -e nf_conntrack_ipv4

3. 安装Docker和Kubernetes组件

3.1 安装Docker 18.06.3

1
2
3
4
5
6
7
8
9
10
11
12
wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
yum -y install docker-ce-18.06.3.ce-3.el7

mkdir -p /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "registry-mirrors": ["https://b9pmyelo.mirror.aliyuncs.com"]
}
EOF

systemctl enable docker && systemctl start docker

3.2 安装Kubernetes组件

1
2
3
4
5
6
7
8
9
10
11
12
13
cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
EOF

yum install -y kubelet-1.18.0 kubeadm-1.18.0 kubectl-1.18.0

echo 'KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"' > /etc/sysconfig/kubelet
systemctl enable kubelet

4. 初始化Master节点

4.1 初始化集群

1
2
3
4
5
6
kubeadm init \
  --apiserver-advertise-address=172.20.3.32 \
  --image-repository registry.aliyuncs.com/google_containers \
  --kubernetes-version v1.18.0 \
  --service-cidr=10.96.0.0/12 \
  --pod-network-cidr=10.244.0.0/16

4.2 配置kubectl

1
2
3
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

4.3 记录Node加入命令

1
2
kubeadm join 172.20.3.32:6443 --token ntxv5o.hxhljd63gskip2y9 \
  --discovery-token-ca-cert-hash sha256:e8f3f469eff4931066679909ce7916c196e25b78a86a379f9303ecc5951a2751

5. 部署网络插件

5.1 部署Flannel网络

1
2
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
kubectl apply -f kube-flannel.yml

5.2 验证集群状态

1
2
3
kubectl get nodes
kubectl get pods -A
kubectl get cs

6. Node节点加入集群

6.1 在Node节点执行加入命令

1
2
kubeadm join 172.20.3.32:6443 --token ntxv5o.hxhljd63gskip2y9 \
  --discovery-token-ca-cert-hash sha256:e8f3f469eff4931066679909ce7916c196e25b78a86a379f9303ecc5951a2751

6.2 配置Node节点kubectl

1
2
3
mkdir -p $HOME/.kube
scp root@172.20.3.32:/etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

7. 部署NFS存储

7.1 安装NFS服务

1
2
3
4
5
6
7
8
9
10
11
12
yum install -y nfs-utils rpcbind
mkdir -p /nfs/data
chmod 755 /nfs/data

cat >> /etc/exports <<EOF
/nfs/ *(async,insecure,no_root_squash,no_subtree_check,rw)
/nfs/data/ *(async,insecure,no_root_squash,no_subtree_check,rw)
EOF

exportfs -r
systemctl start rpcbind nfs
systemctl enable rpcbind nfs-server

7.2 部署NFS Provisioner

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
# 创建RBAC
cat > cluster-admin.rbac.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: dcm-rbac
  name: k8s-admin
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: k8s-admin-crb
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: k8s-admin
  namespace: kube-system
EOF

kubectl apply -f cluster-admin.rbac.yaml

# 部署NFS Provisioner
cat > nfs-client-provisioner.yaml <<EOF
kind: Deployment
apiVersion: apps/v1
metadata:
  name: nfs-client-provisioner
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nfs-client-provisioner
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: k8s-admin
      containers:
        - name: nfs-client-provisioner
          image: registry.cn-hangzhou.aliyuncs.com/open-ali/nfs-client-provisioner:latest
          imagePullPolicy: Never
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: fuseim.pri/ifs
            - name: NFS_SERVER
              value: 172.20.3.32
            - name: NFS_PATH
              value: /nfs/data
      volumes:
        - name: nfs-client-root
          nfs:
            server: 172.20.3.32
            path: /nfs/data
EOF

kubectl apply -f nfs-client-provisioner.yaml

# 创建StorageClass
cat > nfs-client-class.yaml <<EOF
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-nfs-storage
provisioner: fuseim.pri/ifs
EOF

kubectl apply -f nfs-client-class.yaml

7.3 验证NFS存储

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
cat > nfs-client-pvc.yaml <<EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-client-pvc
  namespace: kube-system
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: managed-nfs-storage
  resources:
    requests:
      storage: 100Mi
EOF

kubectl apply -f nfs-client-pvc.yaml
kubectl get pvc -n kube-system

8. 集群管理命令

8.1 删除节点

1
2
kubectl drain k8s-node1 --delete-local-data --force --ignore-daemonsets
kubectl delete node k8s-node1

8.2 查看集群状态

1
2
3
kubectl get nodes
kubectl cluster-info
kubectl get pods -A