sudo cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

使其生效

sudo sysctl --system

sudo cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

sudo vim /etc/containerd/config.toml
在 [plugins."io.containerd.grpc.v1.cri".registry] 下把 config_path 的值修改為 "/etc/containerd/certs.d"

sudo mkdir -p /etc/containerd/certs.d/docker.io
sudo vim /etc/containerd/certs.d/docker.io/config.toml
寫入：
server = "https://docker.io"
[host."https://registry.docker-cn.com"]

sudo systemctl restart containerd

初始化k8s ,使用中國鏡像源拉取鏡像apiserver-advertise-address為自己的主機地址

sudo kubeadm init --apiserver-advertise-address=192.168.190.103 --image-repository registry.aliyuncs.com/google_containers --kubernetes-version v1.25.0 --service-cidr=10.96.0.0/12 --pod-network-cidr=10.244.0.0/16

如果出現以下字符，說明初始化成功。

Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.56.11:6443 --token s0d1qo.dbcwcms72y4y3j7x
--discovery-token-ca-cert-hash sha256:5ad74702eebb9f3c254c3e48a68792e93e07f97c104cdd9b7f1fcf

kubeadm真不行

停止K8S

systemctl stop kubelet
systemctl stop etcd
systemctl stop docker

清空K8S集群設置

kubeadm reset -f

刪除K8S相關軟件

羅列kube關鍵字的軟件

yum list installed | grep kube

卸載相關軟件

yum -y remove kube*

再次查看確保都卸載完

yum list installed | grep kube
cri-tools.x86_64 1.26.0-0 @kubernetes

單獨卸載

yum -y remove cri-tools.x86_64

最終確認已經完全卸載掉

yum list installed | grep kube

刪除docker

卸載Docker Engine、CLI、Containerd和Docker合成包

yum -y remove docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras

手動刪除所有鏡像、容器和卷

rm -rf /var/lib/docker
rm -rf /var/lib/containerd

徹底刪除相關文件

rm -rvf $HOME/.kube
rm -rvf ~/.kube/
rm -rvf /etc/kubernetes/
rm -rvf /etc/systemd/system/kubelet.service.d
rm -rvf /etc/systemd/system/kubelet.service
rm -rvf /usr/bin/kube*
rm -rvf /etc/cni
rm -rvf /opt/cni
rm -rvf /var/lib/etcd
rm -rvf /var/etcd

驗證清除完成

systemctl status docker
systemctl | grep kube

[root@master ~]# systemctl | grep kube
kubepods-besteffort.slice                                                                                        loaded active active    libcontainer container kubepods-besteffort.slice
kubepods-burstable-pod4a7a439e74bb0629b2485e8b7d67ad0e.slice                                                     loaded active active    libcontainer container kubepods-burstable-pod4a7a439e74bb0629b2485e8b7d67ad0e.slice
kubepods-burstable-pod727b102337e040561646211b49b1ca56.slice                                                     loaded active active    libcontainer container kubepods-burstable-pod727b102337e040561646211b49b1ca56.slice
kubepods-burstable-podc208507e4755f039b185a84e6eb21426.slice                                                     loaded active active    libcontainer container kubepods-burstable-podc208507e4755f039b185a84e6eb21426.slice
kubepods-burstable-podc6d00f20c83577f6c3f4a5091cabb3b6.slice                                                     loaded active active    libcontainer container kubepods-burstable-podc6d00f20c83577f6c3f4a5091cabb3b6.slice
kubepods-burstable.slice                                                                                         loaded active active    libcontainer container kubepods-burstable.slice
kubepods.slice                                                                                                   loaded active active    libcontainer container kubepods.slice

沒刪除完成

停止服務：運行以下命令停止相關的 systemd 服務
sudo systemctl stop kubepods-besteffort.slice
sudo systemctl stop kubepods-burstable-pod4a7a439e74bb0629b2485e8b7d67ad0e.slice
sudo systemctl stop kubepods-burstable-pod727b102337e040561646211b49b1ca56.slice
sudo systemctl stop kubepods-burstable-podc208507e4755f039b185a84e6eb21426.slice
sudo systemctl stop kubepods-burstable-podc6d00f20c83577f6c3f4a5091cabb3b6.slice
sudo systemctl stop kubepods-burstable.slice
sudo systemctl stop kubepods.slice

禁用服務：運行以下命令禁用相關的 systemd 服務：
sudo systemctl disable kubepods-besteffort.slice
sudo systemctl disable kubepods-burstable-pod4a7a439e74bb0629b2485e8b7d67ad0e.slice
sudo systemctl disable kubepods-burstable-pod727b102337e040561646211b49b1ca56.slice
sudo systemctl disable kubepods-burstable-podc208507e4755f039b185a84e6eb21426.slice
sudo systemctl disable kubepods-burstable-podc6d00f20c83577f6c3f4a5091cabb3b6.slice
sudo systemctl disable kubepods-burstable.slice
sudo systemctl disable kubepods.slice

刪除相關服務文件：運行以下命令刪除相關的 systemd 服務文件：
sudo rm -rf /etc/systemd/system/kubepods-besteffort.slice
sudo rm -rf /etc/systemd/system/kubepods-burstable-pod4a7a439e74bb0629b2485e8b7d67ad0e.slice
sudo rm -rf /etc/systemd/system/kubepods-burstable-pod727b102337e040561646211b49b1ca56.slice
sudo rm -rf /etc/systemd/system/kubepods-burstable-podc208507e4755f039b185a84e6eb21426.slice
sudo rm -rf /etc/systemd/system/kubepods-burstable-podc6d00f20c83577f6c3f4a5091cabb3b6.slice
sudo rm -rf /etc/systemd/system/kubepods-burstable.slice
sudo rm -rf /etc/systemd/system/kubepods.slice

yum list installed | grep kube
yum list installed | docker

rpm -qa | grep kube
rpm -qa | grep docker

本小節(jié)命令在所有節(jié)點執(zhí)行

關閉防火墻

systemctl stop firewalld # 臨時
systemctl disable firewalld # 永久
systemctl status firewalld # 檢查

關閉selinux

setenforce 0 # 臨時
sed -i 's/enforcing/disabled/' /etc/selinux/config # 永久

關閉swap

swapoff -a # 臨時
sed -ri 's/.swap./#&/' /etc/fstab # 永久

根據規(guī)劃設置主機名（以下兩個命令都可以）

hostnamectl set-hostname master # master節(jié)點執(zhí)行
hostnamectl set-hostname node01 # node01節(jié)點執(zhí)行
hostnamectl set-hostname node02 # node02節(jié)點執(zhí)行
hostname # 查看主機名看是否修改成功

添加hosts(root用戶)

cat >> /etc/hosts << EOF
192.168.31.102 master
192.168.31.103 node01
192.168.31.104 node02
EOF

將橋接的IPv4流量傳遞到iptables的鏈(root用戶)

cat >/etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system  # 使得橋接配置生效

# 設置時間同步(root用戶)
yum install ntpdate -y
timedatectl set-timezone Asia/Shanghai
ntpdate ntp1.aliyun.com

為etcd和API Server生成自簽證書

設置信任證書

yum install -y ca-certificates

下載證書

mkdir /root/cfssl
cd /root/cfssl
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64

移動文件并重命名

mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

授予執(zhí)行權限

chmod +x /usr/local/bin/cfssl*
chmod +x /usr/bin/cfssl-certinfo

接著，創(chuàng)建文件夾：
mkdir -p /root/TLS/{etcd,k8s}
cd /root/TLS/etcd

自簽證書頒發(fā)機構CA

編寫ca-config.json文件
cat > ca-config.json<< EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF

編寫ca-csr.json文件
cat > ca-csr.json<< EOF
{
"CN": "etcd CA",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing"
}
]
}
EOF

生成證書

生成證書

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

查看生成文件，如下兩個

ls *.pem
ca-key.pem ca.pem

使用自簽 CA 簽發(fā) Etcd HTTPS 證書

cat > server-csr.json<< EOF
{
"CN": "etcd",
"hosts": [
"192.168.31.102",
"192.168.31.103",
"192.168.31.104"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF

部署etcd集群

創(chuàng)建文件夾

mkdir -p /opt/etcd/{bin,cfg,ssl}
cd /opt/

下載二進制包

wget https://github.com/etcd-io/etcd/releases/download/v3.4.9/etcd-v3.4.9-linux-amd64.tar.gz

解壓二進制壓縮包

tar zxvf etcd-v3.4.9-linux-amd64.tar.gz

移動到相應目錄

mv ./etcd-v3.4.9-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/

創(chuàng)建etcd配置文件
cat > /opt/etcd/cfg/etcd.conf << EOF

[Member]

ETCD_NAME="etcd-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.31.102:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.31.102:2379"

[Clustering]

ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.31.102:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.31.102:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.31.102:2380,etcd-2=https://192.168.31.103:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

systemd管理etcd
cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd.conf
ExecStart=/opt/etcd/bin/etcd \
--cert-file=/opt/etcd/ssl/server.pem \
--key-file=/opt/etcd/ssl/server-key.pem \
--peer-cert-file=/opt/etcd/ssl/server.pem \
--peer-key-file=/opt/etcd/ssl/server-key.pem \
--trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--logger=zap
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

移動證書
cp /root/TLS/etcd/capem /root/TLS/etcd/serverpem /opt/etcd/ssl

拷貝配置到其他節(jié)點
scp -r /opt/etcd/ root@192.168.31.103:/opt/
scp /usr/lib/systemd/system/etcd.service root@192.168.31.103:/usr/lib/systemd/system/

修改其他節(jié)點的配置
vi /opt/etcd/cfg/etcd.conf

啟動etcd集群
systemctl daemon-reload && systemctl start etcd
systemctl status etcd

設置開機自啟動

systemctl enable etcd

檢查集群狀態(tài)

命令一（以列表形式呈現）

/opt/etcd/bin/etcdctl --write-out=table --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem --endpoints=https://192.168.31.102:2379,https://192.168.31.103:2379 endpoint health

示例如下

[root@master opt]# /opt/etcd/bin/etcdctl --write-out=table --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem --endpoints=https://192.168.31.102:2379,https://192.168.31.103:2379 endpoint health
+-----------------------------+--------+-------------+-------+
| ENDPOINT | HEALTH | TOOK | ERROR |
+-----------------------------+--------+-------------+-------+
| https://192.168.31.102:2379 | true | 16.521258ms | |
| https://192.168.31.103:2379 | true | 19.867578ms | |
+-----------------------------+--------+-------------+-------+

命令二（直接呈現）

/opt/etcd/bin/etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.31.102:2379,https://192.168.31.103:2379" endpoint health
示例如下：
[root@master etcd]# /opt/etcd/bin/etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.31.102:2379,https://192.168.31.103:2379" endpoint health
https://192.168.31.102:2379 is healthy: successfully committed proposal: took = 15.879209ms
https://192.168.31.103:2379 is healthy: successfully committed proposal: took = 16.898771ms

部署master組件

自簽證書頒發(fā)機構（CA）

切換目錄

cd /root/TLS/k8s

cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF

cat > ca-csr.json<< EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing",
"O": "k8s",
"OU": "System"
}
]
}
EOF

、生成自簽機構證書

生成證書

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

查看生成的證書

ls *pem

使用自簽CA簽發(fā)kube-apiserver的HTTPS證書
創(chuàng)建kube-apiserver證書申請文件：
cat > server-csr.json << EOF
{
"CN": "kubernetes",
"hosts": [
"10.0.0.1",
"127.0.0.1",
"192.168.31.102",
"192.168.31.103",
"192.168.31.104",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF

生成kube-apiserver證書

生成證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

查看證書

ls server*pem

下載二進制文件
cd ~

下載解壓，也可以從Windows物理機下載再上傳到虛擬機中

wget https://dl.k8s.io/v1.18.3/kubernetes-server-linux-amd64.tar.gz
tar zxvf kubernetes-server-linux-amd64.tar.gz

創(chuàng)建kubernetes目錄

mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}
cd kubernetes/server/bin

移動文件

cp /root/kubernetes/server/bin/{kube-apiserver,kube-scheduler,kube-controller-manager} /opt/kubernetes/bin
cp /root/kubernetes/server/bin/kubectl /usr/bin/

創(chuàng)建配置文件

cat > /opt/kubernetes/cfg/kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--etcd-servers=https://192.168.31.102:2379,<https://192.168.31.103:2379> \\
--bind-address=192.168.31.102 \\
--secure-port=6443 \\
--advertise-address=192.168.31.102 \\
--allow-privileged=true \\
--service-cluster-ip-range=10.0.0.0/24 \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \\
--authorization-mode=RBAC,Node \\
--enable-bootstrap-token-auth=true \\
--token-auth-file=/opt/kubernetes/cfg/token.csv \\
--service-node-port-range=30000-32767 \\
--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem  \\
--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \\
--tls-cert-file=/opt/kubernetes/ssl/server.pem \\
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \\
--client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--etcd-cafile=/opt/etcd/ssl/ca.pem \\
--etcd-certfile=/opt/etcd/ssl/server.pem \\
--etcd-keyfile=/opt/etcd/ssl/server-key.pem \\
--audit-log-maxage=30 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"
EOF

字段說明：
– logtostderr：啟用日志
– v：日志等級

– log-dir：日志目錄
– etcd-servers：etcd集群地址
– bind-address：監(jiān)聽地址
– secure-port：https 安全端口
– advertise-address：集群通告地址
– allow-privileged：啟用授權
– service-cluster-ip-range：Service虛擬 IP地址段
– enable-admission-plugins：準入控制模塊
– authorization-mode：認證授權，啟用 RBAC 授權和節(jié)點自管理
– enable-bootstrap-token-auth：啟用 TLS bootstrap 機制
– token-auth-file：bootstrap token文件
– service-node-port-range：Service nodeport類型默認分配端口范圍

– kubelet-client-xxx：apiserver 訪問 kubelet客戶端證書
– tls-xxx-file：apiserver https 證書
– etcd-xxxfile：連接 Etcd 集群證書
– audit-log-xxx：審計日志
————————————————

拷貝證書

cp /root/TLS/k8s/ca*pem /root/TLS/k8s/server*pem /opt/kubernetes/ssl/

啟用TLS Bootstrapping機制

cat > /opt/kubernetes/cfg/token.csv << EOF
c47ffb939f5ca36231d9e3121a252940,kubelet-bootstrap,10001,"system:node-bootstrapper"
EOF

systemd管理api-server

cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-apiserver.conf
ExecStart=/opt/kubernetes/bin/kube-apiserver \\$KUBE_APISERVER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF

啟動并設置開機自啟動

systemctl daemon-reload
systemctl start kube-apiserver
systemctl enable kube-apiserver

systemctl status kube-apiserver

kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

56180583523452668480480472597250291928469194654