做个网站要花多少钱_ios开发入门_绍兴seo排名公司_明星百度指数排行

k8s部署方案

1. 环境设置

1.1 虚拟机环境

远程操作环境: MacOS - bash
虚拟机-环境: Windows 10
虚拟机-平台: Oracle VM VirtualBox
虚拟机-系统: CentOS-Stream-9-latest-x86_64
虚拟机-网卡-1: enp0s3，桥接网卡，网段: 192.168.0.1/24
虚拟机-网卡-2: enp0s8，仅内网通信，网段: 169.0.0.0/8

1.2 服务器环境

执行initserver自动化脚本
网卡设置固定的ip
~/.bash_profile设置代理

网络环境

主机	enp0s3	enp0s8	service
sre-lo-test-vm-master-001	192.168.0.22,192.168.0.28(vip)	169.0.0.100	etcd, proxy, k8s-master
sre-lo-test-vm-master-002	192.168.0.23	169.0.0.101	etcd, proxy, k8s-master
sre-lo-test-vm-master-003	192.168.0.24	169.0.0.102	etcd, proxy, k8s-master
sre-lo-test-vm-node-001	192.168.0.25	169.0.0.103	k8s-node
sre-lo-test-vm-node-002	192.168.0.26	169.0.0.104	k8s-node
sre-lo-test-vm-node-003	192.168.0.27	169.0.0.105	k8s-node
host subnet		169.0.0.0/16
k8s pod subnet		169.1.0.0/16
k8s service subnet		169.2.0.0/16

1.3 配置 Mac OS 远程操作环境

echo '# k8s cluster node
192.168.0.22 sre-lo-test-vm-master-001
192.168.0.23 sre-lo-test-vm-master-002
192.168.0.24 sre-lo-test-vm-master-003
192.168.0.25 sre-lo-test-vm-node-001
192.168.0.26 sre-lo-test-vm-node-002
192.168.0.27 sre-lo-test-vm-node-003
' | sudo tee -a /etc/hostsssh-keyscan sre-lo-test-vm-master-001 >> ~/.ssh/known_hosts
ssh-keyscan sre-lo-test-vm-master-002 >> ~/.ssh/known_hosts
ssh-keyscan sre-lo-test-vm-master-003 >> ~/.ssh/known_hosts
ssh-keyscan sre-lo-test-vm-node-001 	>> ~/.ssh/known_hosts
ssh-keyscan sre-lo-test-vm-node-002 	>> ~/.ssh/known_hosts
ssh-keyscan sre-lo-test-vm-node-003 	>> ~/.ssh/known_hostsscp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-master-001:~/.ssh
scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-master-002:~/.ssh
scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-master-003:~/.ssh
scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-node-001:~/.ssh
scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-node-002:~/.ssh
scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-node-003:~/.ssh

1.4 服务器环境初始化（部分）

# ALL
echo '
169.0.0.100 sre-lo-test-vm-master-001 # etcd, proxy, k8s-master
169.0.0.101 sre-lo-test-vm-master-002 # etcd, proxy, k8s-master
169.0.0.102 sre-lo-test-vm-master-003 # etcd, proxy, k8s-master
169.0.0.103 sre-lo-test-vm-node-001 	 # k8s-node
169.0.0.104 sre-lo-test-vm-node-002	 # k8s-node
169.0.0.105 sre-lo-test-vm-node-003   # k8s-node
' | sudo tee -a /etc/hostsecho '
export proxy_ip="192.168.0.10"
export proxy_port="9527"
export http_proxy="http://${proxy_ip}:${proxy_port}"
export https_proxy="http://${proxy_ip}:${proxy_port}"
export socks_proxy="http://${proxy_ip}:${proxy_port}"
export ftp_proxy="http://${proxy_ip}:${proxy_port}"
export no_proxy=".cluster.local,cluster.local,localhost,127.0.0.1,localaddress,.localdomain.com,192.168.0.0/16,169.0.0.0/8,172.16.0.0/12"
' | tee -a ~/.bash_profilesource ~/.bash_profile

2. Etcd集群部署

2.1 下载软件

2.1.1 下载cfssl安装包

# MacOS
cd ~/Downloads
curl -L -O https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl_1.6.4_linux_amd64
curl -L -O https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssljson_1.6.4_linux_amd64
curl -L -O https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl-certinfo_1.6.4_linux_amd64mv cfssl_1.6.4_linux_amd64 cfssl
mv cfssljson_1.6.4_linux_amd64 cfssljson
mv cfssl-certinfo_1.6.4_linux_amd64 cfssl-certinfoscp cfssl root@sre-lo-test-vm-master-001:/usr/local/bin/cfssl
scp cfssljson root@sre-lo-test-vm-master-001:/usr/local/bin/cfssljson
scp cfssl-certinfo root@sre-lo-test-vm-master-001:/usr/bin/cfssl-certinfo# sre-lo-test-vm-master-001
sudo chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/bin/cfssl-certinfo

2.1.2 下载etcd安装包

# MacOS
cd ~/Downloads
curl -L -O https://github.com/etcd-io/etcd/releases/download/v3.5.16/etcd-v3.5.16-linux-amd64.tar.gz
scp etcd-v3.5.16-linux-amd64.tar.gz root@sre-lo-test-vm-master-001:~/
scp etcd-v3.5.16-linux-amd64.tar.gz root@sre-lo-test-vm-master-002:~/
scp etcd-v3.5.16-linux-amd64.tar.gz root@sre-lo-test-vm-master-003:~/

2.2 部署etcd

2.2.1 生成密钥文件

# sre-lo-test-vm-master-001
mkdir -p /tmp/etcd/tls
cd /tmp/etcd/tlsecho '{"signing": {"default": {"expiry": "87600h"},"profiles": {"www": {"expiry": "87600h","usages": ["signing","key encipherment","server auth","client auth"]}}}
}' | sudo tee /tmp/etcd/tls/ca-config.jsonecho '{"CN": "etcd CA","key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","L": "Beijing","ST": "Beijing"}]
}' | sudo tee /tmp/etcd/tls/ca-csr.jsonecho '{"CN": "etcd","hosts": ["169.0.0.100","169.0.0.101","169.0.0.102"],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","L": "BeiJing","ST": "BeiJing"}]
}' | sudo tee /tmp/etcd/tls/server-csr.jsoncfssl gencert -initca ca-csr.json | cfssljson -bare ca -
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem \-config=ca-config.json -profile=www server-csr.json \| cfssljson -bare server

2.2.2 安装etcd应用

# 创建目录 sre-lo-test-vm-master-*
sudo mkdir -p /data/etcd/{bin,cfg,ssl,data}
sudo tar -zxvf ~/etcd-v3.5.16-linux-amd64.tar.gz -C ~/
sudo mv -f ~/etcd-v3.5.16-linux-amd64/{etcd,etcdctl,etcdutl} /data/etcd/bin/# 复制密钥 sre-lo-test-vm-master-001
sudo /bin/cp -rf /tmp/etcd/tls/ca*pem /data/etcd/ssl/
sudo /bin/cp -rf /tmp/etcd/tls/server*pem /data/etcd/ssl/
scp /data/etcd/ssl/* root@sre-lo-test-vm-master-002:/data/etcd/ssl/
scp /data/etcd/ssl/* root@sre-lo-test-vm-master-003:/data/etcd/ssl/# sre-lo-test-vm-master-001
echo '#[Member]
ETCD_NAME="sre-lo-test-vm-master-001"
ETCD_DATA_DIR="/data/etcd/data"
ETCD_LISTEN_PEER_URLS="https://169.0.0.100:2380"
ETCD_LISTEN_CLIENT_URLS="https://169.0.0.100:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://169.0.0.100:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://169.0.0.100:2379"
ETCD_INITIAL_CLUSTER="sre-lo-test-vm-master-001=https://169.0.0.100:2380,sre-lo-test-vm-master-002=https://169.0.0.101:2380,sre-lo-test-vm-master-003=https://169.0.0.102:2380"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"' | sudo tee /data/etcd/cfg/etcd.conf# sre-lo-test-vm-master-002
echo '#[Member]
ETCD_NAME="sre-lo-test-vm-master-002"
ETCD_DATA_DIR="/data/etcd/data"
ETCD_LISTEN_PEER_URLS="https://169.0.0.101:2380"
ETCD_LISTEN_CLIENT_URLS="https://169.0.0.101:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://169.0.0.101:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://169.0.0.101:2379"
ETCD_INITIAL_CLUSTER="sre-lo-test-vm-master-001=https://169.0.0.100:2380,sre-lo-test-vm-master-002=https://169.0.0.101:2380,sre-lo-test-vm-master-003=https://169.0.0.102:2380"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"' | sudo tee /data/etcd/cfg/etcd.conf# sre-lo-test-vm-master-003
echo '#[Member]
ETCD_NAME="sre-lo-test-vm-master-003"
ETCD_DATA_DIR="/data/etcd/data"
ETCD_LISTEN_PEER_URLS="https://169.0.0.102:2380"
ETCD_LISTEN_CLIENT_URLS="https://169.0.0.102:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://169.0.0.102:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://169.0.0.102:2379"
ETCD_INITIAL_CLUSTER="sre-lo-test-vm-master-001=https://169.0.0.100:2380,sre-lo-test-vm-master-002=https://169.0.0.101:2380,sre-lo-test-vm-master-003=https://169.0.0.102:2380"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"' | sudo tee /data/etcd/cfg/etcd.conf# sre-lo-test-vm-master-*
echo '[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/data/etcd/cfg/etcd.conf
ExecStart=/data/etcd/bin/etcd \--cert-file=/data/etcd/ssl/server.pem \--key-file=/data/etcd/ssl/server-key.pem \--peer-cert-file=/data/etcd/ssl/server.pem \--peer-key-file=/data/etcd/ssl/server-key.pem \--trusted-ca-file=/data/etcd/ssl/ca.pem \--peer-trusted-ca-file=/data/etcd/ssl/ca.pem \--logger=zap
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target' | sudo tee /usr/lib/systemd/system/etcd.service

2.2.4 启动

# 至少两台同时执行，才能正常启动
sudo systemctl enable etcd --now
sudo systemctl status etcd# 检查状态
sudo ETCDCTL_API=3 /data/etcd/bin/etcdctl --cacert=/data/etcd/ssl/ca.pem --cert=/data/etcd/ssl/server.pem --key=/data/etcd/ssl/server-key.pem --endpoints="https://169.0.0.100:2379,https://169.0.0.101:2379,https://169.0.0.102:2379" endpoint health

2.3 备份

# sre-lo-test-vm-master-001
echo '#!/bin/bash
etcd_backup_path="/data/etcd/backup"
if [[ ! -d $etcd_backup_path ]];then mkdir -p $etcd_backup_path; 
fi
sudo ETCDCTL_API=3 /data/etcd/bin/etcdctl --cacert=/data/etcd/ssl/ca.pem --cert=/data/etcd/ssl/server.pem --key=/data/etcd/ssl/server-key.pem --endpoints="https://169.0.0.100:2379,https://169.0.0.101:2379,https://169.0.0.102:2379" snapshot save $etcd_backup_path/etcd-snapshot.`date +%Y%m%d%H%M%S`.db
' | sudo tee /data/etcd/bin/backup.shchmod +x /data/etcd/bin/backup.shecho '# etcd data backup
30 3 * * * /data/etcd/bin/backup.sh' |sudo tee -a /var/spool/cron/root

3. 负载均衡

3.1 注意事项

由于先在一台服务器部署kubeadm，再部署另外两台，所以后端服务器也要先设置一台，再设置另外两台
前端端口，如果负载均衡与k8s-master在同一节点上，kubelet监听的6443端口会发生冲突，因此前端端口可以改成16443，监听后端6443端口

3.2 部署haproxy与keepalived

参考文档：https://github.com/kubernetes/kubeadm/blob/main/docs/ha-considerations.md#options-for-software-load-balancing

# ALL
sudo dnf install -y haproxy keepalived
sudo mv /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bakecho 'globallog /dev/log local0log /dev/log local1 noticedaemondefaultsmode                    httplog                     globaloption                  httplogoption                  dontlognulloption http-server-closeoption forwardfor       except 127.0.0.0/8option                  redispatchretries                 1timeout http-request    10stimeout queue           20stimeout connect         5stimeout client          20stimeout server          20stimeout http-keep-alive 10stimeout check           10sfrontend apiserverbind *:16443mode tcpoption tcplogdefault_backend apiserverbackendbackend apiserverbackendoption httpchk GET /healthzhttp-check expect status 200mode tcpoption ssl-hello-chkbalance     roundrobinserver      sre-lo-test-vm-master-001   169.0.0.100:6443 checkserver      sre-lo-test-vm-master-002   169.0.0.101:6443 checkserver      sre-lo-test-vm-master-003   169.0.0.102:6443 check
' |sudo tee /etc/haproxy/haproxy.cfg# sre-lo-test-vm-master-001
echo 'global_defs {router_id LVS_DEVEL
}
vrrp_script check_apiserver {script "/etc/keepalived/check_apiserver.sh"interval 3weight -2fall 10rise 2
}vrrp_instance VI_1 {state MASTERinterface enp0s3virtual_router_id 51priority 100authentication {auth_type PASSauth_pass ceb1b3ec013d66163d6ab}virtual_ipaddress {192.168.0.28}track_script {check_apiserver}
}
' |sudo tee /etc/keepalived/keepalived.conf# sre-lo-test-vm-master-002
echo 'global_defs {router_id LVS_DEVEL
}
vrrp_script check_apiserver {script "/etc/keepalived/check_apiserver.sh"interval 3weight -2fall 10rise 2
}vrrp_instance VI_1 {state BACKUPinterface enp0s3virtual_router_id 51priority 100authentication {auth_type PASSauth_pass ceb1b3ec013d66163d6ab}virtual_ipaddress {192.168.0.28}track_script {check_apiserver}
}
' |sudo tee /etc/keepalived/keepalived.conf# sre-lo-test-vm-master-003
echo 'global_defs {router_id LVS_DEVEL
}
vrrp_script check_apiserver {script "/etc/keepalived/check_apiserver.sh"interval 3weight -2fall 10rise 2
}vrrp_instance VI_1 {state BACKUPinterface enp0s3virtual_router_id 51priority 100authentication {auth_type PASSauth_pass ceb1b3ec013d66163d6ab}virtual_ipaddress {192.168.0.28}track_script {check_apiserver}
}
' |sudo tee /etc/keepalived/keepalived.conf# sre-lo-test-vm-master-*
echo '#!/bin/sh
errorExit() {echo "*** $*" 1>&2exit 1
}curl --silent --max-time 2 --insecure https://localhost:16443/ -o /dev/null || errorExit "Error GET https://localhost:16443/"
if ip addr | grep -q 192.168.0.28; thencurl --silent --max-time 2 --insecure https://192.168.0.28:16443/ -o /dev/null || errorExit "Error GET https://192.168.0.28:16443/"
fi' |sudo tee /etc/keepalived/check_apiserver.sh

3.3 启动

# sre-lo-test-vm-master-*
sudo systemctl enable haproxy --now
sudo systemctl enable keepalived --nowsudo systemctl status haproxy
sudo systemctl status keepalivedsudo systemctl restart haproxy
sudo systemctl restart keepalived

4. container部署

参考文档：https://kubernetes.io/zh-cn/docs/setup/production-environment/container-runtimes/

4.1 设置系统环境

# ALL k8s master, k8s node 
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOFsudo modprobe overlay
sudo modprobe br_netfiltercat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOFsudo sysctl --systemswapoff -a 
sed -ri 's/.*swap.*/#&/' /etc/fstab 
free -h

4.2 下载containerd

# MacOS
curl -L -O https://github.com/containerd/containerd/releases/download/v1.6.36/containerd-1.6.36-linux-amd64.tar.gz
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-master-001:~/
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-master-002:~/
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-master-003:~/
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-node-001:~/
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-node-002:~/
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-node-003:~/curl -L -O https://raw.githubusercontent.com/containerd/containerd/main/containerd.service
scp containerd.service root@sre-lo-test-vm-master-001:/usr/lib/systemd/system/
scp containerd.service root@sre-lo-test-vm-master-002:/usr/lib/systemd/system/
scp containerd.service root@sre-lo-test-vm-master-003:/usr/lib/systemd/system/
scp containerd.service root@sre-lo-test-vm-node-001:/usr/lib/systemd/system/
scp containerd.service root@sre-lo-test-vm-node-002:/usr/lib/systemd/system/
scp containerd.service root@sre-lo-test-vm-node-003:/usr/lib/systemd/system/curl -L -O https://github.com/opencontainers/runc/releases/download/v1.2.1/runc.amd64
scp runc.amd64 root@sre-lo-test-vm-master-001:~/
scp runc.amd64 root@sre-lo-test-vm-master-002:~/
scp runc.amd64 root@sre-lo-test-vm-master-003:~/
scp runc.amd64 root@sre-lo-test-vm-node-001:~/
scp runc.amd64 root@sre-lo-test-vm-node-002:~/
scp runc.amd64 root@sre-lo-test-vm-node-003:~/curl -L -O https://github.com/containernetworking/plugins/releases/download/v1.6.0/cni-plugins-linux-amd64-v1.6.0.tgz
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-master-001:~/
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-master-002:~/
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-master-003:~/
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-node-001:~/
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-node-002:~/
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-node-003:~/

4.3 安装containerd

# ALL k8s master, k8s node
sudo dnf remove -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugincd ~mkdir -p /data/containerd/opt
mkdir -p /data/containerd/data
mkdir -p /opt/cni/bintar -xzvf containerd-1.6.36-linux-amd64.tar.gz -C /usr/localinstall -m 755 runc.amd64 /usr/local/sbin/runctar -xzvf cni-plugins-linux-amd64-v1.6.0.tgz -C /opt/cni/bin

4.4 设置containerd

# ALL k8s master, k8s node## 创建目录
mkdir /etc/containerd
containerd config default > /etc/containerd/config.toml
## 配置 systemd cgroup 驱动
sed -i 's|SystemdCgroup = false|SystemdCgroup = true|g' /etc/containerd/config.toml
## 修改root目录
sed -i 's|/var/lib/containerd|/data/containerd/data|g' /etc/containerd/config.toml
## 修改依赖项目录
sed -i 's|/opt/containerd|/data/containerd/opt|g' /etc/containerd/config.toml
## 修改基础容器
sed -i 's|registry.k8s.io/pause:3.6|registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.10|g' /etc/containerd/config.toml
## 修改镜像源
sed -i '/^\s*\[plugins\."io\.containerd\.grpc\.v1\.cri"\.registry\.mirrors\]/a\      [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]\n        endpoint = ["https://dockerpull.org"]\n      [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]\n        endpoint = ["registry.aliyuncs.com/google_containers"]' /etc/containerd/config.toml
## 镜像源登录账户，用于第三方需要登录的地址（可选）
sed -i '/^\s*\[plugins\."io\.containerd\.grpc\.v1\.cri"\.registry\.configs\]/a\      [plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io".auth]\n        username = ""\n        password = ""\n' /etc/containerd/config.toml## 镜像源、镜像源登录账户，效果如下所示
[plugins."io.containerd.grpc.v1.cri".registry][plugins."io.containerd.grpc.v1.cri".registry.configs][plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io".auth]username = ""password = ""[plugins."io.containerd.grpc.v1.cri".registry.mirrors][plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]endpoint = ["https://dockerpull.org"][plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]endpoint = ["registry.aliyuncs.com/google_containers"]

4.5 启动containerd

# ALL k8s master, k8s node 
systemctl daemon-reloadsystemctl enable --now containerdsudo systemctl restart containerdsudo systemctl status containerdsudo systemctl show --property=Environment containerd

5. 安装kubernetes

5.1 下载应用

# ALL k8s master, k8s node
cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.31/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.31/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOFsudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
sudo systemctl enable --now kubeletsudo crictl config --set runtime-endpoint=unix:///var/run/containerd/containerd.sock

5.2 下载镜像

# ALL k8s master, k8s node
sudo kubeadm config images listcrictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.31.3
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.31.3
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.31.3
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.31.3
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.11.3
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.10
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.5.15-0
crictl pull dockerpull.org/calico/cni:v3.25.0
crictl pull dockerpull.org/calico/kube-controllers:v3.25.0
crictl pull dockerpull.org/calico/node:v3.25.0

5.3 设置kubeadm配置文件

# sre-lo-test-vm-master-001
# ~/kubeadm-config.yaml
kind: InitConfiguration
apiVersion: kubeadm.k8s.io/v1beta4
bootstrapTokens:
- groups:- system:bootstrappers:kubeadm:default-node-tokentoken: abcdef.0123456789abcdefttl: 8760h0m0susages:- signing- authentication
certificateKey: 07ef165f6723337d68b0eca1c6a29222a44aecadabbbbb79016cd160a397782c
localAPIEndpoint:advertiseAddress: 169.0.0.100bindPort: 6443
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresentimagePullSerial: truename: sre-lo-test-vm-master-001---
kind: ClusterConfiguration
apiVersion: kubeadm.k8s.io/v1beta4
kubernetesVersion: v1.31.3
clusterName: k8s-dog
caCertificateValidityPeriod: 87600h0m0s
certificateValidityPeriod: 8760h0m0s
certificatesDir: /etc/kubernetes/pki
encryptionAlgorithm: RSA-2048
dns: {}
proxy: {}
scheduler: {}
apiServer: {}
controllerManager: {}
etcd:external:endpoints:- https://169.0.0.100:2379- https://169.0.0.101:2379- https://169.0.0.102:2379caFile: /etc/kubernetes/pki/etcd/ca.crtcertFile: /etc/kubernetes/pki/apiserver-etcd-client.crtkeyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
controlPlaneEndpoint: 192.168.0.28:16443
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
networking:dnsDomain: cluster.localpodSubnet: 169.1.0.0/16serviceSubnet: 169.2.0.0/16---
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
cgroupDriver: systemd

5.4 拷贝etcd密钥到k8s服务器

# sre-lo-test-vm-master-001
sudo mkdir -p /etc/kubernetes/pki
sudo mkdir -p /etc/kubernetes/pki/etcdsudo /bin/cp -rf /data/etcd/ssl/ca.pem /etc/kubernetes/pki/etcd/ca.crt
sudo /bin/cp -rf /data/etcd/ssl/server.pem /etc/kubernetes/pki/apiserver-etcd-client.crt
sudo /bin/cp -rf /data/etcd/ssl/server-key.pem /etc/kubernetes/pki/apiserver-etcd-client.keysudo chown root:root /etc/kubernetes/pki/etcd/ca.crt
sudo chown root:root /etc/kubernetes/pki/apiserver-etcd-client.crt
sudo chown root:root /etc/kubernetes/pki/apiserver-etcd-client.key

5.5 首台服务器开始部署

sudo kubeadm init --config ~/kubeadm-config.yaml --upload-certs

# 补充说明## 2小时以后添加新的节点
## sre-lo-test-vm-master-001
## 从kubeadm-config.yaml中生成新的certificateKey
kubeadm --config kubeadm-config.yaml init phase upload-certs --upload-certs## 控制平面 certificateKey
kubeadm init phase upload-certs --upload-certs## CA证书 caCertHashes
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | \
openssl rsa -pubin -outform der 2>/dev/null | \
sha256sum | \
awk '{print $1}'## token
kubeadm token create
kubeadm token list

5.6 部署其他节点

# sre-lo-test-vm-master-002 注意修改: caCertHashes
echo 'apiVersion: kubeadm.k8s.io/v1beta4
kind: JoinConfiguration
discovery:bootstrapToken:token: "abcdef.0123456789abcdef"apiServerEndpoint: "192.168.0.28:16443"caCertHashes:- "sha256:d1dc176b72a417c0130da63a4bc12a7b4ec32f68d960d2d3b7037f0304752ec4"
controlPlane:localAPIEndpoint:advertiseAddress: 169.0.0.101bindPort: 6443certificateKey: 07ef165f6723337d68b0eca1c6a29222a44aecadabbbbb79016cd160a397782c
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresentimagePullSerial: truename: sre-lo-test-vm-master-002
' | tee ~/kubeadm-config.yaml
sudo kubeadm join --config ~/kubeadm-config.yaml# sre-lo-test-vm-master-003 注意修改: caCertHashes
echo 'apiVersion: kubeadm.k8s.io/v1beta4
kind: JoinConfiguration
discovery:bootstrapToken:token: "abcdef.0123456789abcdef"apiServerEndpoint: "192.168.0.28:16443"caCertHashes:- "sha256:d1dc176b72a417c0130da63a4bc12a7b4ec32f68d960d2d3b7037f0304752ec4"
controlPlane:localAPIEndpoint:advertiseAddress: 169.0.0.102bindPort: 6443certificateKey: 07ef165f6723337d68b0eca1c6a29222a44aecadabbbbb79016cd160a397782c
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresentimagePullSerial: truename: sre-lo-test-vm-master-003
' | tee ~/kubeadm-config.yaml
sudo kubeadm join --config ~/kubeadm-config.yaml# sre-lo-test-vm-node-001 注意修改: caCertHashes
echo 'apiVersion: kubeadm.k8s.io/v1beta4
kind: JoinConfiguration
discovery:bootstrapToken:token: "abcdef.0123456789abcdef"apiServerEndpoint: "192.168.0.28:16443"caCertHashes:- "sha256:d1dc176b72a417c0130da63a4bc12a7b4ec32f68d960d2d3b7037f0304752ec4"
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresentimagePullSerial: truename: sre-lo-test-vm-node-001
' | tee ~/kubeadm-config.yaml
sudo kubeadm join --config ~/kubeadm-config.yaml# sre-lo-test-vm-node-002 注意修改: caCertHashes
echo 'apiVersion: kubeadm.k8s.io/v1beta4
kind: JoinConfiguration
discovery:bootstrapToken:token: "abcdef.0123456789abcdef"apiServerEndpoint: "192.168.0.28:16443"caCertHashes:- "sha256:d1dc176b72a417c0130da63a4bc12a7b4ec32f68d960d2d3b7037f0304752ec4"
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresentimagePullSerial: truename: sre-lo-test-vm-node-002
' | tee ~/kubeadm-config.yaml
sudo kubeadm join --config ~/kubeadm-config.yaml# sre-lo-test-vm-node-003 注意修改: caCertHashes
echo 'apiVersion: kubeadm.k8s.io/v1beta4
kind: JoinConfiguration
discovery:bootstrapToken:token: "abcdef.0123456789abcdef"apiServerEndpoint: "192.168.0.28:16443"caCertHashes:- "sha256:d1dc176b72a417c0130da63a4bc12a7b4ec32f68d960d2d3b7037f0304752ec4"
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresentimagePullSerial: truename: sre-lo-test-vm-node-003
' | tee ~/kubeadm-config.yaml
sudo kubeadm join --config ~/kubeadm-config.yaml

5.7 设置kubectl

# 检查
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/configkubectl get nodes

5.8 安装网络插件

# calico
wget https://docs.projectcalico.org/manifests/calico.yaml# update CALICO_IPV4POOL_CIDR
CALICO_IPV4POOL_CIDR = pod subnet
# update image registry
sed -i 's|docker.io/calico|dockerpull.org/calico|g' calico.yaml# apply
kubectl apply -f calico.yamlkubectl -n kube-system get pods