一、准备机器准备3台虚拟机都是4核4GIP地址主机名角色192.168.11.101k8s-masterMaster192.168.11.102k8s-node1Node192.168.11.103k8s-node2Node二、环境初始化1、配置网卡配置master网卡[rootk8s-master01 run]# cat /etc/NetworkManager/system-connections/ens160.nmconnection [connection] idens160 uuid5bd99cff-3edc-3223-ac66-4c076f11dd34 typeethernet autoconnect-priority-999 interface-nameens160 timestamp1786099079 [ethernet] [ipv4] methodmanual address1192.168.66.11/24,192.168.66.200 #配置ip等信息备注的需要配置 dns114.114.114.114;8.8.8.8 #配置DNS信息 [ipv6] addr-gen-modeeui64 methodauto [proxy] [rootk8s-master01 run]# cat /etc/NetworkManager/system-connections/ens192.nmconnection [connection] idens192 uuidbde7b363-5909-3b24-87eb-cea62158af49 typeethernet autoconnect-priority-999 autoconnectfalse #配置 interface-nameens192 timestamp1786099079 [ethernet] [ipv4] methodauto [ipv6] addr-gen-modeeui64 methodauto [proxy]配置node1网卡[rootk8s-node01 kubernetes-1.29.2-150500.1.1]# cat /etc/NetworkManager/system-connections/ens160.nmconnection [connection] idens160 uuid7a754f01-804a-3381-8242-f3429643f1ec typeethernet autoconnect-priority-999 interface-nameens160 timestamp1786104241 [ethernet] [ipv4] methodmanual address1192.168.66.12/24,192.168.66.200 #配置 dns114.114.114.114;8.8.8.8 #配置 [ipv6] addr-gen-modeeui64 methodauto [proxy] [rootk8s-node01 kubernetes-1.29.2-150500.1.1]# cat /etc/NetworkManager/system-connections/ens192.nmconnection [connection] idens192 uuid394b3ad7-a691-394d-a628-0bc4a004c86c typeethernet autoconnect-priority-999 autoconnectfalse #配置 interface-nameens192 timestamp1786104241 [ethernet] [ipv4] methodauto [ipv6] addr-gen-modeeui64 methodauto [proxy]配置node2网卡[rootk8s-node02 kubernetes-1.29.2-150500.1.1]# cat /etc/NetworkManager/system-connections/ens160.nmconnection [connection] idens160 uuid7bbd2e55-d0d5-3103-8c6e-de781c58cbe2 typeethernet autoconnect-priority-999 interface-nameens160 timestamp1786103511 [ethernet] [ipv4] methodmanual address1192.168.66.13/24,192.168.66.200 #配置 dns114.114.114.114;8.8.8.8 #配置 [ipv6] addr-gen-modeeui64 methodauto [proxy] [rootk8s-node02 kubernetes-1.29.2-150500.1.1]# cat /etc/NetworkManager/system-connections/ens192.nmconnection [connection] idens192 uuidae4c7050-520d-30da-9231-7234d8bd736f typeethernet autoconnect-priority-999 autoconnectfalse #配置 interface-nameens192 timestamp1786103511 [ethernet] [ipv4] methodauto [ipv6] addr-gen-modeeui64 methodauto [proxy]2、其他配置1、# Rocky 系统软件源更换 sed -e s|^mirrorlist|#mirrorlist|g \ -e s|^#baseurlhttp://dl.rockylinux.org/$contentdir|baseurlhttps://mirrors.aliyun.com/rockylinux|g \ -i.bak \ /etc/yum.repos.d/[Rr]ocky*.repo dnf makecache 2、# 防火墙修改 firewalld 为 iptables systemctl stop firewalld systemctl disable firewalld yum -y install iptables-services systemctl start iptables iptables -F systemctl enable iptables service iptables save 3、# 禁用 Selinux setenforce 0 sed -i s/SELINUXenforcing/SELINUXdisabled/g /etc/selinux/config grubby --update-kernel ALL --args selinux0 # 查看是否禁用grubby --info DEFAULT # 回滚内核层禁用操作grubby --update-kernel ALL --remove-args selinux 4、# 设置时区 timedatectl set-timezone Asia/Shanghai # 关闭 swap 分区 # 临时关闭立即生效 swapoff -a # 永久关闭修改 fstab注释掉 swap 相关行 sed -i s/.*swap.*/#/ /etc/fstab # 验证 swap 是否已关闭 free -h 5、# 修改主机名 hostnamectl set-hostname k8s-node013个机器分别设置为k8s-master01、k8s-node01、k8s-node02 6、# 安装 ipvs yum install -y ipvsadm 7、# 开启路由转发 echo net.ipv4.ip_forward1 /etc/sysctl.conf sysctl -p 8、# 加载 bridge yum install -y epel-release yum install -y bridge-utils modprobe br_netfilter echo br_netfilter /etc/modules-load.d/bridge.conf echo net.bridge.bridge-nf-call-iptables1 /etc/sysctl.conf echo net.bridge.bridge-nf-call-ip6tables1 /etc/sysctl.conf sysctl -p3、三个机器都修改hosts如下[rootk8s-node02 kubernetes-1.29.2-150500.1.1]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.66.11 k8s-master01 m1 192.168.66.12 k8s-node01 n1 192.168.66.13 k8s-node02 n2 192.168.66.14 harbor三、3个机器都装docker和如下配置1、# 添加 docker-ce yum 源 # 中科大(ustc) sudo dnf config-manager --add-repo https://mirrors.ustc.edu.cn/docker-ce/linux/centos/docker-ce.repo cd /etc/yum.repos.d # 切换中科大源 sed -e s|download.docker.com|mirrors.ustc.edu.cn/docker-ce|g docker-ce.repo 2、# 安装 docker-ce yum -y install docker-ce 3、# 配置 daemon. cat /etc/docker/daemon.json EOF { data-root: /data/docker, exec-opts: [native.cgroupdriversystemd], log-driver: json-file, log-opts: { max-size: 100m, max-file: 100 }, insecure-registries: [harbor.xinxainghf.com], registry-mirrors: [ https://kfp63jaj.mirror.aliyuncs.com, https://hub-mirror.c.163.com, https://mirror.baidubce.com, https://registry.docker-cn.com, https://mirror.ccs.tencentyun.com, https://docker.mirrors.ustc.edu.cn, https://docker.1ms.run, https://hub.rat.dev, https://docker.1panel.live ] } EOF 4、mkdir -p /etc/systemd/system/docker.service.d 5、# 重启docker服务 systemctl daemon-reload systemctl restart docker systemctl enable docker reboot 6、# 安装 cri-docker wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.9/cri-dockerd-0.3.9.amd64.tgz tar -xf cri-dockerd-0.3.9.amd64.tgz cp cri-dockerd/cri-dockerd /usr/bin/ chmod x /usr/bin/cri-dockerd 7、# 配置 cri-docker 服务 cat EOF /usr/lib/systemd/system/cri-docker.service [Unit] DescriptionCRI Interface for Docker Application Container Engine Documentationhttps://docs.mirantis.com Afternetwork-online.target firewalld.service docker.service Wantsnetwork-online.target Requirescri-docker.socket [Service] Typenotify ExecStart/usr/bin/cri-dockerd --network-plugincni --pod-infra-container-imageregistry.aliyuncs.com/google_containers/pause:3.8 ExecReload/bin/kill -s HUP $MAINPID TimeoutSec0 RestartSec2 Restartalways StartLimitBurst3 StartLimitInterval60s LimitNOFILEinfinity LimitNPROCinfinity LimitCOREinfinity TasksMaxinfinity Delegateyes KillModeprocess [Install] WantedBymulti-user.target EOF 8、# 添加 cri-docker 套接字 cat EOF /usr/lib/systemd/system/cri-docker.socket [Unit] DescriptionCRI Docker Socket for the API PartOfcri-docker.service [Socket] ListenStream%t/cri-dockerd.sock SocketMode0660 SocketUserroot SocketGroupdocker [Install] WantedBysockets.target EOF 9、# 启动 cri-docker 对应服务 systemctl daemon-reload systemctl enable cri-docker systemctl start cri-docker systemctl is-active cri-docker四、安装kubenates说明我的安装也是根据bilibili的薪享宏福学习的详细教程可以关注他。安装包下载地址如下百度网盘 请输入提取码#把安装包kubernetes-1.29.2-150500.1.1.tar.gz放在master机器的/home目录下 cd /home/ #解压 tar -zxvf kubernetes-1.29.2-150500.1.1.tar.gz #进入该目录 cd kubernetes-1.29.2-150500.1.1 #安装所有rpm安装包 yum -y install * #把安装包同步到k8s-node01、k8s-node02机器上注意同步完也记得在其他2机器上执行安装和自启动命令 scp -r /home/kubernetes-1.29.2-150500.1.1 rootn1:/home/ scp -r /home/kubernetes-1.29.2-150500.1.1 rootn2:/home/ #开机自启动 systemctl enable kubelet.service五、初始化master和node初始化kubeadm init --apiserver-advertise-address192.168.66.11 --image-repository registry.aliyuncs.com/google_containers --kubernetes-version 1.29.2 --service-cidr10.10.0.0/12 --pod-network-cidr10.244.0.0/16 --ignore-preflight-errorsall --cri-socket unix:///var/run/cri-dockerd.sock注意如果初始化失败需要删除缓存的初始化文件# 使用 kubeadm reset 清理所有 K8s 数据 kubeadm reset --cri-socket unix:///var/run/cri-dockerd.sock -f # 删除所有残留目录 rm -rf /etc/kubernetes rm -rf /var/lib/kubelet rm -rf /var/lib/etcd rm -rf /var/lib/cni/ rm -rf /etc/cni/ rm -rf /var/lib/etcd/member添加master01的节点#注意下面3行命令是初始化时生成的 [rootk8s-master01 ~]# mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config [rootk8s-master01 ~]# #查看已添加进来 [rootk8s-master01 ~]# kubectl get node NAME STATUS ROLES AGE VERSION k8s-master01 NotReady control-plane 60m v1.29.2添加k8s-node01、k8s-node02节点分别在k8s-node01、k8s-node02节点执行如下命令这个命令也是初始化时生成的后面的--cri-socket unix:///var/run/cri-dockerd.sock是我自己添加的作用 kubeadm join 192.168.66.11:6443 --token e5nq79.aiixe3wg9mjrv2tw \ --discovery-token-ca-cert-hash sha256:86ad9c50a7037a8824d83cef6d074258a8e77b0dafcf1bdefccc1a2813ec22cf --cri-socket unix:///var/run/cri-dockerd.sockk8s-node执行如上命令时遇到master防火墙设置的坑#这个规则是我node无法telnet master 9 203 18745 REJECT all -- * * 0.0.0.0/0 0.0.0.0/0 reject-with icmp-host-prohibited 这是默认防火墙firewalld的遗留规则即使 firewalld 服务已停止但 iptables 规则依然存在。 处理: # 清空 INPUT 链 iptables -F INPUT # 重新设置默认策略为 ACCEPT iptables -P INPUT ACCEPT2个node执行完命令后master查看node就可以看到2个node了[rootk8s-master01 ~]# kubectl get node NAME STATUS ROLES AGE VERSION k8s-master01 NotReady control-plane 3h53m v1.29.2 k8s-node01 NotReady none 32s v1.29.2 k8s-node02 NotReady none 4s v1.29.2 [rootk8s-master01 ~]#六、部署网络插件calico.zip可以通过上面网盘获得然后3台机器都执行如下操作#去网盘了找到calico.zip,这里有calico的镜像 #解压 #unzip calico.zip #cd calico #tar -zxvf calico-images.tar.gz #cd calico-images #导入镜像 #docker load -i calico-cni-v3.26.3.tar #docker load -i calico-kube-controllers-v3.26.3.tar #docker load -i calico-node-v3.26.3.tar #docker load -i calico-typha-v3.26.3.tar再master节点把calico的部署文件应用在该主机上。#注意这个calico-typha.yaml要关闭ipip,以及配置pod的网段时和初始化master设置的网段一样网盘里的这个yaml文件已经设置过了 kubectl apply -f calico-typha.yaml到此k8s就算部署成功了但是我的有controllers、coredns不正常#查看pod是否都正常 [rootk8s-master01 calico]# kubectl get pod -A NAMESPACE NAME READY STATUS RESTARTS AGE kube-system calico-kube-controllers-558d465845-jr76v 0/1 Running 6 (3m38s ago) 10m kube-system calico-node-4ktgm 1/1 Running 0 10m kube-system calico-node-l5jbs 1/1 Running 0 10m kube-system calico-node-x7bb2 1/1 Running 0 10m kube-system calico-typha-5b56944f9b-9tx54 1/1 Running 0 10m kube-system coredns-857d9ff4c9-k7p6s 0/1 Running 0 7h14m kube-system coredns-857d9ff4c9-v5cv8 0/1 Running 0 7h14m kube-system etcd-k8s-master01 1/1 Running 0 7h14m kube-system kube-apiserver-k8s-master01 1/1 Running 0 7h14m kube-system kube-controller-manager-k8s-master01 1/1 Running 0 7h14m kube-system kube-proxy-f7sl4 1/1 Running 0 3h20m kube-system kube-proxy-mc85r 1/1 Running 0 7h14m kube-system kube-proxy-sd9sd 1/1 Running 0 3h20m kube-system kube-scheduler-k8s-master01 1/1 Running 0 7h14m原因因为我的node01、node02没清理好防火墙规则导致calico、dns、controllers没成功#重置node01、node02防火墙规则 # 清空 INPUT 链 iptables -F INPUT # 重新设置默认策略为 ACCEPT iptables -P INPUT ACCEPT # 删除 CoreDNS Pod会自动重建 kubectl delete pods -n kube-system -l k8s-appkube-dns # 删除所有 Calico-node Pod会自动重建之前calico虽然正常但是我node1、node2网络规则重置了所以重新弄一下calico kubectl delete pods -n kube-system -l k8s-appcalico-node # 删除 calico-kube-controllers会自动重建 kubectl delete pods -n kube-system -l k8s-appcalico-kube-controllers # 等待 10 秒 sleep 10重启电脑后k8s集群有的colico等没启动。执行如下脚本整理3台机器的防火墙规则。#!/bin/bash echo echo 永久固化防火墙规则 - Kubernetes 集群 echo echo 1. 禁用 firewalld systemctl stop firewalld 2/dev/null systemctl disable firewalld 2/dev/null systemctl mask firewalld 2/dev/null echo 2. 禁用 nftables systemctl stop nftables 2/dev/null systemctl disable nftables 2/dev/null systemctl mask nftables 2/dev/null echo 3. 禁用 iptables 服务 systemctl stop iptables 2/dev/null systemctl disable iptables 2/dev/null systemctl mask iptables 2/dev/null echo 4. 清空所有 iptables 规则 iptables -F iptables -X iptables -t nat -F iptables -t nat -X iptables -t mangle -F iptables -t mangle -X iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT echo 5. 保存 iptables 规则 iptables-save /etc/iptables/rules.v4 2/dev/null iptables-save /etc/sysconfig/iptables 2/dev/null service iptables save 2/dev/null echo 6. 重启 kubelet systemctl restart kubelet echo 7. 验证防火墙状态 echo firewalld: $(systemctl is-active firewalld 2/dev/null || echo inactive) echo iptables INPUT 链策略: $(iptables -L INPUT -n | head -1 | awk {print $4}) echo echo 防火墙规则已永久固化 echo 七、启动顺序3台服务器重启k8s启动顺序如下 Kubernetes 组件完整启动顺序6个阶段第一阶段操作系统初始化systemd服务器开机后Linux 内核启动systemdPID 1作为所有进程的“老祖宗”率先运行。它负责根据系统配置拉起后续的各种系统服务。第二阶段容器运行时启动Docker cri-dockerd此时还没有 Kubernetes但必须先把“装东西的仓库”建好。systemd 启动 Dockerdockerd守护进程开始运行但此时 kubelet 还不能直接调用它因为 kubelet 默认使用 containerd 的接口标准 CRI。systemd 启动 cri-dockerdcri-dockerd作为一个适配器垫片启动它监听在unix:///var/run/cri-dockerd.sock让 kubelet 能通过这个接口来指挥 Docker 创建容器。第三阶段Kubelet 进程启动关键节点systemd 启动 kubelet 进程你之前ps看到的 PID 2209。这是第一个也是唯一一个以宿主机进程形式运行的 K8s 组件。此时kubelet 并不知道自己要管什么它只是连接上了刚才的cri-dockerd.sock并开始持续监控/etc/kubernetes/manifests/这个目录。第四阶段控制平面静态 Pod 启动核心组件当你执行kubeadm init后会在/etc/kubernetes/manifests/目录生成etcd.yaml、kube-apiserver.yaml等文件。kubelet 发现清单文件kubelet 监控到目录变化立即通过 cri-dockerd 调用 Docker 创建容器。容器启动顺序存在内部依赖但 kubelet 会同时拉起etcd 容器优先启动因为 API Server 依赖它存数据。kube-apiserver 容器启动集群的大脑所有操作都要经过它。当 API Server 确认 etcd 已连接后kube-controller-manager 和 kube-scheduler 容器才真正开始工作它们需要监听 API Server 来干活。此时你执行docker ps就能看到这些容器了控制平面就算“活”了。第五阶段Node 节点向集群注册API Server 启动后kubelet 进程会拿着自己的证书/etc/kubernetes/kubelet.conf向 API Server 发起请求把自己这个节点注册到集群中。此时kubectl get nodes才能看到这台机器。第六阶段网络插件与工作负载 Pod 启动控制平面运行后你通常会执行kubectl apply -f calico.yaml安装网络插件CNI。Calico、CoreDNS 等作为普通的 Pod非静态被调度起来。这时集群才完全就绪可以部署业务应用了。
一、准备机器准备3台虚拟机都是4核4GIP地址主机名角色192.168.11.101k8s-masterMaster192.168.11.102k8s-node1Node192.168.11.103k8s-node2Node二、环境初始化1、配置网卡配置master网卡[rootk8s-master01 run]# cat /etc/NetworkManager/system-connections/ens160.nmconnection [connection] idens160 uuid5bd99cff-3edc-3223-ac66-4c076f11dd34 typeethernet autoconnect-priority-999 interface-nameens160 timestamp1786099079 [ethernet] [ipv4] methodmanual address1192.168.66.11/24,192.168.66.200 #配置ip等信息备注的需要配置 dns114.114.114.114;8.8.8.8 #配置DNS信息 [ipv6] addr-gen-modeeui64 methodauto [proxy] [rootk8s-master01 run]# cat /etc/NetworkManager/system-connections/ens192.nmconnection [connection] idens192 uuidbde7b363-5909-3b24-87eb-cea62158af49 typeethernet autoconnect-priority-999 autoconnectfalse #配置 interface-nameens192 timestamp1786099079 [ethernet] [ipv4] methodauto [ipv6] addr-gen-modeeui64 methodauto [proxy]配置node1网卡[rootk8s-node01 kubernetes-1.29.2-150500.1.1]# cat /etc/NetworkManager/system-connections/ens160.nmconnection [connection] idens160 uuid7a754f01-804a-3381-8242-f3429643f1ec typeethernet autoconnect-priority-999 interface-nameens160 timestamp1786104241 [ethernet] [ipv4] methodmanual address1192.168.66.12/24,192.168.66.200 #配置 dns114.114.114.114;8.8.8.8 #配置 [ipv6] addr-gen-modeeui64 methodauto [proxy] [rootk8s-node01 kubernetes-1.29.2-150500.1.1]# cat /etc/NetworkManager/system-connections/ens192.nmconnection [connection] idens192 uuid394b3ad7-a691-394d-a628-0bc4a004c86c typeethernet autoconnect-priority-999 autoconnectfalse #配置 interface-nameens192 timestamp1786104241 [ethernet] [ipv4] methodauto [ipv6] addr-gen-modeeui64 methodauto [proxy]配置node2网卡[rootk8s-node02 kubernetes-1.29.2-150500.1.1]# cat /etc/NetworkManager/system-connections/ens160.nmconnection [connection] idens160 uuid7bbd2e55-d0d5-3103-8c6e-de781c58cbe2 typeethernet autoconnect-priority-999 interface-nameens160 timestamp1786103511 [ethernet] [ipv4] methodmanual address1192.168.66.13/24,192.168.66.200 #配置 dns114.114.114.114;8.8.8.8 #配置 [ipv6] addr-gen-modeeui64 methodauto [proxy] [rootk8s-node02 kubernetes-1.29.2-150500.1.1]# cat /etc/NetworkManager/system-connections/ens192.nmconnection [connection] idens192 uuidae4c7050-520d-30da-9231-7234d8bd736f typeethernet autoconnect-priority-999 autoconnectfalse #配置 interface-nameens192 timestamp1786103511 [ethernet] [ipv4] methodauto [ipv6] addr-gen-modeeui64 methodauto [proxy]2、其他配置1、# Rocky 系统软件源更换 sed -e s|^mirrorlist|#mirrorlist|g \ -e s|^#baseurlhttp://dl.rockylinux.org/$contentdir|baseurlhttps://mirrors.aliyun.com/rockylinux|g \ -i.bak \ /etc/yum.repos.d/[Rr]ocky*.repo dnf makecache 2、# 防火墙修改 firewalld 为 iptables systemctl stop firewalld systemctl disable firewalld yum -y install iptables-services systemctl start iptables iptables -F systemctl enable iptables service iptables save 3、# 禁用 Selinux setenforce 0 sed -i s/SELINUXenforcing/SELINUXdisabled/g /etc/selinux/config grubby --update-kernel ALL --args selinux0 # 查看是否禁用grubby --info DEFAULT # 回滚内核层禁用操作grubby --update-kernel ALL --remove-args selinux 4、# 设置时区 timedatectl set-timezone Asia/Shanghai # 关闭 swap 分区 # 临时关闭立即生效 swapoff -a # 永久关闭修改 fstab注释掉 swap 相关行 sed -i s/.*swap.*/#/ /etc/fstab # 验证 swap 是否已关闭 free -h 5、# 修改主机名 hostnamectl set-hostname k8s-node013个机器分别设置为k8s-master01、k8s-node01、k8s-node02 6、# 安装 ipvs yum install -y ipvsadm 7、# 开启路由转发 echo net.ipv4.ip_forward1 /etc/sysctl.conf sysctl -p 8、# 加载 bridge yum install -y epel-release yum install -y bridge-utils modprobe br_netfilter echo br_netfilter /etc/modules-load.d/bridge.conf echo net.bridge.bridge-nf-call-iptables1 /etc/sysctl.conf echo net.bridge.bridge-nf-call-ip6tables1 /etc/sysctl.conf sysctl -p3、三个机器都修改hosts如下[rootk8s-node02 kubernetes-1.29.2-150500.1.1]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.66.11 k8s-master01 m1 192.168.66.12 k8s-node01 n1 192.168.66.13 k8s-node02 n2 192.168.66.14 harbor三、3个机器都装docker和如下配置1、# 添加 docker-ce yum 源 # 中科大(ustc) sudo dnf config-manager --add-repo https://mirrors.ustc.edu.cn/docker-ce/linux/centos/docker-ce.repo cd /etc/yum.repos.d # 切换中科大源 sed -e s|download.docker.com|mirrors.ustc.edu.cn/docker-ce|g docker-ce.repo 2、# 安装 docker-ce yum -y install docker-ce 3、# 配置 daemon. cat /etc/docker/daemon.json EOF { data-root: /data/docker, exec-opts: [native.cgroupdriversystemd], log-driver: json-file, log-opts: { max-size: 100m, max-file: 100 }, insecure-registries: [harbor.xinxainghf.com], registry-mirrors: [ https://kfp63jaj.mirror.aliyuncs.com, https://hub-mirror.c.163.com, https://mirror.baidubce.com, https://registry.docker-cn.com, https://mirror.ccs.tencentyun.com, https://docker.mirrors.ustc.edu.cn, https://docker.1ms.run, https://hub.rat.dev, https://docker.1panel.live ] } EOF 4、mkdir -p /etc/systemd/system/docker.service.d 5、# 重启docker服务 systemctl daemon-reload systemctl restart docker systemctl enable docker reboot 6、# 安装 cri-docker wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.9/cri-dockerd-0.3.9.amd64.tgz tar -xf cri-dockerd-0.3.9.amd64.tgz cp cri-dockerd/cri-dockerd /usr/bin/ chmod x /usr/bin/cri-dockerd 7、# 配置 cri-docker 服务 cat EOF /usr/lib/systemd/system/cri-docker.service [Unit] DescriptionCRI Interface for Docker Application Container Engine Documentationhttps://docs.mirantis.com Afternetwork-online.target firewalld.service docker.service Wantsnetwork-online.target Requirescri-docker.socket [Service] Typenotify ExecStart/usr/bin/cri-dockerd --network-plugincni --pod-infra-container-imageregistry.aliyuncs.com/google_containers/pause:3.8 ExecReload/bin/kill -s HUP $MAINPID TimeoutSec0 RestartSec2 Restartalways StartLimitBurst3 StartLimitInterval60s LimitNOFILEinfinity LimitNPROCinfinity LimitCOREinfinity TasksMaxinfinity Delegateyes KillModeprocess [Install] WantedBymulti-user.target EOF 8、# 添加 cri-docker 套接字 cat EOF /usr/lib/systemd/system/cri-docker.socket [Unit] DescriptionCRI Docker Socket for the API PartOfcri-docker.service [Socket] ListenStream%t/cri-dockerd.sock SocketMode0660 SocketUserroot SocketGroupdocker [Install] WantedBysockets.target EOF 9、# 启动 cri-docker 对应服务 systemctl daemon-reload systemctl enable cri-docker systemctl start cri-docker systemctl is-active cri-docker四、安装kubenates说明我的安装也是根据bilibili的薪享宏福学习的详细教程可以关注他。安装包下载地址如下百度网盘 请输入提取码#把安装包kubernetes-1.29.2-150500.1.1.tar.gz放在master机器的/home目录下 cd /home/ #解压 tar -zxvf kubernetes-1.29.2-150500.1.1.tar.gz #进入该目录 cd kubernetes-1.29.2-150500.1.1 #安装所有rpm安装包 yum -y install * #把安装包同步到k8s-node01、k8s-node02机器上注意同步完也记得在其他2机器上执行安装和自启动命令 scp -r /home/kubernetes-1.29.2-150500.1.1 rootn1:/home/ scp -r /home/kubernetes-1.29.2-150500.1.1 rootn2:/home/ #开机自启动 systemctl enable kubelet.service五、初始化master和node初始化kubeadm init --apiserver-advertise-address192.168.66.11 --image-repository registry.aliyuncs.com/google_containers --kubernetes-version 1.29.2 --service-cidr10.10.0.0/12 --pod-network-cidr10.244.0.0/16 --ignore-preflight-errorsall --cri-socket unix:///var/run/cri-dockerd.sock注意如果初始化失败需要删除缓存的初始化文件# 使用 kubeadm reset 清理所有 K8s 数据 kubeadm reset --cri-socket unix:///var/run/cri-dockerd.sock -f # 删除所有残留目录 rm -rf /etc/kubernetes rm -rf /var/lib/kubelet rm -rf /var/lib/etcd rm -rf /var/lib/cni/ rm -rf /etc/cni/ rm -rf /var/lib/etcd/member添加master01的节点#注意下面3行命令是初始化时生成的 [rootk8s-master01 ~]# mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config [rootk8s-master01 ~]# #查看已添加进来 [rootk8s-master01 ~]# kubectl get node NAME STATUS ROLES AGE VERSION k8s-master01 NotReady control-plane 60m v1.29.2添加k8s-node01、k8s-node02节点分别在k8s-node01、k8s-node02节点执行如下命令这个命令也是初始化时生成的后面的--cri-socket unix:///var/run/cri-dockerd.sock是我自己添加的作用 kubeadm join 192.168.66.11:6443 --token e5nq79.aiixe3wg9mjrv2tw \ --discovery-token-ca-cert-hash sha256:86ad9c50a7037a8824d83cef6d074258a8e77b0dafcf1bdefccc1a2813ec22cf --cri-socket unix:///var/run/cri-dockerd.sockk8s-node执行如上命令时遇到master防火墙设置的坑#这个规则是我node无法telnet master 9 203 18745 REJECT all -- * * 0.0.0.0/0 0.0.0.0/0 reject-with icmp-host-prohibited 这是默认防火墙firewalld的遗留规则即使 firewalld 服务已停止但 iptables 规则依然存在。 处理: # 清空 INPUT 链 iptables -F INPUT # 重新设置默认策略为 ACCEPT iptables -P INPUT ACCEPT2个node执行完命令后master查看node就可以看到2个node了[rootk8s-master01 ~]# kubectl get node NAME STATUS ROLES AGE VERSION k8s-master01 NotReady control-plane 3h53m v1.29.2 k8s-node01 NotReady none 32s v1.29.2 k8s-node02 NotReady none 4s v1.29.2 [rootk8s-master01 ~]#六、部署网络插件calico.zip可以通过上面网盘获得然后3台机器都执行如下操作#去网盘了找到calico.zip,这里有calico的镜像 #解压 #unzip calico.zip #cd calico #tar -zxvf calico-images.tar.gz #cd calico-images #导入镜像 #docker load -i calico-cni-v3.26.3.tar #docker load -i calico-kube-controllers-v3.26.3.tar #docker load -i calico-node-v3.26.3.tar #docker load -i calico-typha-v3.26.3.tar再master节点把calico的部署文件应用在该主机上。#注意这个calico-typha.yaml要关闭ipip,以及配置pod的网段时和初始化master设置的网段一样网盘里的这个yaml文件已经设置过了 kubectl apply -f calico-typha.yaml到此k8s就算部署成功了但是我的有controllers、coredns不正常#查看pod是否都正常 [rootk8s-master01 calico]# kubectl get pod -A NAMESPACE NAME READY STATUS RESTARTS AGE kube-system calico-kube-controllers-558d465845-jr76v 0/1 Running 6 (3m38s ago) 10m kube-system calico-node-4ktgm 1/1 Running 0 10m kube-system calico-node-l5jbs 1/1 Running 0 10m kube-system calico-node-x7bb2 1/1 Running 0 10m kube-system calico-typha-5b56944f9b-9tx54 1/1 Running 0 10m kube-system coredns-857d9ff4c9-k7p6s 0/1 Running 0 7h14m kube-system coredns-857d9ff4c9-v5cv8 0/1 Running 0 7h14m kube-system etcd-k8s-master01 1/1 Running 0 7h14m kube-system kube-apiserver-k8s-master01 1/1 Running 0 7h14m kube-system kube-controller-manager-k8s-master01 1/1 Running 0 7h14m kube-system kube-proxy-f7sl4 1/1 Running 0 3h20m kube-system kube-proxy-mc85r 1/1 Running 0 7h14m kube-system kube-proxy-sd9sd 1/1 Running 0 3h20m kube-system kube-scheduler-k8s-master01 1/1 Running 0 7h14m原因因为我的node01、node02没清理好防火墙规则导致calico、dns、controllers没成功#重置node01、node02防火墙规则 # 清空 INPUT 链 iptables -F INPUT # 重新设置默认策略为 ACCEPT iptables -P INPUT ACCEPT # 删除 CoreDNS Pod会自动重建 kubectl delete pods -n kube-system -l k8s-appkube-dns # 删除所有 Calico-node Pod会自动重建之前calico虽然正常但是我node1、node2网络规则重置了所以重新弄一下calico kubectl delete pods -n kube-system -l k8s-appcalico-node # 删除 calico-kube-controllers会自动重建 kubectl delete pods -n kube-system -l k8s-appcalico-kube-controllers # 等待 10 秒 sleep 10重启电脑后k8s集群有的colico等没启动。执行如下脚本整理3台机器的防火墙规则。#!/bin/bash echo echo 永久固化防火墙规则 - Kubernetes 集群 echo echo 1. 禁用 firewalld systemctl stop firewalld 2/dev/null systemctl disable firewalld 2/dev/null systemctl mask firewalld 2/dev/null echo 2. 禁用 nftables systemctl stop nftables 2/dev/null systemctl disable nftables 2/dev/null systemctl mask nftables 2/dev/null echo 3. 禁用 iptables 服务 systemctl stop iptables 2/dev/null systemctl disable iptables 2/dev/null systemctl mask iptables 2/dev/null echo 4. 清空所有 iptables 规则 iptables -F iptables -X iptables -t nat -F iptables -t nat -X iptables -t mangle -F iptables -t mangle -X iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT echo 5. 保存 iptables 规则 iptables-save /etc/iptables/rules.v4 2/dev/null iptables-save /etc/sysconfig/iptables 2/dev/null service iptables save 2/dev/null echo 6. 重启 kubelet systemctl restart kubelet echo 7. 验证防火墙状态 echo firewalld: $(systemctl is-active firewalld 2/dev/null || echo inactive) echo iptables INPUT 链策略: $(iptables -L INPUT -n | head -1 | awk {print $4}) echo echo 防火墙规则已永久固化 echo 七、启动顺序3台服务器重启k8s启动顺序如下 Kubernetes 组件完整启动顺序6个阶段第一阶段操作系统初始化systemd服务器开机后Linux 内核启动systemdPID 1作为所有进程的“老祖宗”率先运行。它负责根据系统配置拉起后续的各种系统服务。第二阶段容器运行时启动Docker cri-dockerd此时还没有 Kubernetes但必须先把“装东西的仓库”建好。systemd 启动 Dockerdockerd守护进程开始运行但此时 kubelet 还不能直接调用它因为 kubelet 默认使用 containerd 的接口标准 CRI。systemd 启动 cri-dockerdcri-dockerd作为一个适配器垫片启动它监听在unix:///var/run/cri-dockerd.sock让 kubelet 能通过这个接口来指挥 Docker 创建容器。第三阶段Kubelet 进程启动关键节点systemd 启动 kubelet 进程你之前ps看到的 PID 2209。这是第一个也是唯一一个以宿主机进程形式运行的 K8s 组件。此时kubelet 并不知道自己要管什么它只是连接上了刚才的cri-dockerd.sock并开始持续监控/etc/kubernetes/manifests/这个目录。第四阶段控制平面静态 Pod 启动核心组件当你执行kubeadm init后会在/etc/kubernetes/manifests/目录生成etcd.yaml、kube-apiserver.yaml等文件。kubelet 发现清单文件kubelet 监控到目录变化立即通过 cri-dockerd 调用 Docker 创建容器。容器启动顺序存在内部依赖但 kubelet 会同时拉起etcd 容器优先启动因为 API Server 依赖它存数据。kube-apiserver 容器启动集群的大脑所有操作都要经过它。当 API Server 确认 etcd 已连接后kube-controller-manager 和 kube-scheduler 容器才真正开始工作它们需要监听 API Server 来干活。此时你执行docker ps就能看到这些容器了控制平面就算“活”了。第五阶段Node 节点向集群注册API Server 启动后kubelet 进程会拿着自己的证书/etc/kubernetes/kubelet.conf向 API Server 发起请求把自己这个节点注册到集群中。此时kubectl get nodes才能看到这台机器。第六阶段网络插件与工作负载 Pod 启动控制平面运行后你通常会执行kubectl apply -f calico.yaml安装网络插件CNI。Calico、CoreDNS 等作为普通的 Pod非静态被调度起来。这时集群才完全就绪可以部署业务应用了。
