K8S V1.23 安装 --Kubeadm+contained+ 公网 IP 多节点部署

cat <<EOF | sudo tee /etc/modules-load.d/k8s.confbr_netfilterEOF
cat <<EOF | sudo tee /etc/sysctl.d/k8s.confnet.bridge.bridge-nf-call-ip6tables = 1net.bridge.bridge-nf-call-iptables = 1EOF
sysctl --system
# 修改Hosts文件，添加相关的配置，示例如下：[root@crio-master k8s]# cat /etc/hosts121.4.190.84 crio-master106.55.227.160 VM-20-11-centos

# 安装软件包modprobe tunlsmod | grep tun
# 编辑文件vim /etc/yum.repos.d/nux-misc.repo# 填入下面的内容[nux-misc]name=Nux Miscbaseurl=http://li.nux.ro/download/nux/misc/el7/x86_64/enabled=0gpgcheck=1gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
# 安装yum --enablerepo=nux-misc install tunctl
# 新建虚拟网卡tunctl -t publick -u root# 配置网卡的IP，注意替换ip成自己机器对应的公网IPifconfig publick 121.37.246.218 netmask 255.255.255.0 promisc

# 将机器人上的docker清理下，不然会有影响yum remove docker \                  docker-client \                  docker-client-latest \                  docker-common \                  docker-latest \                  docker-latest-logrotate \                  docker-logrotate \                  docker-engineyum remove docker-ce
# 我们单独安装contained即可yum install -y yum-utilsyum-config-manager \    --add-repo \    https://download.docker.com/linux/centos/docker-ce.repoyum install containerd.io
# 加载systemctl daemon-reload
# 启动服务systemctl enable containerdsystemctl start containerdsystemctl status containerd

# 下载安装,下载不了，到github上找个版本：https://github.com/kubernetes-sigs/cri-tools/releases?q=v1.23.1&expanded=true，把下面的version改改就行了VERSION="v1.24.1"wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-$VERSION-linux-amd64.tar.gzsudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/binrm -f crictl-$VERSION-linux-amd64.tar.gz
# 编辑配置文件vim /etc/crictl.yaml# 填入下面的内容runtime-endpoint: unix:///var/run/containerd/containerd.sockimage-endpoint: unix:///var/run/containerd/containerd.socktimeout: 2debug: falsepull-image-on-create: false

crictl pscrictl images

➜  ~ crictl psFATA[0000] listing containers: rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.RuntimeService

rm /etc/containerd/config.toml$ systemctl restart containerd

# 增加软件源cat <<EOF > /etc/yum.repos.d/kubernetes.repo[kubernetes]name=Kubernetesbaseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64enabled=1gpgcheck=0repo_gpgcheck=0gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpgEOF
# 设置下setenforce 0sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
# 清理以前的版本，如果有的话yum remove kubelet kubeadm kubectlyum install -y kubelet-1.23.1 kubeadm-1.23.1 kubectl-1.23.1
# 启动systemctl enable --now kubelet
# 编辑下配置文件mkdir -p /etc/systemd/system/kubelet.service.d/vi /etc/systemd/system/kubelet.service.d/10-kubeadm.conf# 填入下面的内容[Service]Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"# 这是 "kubeadm init" 和 "kubeadm join" 运行时生成的文件，动态地填充 KUBELET_KUBEADM_ARGS 变量EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env# 这是一个文件，用户在不得已下可以将其用作替代 kubelet args。# 用户最好使用 .NodeRegistration.KubeletExtraArgs 对象在配置文件中替代。# KUBELET_EXTRA_ARGS 应该从此文件中获取。EnvironmentFile=-/etc/default/kubeletExecStart=ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS --node-ip=159.138.89.50
# 重新启动，应用配置systemctl daemon-reloadsystemctl restart kubeletsystemctl status kubelet
# 查看下状态[root@VM-16-14-centos k8s]# systemctl status kubelet● kubelet.service - kubelet: The Kubernetes Node Agent   Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; vendor preset: disabled)  Drop-In: /etc/systemd/system/kubelet.service.d           └─10-kubeadm.conf   Active: active (running) since Sun 2022-05-08 14:00:11 CST; 54s ago     Docs: https://kubernetes.io/docs/ Main PID: 12618 (kubelet)   CGroup: /system.slice/kubelet.service           └─12618 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime=remot...
# 查看日志journalctl -xeu kubelet

# 查看相关的版本[root@VM-16-14-centos ~]# kubeadm versionkubeadm version: &version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.0", GitCommit:"ab69524f795c42094a6630298ff53f3c3ebab7f4", GitTreeState:"clean", BuildDate:"2021-12-07T18:15:11Z", GoVersion:"go1.17.3", Compiler:"gc", Platform:"linux/amd64"}
[root@VM-16-14-centos ~]# kubectl versionClient Version: version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.0", GitCommit:"ab69524f795c42094a6630298ff53f3c3ebab7f4", GitTreeState:"clean", BuildDate:"2021-12-07T18:16:20Z", GoVersion:"go1.17.3", Compiler:"gc", Platform:"linux/amd64"}The connection to the server localhost:8080 was refused - did you specify the right host or port?

crictl pull registry.aliyuncs.com/google_containers/pause:3.6ctr i pull registry.aliyuncs.com/google_containers/pause:3.6ctr -n k8s.io i tag registry.aliyuncs.com/google_containers/pause:3.6 k8s.gcr.io/pause:3.6

# --apiserver-advertise-address 填写公网ip# --service-cidr --pod-network-cidr 照抄填写即可# --image-repository 配置国内镜像源[root@crio-master k8s]# kubeadm init --apiserver-advertise-address=121.4.190.84 --service-cidr=10.1.0.0/16 --pod-network-cidr=192.168.0.0/16 --image-repository='registry.cn-hangzhou.aliyuncs.com/google_containers'
# 下面是安装中生成的日志I0626 12:19:55.601776   10881 version.go:255] remote version is much newer: v1.24.2; falling back to: stable-1.23[init] Using Kubernetes version: v1.23.8[preflight] Running pre-flight checks[preflight] Pulling images required for setting up a Kubernetes cluster[preflight] This might take a minute or two, depending on the speed of your internet connection[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'[certs] Using certificateDir folder "/etc/kubernetes/pki"[certs] Generating "ca" certificate and key[certs] Generating "apiserver" certificate and key[certs] apiserver serving cert is signed for DNS names [crio-master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.1.0.1 121.4.190.84][certs] Generating "apiserver-kubelet-client" certificate and key[certs] Generating "front-proxy-ca" certificate and key[certs] Generating "front-proxy-client" certificate and key[certs] Generating "etcd/ca" certificate and key[certs] Generating "etcd/server" certificate and key[certs] etcd/server serving cert is signed for DNS names [crio-master localhost] and IPs [121.4.190.84 127.0.0.1 ::1][certs] Generating "etcd/peer" certificate and key[certs] etcd/peer serving cert is signed for DNS names [crio-master localhost] and IPs [121.4.190.84 127.0.0.1 ::1][certs] Generating "etcd/healthcheck-client" certificate and key[certs] Generating "apiserver-etcd-client" certificate and key[certs] Generating "sa" key and public key[kubeconfig] Using kubeconfig folder "/etc/kubernetes"[kubeconfig] Writing "admin.conf" kubeconfig file[kubeconfig] Writing "kubelet.conf" kubeconfig file[kubeconfig] Writing "controller-manager.conf" kubeconfig file[kubeconfig] Writing "scheduler.conf" kubeconfig file[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"[kubelet-start] Starting the kubelet[control-plane] Using manifest folder "/etc/kubernetes/manifests"[control-plane] Creating static Pod manifest for "kube-apiserver"[control-plane] Creating static Pod manifest for "kube-controller-manager"[control-plane] Creating static Pod manifest for "kube-scheduler"[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s[apiclient] All control plane components are healthy after 12.002365 seconds[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace[kubelet] Creating a ConfigMap "kubelet-config-1.23" in namespace kube-system with the configuration for the kubelets in the clusterNOTE: The "kubelet-config-1.23" naming of the kubelet ConfigMap is deprecated. Once the UnversionedKubeletConfigMap feature gate graduates to Beta the default name will become just "kubelet-config". Kubeadm upgrade will handle this transition transparently.[upload-certs] Skipping phase. Please see --upload-certs[mark-control-plane] Marking the node crio-master as control-plane by adding the labels: [node-role.kubernetes.io/master(deprecated) node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers][mark-control-plane] Marking the node crio-master as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule][bootstrap-token] Using token: m5yedd.clmq0lw4s4d961yu[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key[addons] Applied essential addon: CoreDNS[addons] Applied essential addon: kube-proxy
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
  mkdir -p $HOME/.kube  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config  sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
  export KUBECONFIG=/etc/kubernetes/admin.conf
You should now deploy a pod network to the cluster.Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:  https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
# 这段日志很关键，工作节点加入是，复制下面这个运行即可kubeadm join 121.4.190.84:6443 --token m5yedd.clmq0lw4s4d961yu \        --discovery-token-ca-cert-hash sha256:7c3ac34b89c2c4a079dd5684286c6306abc8b5dd98fdd1ea8f1f1df8f254f256

# 运行下面的命令[root@crio-master k8s]# mkdir -p $HOME/.kube[root@crio-master k8s]#  cp -i /etc/kubernetes/admin.conf $HOME/.kube/configcp: overwrite ‘/root/.kube/config’?  chown $(id -u):$(id -g) $HOME/.kube/config[root@crio-master k8s]# mkdir -p $HOME/.kube[root@crio-master k8s]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/configcp: overwrite ‘/root/.kube/config’? y[root@crio-master k8s]# chown $(id -u):$(id -g) $HOME/.kube/config

# 运行下面的命令kubectl create -f https://projectcalico.docs.tigera.io/manifests/tigera-operator.yamlkubectl create -f https://projectcalico.docs.tigera.io/manifests/custom-resources.yaml
# 查看部署状态watch kubectl get pods -n calico-system

[root@crio-master k8s]# kubectl get nodesNAME              STATUS   ROLES                  AGE     VERSIONcrio-master       Ready    control-plane,master   3d1h    v1.23.1
[root@crio-master k8s]# crictl psCONTAINER           IMAGE               CREATED             STATE               NAME                      ATTEMPT             POD ID386c53701c331       6cba34c44d478       3 days ago          Running             calico-apiserver          0                   c96c43e126a15e822cc35cc1c7       6cba34c44d478       3 days ago          Running             calico-apiserver          0                   b439d9c51353c335493793963d       a4ca41631cc7a       3 days ago          Running             coredns                   0                   f8b273d0b213a8e25629c36fe5       ec95788d0f725       3 days ago          Running             calico-kube-controllers   0                   706ceebf584f7f8bbdbc140cdb       a4ca41631cc7a       3 days ago          Running             coredns                   0                   e6a73ececa3845be790a6dd5a9       a3447b26d32c7       3 days ago          Running             calico-node               0                   1728a55642148f77615c9ea6a8       22336acac6bba       3 days ago          Running             calico-typha              0                   dd612b2d122b8a887f081de0f5       9735044632553       3 days ago          Running             tigera-operator           0                   e7d02d9eb77c5df1d4d706ad2d       db4da8720bcb9       3 days ago          Running             kube-proxy                0                   4b7a385e851a7142aedbdc00c1       09d62ad3189b4       3 days ago          Running             kube-apiserver            0                   29033f38ec856e457ace9d5f03       25f8c7f3da61c       3 days ago          Running             etcd                      0                   b9307128a614228b0581787520       2b7c5a0399844       3 days ago          Running             kube-controller-manager   7                   fa0f2ccf4585cfc4f7f3d1bb6f       afd180ec7435a       3 days ago          Running             kube-scheduler            7                   b7c2064e73acf

kubeadm token list
# 如果是过期了，需要重新生成kubeadm token create --print-join-command

modprobe br_netfilterecho 1 > /proc/sys/net/ipv4/ip_forward
kubeadm join 121.4.190.84:6443 --token m5yedd.clmq0lw4s4d961yu \        --discovery-token-ca-cert-hash sha256:7c3ac34b89c2c4a079dd5684286c6306abc8b5dd98fdd1ea8f1f1df8f254f256

[root@crio-master k8s]# kubectl get nodesNAME              STATUS   ROLES                  AGE    VERSIONcrio-master       Ready    control-plane,master   3d2h   v1.23.1vm-20-11-centos   Ready    <none>                 3d1h   v1.23.1

[root@crio-master k8s]# cat /etc/hosts121.4.190.84 crio-master106.55.227.160 VM-20-11-centos

# 运行后即部署kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.0/aio/deploy/recommended.yaml
# 查看状态，等待状态完毕[root@crio-master k8s]# kubectl get pods --namespace=kubernetes-dashboard -o wideNAME                                         READY   STATUS    RESTARTS   AGE   IP              NODE              NOMINATED NODE   READINESS GATESdashboard-metrics-scraper-799d786dbf-prrgr   1/1     Running   0          12m   192.168.3.194   vm-20-11-centos   <none>           <none>kubernetes-dashboard-546cbc58cd-c5rs9        1/1     Running   0          12m   192.168.3.193   vm-20-11-centos   <none>           <none>
# 查看详细信息kubectl describe pod dashboard-metrics-scraper-799d786dbf-prrgr --namespace=kubernetes-dashboard# 查看节点信息[root@crio-master ~]# kubectl --namespace=kubernetes-dashboard get service kubernetes-dashboardNAME                   TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGEkubernetes-dashboard   ClusterIP   10.1.45.51   <none>        443/TCP   5m24s
# 编辑，将最后的type：ClusterIP改为type：NodePortkubectl --namespace=kubernetes-dashboard edit service kubernetes-dashboard# 再次运行命令查看，得到其映射到主机的端口32625，这样既可通过注解ip+端口访问[root@crio-master ~]# kubectl --namespace=kubernetes-dashboard get service kubernetes-dashboardNAME                   TYPE       CLUSTER-IP   EXTERNAL-IP   PORT(S)         AGEkubernetes-dashboard   NodePort   10.1.45.51   <none>        443:32652/TCP   7m31s

# 新建用户vim admin-user.yaml# 填入下面的内容apiVersion: v1kind: ServiceAccountmetadata:  name: admin-user  namespace: kubernetes-dashboard# 执行下面的命令kubectl create -f admin-user.yaml
# 绑定用户关系vim admin-user-role-binding.yaml# 填入下面的内容apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRoleBindingmetadata:  name: admin-userroleRef:  apiGroup: rbac.authorization.k8s.io  kind: ClusterRole  name: cluster-adminsubjects:- kind: ServiceAccount  name: admin-user  namespace: kubernetes-dashboard# 执行下面的命令kubectl create -f admin-user-role-binding.yaml

kubectl -n kubernetes-dashboard describe secret $(kubectl -n kubernetes-dashboard get secret | grep admin-user | awk '{print $1}')

[root@crio-master k8s]# kubectl get pods --namespace=kubernetes-dashboard -o wideNAME                                         READY   STATUS    RESTARTS   AGE   IP              NODE              NOMINATED NODE   READINESS GATESdashboard-metrics-scraper-799d786dbf-prrgr   1/1     Running   0          12m   192.168.3.194   vm-20-11-centos   <none>           <none>kubernetes-dashboard-546cbc58cd-c5rs9        1/1     Running   0          12m   192.168.3.193   vm-20-11-centos   <none>           <none>

➜  k8s crictl psCONTAINER           IMAGE               CREATED             STATE               NAME                        ATTEMPT             POD ID93c138b35118e       7801cfc6d5c07       29 hours ago        Running             dashboard-metrics-scraper   0                   87f8f420a8614c1e3fc2d6c6d9       57446aa2002e1       29 hours ago        Running             kubernetes-dashboard        0                   68897036dab5fdb3684cf5e6a2       a3447b26d32c7       2 days ago          Running             calico-node                 0                   cb13bab18abe75811551b53ad0       db4da8720bcb9       2 days ago          Running             kube-proxy                  1                   2960526e1e594

➜  k8s curl 192.168.3.193:8443Client sent an HTTP request to an HTTPS server.

user  nginx;worker_processes  auto;
error_log  /var/log/nginx/error.log notice;pid        /var/run/nginx.pid;

events {    worker_connections  1024;}

http {    include       /etc/nginx/mime.types;    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '                      '$status $body_bytes_sent "$http_referer" '                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile        on;    #tcp_nopush     on;
    keepalive_timeout  65;
    #gzip  on;
    include /etc/nginx/conf.d/*.conf;
    server {        listen       443 ssl http2;        listen       [::]:443 ssl http2;        server_name  _;        root         /usr/share/nginx/html;
        ssl_certificate "/etc/nginx/cert/selfgrowth.club_bundle.crt";        ssl_certificate_key "/etc/nginx/cert/selfgrowth.club.key";        ssl_session_cache shared:SSL:1m;        ssl_session_timeout  10m;        ssl_ciphers HIGH:!aNULL:!MD5;        ssl_prefer_server_ciphers on;
        # Load configuration files for the default server block.        include /etc/nginx/default.d/*.conf;
        location /k8sadmin/ {            proxy_pass https://192.168.3.193:8443/;        }
        error_page 404 /404.html;            location = /40x.html {        }
        error_page 500 502 503 504 /50x.html;            location = /50x.html {        }    }}