02-kubernetes 自建 CA 及双向 TLS 认证

 关注
发布于: 2020 年 06 月 11 日
在上一篇kubernetes安装部署（手动）中介绍了手动安装kubernetes集群的方法，本文在上一篇的基础上，讲解如何创建集群的安全证书。
在上生产环境前，集群的安全证书是十分有必要使用的， 主要步骤是在master节点上创建证书，然后分发到各个node节点，如果有新node节点的加入，也要给新节点分发一下。本文代码居多，整体配置下来耗时也就大概15分钟。
master主机
使用自建的CA签署证书
#新建CA
[root@master ~]# cd key/
[root@master key]# openssl genrsa -out ca-key.pem 2048
[root@master key]# openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=k8s-master"
[root@master key]# ll
total 8
-rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
-rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem
﻿
#新建openssl.conf文件
[root@master key]# vi openssl.cnf 
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.cluster.local
IP.1 = 10.254.0.1
IP.2 = <你的master的ip> 
﻿
#制作apiserver key
[root@master key]# openssl genrsa -out apiserver-key.pem 2048
[root@master key]# openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=k8s-master" -config openssl.cnf
﻿
#签署apiserver证书
[root@master key]# openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf
[root@master key]# ll
total 28
-rw-r--r-- 1 root root 1115 Sep 21 11:11 apiserver.csr
-rw-r--r-- 1 root root 1679 Sep 21 11:11 apiserver-key.pem
-rw-r--r-- 1 root root 1184 Sep 21 11:12 apiserver.pem
-rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
-rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem
-rw-r--r-- 1 root root 17 Sep 21 11:12 ca.srl
-rw-r--r-- 1 root root 398 Sep 21 11:10 openssl.cnf
﻿
#新建node01的openssl.conf
[root@master key]# vi worker-openssl-001.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
IP.1 = <你的node01节点的ip>
﻿
#制作node01 key
[root@master key]# openssl genrsa -out k8s-node001-worker-key.pem 2048
[root@master key]# openssl req -new -key k8s-node001-worker-key.pem -out k8s-node001-worker.csr -subj "/CN=k8s-node-1" -config worker-openssl-001.cnf
﻿
#签署node01证书
[root@master key]# openssl x509 -req -in k8s-node001-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out k8s-node001-worker.pem -days 365 -extensions v3_req -extfile worker-openssl-001.cnf
[root@master key]# ll
total 44
-rw-r--r-- 1 root root 1115 Sep 21 11:11 apiserver.csr
-rw-r--r-- 1 root root 1679 Sep 21 11:11 apiserver-key.pem
-rw-r--r-- 1 root root 1184 Sep 21 11:12 apiserver.pem
-rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
-rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem
-rw-r--r-- 1 root root 17 Sep 21 11:15 ca.srl
-rw-r--r-- 1 root root 972 Sep 21 11:14 k8s-node001-worker.csr
-rw-r--r-- 1 root root 1679 Sep 21 11:14 k8s-node001-worker-key.pem
-rw-r--r-- 1 root root 1046 Sep 21 11:15 k8s-node001-worker.pem
-rw-r--r-- 1 root root 398 Sep 21 11:10 openssl.cnf
-rw-r--r-- 1 root root 262 Sep 21 11:13 worker-openssl-001.cnf
master主机修改相关配置
#复制apiserver的pem文件到ssl目录
[root@master key]# mkdir -p /etc/kubernetes/ssl
[root@master key]# cd /etc/kubernetes/ssl
[root@master ssl]# cp /root/key/ca.pem .
[root@master ssl]# cp /root/key/apiserver.pem .
[root@master ssl]# cp /root/key/apiserver-key.pem .
﻿
#新建kubeconfig文件
[root@master ~]# vi /etc/kubernetes/cm-kubeconfig.yaml 
apiVersion: v1
kind: Config
clusters:
- name: local
  cluster:
    certificate-authority: /etc/kubernetes/ssl/ca.pem
users:
- name: controllermanager
  user:
    client-certificate: /etc/kubernetes/ssl/apiserver.pem
    client-key: /etc/kubernetes/ssl/apiserver-key.pem
contexts:
- context:
    cluster: local
    user: controllermanager
  name: kubelet-context
current-context: kubelet-context
﻿
#按下面配置，修改各组件
#因为controllmanager、scheduler与apiserver在同一台主机上，也可以使用非安全端口8080
[root@master ~]# grep -v ^# /etc/kubernetes/config 
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=https://<你的master的ip>:6443"
﻿
[root@master ~]# grep -v ^# /etc/kubernetes/apiserver 
KUBE_API_ADDRESS="--bind-address=0.0.0.0 --insecure-bind-address=127.0.0.1"
KUBE_API_PORT="--secure-port=6443 --insecure-port=8080"
KUBE_ETCD_SERVERS="--etcd-servers=http://etcd:2379"
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
KUBE_API_ARGS="--client-ca-file=/etc/kubernetes/ssl/ca.pem \
 --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \
 --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem \
 --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem"
﻿
[root@master ~]# grep -v ^# /etc/kubernetes/controller-manager 
KUBE_CONTROLLER_MANAGER_ARGS="--service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \
 --root-ca-file=/etc/kubernetes/ssl/ca.pem \
 --master=https://<你的master的ip>:6443 \
 --kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml"
﻿
[root@master ~]# grep -v ^# /etc/kubernetes/scheduler 
KUBE_SCHEDULER_ARGS="--master=https://<你的master的ip>:6443 \
 --kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml"
﻿
#重启各组件
[root@master kubernetes]# systemctl restart kube-controller-manager
[root@master kubernetes]# systemctl restart kube-scheduler
[root@master kubernetes]# systemctl restart kube-apiserver
﻿
#查看状态
[root@master kubernetes]# systemctl status kube-controller-manager -l 
[root@master kubernetes]# systemctl status kube-scheduler -l 
[root@master kubernetes]# systemctl status kube-apiserver -l
﻿
#验证证书
[root@master kubernetes]# curl https://<你的master的ip>:6443/api/v1/nodes --cert /etc/kubernetes/ssl/apiserver.pem --key /etc/kubernetes/ssl/apiserver-key.pem --cacert /etc/kubernetes/ssl/ca.pem
此时，在master上使用kubectl命令需要附带证书参数，不带证书去访问就已经访问不通了。例：
kubectl get pods --all-namespaces --server=https://<你的master的ip>:6443 --certificate-authority=/etc/kubernetes/ssl/ca.pem --client-certificate=/etc/kubernetes/ssl/apiserver.pem --client-key=/etc/kubernetes/ssl/apiserver-key.pem
编辑kubernetes默认的kubeconfig文件（使用kubectl命令时如果不指定kubeconfig文件，则默认使用此配置文件信息，配置此文件后再使用kubectl命令，则无需附带证书参数信息）：
vi /root/.kube/config
﻿
apiVersion: v1
clusters:
- cluster:
    server: https://<你的master的ip>:6443
    certificate-authority: /etc/kubernetes/ssl/ca.pem
  name: default
contexts:
- context:
    cluster: default
    user: admin
  name: default
current-context: default
kind: Config
preferences: {}
users:
- name: admin
  user:
    client-certificate: /etc/kubernetes/ssl/apiserver.pem
    client-key: /etc/kubernetes/ssl/apiserver-key.pem
﻿
node01主机
接下来在node节点上进行操作，所有的node节点操作都相同
#将master主机生成的node01证书复制到ssl目录
[root@node01 ~]# mkdir -p /etc/kubernetes/ssl/
[root@node01 ~]# cd /etc/kubernetes/ssl/
[root@node01 ~]# scp root@<你的master的ip>:/root/key/k8s-node001-worker-key.pem /etc/kubernetes/ssl/
[root@node01 ~]# scp root@<你的master的ip>:/root/key/ca.pem /etc/kubernetes/ssl/
[root@node01 ~]# scp root@<你的master的ip>:/root/key/k8s-node001-worker.pem /etc/kubernetes/ssl/
[root@node01 ssl]# ll
total 12
-rw-r--r-- 1 root root 1090 Sep 21 11:49 ca.pem
-rw-r--r-- 1 root root 1679 Sep 21 11:49 k8s-node001-worker-key.pem
-rw-r--r-- 1 root root 1046 Sep 21 11:50 k8s-node001-worker.pem
﻿
#做一个超链接
[root@node01 ssl]# ln -s k8s-node001-worker.pem worker.pem
[root@node01 ssl]# ln -s k8s-node001-worker-key.pem worker-key.pem
﻿
#新建kubeconfig
[root@node01 ssl]# vi /etc/kubernetes/worker-kubeconfig.yaml
apiVersion: v1
kind: Config
clusters:
- name: local
  cluster:
    certificate-authority: /etc/kubernetes/ssl/ca.pem
users:
- name: kubelet
  user:
    client-certificate: /etc/kubernetes/ssl/worker.pem
    client-key: /etc/kubernetes/ssl/worker-key.pem
contexts:
- context:
    cluster: local
    user: kubelet
  name: kubelet-context
current-context: kubelet-context
﻿
#按照如下配置，修改各配置文件
[root@node01 ssl]# vi /etc/kubernetes/config 
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=https://<你的master的ip>:6443"
﻿
[root@node01 ssl]# vi /etc/kubernetes/kubelet 
KUBELET_ADDRESS="--address=0.0.0.0"
KUBELET_HOSTNAME="--hostname-override=k8s-node-1"
KUBELET_API_SERVER="--api-servers=https://<你的master的ip>:6443"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=<你的docker私服库的ip>:<你的docker私服库的port>/registry.access.redhat.com/rhel7/pod-infrastructure:latest"
KUBELET_ARGS="--cluster-dns=10.254.0.2 \
 --cluster-domain=cluster.local \
 --tls-cert-file=/etc/kubernetes/ssl/worker.pem \
 --tls-private-key-file=/etc/kubernetes/ssl/worker-key.pem \
 --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"
﻿
[root@node01 ssl]# vi /etc/kubernetes/proxy 
KUBE_PROXY_ARGS="--master=https://<你的master的ip>:6443 \
--kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"
﻿
#重启服务
[root@node01 ssl]# systemctl restart docker kubelet kube-proxy
﻿
#查看
[root@node01 ssl]# systemctl status docker kubelet kube-proxy -l
﻿
#验证证书
[root@node01 ssl]# curl https://<你的master的ip>:6443/api/v1/nodes --cert /etc/kubernetes/ssl/worker.pem --key /etc/kubernetes/ssl/worker-key.pem --cacert /etc/kubernetes/ssl/ca.pem
至此，k8s集群的证书已经配置完毕
Dashboard证书问题
由于dashboard通过调用apiserver接口获取集群信息，加证书后会导致dashboard出错，查看dashboard的pod日志发现：
Error while initializing connection to Kubernetes apiserver. This most likely means that the cluster is misconfigured (e.g., it has invalid apiserver certificates or service accounts configuration) or the --apiserver-host param points to a server that does not exist. Reason: Get https://ip:6443/version: x509: failed to load system roots and no roots provided
需要修改一下dashboard的yaml信息，添加证书地址和kubeconfig路径的挂载点，完整的yaml文件如下：
apiVersion: v1
kind: ReplicationController
metadata:
  name: kubernetes-dashboard-latest
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    version: latest
    kubernetes.io/cluster-service: "true"
spec:
  replicas: 1
  selector:
    k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
        version: latest
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
      - name: kubernetes-dashboard
        image: docker.io/ist0ne/kubernetes-dashboard-amd64:v1.5.0
        resources:
          limits:
            cpu: 100m
            memory: 50Mi
          requests:
            cpu: 100m
            memory: 50Mi
        ports:
        - containerPort: 9090
        args:
           - --apiserver-host=https://<你的master的ip>:6443
           - --kubeconfig=/root/.kube/config
        livenessProbe:
          httpGet:
            path: /
            port: 9090
          initialDelaySeconds: 30
          timeoutSeconds: 30
        volumeMounts:
        - name: auth
          mountPath: /etc/kubernetes/ssl
        - name: config
          mountPath: /root/.kube
      volumes:
      - name: auth
        hostPath:
          path: /etc/kubernetes/ssl
      - name: config
        hostPath:
          path: /root/.kube
﻿
apiVersion: v1
kind: Service
metadata:
  name: kubernetes-dashboard
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    kubernetes.io/cluster-service: "true"
spec:
  selector:
    k8s-app: kubernetes-dashboard
  type: NodePort
  ports:
  - port: 80
    nodePort: 30080
targetPort: 9090
其中，rc文件添加了kubeconfig文件路径和volumn挂载（增加SSL认证）。
service文件添加了nodeport暴露端口（由于windows下无法加载自建ca证书而无法访问dashboard，所以添加nodeport）。
修改过程需要注意两点：
（1） kubernetes-dashboard-amd64镜像版本使用1.5.0版本及以上
（2） rc文件中的kubeconfig路径指向的是节点机器的路径，所以需要在节点机器下创建config，内容与master节点下的/root/.kube/config文件类似，证书可以使用节点自己的证书，例：
apiVersion: v1
clusters:
- cluster:
    server: https://<你的master的ip>:6443
    certificate-authority: /etc/kubernetes/ssl/ca.pem
  name: default
contexts:
- context:
    cluster: default
    user: admin
  name: default
current-context: default
kind: Config
preferences: {}
users:
- name: admin
  user:
    client-certificate: /etc/kubernetes/ssl/worker.pem
    client-key: /etc/kubernetes/ssl/worker-key.pem
完毕。
发布于: 2020 年 06 月 11 日阅读数: 73
原文链接:【http://xie.infoq.cn/article/48ab321f364223177462f4f45】。文章转载请联系作者。
绿星雪碧

 关注
一直在寻找赚钱的方法 2018.03.03 加入
擅长：有问题，上网查
发布
暂无评论