02-kubernetes 自建 CA 及双向 TLS 认证

用户头像
绿星雪碧
关注
发布于: 2020 年 06 月 11 日
02-kubernetes自建CA及双向TLS认证

在上一篇kubernetes安装部署(手动)中介绍了手动安装kubernetes集群的方法,本文在上一篇的基础上,讲解如何创建集群的安全证书。

在上生产环境前,集群的安全证书是十分有必要使用的, 主要步骤是在master节点上创建证书,然后分发到各个node节点,如果有新node节点的加入,也要给新节点分发一下。本文代码居多,整体配置下来耗时也就大概15分钟。


  1. master主机

  • 使用自建的CA签署证书

#新建CA
[root@master ~]# cd key/
[root@master key]# openssl genrsa -out ca-key.pem 2048
[root@master key]# openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=k8s-master"
[root@master key]# ll
total 8
-rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
-rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem

#新建openssl.conf文件
[root@master key]# vi openssl.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.cluster.local
IP.1 = 10.254.0.1
IP.2 = <你的master的ip>



#制作apiserver key
[root@master key]# openssl genrsa -out apiserver-key.pem 2048
[root@master key]# openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=k8s-master" -config openssl.cnf

#签署apiserver证书
[root@master key]# openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf
[root@master key]# ll
total 28
-rw-r--r-- 1 root root 1115 Sep 21 11:11 apiserver.csr
-rw-r--r-- 1 root root 1679 Sep 21 11:11 apiserver-key.pem
-rw-r--r-- 1 root root 1184 Sep 21 11:12 apiserver.pem
-rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
-rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem
-rw-r--r-- 1 root root 17 Sep 21 11:12 ca.srl
-rw-r--r-- 1 root root 398 Sep 21 11:10 openssl.cnf

#新建node01的openssl.conf
[root@master key]# vi worker-openssl-001.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
IP.1 = <你的node01节点的ip>



#制作node01 key
[root@master key]# openssl genrsa -out k8s-node001-worker-key.pem 2048
[root@master key]# openssl req -new -key k8s-node001-worker-key.pem -out k8s-node001-worker.csr -subj "/CN=k8s-node-1" -config worker-openssl-001.cnf



#签署node01证书
[root@master key]# openssl x509 -req -in k8s-node001-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out k8s-node001-worker.pem -days 365 -extensions v3_req -extfile worker-openssl-001.cnf
[root@master key]# ll
total 44
-rw-r--r-- 1 root root 1115 Sep 21 11:11 apiserver.csr
-rw-r--r-- 1 root root 1679 Sep 21 11:11 apiserver-key.pem
-rw-r--r-- 1 root root 1184 Sep 21 11:12 apiserver.pem
-rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
-rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem
-rw-r--r-- 1 root root 17 Sep 21 11:15 ca.srl
-rw-r--r-- 1 root root 972 Sep 21 11:14 k8s-node001-worker.csr
-rw-r--r-- 1 root root 1679 Sep 21 11:14 k8s-node001-worker-key.pem
-rw-r--r-- 1 root root 1046 Sep 21 11:15 k8s-node001-worker.pem
-rw-r--r-- 1 root root 398 Sep 21 11:10 openssl.cnf
-rw-r--r-- 1 root root 262 Sep 21 11:13 worker-openssl-001.cnf
  • master主机修改相关配置

#复制apiserver的pem文件到ssl目录
[root@master key]# mkdir -p /etc/kubernetes/ssl
[root@master key]# cd /etc/kubernetes/ssl
[root@master ssl]# cp /root/key/ca.pem .
[root@master ssl]# cp /root/key/apiserver.pem .
[root@master ssl]# cp /root/key/apiserver-key.pem .



#新建kubeconfig文件
[root@master ~]# vi /etc/kubernetes/cm-kubeconfig.yaml
apiVersion: v1
kind: Config
clusters:
- name: local
cluster:
certificate-authority: /etc/kubernetes/ssl/ca.pem
users:
- name: controllermanager
user:
client-certificate: /etc/kubernetes/ssl/apiserver.pem
client-key: /etc/kubernetes/ssl/apiserver-key.pem
contexts:
- context:
cluster: local
user: controllermanager
name: kubelet-context
current-context: kubelet-context



#按下面配置,修改各组件
#因为controllmanager、scheduler与apiserver在同一台主机上,也可以使用非安全端口8080
[root@master ~]# grep -v ^# /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=https://<你的master的ip>:6443"

[root@master ~]# grep -v ^# /etc/kubernetes/apiserver
KUBE_API_ADDRESS="--bind-address=0.0.0.0 --insecure-bind-address=127.0.0.1"
KUBE_API_PORT="--secure-port=6443 --insecure-port=8080"
KUBE_ETCD_SERVERS="--etcd-servers=http://etcd:2379"
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
KUBE_API_ARGS="--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \
--tls-cert-file=/etc/kubernetes/ssl/apiserver.pem \
--service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem"

[root@master ~]# grep -v ^# /etc/kubernetes/controller-manager
KUBE_CONTROLLER_MANAGER_ARGS="--service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--master=https://<你的master的ip>:6443 \
--kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml"

[root@master ~]# grep -v ^# /etc/kubernetes/scheduler
KUBE_SCHEDULER_ARGS="--master=https://<你的master的ip>:6443 \
--kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml"



#重启各组件
[root@master kubernetes]# systemctl restart kube-controller-manager
[root@master kubernetes]# systemctl restart kube-scheduler
[root@master kubernetes]# systemctl restart kube-apiserver



#查看状态
[root@master kubernetes]# systemctl status kube-controller-manager -l
[root@master kubernetes]# systemctl status kube-scheduler -l
[root@master kubernetes]# systemctl status kube-apiserver -l



#验证证书
[root@master kubernetes]# curl https://<你的master的ip>:6443/api/v1/nodes --cert /etc/kubernetes/ssl/apiserver.pem --key /etc/kubernetes/ssl/apiserver-key.pem --cacert /etc/kubernetes/ssl/ca.pem

此时,在master上使用kubectl命令需要附带证书参数,不带证书去访问就已经访问不通了。例:

kubectl get pods --all-namespaces --server=https://<你的master的ip>:6443 --certificate-authority=/etc/kubernetes/ssl/ca.pem --client-certificate=/etc/kubernetes/ssl/apiserver.pem --client-key=/etc/kubernetes/ssl/apiserver-key.pem

编辑kubernetes默认的kubeconfig文件(使用kubectl命令时如果不指定kubeconfig文件,则默认使用此配置文件信息,配置此文件后再使用kubectl命令,则无需附带证书参数信息):

vi /root/.kube/config

apiVersion: v1
clusters:
- cluster:
server: https://<你的master的ip>:6443
certificate-authority: /etc/kubernetes/ssl/ca.pem
name: default
contexts:
- context:
cluster: default
user: admin
name: default
current-context: default
kind: Config
preferences: {}
users:
- name: admin
user:
client-certificate: /etc/kubernetes/ssl/apiserver.pem
client-key: /etc/kubernetes/ssl/apiserver-key.pem




  1. node01主机

接下来在node节点上进行操作,所有的node节点操作都相同

#将master主机生成的node01证书复制到ssl目录
[root@node01 ~]# mkdir -p /etc/kubernetes/ssl/
[root@node01 ~]# cd /etc/kubernetes/ssl/
[root@node01 ~]# scp root@<你的master的ip>:/root/key/k8s-node001-worker-key.pem /etc/kubernetes/ssl/
[root@node01 ~]# scp root@<你的master的ip>:/root/key/ca.pem /etc/kubernetes/ssl/
[root@node01 ~]# scp root@<你的master的ip>:/root/key/k8s-node001-worker.pem /etc/kubernetes/ssl/
[root@node01 ssl]# ll
total 12
-rw-r--r-- 1 root root 1090 Sep 21 11:49 ca.pem
-rw-r--r-- 1 root root 1679 Sep 21 11:49 k8s-node001-worker-key.pem
-rw-r--r-- 1 root root 1046 Sep 21 11:50 k8s-node001-worker.pem



#做一个超链接
[root@node01 ssl]# ln -s k8s-node001-worker.pem worker.pem
[root@node01 ssl]# ln -s k8s-node001-worker-key.pem worker-key.pem



#新建kubeconfig
[root@node01 ssl]# vi /etc/kubernetes/worker-kubeconfig.yaml
apiVersion: v1
kind: Config
clusters:
- name: local
cluster:
certificate-authority: /etc/kubernetes/ssl/ca.pem
users:
- name: kubelet
user:
client-certificate: /etc/kubernetes/ssl/worker.pem
client-key: /etc/kubernetes/ssl/worker-key.pem
contexts:
- context:
cluster: local
user: kubelet
name: kubelet-context
current-context: kubelet-context



#按照如下配置,修改各配置文件
[root@node01 ssl]# vi /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=https://<你的master的ip>:6443"

[root@node01 ssl]# vi /etc/kubernetes/kubelet
KUBELET_ADDRESS="--address=0.0.0.0"
KUBELET_HOSTNAME="--hostname-override=k8s-node-1"
KUBELET_API_SERVER="--api-servers=https://<你的master的ip>:6443"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=<你的docker私服库的ip>:<你的docker私服库的port>/registry.access.redhat.com/rhel7/pod-infrastructure:latest"
KUBELET_ARGS="--cluster-dns=10.254.0.2 \
--cluster-domain=cluster.local \
--tls-cert-file=/etc/kubernetes/ssl/worker.pem \
--tls-private-key-file=/etc/kubernetes/ssl/worker-key.pem \
--kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"

[root@node01 ssl]# vi /etc/kubernetes/proxy
KUBE_PROXY_ARGS="--master=https://<你的master的ip>:6443 \
--kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"



#重启服务
[root@node01 ssl]# systemctl restart docker kubelet kube-proxy

#查看
[root@node01 ssl]# systemctl status docker kubelet kube-proxy -l



#验证证书
[root@node01 ssl]# curl https://<你的master的ip>:6443/api/v1/nodes --cert /etc/kubernetes/ssl/worker.pem --key /etc/kubernetes/ssl/worker-key.pem --cacert /etc/kubernetes/ssl/ca.pem

至此,k8s集群的证书已经配置完毕


  1. Dashboard证书问题

由于dashboard通过调用apiserver接口获取集群信息,加证书后会导致dashboard出错,查看dashboard的pod日志发现:

Error while initializing connection to Kubernetes apiserver. This most likely means that the cluster is misconfigured (e.g., it has invalid apiserver certificates or service accounts configuration) or the --apiserver-host param points to a server that does not exist. Reason: Get https://ip:6443/version: x509: failed to load system roots and no roots provided

需要修改一下dashboard的yaml信息,添加证书地址和kubeconfig路径的挂载点,完整的yaml文件如下:

apiVersion: v1
kind: ReplicationController
metadata:
name: kubernetes-dashboard-latest
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
version: latest
kubernetes.io/cluster-service: "true"
spec:
replicas: 1
selector:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
version: latest
kubernetes.io/cluster-service: "true"
spec:
containers:
- name: kubernetes-dashboard
image: docker.io/ist0ne/kubernetes-dashboard-amd64:v1.5.0
resources:
limits:
cpu: 100m
memory: 50Mi
requests:
cpu: 100m
memory: 50Mi
ports:
- containerPort: 9090
args:
- --apiserver-host=https://<你的master的ip>:6443
- --kubeconfig=/root/.kube/config
livenessProbe:
httpGet:
path: /
port: 9090
initialDelaySeconds: 30
timeoutSeconds: 30
volumeMounts:
- name: auth
mountPath: /etc/kubernetes/ssl
- name: config
mountPath: /root/.kube
volumes:
- name: auth
hostPath:
path: /etc/kubernetes/ssl
- name: config
hostPath:
path: /root/.kube

apiVersion: v1
kind: Service
metadata:
name: kubernetes-dashboard
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
spec:
selector:
k8s-app: kubernetes-dashboard
type: NodePort
ports:
- port: 80
nodePort: 30080
targetPort: 9090

其中,rc文件添加了kubeconfig文件路径和volumn挂载(增加SSL认证)。

service文件添加了nodeport暴露端口(由于windows下无法加载自建ca证书而无法访问dashboard,所以添加nodeport)。

修改过程需要注意两点:

(1) kubernetes-dashboard-amd64镜像版本使用1.5.0版本及以上

(2) rc文件中的kubeconfig路径指向的是节点机器的路径,所以需要在节点机器下创建config,内容与master节点下的/root/.kube/config文件类似,证书可以使用节点自己的证书,例:

apiVersion: v1
clusters:
- cluster:
server: https://<你的master的ip>:6443
certificate-authority: /etc/kubernetes/ssl/ca.pem
name: default
contexts:
- context:
cluster: default
user: admin
name: default
current-context: default
kind: Config
preferences: {}
users:
- name: admin
user:
client-certificate: /etc/kubernetes/ssl/worker.pem
client-key: /etc/kubernetes/ssl/worker-key.pem

完毕。

发布于: 2020 年 06 月 11 日 阅读数: 66
用户头像

绿星雪碧

关注

一直在寻找赚钱的方法 2018.03.03 加入

擅长:有问题,上网查

评论

发布
暂无评论
02-kubernetes自建CA及双向TLS认证