[TOC] # 3.1、证书规划 | 证书名称 | MASTER01 | MASTER02 | MASTER03 | NODE01 | NODE02 | |---|---|---|---|---|---|---| | ETCD CA 证书 | Y | Y | Y | | | | ETCD Server 证书 | Y | Y | Y | | | | ETCD Member 1 证书 | Y | | | | | | ETCD Member 2 证书 | | Y | | | | | ETCD Member 3 证书 | | | Y | | | | ETCD Client 证书 | Y | Y | Y | | | | K8S CA 证书 | Y | Y | Y | | | | K8S API Server 证书 | Y | Y | Y | | | | K8S Front Proxy CA 证书 | Y | Y | Y | | | | K8S Front Proxy Client 证书 | Y | Y | Y | | | | K8S Service Account 键值对 | Y | Y | Y | | | | K8S Controller Manager 鉴权证书 | Y | Y | Y | | | | K8S Scheduler 鉴权证书 | Y | Y | Y | | | ***** # 3.2、安装及配置CFSSL ``` yum install go ​ vi ~/.bash_profile GOBIN=/root/go/bin/ PATH=$PATH:$GOBIN:$HOME/bin ​ export PATH ​ go get -u github.com/cloudflare/cfssl/cmd/cfssl go get -u github.com/cloudflare/cfssl/cmd/cfssljson ## 创建证书配置文件 mkdir -p /etc/cfssl/ && \ cat << EOF | tee /etc/cfssl/ca-config.json { "signing": { "default": { "expiry": "87600h" }, "profiles": { "kubernetes": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "87600h" } } } } EOF ## 创建证书配置文件保存路径 mkdir -p /etc/cfssl/etcd /etc/cfssl/k8s ## 创建证书保存路径 mkdir -p /etc/pki/etcd /etc/pki/k8s ssh root@10.10.10.232 mkdir -p /etc/pki/etcd/ /etc/pki/k8s/ ssh root@10.10.10.233 mkdir -p /etc/pki/etcd/ /etc/pki/k8s/ ``` ***** # 3.3、创建 ETCD 证书 ## 3.3.1、创建 ETCD CA 证书 ``` ## 创建 ETCD CA 配置文件 cat << EOF | tee /etc/cfssl/etcd/etcd-ca-csr.json { "CN": "etcd", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "cbx", "OU": "cbxhome" } ] } EOF ## 生成 ETCD CA 证书和私钥 cfssl gencert -initca /etc/cfssl/etcd/etcd-ca-csr.json | \ cfssljson -bare /etc/pki/etcd/etcd-ca ## 分发 ETCD CA 证书至其他 ETCD 节点 scp /etc/pki/etcd/etcd-ca* root@10.10.10.232:/etc/pki/etcd/ scp /etc/pki/etcd/etcd-ca* root@10.10.10.233:/etc/pki/etcd/ ``` ***** ## 3.3.2、创建 ETCD Server 证书 ``` ## 创建 ETCD Server 配置文件 export ETCD_SERVER_IPS=" \ \"10.10.10.231\", \ \"10.10.10.232\", \ \"10.10.10.233\" \ " && \ export ETCD_SERVER_HOSTNAMES=" \ \"etcd01.cbxhome.local\", \ \"etcd02.cbxhome.local\", \ \"etcd03.cbxhome.local\" \ " && \ cat << EOF | tee /etc/cfssl/etcd/etcd_server.json { "CN": "etcd", "hosts": [ "127.0.0.1", ${ETCD_SERVER_IPS}, ${ETCD_SERVER_HOSTNAMES} ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "cbx", "OU": "cbxhome" } ] } EOF ## 生成 ETCD Server 证书和私钥 cfssl gencert \ -ca=/etc/pki/etcd/etcd-ca.pem \ -ca-key=/etc/pki/etcd/etcd-ca-key.pem \ -config=/etc/cfssl/ca-config.json \ -profile=kubernetes \ /etc/ssl/etcd/etcd_server.json | \ cfssljson -bare /etc/pki/etcd/etcd_server ## 分发 ETCD Server 证书至其他 ETCD 节点 scp /etc/pki/etcd/etcd_server* root@10.10.10.232:/etc/pki/etcd/ scp /etc/pki/etcd/etcd_server* root@10.10.10.233:/etc/pki/etcd/ ``` ***** ## 3.3.3、创建 ETCD Member 1 证书 ``` ## 创建 ETCD Member 1 配置文件 export ETCD_MEMBER_1_IP=" \ \"10.10.10.231\" \ " && \ export ETCD_MEMBER_1_HOSTNAMES=" \ \"etcd01.cbxhome.local\" \ " && \ cat << EOF | tee /etc/ssl/etcd/etcd_member01.json { "CN": "etcd", "hosts": [ "127.0.0.1", ${ETCD_MEMBER_1_IP}, ${ETCD_MEMBER_1_HOSTNAMES} ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "cbx", "OU": "cbxhome" } ] } ## 生成 ETCD Member 1 证书和私钥 cfssl gencert \ -ca=/etc/pki/etcd/etcd-ca.pem \ -ca-key=/etc/pki/etcd/etcd-ca-key.pem \ -config=/etc/cfssl/ca-config.json \ -profile=kubernetes \ /etc/ssl/etcd/etcd_member01.json | \ cfssljson -bare /etc/pki/etcd/etcd_member01 EOF ``` ***** ## 3.3.4、创建 ETCD Member 2 证书 ``` ## 创建 ETCD Member 2 配置文件 export ETCD_MEMBER_2_IP=" \ \"10.10.10.232\" \ " && \ export ETCD_MEMBER_2_HOSTNAMES=" \ \"etcd02.cbxhome.local\" \ " && \ cat << EOF | tee /etc/ssl/etcd/etcd_member02.json { "CN": "etcd", "hosts": [ "127.0.0.1", ${ETCD_MEMBER_2_IP}, ${ETCD_MEMBER_2_HOSTNAMES} ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "cbx", "OU": "cbxhome" } ] } ## 生成 ETCD Member 2 证书和私钥 cfssl gencert \ -ca=/etc/pki/etcd/etcd-ca.pem \ -ca-key=/etc/pki/etcd/etcd-ca-key.pem \ -config=/etc/cfssl/ca-config.json \ -profile=kubernetes \ /etc/ssl/etcd/etcd_member02.json | \ cfssljson -bare /etc/pki/etcd/etcd_member02 EOF ## 分发密钥至 ETCD Member 2 scp /etc/pki/etcd/etcd_member02* root@10.10.10.232:/etc/pki/etcd/ ``` ***** ## 3.3.5、创建 ETCD Member 3 证书 ``` ## 创建 ETCD Member 3 配置文件 export ETCD_MEMBER_3_IP=" \ \"10.10.10.233\" \ " && \ export ETCD_MEMBER_3_HOSTNAMES=" \ \"etcd03.cbxhome.local\" \ " && \ cat << EOF | tee /etc/ssl/etcd/etcd_member03.json { "CN": "etcd", "hosts": [ "127.0.0.1", ${ETCD_MEMBER_3_IP}, ${ETCD_MEMBER_3_HOSTNAMES} ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "cbx", "OU": "cbxhome" } ] } ## 生成 ETCD Member 3 证书和私钥 cfssl gencert \ -ca=/etc/pki/etcd/etcd-ca.pem \ -ca-key=/etc/pki/etcd/etcd-ca-key.pem \ -config=/etc/cfssl/ca-config.json \ -profile=kubernetes \ /etc/ssl/etcd/etcd_member03.json | \ cfssljson -bare /etc/pki/etcd/etcd_member03 EOF ## 分发密钥至 ETCD Member 3 scp /etc/pki/etcd/etcd_member03* root@10.10.10.232:/etc/pki/etcd/ ``` ***** ## 3.3.6、创建 ETCD Client 证书 ``` ## 创建 ETCD Client 配置文件 cat << EOF | tee /etc/cfssl/etcd/etcd_client.json { "CN": "client", "hosts": [], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "cbx", "OU": "cbxhome" } ] } EOF ## 生成 ETCD Client 证书和私钥 cfssl gencert \ -ca=/etc/pki/etcd/etcd-ca.pem \ -ca-key=/etc/pki/etcd/etcd-ca-key.pem \ -config=/etc/cfssl/ca-config.json \ -profile=kubernetes \ /etc/ssl/etcd/etcd_client.json | \ cfssljson -bare /etc/pki/etcd/etcd_client ## 分发 ETCD Client 证书至其他 ETCD 节点 scp /etc/pki/etcd/etcd_client* root@10.10.10.232:/etc/pki/etcd/ scp /etc/pki/etcd/etcd_client* root@10.10.10.233:/etc/pki/etcd/ ``` ***** # 3.4、创建 Kubernetes 证书 ## 3.4.1、创建 Kubernetes CA 证书 ``` ## 创建 Kubernetes CA 配置文件 cat << EOF | tee /etc/cfssl/k8s/k8s-ca-csr.json { "CN": "kubernetes", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "cbx", "OU": "cbxhome" } ] } EOF ## 生成 Kubernetes CA 证书和私钥 cfssl gencert -initca /etc/cfssl/k8s/k8s-ca-csr.json | \ cfssljson -bare /etc/pki/k8s/k8s-ca ## 分发 Kubernetes CA 证书至其他 Kube-Master 节点 scp /etc/pki/k8s/k8s-ca* root@10.10.10.232:/etc/pki/k8s/ scp /etc/pki/k8s/k8s-ca* root@10.10.10.233:/etc/pki/k8s/ ``` ***** ## 3.4.2、创建 Kubernetes API Server 证书 ``` ## 创建 Kubernetes API Server 配置文件 export K8S_APISERVER_VIP="10.10.10.230" && \ export K8S_APISERVER_SERVICE_CLUSTER_IP="10.253.0.1" && \ export K8S_APISERVER_HOSTNAME="api.k8s.cbxhome.local" && \ export K8S_CLUSTER_DOMAIN_SHORTNAME="cbxhome" && \ export K8S_CLUSTER_DOMAIN_FULLNAME="cbxhome.local" && \ cat << EOF | tee /etc/cfssl/k8s/k8s_apiserver.json { "CN": "kubernetes", "hosts": [ "127.0.0.1", "${K8S_APISERVER_VIP}", "${K8S_APISERVER_SERVICE_CLUSTER_IP}", "${K8S_APISERVER_HOSTNAME}", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.${K8S_CLUSTER_DOMAIN_SHORTNAME}", "kubernetes.default.svc.${K8S_CLUSTER_DOMAIN_FULLNAME}" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "cbx", "OU": "cbxhome" } ] } EOF ## 生成 Kubernetes API Server 证书和私钥 cfssl gencert \ -ca=/etc/pki/k8s/k8s-ca.pem \ -ca-key=/etc/pki/k8s/k8s-ca-key.pem \ -config=/etc/cfssl/ca-config.json \ -profile=kubernetes \ /etc/cfssl/k8s/k8s_apiserver.json | \ cfssljson -bare /etc/pki/k8s/k8s_server ## 分发 Kubernetes API Server 证书至其他 Kube-Master 节点 scp /etc/pki/k8s/k8s_server* root@10.10.10.232:/etc/pki/k8s/ scp /etc/pki/k8s/k8s_server* root@10.10.10.233:/etc/pki/k8s/ ``` ***** ## 3.4.3、创建 Kubernetes Front Proxy CA 证书 ``` ## 创建 Kubernetes Front Proxy CA 配置文件 cat << EOF | tee /etc/cfssl/k8s/front-proxy-ca-csr.json { "CN": "kubernetes", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "cbx", "OU": "cbxhome" } ] } EOF ## 生成 Kubernetes Front Proxy CA 证书和私钥 cfssl gencert -initca /etc/cfssl/k8s/front-proxy-ca-csr.json | \ cfssljson -bare /etc/pki/k8s/k8s-front-proxy-ca ## 分发 Kubernetes Front Proxy CA 证书至其他 Kube-Master 节点 scp /etc/pki/k8s/k8s-front-proxy-ca* root@10.10.10.232:/etc/pki/k8s/ scp /etc/pki/k8s/k8s-front-proxy-ca* root@10.10.10.233:/etc/pki/k8s/ ``` ## 3.4.4、创建 Kubernetes Front Proxy Client 证书 ``` ## 创建 Kubernetes Front Proxy Client 配置文件 cat << EOF | tee /etc/cfssl/k8s/front-proxy-client-csr.json { "CN": "front-proxy-client", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "cbx", "OU": "cbxhome" } ] } EOF ## 生成 Kubernetes Front Proxy Client 证书和私钥 cfssl gencert \ -ca=/etc/pki/k8s/k8s-front-proxy-ca.pem \ -ca-key=/etc/pki/k8s/k8s-front-proxy-ca-key.pem \ -config=/etc/cfssl/ca-config.json \ -profile=kubernetes \ /etc/cfssl/k8s/front-proxy-client-csr.json | \ cfssljson -bare /etc/pki/k8s/k8s_front_proxy_client ## 分发 Kubernetes Front Proxy Client 证书至其他 Kube-Master 节点 scp /etc/pki/k8s/k8s_front_proxy_client* root@10.10.10.232:/etc/pki/k8s/ scp /etc/pki/k8s/k8s_front_proxy_client* root@10.10.10.233:/etc/pki/k8s/ ``` ***** ## 3.4.5、创建 Kubernetes Service Account 键值对 ``` ## 生成 Kubernetes Service Account 键值对 openssl genrsa -out /etc/pki/k8s/sa.key 2048 openssl rsa -in /etc/pki/k8s/sa.key -pubout -out /etc/pki/k8s/sa.pub ## 分发 Kubernetes Service Account 键值对至其他 Kube-Master 节点 scp /etc/pki/k8s/sa.* root@10.10.10.232:/etc/pki/k8s/ scp /etc/pki/k8s/sa.* root@10.10.10.233:/etc/pki/k8s/ ``` ***** ## 3.4.6、创建 Kubernetes Controller Manager 证书 ``` ## 创建 Kubernetes Controller Manager 配置文件 cat << EOF | tee /etc/cfssl/k8s/k8s_controller_manager.json { "CN": "system:kube-controller-manager", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "system:kube-controller-manager", "OU": "Kubernetes-manual" } ] } EOF ## 生成 Kubernetes Controller Manager 证书和私钥 cfssl gencert \ -ca=/etc/pki/k8s/k8s-ca.pem \ -ca-key=/etc/pki/k8s/k8s-ca-key.pem \ -config=/etc/cfssl/ca-config.json \ -profile=kubernetes \ /etc/cfssl/k8s/k8s_controller_manager.json | \ cfssljson -bare /etc/pki/k8s/k8s_controller_manager ## 分发 Kubernetes Controller Manager 证书至其他 Kube-Master 节点 scp /etc/pki/k8s/k8s_controller_manager* root@10.10.10.232:/etc/pki/k8s/ scp /etc/pki/k8s/k8s_controller_manager* root@10.10.10.233:/etc/pki/k8s/ ``` ***** ## 3.4.7、创建 Kubernetes Scheduler 证书 ``` ## 创建 Kubernetes Scheduler 配置文件 cat << EOF | tee /etc/cfssl/k8s/k8s_scheduler.json { "CN": "system:kube-scheduler", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangXi", "L": "Nanning", "O": "system:kube-scheduler", "OU": "Kubernetes-manual" } ] } EOF ## 生成 Kubernetes Scheduler 证书和私钥 cfssl gencert \ -ca=/etc/pki/k8s/k8s-ca.pem \ -ca-key=/etc/pki/k8s/k8s-ca-key.pem \ -config=/etc/cfssl/ca-config.json \ -profile=kubernetes \ /etc/cfssl/k8s/k8s_scheduler.json | \ cfssljson -bare /etc/pki/k8s/k8s_scheduler ## 分发 Kubernetes Scheduler 证书至其他 Kube-Master 节点 scp /etc/pki/k8s/k8s_scheduler* root@10.10.10.232:/etc/pki/k8s/ scp /etc/pki/k8s/k8s_scheduler* root@10.10.10.233:/etc/pki/k8s/ ``` *****