Dannyvon 2020-05-07
ssh 192.168.2.175 hostnamectl set-hostname k8s-master-1 ssh 192.168.2.176 hostnamectl set-hostname k8s-master-2 ssh 192.168.2.177 hostnamectl set-hostname k8s-master-3 ssh 192.168.2.185 hostnamectl set-hostname k8s-node-1 ssh 192.168.2.187 hostnamectl set-hostname k8s-node-2
# centosx sed -i ‘s/SELINUX=.*/SELINUX=disabled/g‘ /etc/selinux/config systemctl stop firewalld && systemctl disable firewalld setenforce 0 # Ubuntu systemctl stop ufw.service systemctl disable ufw.service
#go 环境部署 yum install go vi ~/.bash_profile GOBIN=/root/go/bin/ PATH=$PATH:$GOBIN:$HOME/bin export PATH go get github.com/cloudflare/cfssl/cmd/cfssl go get github.com/cloudflare/cfssl/cmd/cfssljson
# 设置证书环境变量 # 设置证书使用时间87600h 10年 export EXPIRY_TIME="87600h" # 签发证书IP export ETCD_MEMBER_1_IP="192.168.2.175" export ETCD_MEMBER_2_IP="192.168.2.176" export ETCD_MEMBER_3_IP="192.168.2.177" # 机器名 export ETCD_MEMBER_1_HOSTNAMES="k8s-master-1" export ETCD_MEMBER_2_HOSTNAMES="k8s-master-2" export ETCD_MEMBER_3_HOSTNAMES="k8s-master-3" # etcd 集群通讯证书 export ETCD_SERVER_HOSTNAMES="\"${ETCD_MEMBER_1_HOSTNAMES}\",\"${ETCD_MEMBER_2_HOSTNAMES}\",\"${ETCD_MEMBER_3_HOSTNAMES}\"" export ETCD_SERVER_IPS="\"${ETCD_MEMBER_1_IP}\",\"${ETCD_MEMBER_2_IP}\",\"${ETCD_MEMBER_3_IP}\"" #证书所需要的配置参数 export CERT_ST="GuangDong" export CERT_L="GuangZhou" export CERT_O="k8s" export CERT_OU="Qist" export CERT_PROFILE="kubernetes" # 设置工作目录 export HOST_PATH=`pwd`
# 创建etcd K8S 证书json 存放目录 mkdir -p ${HOST_PATH}/cfssl/{k8s,etcd} # 创建签发证书存放目录 mkdir -p ${HOST_PATH}/cfssl/pki/{k8s,etcd} # CA 配置文件用于配置根证书的使用场景 (profile) 和具体参数 (usage,过期时间、服务端认证、客户端认证、加密等),后续在签名其它证书时需要指定特定场景。 cat << EOF | tee ${HOST_PATH}/cfssl/ca-config.json { "signing": { "default": { "expiry": "${EXPIRY_TIME}" }, "profiles": { "${CERT_PROFILE}": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "${EXPIRY_TIME}" } } } } EOF # 创建 ETCD CA 配置文件 cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-ca-csr.json { "CN": "etcd", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "$CERT_ST", "L": "$CERT_L", "O": "$CERT_O", "OU": "$CERT_OU" } ], "ca": { "expiry": "${EXPIRY_TIME}" } } EOF # etcd ca 证书签发 cfssl gencert -initca ${HOST_PATH}/cfssl/etcd/etcd-ca-csr.json | cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-ca # 创建 ETCD Server 配置文件 cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-server.json { "CN": "etcd", "hosts": [ "127.0.0.1", ${ETCD_SERVER_IPS}, ${ETCD_SERVER_HOSTNAMES} ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "$CERT_ST", "L": "$CERT_L", "O": "$CERT_O", "OU": "$CERT_OU" } ] } EOF # 生成 ETCD Server 证书和私钥 cfssl gencert -ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem -ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem -config=${HOST_PATH}/cfssl/ca-config.json -profile=${CERT_PROFILE} ${HOST_PATH}/cfssl/etcd/etcd-server.json | cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-server # 创建 ETCD Member 1 配置文件 cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json { "CN": "etcd", "hosts": [ "127.0.0.1", "${ETCD_MEMBER_1_IP}", "${ETCD_MEMBER_1_HOSTNAMES}" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "$CERT_ST", "L": "$CERT_L", "O": "$CERT_O", "OU": "$CERT_OU" } ] } EOF # 生成 ETCD Member 1 证书和私钥 cfssl gencert -ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem -ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem -config=${HOST_PATH}/cfssl/ca-config.json -profile=${CERT_PROFILE} ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json | cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_1_HOSTNAMES} # 创建 ETCD Member 2 配置文件 cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json { "CN": "etcd", "hosts": [ "127.0.0.1", "${ETCD_MEMBER_2_IP}", "${ETCD_MEMBER_2_HOSTNAMES}" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "$CERT_ST", "L": "$CERT_L", "O": "$CERT_O", "OU": "$CERT_OU" } ] } EOF # 生成 ETCD Member 2 证书和私钥 cfssl gencert -ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem -ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem -config=${HOST_PATH}/cfssl/ca-config.json -profile=${CERT_PROFILE} ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json | cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_2_HOSTNAMES} # 创建 ETCD Member 3 配置文件 cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json { "CN": "etcd", "hosts": [ "127.0.0.1", "${ETCD_MEMBER_3_IP}", "${ETCD_MEMBER_3_HOSTNAMES}" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "$CERT_ST", "L": "$CERT_L", "O": "$CERT_O", "OU": "$CERT_OU" } ] } EOF # 生成 ETCD Member 3 证书和私钥 cfssl gencert -ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem -ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem -config=${HOST_PATH}/cfssl/ca-config.json -profile=${CERT_PROFILE} ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json | cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_3_HOSTNAMES} # 创建 ETCD Client 配置文件 cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-client.json { "CN": "client", "hosts": [""], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "$CERT_ST", "L": "$CERT_L", "O": "$CERT_O", "OU": "$CERT_OU" } ] } EOF # 生成 ETCD Client 证书和私钥 cfssl gencert -ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem -ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem -config=${HOST_PATH}/cfssl/ca-config.json -profile=${CERT_PROFILE} ${HOST_PATH}/cfssl/etcd/etcd-client.json | cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-client # 分发生成的证书到所有需要部署etcd 节点 ssh 192.168.2.175 mkdir -p /apps/etcd/ssl ssh 192.168.2.176 mkdir -p /apps/etcd/ssl ssh 192.168.2.177 mkdir -p /apps/etcd/ssl # 分发文件 scp -r ./cfssl/pki/etcd/* 192.168.2.175:/apps/etcd/ssl/ scp -r ./cfssl/pki/etcd/* 192.168.2.176:/apps/etcd/ssl/ scp -r ./cfssl/pki/etcd/* 192.168.2.177:/apps/etcd/ssl/
wget https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-linux-amd64.tar.gz # 解压下载好文件 tar -xvf etcd-v3.4.7-linux-amd64.tar.gz # 创建二进制远程存放目录 ssh 192.168.2.175 mkdir -p /apps/etcd/bin ssh 192.168.2.176 mkdir -p /apps/etcd/bin ssh 192.168.2.177 mkdir -p /apps/etcd/bin # 分发解压好二进制文件 cd etcd-v3.4.7-linux-amd64/ scp -r etcd* 192.168.2.175:/apps/etcd/bin scp -r etcd* 192.168.2.176:/apps/etcd/bin scp -r etcd* 192.168.2.177:/apps/etcd/bin
# 创建配置文件存放目录 ssh 192.168.2.175 mkdir -p /apps/etcd/conf ssh 192.168.2.176 mkdir -p /apps/etcd/conf ssh 192.168.2.177 mkdir -p /apps/etcd/conf # 192.168.2.175 配置 ssh 192.168.2.175 cat << EOF | tee /apps/etcd/conf/etcd ETCD_OPTS="--name=k8s-master-1 \ --data-dir=/apps/etcd/data/default.etcd \ --wal-dir=/apps/etcd/data/default.etcd/wal \ --listen-peer-urls=https://192.168.2.175:2380 \ --listen-client-urls=https://192.168.2.175:2379,https://127.0.0.1:2379 \ --advertise-client-urls=https://192.168.2.175:2379 \ --initial-advertise-peer-urls=https://192.168.2.175:2380 \ --initial-cluster=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \ --initial-cluster-token=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \ --initial-cluster-state=new \ --heartbeat-interval=6000 \ --election-timeout=30000 \ --snapshot-count=5000 \ --auto-compaction-retention=1 \ --max-request-bytes=33554432 \ --quota-backend-bytes=17179869184 \ --trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \ --cert-file=/apps/etcd/ssl/etcd-server.pem \ --key-file=/apps/etcd/ssl/etcd-server-key.pem \ --peer-cert-file=/apps/etcd/ssl/etcd-member-k8s-master-1.pem \ --peer-key-file=/apps/etcd/ssl/etcd-member-k8s-master-1-key.pem \ --peer-client-cert-auth \ --enable-v2=true \ --peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem" EOF # 192.168.2.176 配置 ssh 192.168.2.176 cat << EOF | tee /apps/etcd/conf/etcd ETCD_OPTS="--name=k8s-master-2 \ --data-dir=/apps/etcd/data/default.etcd \ --wal-dir=/apps/etcd/data/default.etcd/wal \ --listen-peer-urls=https://192.168.2.176:2380 \ --listen-client-urls=https://192.168.2.176:2379,https://127.0.0.1:2379 \ --advertise-client-urls=https://192.168.2.176:2379 \ --initial-advertise-peer-urls=https://192.168.2.176:2380 \ --initial-cluster=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \ --initial-cluster-token=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \ --initial-cluster-state=new \ --heartbeat-interval=6000 \ --election-timeout=30000 \ --snapshot-count=5000 \ --auto-compaction-retention=1 \ --max-request-bytes=33554432 \ --quota-backend-bytes=17179869184 \ --trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \ --cert-file=/apps/etcd/ssl/etcd-server.pem \ --key-file=/apps/etcd/ssl/etcd-server-key.pem \ --peer-cert-file=/apps/etcd/ssl/etcd-member-k8s-master-2.pem \ --peer-key-file=/apps/etcd/ssl/etcd-member-k8s-master-2-key.pem \ --peer-client-cert-auth \ --enable-v2=true \ --peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem" EOF # 192.168.2.177 配置 ssh 192.168.2.177 cat << EOF | tee /apps/etcd/conf/etcd ETCD_OPTS="--name=k8s-master-3 \ --data-dir=/apps/etcd/data/default.etcd \ --wal-dir=/apps/etcd/data/default.etcd/wal \ --listen-peer-urls=https://192.168.2.177:2380 \ --listen-client-urls=https://192.168.2.177:2379,https://127.0.0.1:2379 \ --advertise-client-urls=https://192.168.2.177:2379 \ --initial-advertise-peer-urls=https://192.168.2.177:2380 \ --initial-cluster=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \ --initial-cluster-token=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \ --initial-cluster-state=new \ --heartbeat-interval=6000 \ --election-timeout=30000 \ --snapshot-count=5000 \ --auto-compaction-retention=1 \ --max-request-bytes=33554432 \ --quota-backend-bytes=17179869184 \ --trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \ --cert-file=/apps/etcd/ssl/etcd-server.pem \ --key-file=/apps/etcd/ssl/etcd-server-key.pem \ --peer-cert-file=/apps/etcd/ssl/etcd-member-k8s-master-3.pem \ --peer-key-file=/apps/etcd/ssl/etcd-member-k8s-master-3-key.pem \ --peer-client-cert-auth \ --enable-v2=true \ --peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem" EOF
cat << EOF | tee etcd.service [Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target Documentation=https://github.com/etcd-io/etcd [Service] Type=notify LimitNOFILE=65535 LimitNPROC=65535 LimitCORE=infinity LimitMEMLOCK=infinity User=etcd Group=etcd WorkingDirectory=/apps/etcd/data/default.etcd EnvironmentFile=-/apps/etcd/conf/etcd ExecStart=/apps/etcd/bin/etcd \$ETCD_OPTS Restart=on-failure [Install] WantedBy=multi-user.target EOF # 上传启动文件到服务器 scp etcd.service 192.168.2.175:/usr/lib/systemd/system scp etcd.service 192.168.2.176:/usr/lib/systemd/system scp etcd.service 192.168.2.176:/usr/lib/systemd/system
# 创建etcd 用户 ssh 192.168.2.175 useradd etcd -s /sbin/nologin -M ssh 192.168.2.176 useradd etcd -s /sbin/nologin -M ssh 192.168.2.177 useradd etcd -s /sbin/nologin -M # 创建etcd 存储文件目录 ssh 192.168.2.175 mkdir -p /apps/etcd/data/default.etcd/wal ssh 192.168.2.176 mkdir -p /apps/etcd/data/default.etcd/wal ssh 192.168.2.177 mkdir -p /apps/etcd/data/default.etcd/wal # 给/apps/etcd etcd 用户权限 ssh 192.168.2.175 chown -R etcd:etcd /apps/etcd/ ssh 192.168.2.176 chown -R etcd:etcd /apps/etcd/ ssh 192.168.2.177 chown -R etcd:etcd /apps/etcd/
# 刷新service systemctl daemon-reload # 设置开机启动 systemctl enable etcd.service # 启动etcd systemctl start etcd.service # 查看启动状态 systemctl status etcd.service # 验证etcd 集群是否正常 任意节点 vi ~/.bashrc export ETCDCTL_API=3 export ENDPOINTS=https://192.168.2.175:2379,https://192.168.2.176:2379,https://192.168.2.177:2379 alias ctl=‘/apps/etcd/bin/etcdctl --endpoints=${ENDPOINTS} --cacert=/apps/etcd/ssl/etcd-ca.pem --cert=/apps/etcd/ssl/etcd-client.pem --key=/apps/etcd/ssl/etcd-client-key.pem‘ # 保存 source ~/.bashrc # 验证集群是否正常 [ conf]# ctl endpoint status https://192.168.2.175:2379, 5ba4ebb731bb27f7, 3.4.7, 20 kB, false, false, 2, 11, 11, https://192.168.2.176:2379, 2982ccbf31a4b4e, 3.4.7, 20 kB, true, false, 2, 11, 11, https://192.168.2.177:2379, d86131c63ec195be, 3.4.7, 20 kB, false, false, 2, 11, 11, [ conf]# ctl endpoint hashkv https://192.168.2.175:2379, 1084519789 https://192.168.2.176:2379, 1084519789 https://192.168.2.177:2379, 1084519789 [ conf]# ctl endpoint health https://192.168.2.176:2379 is healthy: successfully committed proposal: took = 19.088204ms https://192.168.2.175:2379 is healthy: successfully committed proposal: took = 24.073788ms https://192.168.2.177:2379 is healthy: successfully committed proposal: took = 26.414683ms [ conf]# ctl member list 2982ccbf31a4b4e, started, k8s-master-2, https://192.168.2.176:2380, https://192.168.2.176:2379, false 5ba4ebb731bb27f7, started, k8s-master-1, https://192.168.2.175:2380, https://192.168.2.175:2379, false d86131c63ec195be, started, k8s-master-3, https://192.168.2.177:2380, https://192.168.2.177:2379, false
###host字段指定授权使用该证书的etcd节点IP或子网列表,需要将etcd集群的3个节点都添加其中。cp etcd-v3.3.13-linux-amd64/etcd* /opt/k8s/bin/