# 备份现有 Etcd 数据和manifests
mkdir -p /root/backup/kubernetes/
cp -r /var/lib/etcd/member /root/backup/kubernetes/member-$(date +%F)
cp -r /etc/kubernetes/manifests /root/backup/kubernetes/manifests-$(date +%F)

# 通过运行 Etcd 镜像，并且使用镜像内部的 etcdctl 工具连接 etcd 集群，执行数据快照备份：
docker run --rm --name etcdctl \
  -v /root/backup/kubernetes:/backup \
  -v /etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd:ro \
  --env ETCDCTL_API=3 \
  registry.aliyuncs.com/k8sxio/etcd:3.4.13-0 \
  /bin/sh -c "etcdctl --endpoints=https://172.38.40.212:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --key=/etc/kubernetes/pki/etcd/healthcheck-client.key \
  --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt \
  snapshot save /backup/etcd.db-$(date +%F)"

# 停止所有 master 节点上 k8s 系统组件
# 移除且备份 /etc/kubernetes/manifests 目录
$ mv /etc/kubernetes/manifests /root/backup
# 查看 kube-apiserver、etcd 镜像是否停止
$ docker ps|grep etcd && docker ps|grep kube-apiserver
# 清理已经损坏的 etcd 数据
rm -rf /var/lib/etcd/member

# 下面命令的 TIME 变量改成想要恢复的数据哪个时间的
# 在其中一个节点上恢复数据
docker run --rm              \
-v /root/backup:/backup        \
-v /var/lib/etcd:/var/lib/etcd \
--env ETCDCTL_API=3            \
registry.aliyuncs.com/k8sxio/etcd:3.4.3-0        \
/bin/sh -c "etcdctl snapshot restore /backup/etcd.db-${TIME}; mv /default.etcd/member/ /var/lib/etcd/"
# 在其他 master 节点上，将数据拷贝过来
scp -r 172.38.40.212:/var/lib/etcd/member /var/lib/etcd/

# 将 /etc/kubernetes/manifests 目录恢复，使 Kubernetes 重启 Kube-Apiserver 与 Etcd 镜像：
cp -r /root/backup/manifests/ /etc/kubernetes/

apiVersion: batch/v1
kind: CronJob
metadata:
  name: etcd-backup
  namespace: etcd-backup
spec:
  schedule: "0 0 1 * *"
  jobTemplate:
    spec:
      template:
        metadata:
          labels:
            name: etcd-backup
        spec:
          affinity:
            nodeAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                nodeSelectorTerms:
                  - matchExpressions:
                      - key: node-role.kubernetes.io/master
                        operator: Exists
          containers:
            - command:
                - sh
                - -c
                - "etcdctl --endpoints=https://localhost:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt
                  --key=/etc/kubernetes/pki/etcd/healthcheck-client.key --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt
                  snapshot save /backup/etcd.db-$(printf '%(%Y-%m-%d-%H:%M:%S)T') "
              env:
                - name: ETCDCTL_API
                  value: "3"
              image: registry.aliyuncs.com/dd_k8s/etcd:3.5.1-0
              name: etcd
              volumeMounts:
                - mountPath: /backup
                  name: etcd-backup-db
                - mountPath: /etc/localtime
                  name: host-time
                  readOnly: true
                - mountPath: /etc/kubernetes/pki/etcd
                  name: etcd-certs
                  readOnly: true
          hostNetwork: true
          restartPolicy: Never
          tolerations:
            - effect: NoSchedule
              key: node-role.kubernetes.io/master
              operator: Exists
          volumes:
            - name: etcd-backup-db
              persistentVolumeClaim:
                claimName: etcd-backup
            - hostPath:
                path: /etc/localtime
              name: host-time
            - hostPath:
                path: /etc/kubernetes/pki/etcd
              name: etcd-certs