ETCD备份恢复

2025-04-14

memos

备份

创建测试pod

kubectl run test --image=nginx:1.7.9 --labels="app=test"
kubectl run test --image=nginx:1.7.9 --labels="app=test" -n kube-system

在etcd集群的每个节点上建立备份存放目录
mkdir -p /data/backup/etcd/backup_$(date +%Y%m%d)
使用kubeadm创建的k8s集群，etcd是使用容器运行的，因此备份和还原数据库需要将容器中的etcdctl命令拷贝到操作节点系统下的/usr/bin/目录下
docker cp $(docker ps | grep -v etcd-mirror | grep -w etcd | awk '{print $1}'):/usr/local/bin/etcdctl /usr/bin/

备份ETCDCTL_API为3的etcd数据到步骤2建立的备份目录下（替换ip为主机ip）
只需要备份一个master节点数据就可以了，数据备份后打包传到其他etcd节点上

# 备份
ETCDCTL_API=3 etcdctl snapshot save /data/backup/etcd/backup_$(date +%Y%m%d)/snap-$(date +%Y%m%d).db --endpoints 11.0.1.9:2379 --cert="/etc/kubernetes/pki/etcd/server.crt" --key="/etc/kubernetes/pki/etcd/server.key" --cacert="/etc/kubernetes/pki/etcd/ca.crt"

# 传到其他etcd节点上
scp /data/backup/etcd/backup_$(date +%Y%m%d)/snap-$(date +%Y%m%d).db root@11.0.1.6:/data/backup/etcd/backup_$(date +%Y%m%d)/
scp /data/backup/etcd/backup_$(date +%Y%m%d)/snap-$(date +%Y%m%d).db root@11.0.1.7:/data/backup/etcd/backup_$(date +%Y%m%d)/

定时备份脚本

[root@master ~]# vim /data/backup/etcd/etcd_backup.sh
#!/usr/bin/bash
 
date;
CERT="/etc/kubernetes/pki/etcd/server.crt"
EKY="/etc/kubernetes/pki/etcd/server.key"
CACERT="/etc/kubernetes/pki/etcd/ca.crt"
# 注意替换为主机IP
ENDPOINTS="11.0.1.9:2379"
 
ETCDCTL_API=3 /usr/bin/etcdctl \
--cacert="${CACERT}" --cert="${CERT}" --key="${EKY}" \
--endpoints=${ENDPOINTS} \
snapshot save /data/backup/etcd/backup_`date +%Y%m%d`/snap-`date +%Y%m%d`.db
 
# 备份保留7天
# -mtime -1表示，mtime在1天之内的文件，单位是天
# 而-mtime +10表示mtime在10天以上的文件
# 还有用法：-mmin -10，表示mtime在10分钟内的文件
# 有时候也可以不加+或-，比如-mtime 10，这表示正好为10天，此用法比较少
find /data/backup/etcd/ -name "*.db" -mtime +7 -exec rm -f {} \;
 
# 同步到其他两个etcd节点
/bin/rsync -avpgolr --delete /data/backup/etcd/backup_`date +%Y%m%d`/ root@11.0.1.6:/data/backup/etcd/backup_`date +%Y%m%d`/
/bin/rsync -avpgolr --delete /data/backup/etcd/backup_`date +%Y%m%d`/ root@11.0.1.7:/data/backup/etcd/backup_`date +%Y%m%d`/

[root@master ~]# chmod 755 /data/backup/etcd/etcd_backup.sh
[root@master ~]# crontab -l
#etcd集群数据备份 每天凌晨5点执行备份
0 5 * * * /bin/bash -x /data/backup/etcd/etcd_backup.sh > /dev/null 2>&1

恢复

模拟删除测试pod

kubectl delete pod test
kubectl delete pod test -n kube-system

停掉etcd集群全部节点的kube-apiserver和etcd，防止再有数据写入etcd。
（由于kube-apiserver和etcd属于静态pod，是由kubelet创建，所以需要将/etc/kubernetes/manifests/下的yaml文件移除，让其不可用）
mv /etc/kubernetes/manifests /etc/kubernetes/manifests.bak
查看etcd、api是否up，等待全部停止
docker ps |grep -E "apiserver|etcd"
重命名manifests目录后，无法再获取到pods了
可以发现manifests目录的重要性，因此建议对此目录也进行定期备份
kubectl get pods -A
移除etcd服务实例的数据目录，在etcd集群的全部节点执行
mv /var/lib/etcd /var/lib/etcd.bak

恢复etcd数据，etcd集群用同一份snapshot恢复，在etcd集群的全部节点依次执行

## 单集群演示
ETCDCTL_API=3 etcdctl snapshot restore /data/backup/etcd/backup_20220913/snap-20220913.db --data-dir=/var/lib/etcd --endpoints 11.0.1.9:2379 --cert="/etc/kubernetes/pki/etcd/server.crt" --key="/etc/kubernetes/pki/etcd/server.key" --cacert="/etc/kubernetes/pki/etcd/ca.crt"   

## 多集群
# 在master1执行
ETCDCTL_API=3 etcdctl snapshot restore /data/backup/etcd/backup_20220913/snap-20220913.db \
    --endpoints=11.0.1.9:2379 \
    --name=master1 \
    --cert=/etc/kubernetes/pki/etcd/server.crt \
    --key=/etc/kubernetes/pki/etcd/server.key \
    --cacert=/etc/kubernetes/pki/etcd/ca.crt \
    --initial-advertise-peer-urls=https://11.0.1.9:2380 \
    --initial-cluster-token=etcd-cluster-0 \
    --initial-cluster=master1=https://11.0.1.9:2380,master2=https://11.0.1.6:2380,master3=https://11.0.1.7:2380 \
    --data-dir=/var/lib/etcd
 # 在master2执行
 ETCDCTL_API=3 etcdctl snapshot restore /data/backup/etcd/backup_20220909/snap-202209091110.db \
    --endpoints=11.0.1.6:2379 \
    --name=master2 \
    --cert=/etc/kubernetes/pki/etcd/server.crt \
    --key=/etc/kubernetes/pki/etcd/server.key \
    --cacert=/etc/kubernetes/pki/etcd/ca.crt \
    --initial-advertise-peer-urls=https://11.0.1.7:2380 \
    --initial-cluster-token=etcd-cluster-0 \
    --initial-cluster=master1=https://11.0.1.9:2380,master2=https://11.0.1.6:2380,master3=https://11.0.1.7:2380 \
    --data-dir=/var/lib/etcd
 # 在master3执行
  ETCDCTL_API=3 etcdctl snapshot restore /data/backup/etcd/backup_20220913/snap-20220913.db \
    --endpoints=11.0.1.7:2379 \
    --name=master2 \
    --cert=/etc/kubernetes/pki/etcd/server.crt \
    --key=/etc/kubernetes/pki/etcd/server.key \
    --cacert=/etc/kubernetes/pki/etcd/ca.crt \
    --initial-advertise-peer-urls=https://11.0.1.7:2380 \
    --initial-cluster-token=etcd-cluster-0 \
    --initial-cluster=master1=https://11.0.1.9:2380,master2=https://11.0.1.6:2380,master3=https://11.0.1.7:2380 \
    --data-dir=/var/lib/etcd

恢复manifests，静态pod会自动重建kube-apiserver和etcd
mv /etc/kubernetes/manifests.bak /etc/kubernetes/manifests
查看集群是否恢复正常
kubectl get pod -A
kubectl get cs

etcdctl命令参考

# 一些etcdctl命令供参考
# 以表格的形式查看健康状态
ETCDCTL_API=3 etcdctl endpoint status -w table --endpoints 11.0.1.9:2379 --cert="/etc/kubernetes/pki/etcd/server.crt"  --key="/etc/kubernetes/pki/etcd/server.key"  --cacert="/etc/kubernetes/pki/etcd/ca.crt"
# 查看成员信息
ETCDCTL_API=3 etcdctl member list --write-out=table --endpoints 11.0.1.9:2379 --cert="/etc/kubernetes/pki/etcd/server.crt"  --key="/etc/kubernetes/pki/etcd/server.key"  --cacert="/etc/kubernetes/pki/etcd/ca.crt"
# 查看key
ECDCTL_API=3 etcdctl get / --prefix --keys-only --endpoints 11.0.1.9:2379 --cert="/etc/kubernetes/pki/etcd/server.crt"  --key="/etc/kubernetes/pki/etcd/server.key"  --cacert="/etc/kubernetes/pki/etcd/ca.crt"
# 查看节点
ETCDCTL_API=3 etcdctl get /registry/minions/ --prefix --keys-only --endpoints 11.0.1.9:2379 --cert="/etc/kubernetes/pki/etcd/server.crt"  --key="/etc/kubernetes/pki/etcd/server.key"  --cacert="/etc/kubernetes/pki/etcd/ca.crt" 
# 查看pod
ETCDCTL_API=3 etcdctl get /registry/pods/ --prefix --keys-only --endpoints 11.0.1.9:2379 --cert="/etc/kubernetes/pki/etcd/server.crt"  --key="/etc/kubernetes/pki/etcd/server.key"  --cacert="/etc/kubernetes/pki/etcd/ca.crt" 
# 查看健康状态
ETCDCTL_API=3 etcdctl endpoint health --endpoints 11.0.1.9:2379 --cert="/etc/kubernetes/pki/etcd/server.crt"  --key="/etc/kubernetes/pki/etcd/server.key"  --cacert="/etc/kubernetes/pki/etcd/ca.crt"

etcdctl x86文件

备份

恢复

etcdctl命令参考

说些什么吧！