1Panel/agent/app/service/snapshot_recover.go

package service

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path"
	"strings"
	"sync"

	"github.com/1Panel-dev/1Panel/agent/app/dto"
	"github.com/1Panel-dev/1Panel/agent/app/model"
	"github.com/1Panel-dev/1Panel/agent/app/task"
	"github.com/1Panel-dev/1Panel/agent/constant"
	"github.com/1Panel-dev/1Panel/agent/global"
	"github.com/1Panel-dev/1Panel/agent/i18n"
	"github.com/1Panel-dev/1Panel/agent/utils/cmd"
	"github.com/1Panel-dev/1Panel/agent/utils/compose"
	"github.com/1Panel-dev/1Panel/agent/utils/files"
	"github.com/pkg/errors"
)

type snapRecoverHelper struct {
	FileOp files.FileOp
	Task   *task.Task
}

func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error {
	global.LOG.Info("start to recover panel by snapshot now")
	snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID))
	if err != nil {
		return err
	}
	if hasOs(snap.Name) && !strings.Contains(snap.Name, loadOs()) {
		errInfo := fmt.Sprintf("restoring snapshots(%s) between different server architectures(%s) is not supported", snap.Name, loadOs())
		_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusFailed, "recover_message": errInfo})
		return errors.New(errInfo)
	}
	if len(snap.RollbackStatus) != 0 && snap.RollbackStatus != constant.StatusSuccess {
		req.IsNew = true
	}
	if !req.IsNew && (snap.InterruptStep == "RecoverDownload" || snap.InterruptStep == "RecoverDecompress" || snap.InterruptStep == "BackupBeforeRecover") {
		req.IsNew = true
	}
	_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting})
	_ = settingRepo.Update("SystemStatus", "Recovering")

	if len(snap.InterruptStep) == 0 {
		req.IsNew = true
	}
	if len(snap.TaskRecoverID) != 0 {
		req.TaskID = snap.TaskRecoverID
	} else {
		_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"task_recover_id": req.TaskID})
	}
	taskItem, err := task.NewTaskWithOps(snap.Name, task.TaskRecover, task.TaskScopeSnapshot, req.TaskID, snap.ID)
	if err != nil {
		global.LOG.Errorf("new task for create snapshot failed, err: %v", err)
		return err
	}
	rootDir := path.Join(global.CONF.System.TmpDir, "system", snap.Name)
	if _, err := os.Stat(rootDir); err != nil && os.IsNotExist(err) {
		_ = os.MkdirAll(rootDir, os.ModePerm)
	}
	itemHelper := snapRecoverHelper{Task: taskItem, FileOp: files.NewFileOp()}

	go func() {
		_ = global.Cron.Stop()
		defer func() {
			global.Cron.Start()
		}()

		if req.IsNew || snap.InterruptStep == "RecoverDownload" || req.ReDownload {
			taskItem.AddSubTaskWithAlias(
				"RecoverDownload",
				func(t *task.Task) error { return handleDownloadSnapshot(&itemHelper, snap, rootDir) },
				nil,
			)
			req.IsNew = true
		}
		if req.IsNew || snap.InterruptStep == "RecoverDecompress" {
			taskItem.AddSubTaskWithAlias(
				"RecoverDecompress",
				func(t *task.Task) error {
					itemHelper.Task.Log("######################## 2 / 10 ########################")
					itemHelper.Task.LogStart(i18n.GetWithName("RecoverDecompress", snap.Name))
					err := itemHelper.FileOp.TarGzExtractPro(fmt.Sprintf("%s/%s.tar.gz", rootDir, snap.Name), rootDir, req.Secret)
					itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err)
					return err
				},
				nil,
			)
			req.IsNew = true
		}
		if req.IsNew || snap.InterruptStep == "BackupBeforeRecover" {
			taskItem.AddSubTaskWithAlias(
				"BackupBeforeRecover",
				func(t *task.Task) error { return backupBeforeRecover(snap.Name, &itemHelper) },
				nil,
			)
			req.IsNew = true
		}

		var snapJson SnapshotJson
		taskItem.AddSubTaskWithAlias(
			"Readjson",
			func(t *task.Task) error {
				snapJson, err = readFromJson(path.Join(rootDir, snap.Name), &itemHelper)
				return err
			},
			nil,
		)
		if req.IsNew || snap.InterruptStep == "RecoverApp" {
			taskItem.AddSubTaskWithAlias(
				"RecoverApp",
				func(t *task.Task) error { return recoverAppData(path.Join(rootDir, snap.Name), &itemHelper) },
				nil,
			)
			req.IsNew = true
		}
		if req.IsNew || snap.InterruptStep == "RecoverBaseData" {
			taskItem.AddSubTaskWithAlias(
				"RecoverBaseData",
				func(t *task.Task) error { return recoverBaseData(path.Join(rootDir, snap.Name, "base"), &itemHelper) },
				nil,
			)
			req.IsNew = true
		}
		if req.IsNew || snap.InterruptStep == "RecoverDBData" {
			taskItem.AddSubTaskWithAlias(
				"RecoverDBData",
				func(t *task.Task) error { return recoverDBData(path.Join(rootDir, snap.Name, "db"), &itemHelper) },
				nil,
			)
			req.IsNew = true
		}
		if req.IsNew || snap.InterruptStep == "RecoverBackups" {
			taskItem.AddSubTaskWithAlias(
				"RecoverBackups",
				func(t *task.Task) error {
					itemHelper.Task.Log("######################## 8 / 10 ########################")
					itemHelper.Task.LogStart(i18n.GetWithName("RecoverBackups", snap.Name))
					err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/1panel_backup.tar.gz"), snapJson.BackupDataDir, "")
					itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err)
					return err
				},
				nil,
			)
			req.IsNew = true
		}
		if req.IsNew || snap.InterruptStep == "RecoverPanelData" {
			taskItem.AddSubTaskWithAlias(
				"RecoverPanelData",
				func(t *task.Task) error {
					itemHelper.Task.Log("######################## 9 / 10 ########################")
					itemHelper.Task.LogStart(i18n.GetWithName("RecoverPanelData", snap.Name))
					err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), "")
					itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err)
					return err
				},
				nil,
			)
			req.IsNew = true
		}
		taskItem.AddSubTaskWithAlias(
			"RecoverDBData",
			func(t *task.Task) error {
				return restartCompose(path.Join(snapJson.BaseDir, "1panel/docker/compose"), &itemHelper)
			},
			nil,
		)

		if err := taskItem.Execute(); err != nil {
			_ = settingRepo.Update("SystemStatus", "Free")
			_ = snapshotRepo.Update(req.ID, map[string]interface{}{"recover_status": constant.StatusFailed, "recover_message": err.Error(), "interrupt_step": taskItem.Task.CurrentStep})
			return
		}
		_ = os.RemoveAll(rootDir)
		_, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service")
	}()
	return nil
}

func handleDownloadSnapshot(itemHelper *snapRecoverHelper, snap model.Snapshot, targetDir string) error {
	itemHelper.Task.Log("######################## 1 / 10 ########################")
	itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverDownload"))

	account, client, err := NewBackupClientWithID(snap.DownloadAccountID)
	itemHelper.Task.LogWithStatus(i18n.GetWithName("RecoverDownloadAccount", fmt.Sprintf("%s - %s", account.Type, account.Name)), err)
	pathItem := account.BackupPath
	if account.BackupPath != "/" {
		pathItem = strings.TrimPrefix(account.BackupPath, "/")
	}
	filePath := fmt.Sprintf("%s/%s.tar.gz", targetDir, snap.Name)
	_ = os.RemoveAll(filePath)
	_, err = client.Download(path.Join(pathItem, fmt.Sprintf("system_snapshot/%s.tar.gz", snap.Name)), filePath)
	itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Download"), err)
	return err
}

func backupBeforeRecover(name string, itemHelper *snapRecoverHelper) error {
	itemHelper.Task.Log("######################## 3 / 10 ########################")
	itemHelper.Task.LogStart(i18n.GetMsgByKey("BackupBeforeRecover"))

	rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name)
	baseDir := path.Join(rootDir, "base")
	if _, err := os.Stat(baseDir); err != nil {
		_ = os.MkdirAll(baseDir, os.ModePerm)
	}

	err := itemHelper.FileOp.CopyDirWithExclude(path.Join(global.CONF.System.BaseDir, "1panel"), rootDir, []string{"cache", "tmp"})
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", path.Join(global.CONF.System.BaseDir, "1panel")), err)
	if err != nil {
		return err
	}
	err = itemHelper.FileOp.CopyDirWithExclude(global.CONF.System.Backup, rootDir, []string{"system_snapshot"})
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", global.CONF.System.Backup), err)
	if err != nil {
		return err
	}
	err = itemHelper.FileOp.CopyFile("/usr/local/bin/1pctl", baseDir)
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err)
	if err != nil {
		return err
	}
	err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel", baseDir)
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), err)
	if err != nil {
		return err
	}
	err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel_agent", baseDir)
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), err)
	if err != nil {
		return err
	}
	err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel.service", baseDir)
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err)
	if err != nil {
		return err
	}
	err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel_agent.service", baseDir)
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), err)
	if err != nil {
		return err
	}
	err = itemHelper.FileOp.CopyFile("/etc/docker/daemon.json", baseDir)
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/docker/daemon.json"), err)
	if err != nil {
		return err
	}
	return nil
}

func readFromJson(rootDir string, itemHelper *snapRecoverHelper) (SnapshotJson, error) {
	itemHelper.Task.Log("######################## 4 / 10 ########################")
	itemHelper.Task.LogStart(i18n.GetMsgByKey("Readjson"))

	snapJsonPath := path.Join(rootDir, "base/snapshot.json")
	var snap SnapshotJson
	_, err := os.Stat(snapJsonPath)
	itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonPath"), err)
	if err != nil {
		return snap, err
	}
	fileByte, err := os.ReadFile(snapJsonPath)
	itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonContent"), err)
	if err != nil {
		return snap, err
	}
	err = json.Unmarshal(fileByte, &snap)
	itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonMarshal"), err)
	if err != nil {
		return snap, err
	}
	return snap, nil
}

func recoverAppData(src string, itemHelper *snapRecoverHelper) error {
	itemHelper.Task.Log("######################## 5 / 10 ########################")
	itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverApp"))

	if _, err := os.Stat(path.Join(src, "images.tar.gz")); err != nil {
		itemHelper.Task.Log(i18n.GetMsgByKey("RecoverAppEmpty"))
		return nil
	} else {
		std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz"))
		if err != nil {
			itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverAppImage"), errors.New(std))
			return fmt.Errorf("docker load images failed, err: %v", err)
		}
		itemHelper.Task.LogSuccess(i18n.GetMsgByKey("RecoverAppImage"))
	}

	appInstalls, err := appInstallRepo.ListBy()
	itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverAppList"), err)
	if err != nil {
		return err
	}

	var wg sync.WaitGroup
	for i := 0; i < len(appInstalls); i++ {
		wg.Add(1)
		appInstalls[i].Status = constant.Rebuilding
		_ = appInstallRepo.Save(context.Background(), &appInstalls[i])
		go func(app model.AppInstall) {
			defer wg.Done()
			dockerComposePath := app.GetComposePath()
			out, err := compose.Down(dockerComposePath)
			if err != nil {
				_ = handleErr(app, err, out)
				return
			}
			out, err = compose.Up(dockerComposePath)
			if err != nil {
				_ = handleErr(app, err, out)
				return
			}
			app.Status = constant.Running
			_ = appInstallRepo.Save(context.Background(), &app)
		}(appInstalls[i])
	}
	wg.Wait()
	return nil
}

func recoverBaseData(src string, itemHelper *snapRecoverHelper) error {
	itemHelper.Task.Log("######################## 6 / 10 ########################")
	itemHelper.Task.LogStart(i18n.GetMsgByKey("SnapBaseInfo"))

	err := itemHelper.FileOp.CopyFile(path.Join(src, "1pctl"), "/usr/local/bin")
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err)
	if err != nil {
		return err
	}

	err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel"), "/usr/local/bin")
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), err)
	if err != nil {
		return err
	}
	err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel_agent"), "/usr/local/bin")
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), err)
	if err != nil {
		return err
	}
	err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel.service"), "/etc/systemd/system")
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err)
	if err != nil {
		return err
	}
	err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel_agent.service"), "/etc/systemd/system")
	itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), err)
	if err != nil {
		return err
	}

	daemonJsonPath := "/etc/docker/daemon.json"
	_, errSrc := os.Stat(path.Join(src, "docker/daemon.json"))
	_, errPath := os.Stat(daemonJsonPath)
	if os.IsNotExist(errSrc) && os.IsNotExist(errPath) {
		itemHelper.Task.Log(i18n.GetMsgByKey("RecoverDaemonJsonEmpty"))
		return nil
	}
	if errSrc == nil {
		err = itemHelper.FileOp.CopyFile(path.Join(src, "docker/daemon.json"), "/etc/docker")
		itemHelper.Task.Log(i18n.GetMsgByKey("RecoverDaemonJson"))
		if err != nil {
			return fmt.Errorf("recover docker daemon.json failed, err: %v", err)
		}
	}

	_, _ = cmd.Exec("systemctl restart docker")
	return nil
}

func recoverDBData(src string, itemHelper *snapRecoverHelper) error {
	itemHelper.Task.Log("######################## 7 / 10 ########################")
	itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverDBData"))
	err := itemHelper.FileOp.CopyDirWithExclude(src, path.Join(global.CONF.System.BaseDir, "1panel"), nil)

	itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverDBData"), err)
	return err
}

func restartCompose(composePath string, itemHelper *snapRecoverHelper) error {
	itemHelper.Task.Log("######################## 10 / 10 ########################")
	itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverCompose"))

	composes, err := composeRepo.ListRecord()
	itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverComposeList"), err)
	if err != nil {
		return err
	}

	for _, compose := range composes {
		pathItem := path.Join(composePath, compose.Name, "docker-compose.yml")
		if _, err := os.Stat(pathItem); err != nil {
			continue
		}
		upCmd := fmt.Sprintf("docker compose -f %s up -d", pathItem)
		stdout, err := cmd.Exec(upCmd)
		if err != nil {
			itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverCompose"), errors.New(stdout))
			continue
		}
		itemHelper.Task.LogSuccess(i18n.GetWithName("RecoverComposeItem", pathItem))
	}
	return nil
}