1Panel/backend/app/service/snapshot.go

package service

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/1Panel-dev/1Panel/backend/app/dto"
	"github.com/1Panel-dev/1Panel/backend/app/model"
	"github.com/1Panel-dev/1Panel/backend/constant"
	"github.com/1Panel-dev/1Panel/backend/global"
	"github.com/1Panel-dev/1Panel/backend/utils/cmd"
	"github.com/1Panel-dev/1Panel/backend/utils/compose"
	"github.com/1Panel-dev/1Panel/backend/utils/files"
	"github.com/jinzhu/copier"
	"github.com/pkg/errors"
	"github.com/shirou/gopsutil/v3/host"
)

type SnapshotService struct {
	OriginalPath string
}

type ISnapshotService interface {
	SearchWithPage(req dto.SearchWithPage) (int64, interface{}, error)
	SnapshotCreate(req dto.SnapshotCreate) error
	SnapshotRecover(req dto.SnapshotRecover) error
	SnapshotRollback(req dto.SnapshotRecover) error
	SnapshotImport(req dto.SnapshotImport) error
	Delete(req dto.SnapshotBatchDelete) error

	LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, error)

	UpdateDescription(req dto.UpdateDescription) error
	readFromJson(path string) (SnapshotJson, error)

	HandleSnapshot(isCronjob bool, logPath string, req dto.SnapshotCreate, timeNow string, secret string) (string, error)
}

func NewISnapshotService() ISnapshotService {
	return &SnapshotService{}
}

func (u *SnapshotService) SearchWithPage(req dto.SearchWithPage) (int64, interface{}, error) {
	total, systemBackups, err := snapshotRepo.Page(req.Page, req.PageSize, commonRepo.WithLikeName(req.Info))
	dtoSnap, err := loadSnapSize(systemBackups)
	if err != nil {
		return 0, nil, err
	}
	return total, dtoSnap, err
}

func (u *SnapshotService) SnapshotImport(req dto.SnapshotImport) error {
	if len(req.Names) == 0 {
		return fmt.Errorf("incorrect snapshot request body: %v", req.Names)
	}
	for _, snapName := range req.Names {
		snap, _ := snapshotRepo.Get(commonRepo.WithByName(strings.ReplaceAll(snapName, ".tar.gz", "")))
		if snap.ID != 0 {
			return constant.ErrRecordExist
		}
	}
	for _, snap := range req.Names {
		shortName := strings.TrimPrefix(snap, "snapshot_")
		nameItems := strings.Split(shortName, "_")
		if !strings.HasPrefix(shortName, "1panel_v") || !strings.HasSuffix(shortName, ".tar.gz") || len(nameItems) < 3 {
			return fmt.Errorf("incorrect snapshot name format of %s", shortName)
		}
		if strings.HasSuffix(snap, ".tar.gz") {
			snap = strings.ReplaceAll(snap, ".tar.gz", "")
		}
		itemSnap := model.Snapshot{
			Name:            snap,
			From:            req.From,
			DefaultDownload: req.From,
			Version:         nameItems[1],
			Description:     req.Description,
			Status:          constant.StatusSuccess,
		}
		if err := snapshotRepo.Create(&itemSnap); err != nil {
			return err
		}
	}
	return nil
}

func (u *SnapshotService) UpdateDescription(req dto.UpdateDescription) error {
	return snapshotRepo.Update(req.ID, map[string]interface{}{"description": req.Description})
}

func (u *SnapshotService) LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, error) {
	var data dto.SnapshotStatus
	status, err := snapshotRepo.GetStatus(id)
	if err != nil {
		return nil, err
	}
	if err := copier.Copy(&data, &status); err != nil {
		return nil, errors.WithMessage(constant.ErrStructTransform, err.Error())
	}
	return &data, nil
}

type SnapshotJson struct {
	OldBaseDir       string `json:"oldBaseDir"`
	OldDockerDataDir string `json:"oldDockerDataDir"`
	OldBackupDataDir string `json:"oldBackupDataDir"`
	OldPanelDataDir  string `json:"oldPanelDataDir"`

	BaseDir            string `json:"baseDir"`
	DockerDataDir      string `json:"dockerDataDir"`
	BackupDataDir      string `json:"backupDataDir"`
	PanelDataDir       string `json:"panelDataDir"`
	LiveRestoreEnabled bool   `json:"liveRestoreEnabled"`
}

func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error {
	if _, err := u.HandleSnapshot(false, "", req, time.Now().Format(constant.DateTimeSlimLayout), req.Secret); err != nil {
		return err
	}
	return nil
}

func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error {
	global.LOG.Info("start to recover panel by snapshot now")
	snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID))
	if err != nil {
		return err
	}
	if hasOs(snap.Name) && !strings.Contains(snap.Name, loadOs()) {
		return fmt.Errorf("restoring snapshots(%s) between different server architectures(%s) is not supported", snap.Name, loadOs())
	}
	if !req.IsNew && len(snap.InterruptStep) != 0 && len(snap.RollbackStatus) != 0 {
		return fmt.Errorf("the snapshot has been rolled back and cannot be restored again")
	}

	baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name))
	if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) {
		_ = os.MkdirAll(baseDir, os.ModePerm)
	}

	_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting})
	_ = settingRepo.Update("SystemStatus", "Recovering")
	go u.HandleSnapshotRecover(snap, true, req)
	return nil
}

func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error {
	global.LOG.Info("start to rollback now")
	snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID))
	if err != nil {
		return err
	}
	req.IsNew = false
	snap.InterruptStep = "Readjson"
	go u.HandleSnapshotRecover(snap, false, req)
	return nil
}

func (u *SnapshotService) readFromJson(path string) (SnapshotJson, error) {
	var snap SnapshotJson
	if _, err := os.Stat(path); err != nil {
		return snap, fmt.Errorf("find snapshot json file in recover package failed, err: %v", err)
	}
	fileByte, err := os.ReadFile(path)
	if err != nil {
		return snap, fmt.Errorf("read file from path %s failed, err: %v", path, err)
	}
	if err := json.Unmarshal(fileByte, &snap); err != nil {
		return snap, fmt.Errorf("unmarshal snapjson failed, err: %v", err)
	}
	return snap, nil
}

func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto.SnapshotCreate, timeNow string, secret string) (string, error) {
	localDir, err := loadLocalDir()
	if err != nil {
		return "", err
	}
	var (
		rootDir    string
		snap       model.Snapshot
		snapStatus model.SnapshotStatus
	)

	if req.ID == 0 {
		versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion"))

		name := fmt.Sprintf("1panel_%s_%s_%s", versionItem.Value, loadOs(), timeNow)
		if isCronjob {
			name = fmt.Sprintf("snapshot_1panel_%s_%s_%s", versionItem.Value, loadOs(), timeNow)
		}
		rootDir = path.Join(localDir, "system", name)

		snap = model.Snapshot{
			Name:            name,
			Description:     req.Description,
			From:            req.From,
			DefaultDownload: req.DefaultDownload,
			Version:         versionItem.Value,
			Status:          constant.StatusWaiting,
		}
		_ = snapshotRepo.Create(&snap)
		snapStatus.SnapID = snap.ID
		_ = snapshotRepo.CreateStatus(&snapStatus)
	} else {
		snap, err = snapshotRepo.Get(commonRepo.WithByID(req.ID))
		if err != nil {
			return "", err
		}
		_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusWaiting})
		snapStatus, _ = snapshotRepo.GetStatus(snap.ID)
		if snapStatus.ID == 0 {
			snapStatus.SnapID = snap.ID
			_ = snapshotRepo.CreateStatus(&snapStatus)
		}
		rootDir = path.Join(localDir, fmt.Sprintf("system/%s", snap.Name))
	}

	var wg sync.WaitGroup
	itemHelper := snapHelper{SnapID: snap.ID, Status: &snapStatus, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()}
	backupPanelDir := path.Join(rootDir, "1panel")
	_ = os.MkdirAll(backupPanelDir, os.ModePerm)
	backupDockerDir := path.Join(rootDir, "docker")
	_ = os.MkdirAll(backupDockerDir, os.ModePerm)

	jsonItem := SnapshotJson{
		BaseDir:       global.CONF.System.BaseDir,
		BackupDataDir: localDir,
		PanelDataDir:  path.Join(global.CONF.System.BaseDir, "1panel"),
	}
	loadLogByStatus(snapStatus, logPath)
	if snapStatus.PanelInfo != constant.StatusDone {
		wg.Add(1)
		go snapJson(itemHelper, jsonItem, rootDir)
	}
	if snapStatus.Panel != constant.StatusDone {
		wg.Add(1)
		go snapPanel(itemHelper, backupPanelDir)
	}
	if snapStatus.DaemonJson != constant.StatusDone {
		wg.Add(1)
		go snapDaemonJson(itemHelper, backupDockerDir)
	}
	if snapStatus.AppData != constant.StatusDone {
		wg.Add(1)
		go snapAppData(itemHelper, backupDockerDir)
	}
	if snapStatus.BackupData != constant.StatusDone {
		wg.Add(1)
		go snapBackup(itemHelper, localDir, backupPanelDir)
	}

	if !isCronjob {
		go func() {
			wg.Wait()
			if !checkIsAllDone(snap.ID) {
				_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
				return
			}
			if snapStatus.PanelData != constant.StatusDone {
				snapPanelData(itemHelper, localDir, backupPanelDir)
			}
			if snapStatus.PanelData != constant.StatusDone {
				_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
				return
			}
			if snapStatus.Compress != constant.StatusDone {
				snapCompress(itemHelper, rootDir, secret)
			}
			if snapStatus.Compress != constant.StatusDone {
				_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
				return
			}
			if snapStatus.Upload != constant.StatusDone {
				snapUpload(itemHelper, req.From, fmt.Sprintf("%s.tar.gz", rootDir))
			}
			if snapStatus.Upload != constant.StatusDone {
				_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
				return
			}
			_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess})
		}()
		return "", nil
	}
	wg.Wait()
	if !checkIsAllDone(snap.ID) {
		_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
		loadLogByStatus(snapStatus, logPath)
		return snap.Name, fmt.Errorf("snapshot %s backup failed", snap.Name)
	}
	loadLogByStatus(snapStatus, logPath)
	snapPanelData(itemHelper, localDir, backupPanelDir)
	if snapStatus.PanelData != constant.StatusDone {
		_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
		loadLogByStatus(snapStatus, logPath)
		return snap.Name, fmt.Errorf("snapshot %s 1panel data failed", snap.Name)
	}
	loadLogByStatus(snapStatus, logPath)
	snapCompress(itemHelper, rootDir, secret)
	if snapStatus.Compress != constant.StatusDone {
		_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
		loadLogByStatus(snapStatus, logPath)
		return snap.Name, fmt.Errorf("snapshot %s compress failed", snap.Name)
	}
	loadLogByStatus(snapStatus, logPath)
	snapUpload(itemHelper, req.From, fmt.Sprintf("%s.tar.gz", rootDir))
	if snapStatus.Upload != constant.StatusDone {
		_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
		loadLogByStatus(snapStatus, logPath)
		return snap.Name, fmt.Errorf("snapshot %s upload failed", snap.Name)
	}
	_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess})
	loadLogByStatus(snapStatus, logPath)
	return snap.Name, nil
}

func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error {
	snaps, _ := snapshotRepo.GetList(commonRepo.WithIdsIn(req.Ids))
	for _, snap := range snaps {
		if req.DeleteWithFile {
			targetAccounts, err := loadClientMap(snap.From)
			if err != nil {
				return err
			}
			for _, item := range targetAccounts {
				global.LOG.Debugf("remove snapshot file %s.tar.gz from %s", snap.Name, item.backType)
				_, _ = item.client.Delete(path.Join(item.backupPath, "system_snapshot", snap.Name+".tar.gz"))
			}
		}

		_ = snapshotRepo.DeleteStatus(snap.ID)
		if err := snapshotRepo.Delete(commonRepo.WithByID(snap.ID)); err != nil {
			return err
		}
	}
	return nil
}

func updateRecoverStatus(id uint, isRecover bool, interruptStep, status, message string) {
	if isRecover {
		if status != constant.StatusSuccess {
			global.LOG.Errorf("recover failed, err: %s", message)
		}
		if err := snapshotRepo.Update(id, map[string]interface{}{
			"interrupt_step":    interruptStep,
			"recover_status":    status,
			"recover_message":   message,
			"last_recovered_at": time.Now().Format(constant.DateTimeLayout),
		}); err != nil {
			global.LOG.Errorf("update snap recover status failed, err: %v", err)
		}
		_ = settingRepo.Update("SystemStatus", "Free")
		return
	}
	_ = settingRepo.Update("SystemStatus", "Free")
	if status == constant.StatusSuccess {
		if err := snapshotRepo.Update(id, map[string]interface{}{
			"recover_status":     "",
			"recover_message":    "",
			"interrupt_step":     "",
			"rollback_status":    "",
			"rollback_message":   "",
			"last_rollbacked_at": time.Now().Format(constant.DateTimeLayout),
		}); err != nil {
			global.LOG.Errorf("update snap recover status failed, err: %v", err)
		}
		return
	}
	global.LOG.Errorf("rollback failed, err: %s", message)
	if err := snapshotRepo.Update(id, map[string]interface{}{
		"rollback_status":    status,
		"rollback_message":   message,
		"last_rollbacked_at": time.Now().Format(constant.DateTimeLayout),
	}); err != nil {
		global.LOG.Errorf("update snap recover status failed, err: %v", err)
	}
}

func (u *SnapshotService) handleUnTar(sourceDir, targetDir string, secret string) error {
	if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) {
		if err = os.MkdirAll(targetDir, os.ModePerm); err != nil {
			return err
		}
	}
	commands := ""
	if len(secret) != 0 {
		extraCmd := "openssl enc -d -aes-256-cbc -k '" + secret + "' -in " + sourceDir + " | "
		commands = fmt.Sprintf("%s tar -zxvf - -C %s", extraCmd, targetDir+" > /dev/null 2>&1")
		global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******"))
	} else {
		commands = fmt.Sprintf("tar zxvfC %s %s", sourceDir, targetDir)
		global.LOG.Debug(commands)
	}
	stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute)
	if err != nil {
		if len(stdout) != 0 {
			global.LOG.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err)
			return fmt.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err)
		}
	}
	return nil
}

func rebuildAllAppInstall() error {
	global.LOG.Debug("start to rebuild all app")
	appInstalls, err := appInstallRepo.ListBy()
	if err != nil {
		global.LOG.Errorf("get all app installed for rebuild failed, err: %v", err)
		return err
	}
	var wg sync.WaitGroup
	for i := 0; i < len(appInstalls); i++ {
		wg.Add(1)
		appInstalls[i].Status = constant.Rebuilding
		_ = appInstallRepo.Save(context.Background(), &appInstalls[i])
		go func(app model.AppInstall) {
			defer wg.Done()
			dockerComposePath := app.GetComposePath()
			out, err := compose.Down(dockerComposePath)
			if err != nil {
				_ = handleErr(app, err, out)
				return
			}
			out, err = compose.Up(dockerComposePath)
			if err != nil {
				_ = handleErr(app, err, out)
				return
			}
			app.Status = constant.Running
			_ = appInstallRepo.Save(context.Background(), &app)
		}(appInstalls[i])
	}
	wg.Wait()
	return nil
}

func checkIsAllDone(snapID uint) bool {
	status, err := snapshotRepo.GetStatus(snapID)
	if err != nil {
		return false
	}
	isOK, _ := checkAllDone(status)
	return isOK
}

func checkAllDone(status model.SnapshotStatus) (bool, string) {
	if status.Panel != constant.StatusDone {
		return false, status.Panel
	}
	if status.PanelInfo != constant.StatusDone {
		return false, status.PanelInfo
	}
	if status.DaemonJson != constant.StatusDone {
		return false, status.DaemonJson
	}
	if status.AppData != constant.StatusDone {
		return false, status.AppData
	}
	if status.BackupData != constant.StatusDone {
		return false, status.BackupData
	}
	return true, ""
}

func loadLogByStatus(status model.SnapshotStatus, logPath string) {
	logs := ""
	logs += fmt.Sprintf("Write 1Panel basic information: %s \n", status.PanelInfo)
	logs += fmt.Sprintf("Backup 1Panel system files: %s \n", status.Panel)
	logs += fmt.Sprintf("Backup Docker configuration file: %s \n", status.DaemonJson)
	logs += fmt.Sprintf("Backup installed apps from 1Panel: %s \n", status.AppData)
	logs += fmt.Sprintf("Backup 1Panel data directory: %s \n", status.PanelData)
	logs += fmt.Sprintf("Backup local backup directory for 1Panel: %s \n", status.BackupData)
	logs += fmt.Sprintf("Create snapshot file: %s \n", status.Compress)
	logs += fmt.Sprintf("Snapshot size: %s \n", status.Size)
	logs += fmt.Sprintf("Upload snapshot file: %s \n", status.Upload)

	file, err := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return
	}
	defer file.Close()
	_, _ = file.Write([]byte(logs))
}

func hasOs(name string) bool {
	return strings.Contains(name, "amd64") ||
		strings.Contains(name, "arm64") ||
		strings.Contains(name, "armv7") ||
		strings.Contains(name, "ppc64le") ||
		strings.Contains(name, "s390x")
}

func loadOs() string {
	hostInfo, _ := host.Info()
	switch hostInfo.KernelArch {
	case "x86_64":
		return "amd64"
	case "armv7l":
		return "armv7"
	default:
		return hostInfo.KernelArch
	}
}

func loadSnapSize(records []model.Snapshot) ([]dto.SnapshotInfo, error) {
	var datas []dto.SnapshotInfo
	clientMap := make(map[string]loadSizeHelper)
	var wg sync.WaitGroup
	for i := 0; i < len(records); i++ {
		var item dto.SnapshotInfo
		if err := copier.Copy(&item, &records[i]); err != nil {
			return nil, errors.WithMessage(constant.ErrStructTransform, err.Error())
		}
		itemPath := fmt.Sprintf("system_snapshot/%s.tar.gz", item.Name)
		if _, ok := clientMap[records[i].DefaultDownload]; !ok {
			backup, err := backupRepo.Get(commonRepo.WithByType(records[i].DefaultDownload))
			if err != nil {
				global.LOG.Errorf("load backup model %s from db failed, err: %v", records[i].DefaultDownload, err)
				clientMap[records[i].DefaultDownload] = loadSizeHelper{}
				datas = append(datas, item)
				continue
			}
			client, err := NewIBackupService().NewClient(&backup)
			if err != nil {
				global.LOG.Errorf("load backup client %s from db failed, err: %v", records[i].DefaultDownload, err)
				clientMap[records[i].DefaultDownload] = loadSizeHelper{}
				datas = append(datas, item)
				continue
			}
			item.Size, _ = client.Size(path.Join(strings.TrimLeft(backup.BackupPath, "/"), itemPath))
			datas = append(datas, item)
			clientMap[records[i].DefaultDownload] = loadSizeHelper{backupPath: strings.TrimLeft(backup.BackupPath, "/"), client: client, isOk: true}
			continue
		}
		if clientMap[records[i].DefaultDownload].isOk {
			wg.Add(1)
			go func(index int) {
				item.Size, _ = clientMap[records[index].DefaultDownload].client.Size(path.Join(clientMap[records[index].DefaultDownload].backupPath, itemPath))
				datas = append(datas, item)
				wg.Done()
			}(i)
		} else {
			datas = append(datas, item)
		}
	}
	wg.Wait()
	return datas, nil
}