mirror/1Panel
1
0
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2024-11-27 20:49:03 +08:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

feat: 备份运行环境/静态网站时支持同步备份数据库 (#6618)

Browse Source
This commit is contained in:
zhengkunwang 2024-09-29 16:00:00 +08:00 committed by GitHub
parent 53e9af1a62
commit 4d560a3764
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 243 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

315
agent/app/service/backup_app.go
View File

@ -38,7 +38,7 @@ func (u *BackupService) AppBackup(req dto.CommonBackup) (*model.BackupRecord, er
backupDir := path.Join(global.CONF.System.Backup, itemDir)
fileName := fmt.Sprintf("%s_%s.tar.gz", req.DetailName, timeNow+common.RandStrAndNum(5))
if err := handleAppBackup(&install, nil, backupDir, fileName, "", req.Secret, ""); err != nil {
if err := handleAppBackup(&install, nil, backupDir, fileName, "", req.Secret, req.TaskID); err != nil {
return nil, err
}
@ -76,7 +76,7 @@ func (u *BackupService) AppRecover(req dto.CommonRecover) error {
if _, err := compose.Down(install.GetComposePath()); err != nil {
return err
}
if err := handleAppRecover(&install, req.File, false, req.Secret); err != nil {
if err := handleAppRecover(&install, nil, req.File, false, req.Secret, req.TaskID); err != nil {
return err
}
return nil
@ -164,151 +164,184 @@ func handleAppBackup(install *model.AppInstall, parentTask *task.Task, backupDir
return backupTask.Execute()
}
func handleAppRecover(install *model.AppInstall, recoverFile string, isRollback bool, secret string) error {
isOk := false
fileOp := files.NewFileOp()
if err := handleUnTar(recoverFile, path.Dir(recoverFile), secret); err != nil {
return err
}
tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "")
defer func() {
_, _ = compose.Up(install.GetComposePath())
_ = os.RemoveAll(strings.ReplaceAll(recoverFile, ".tar.gz", ""))
}()
if !fileOp.Stat(tmpPath+"/app.json") || !fileOp.Stat(tmpPath+"/app.tar.gz") {
return errors.New("the wrong recovery package does not have app.json or app.tar.gz files")
}
var oldInstall model.AppInstall
appjson, err := os.ReadFile(tmpPath + "/app.json")
if err != nil {
return err
}
if err := json.Unmarshal(appjson, &oldInstall); err != nil {
return fmt.Errorf("unmarshal app.json failed, err: %v", err)
}
if oldInstall.App.Key != install.App.Key || oldInstall.Name != install.Name {
return errors.New("the current backup file does not match the application")
}
if !isRollback {
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("app/%s_%s.tar.gz", install.Name, time.Now().Format(constant.DateTimeSlimLayout)))
if err := handleAppBackup(install, nil, path.Dir(rollbackFile), path.Base(rollbackFile), "", "", ""); err != nil {
return fmt.Errorf("backup app %s for rollback before recover failed, err: %v", install.Name, err)
}
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := handleAppRecover(install, rollbackFile, true, secret); err != nil {
global.LOG.Errorf("rollback app %s from %s failed, err: %v", install.Name, rollbackFile, err)
return
}
global.LOG.Infof("rollback app %s from %s successful", install.Name, rollbackFile)
_ = os.RemoveAll(rollbackFile)
} else {
_ = os.RemoveAll(rollbackFile)
}
}()
}
newEnvFile := ""
resources, _ := appInstallResourceRepo.GetBy(appInstallResourceRepo.WithAppInstallId(install.ID))
for _, resource := range resources {
var database model.Database
switch resource.From {
case constant.AppResourceRemote:
database, err = databaseRepo.Get(commonRepo.WithByID(resource.LinkId))
if err != nil {
return err
}
case constant.AppResourceLocal:
resourceApp, err := appInstallRepo.GetFirst(commonRepo.WithByID(resource.LinkId))
if err != nil {
return err
}
database, err = databaseRepo.Get(databaseRepo.WithAppInstallID(resourceApp.ID), commonRepo.WithByType(resource.Key), commonRepo.WithByFrom(constant.AppResourceLocal), commonRepo.WithByName(resourceApp.Name))
if err != nil {
return err
}
}
switch database.Type {
case constant.AppPostgresql:
db, err := postgresqlRepo.Get(commonRepo.WithByID(resource.ResourceId))
if err != nil {
return err
}
if err := handlePostgresqlRecover(dto.CommonRecover{
Name: database.Name,
DetailName: db.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name),
}, true); err != nil {
global.LOG.Errorf("handle recover from sql.gz failed, err: %v", err)
return err
}
case constant.AppMysql, constant.AppMariaDB:
db, err := mysqlRepo.Get(commonRepo.WithByID(resource.ResourceId))
if err != nil {
return err
}
newDB, envMap, err := reCreateDB(db.ID, database, oldInstall.Env)
if err != nil {
return err
}
oldHost := fmt.Sprintf("\"PANEL_DB_HOST\":\"%v\"", envMap["PANEL_DB_HOST"].(string))
newHost := fmt.Sprintf("\"PANEL_DB_HOST\":\"%v\"", database.Address)
oldInstall.Env = strings.ReplaceAll(oldInstall.Env, oldHost, newHost)
envMap["PANEL_DB_HOST"] = database.Address
newEnvFile, err = coverEnvJsonToStr(oldInstall.Env)
if err != nil {
return err
}
_ = appInstallResourceRepo.BatchUpdateBy(map[string]interface{}{"resource_id": newDB.ID}, commonRepo.WithByID(resource.ID))
if err := handleMysqlRecover(dto.CommonRecover{
Name: newDB.MysqlName,
DetailName: newDB.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name),
}, true); err != nil {
global.LOG.Errorf("handle recover from sql.gz failed, err: %v", err)
return err
}
}
}
appDir := install.GetPath()
backPath := fmt.Sprintf("%s_bak", appDir)
_ = fileOp.Rename(appDir, backPath)
_ = fileOp.CreateDir(appDir, 0755)
if err := handleUnTar(tmpPath+"/app.tar.gz", install.GetAppPath(), ""); err != nil {
global.LOG.Errorf("handle recover from app.tar.gz failed, err: %v", err)
_ = fileOp.DeleteDir(appDir)
_ = fileOp.Rename(backPath, appDir)
return err
}
_ = fileOp.DeleteDir(backPath)
if len(newEnvFile) != 0 {
envPath := fmt.Sprintf("%s/%s/.env", install.GetAppPath(), install.Name)
file, err := os.OpenFile(envPath, os.O_WRONLY|os.O_TRUNC, 0640)
func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverFile string, isRollback bool, secret, taskID string) error {
var (
err error
recoverTask *task.Task
isOk = false
rollbackFile string
)
recoverTask = parentTask
if parentTask == nil {
recoverTask, err = task.NewTaskWithOps(install.Name, task.TaskRecover, task.TaskScopeApp, taskID, install.ID)
if err != nil {
return err
}
defer file.Close()
_, _ = file.WriteString(newEnvFile)
}
oldInstall.ID = install.ID
oldInstall.Status = constant.StatusRunning
oldInstall.AppId = install.AppId
oldInstall.AppDetailId = install.AppDetailId
oldInstall.App.ID = install.AppId
if err := appInstallRepo.Save(context.Background(), &oldInstall); err != nil {
global.LOG.Errorf("save db app install failed, err: %v", err)
return err
}
isOk = true
recoverApp := func(t *task.Task) error {
fileOp := files.NewFileOp()
if err := handleUnTar(recoverFile, path.Dir(recoverFile), secret); err != nil {
return err
}
tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "")
defer func() {
_, _ = compose.Up(install.GetComposePath())
_ = os.RemoveAll(strings.ReplaceAll(recoverFile, ".tar.gz", ""))
}()
if !fileOp.Stat(tmpPath+"/app.json") || !fileOp.Stat(tmpPath+"/app.tar.gz") {
return errors.New(i18n.GetMsgByKey("AppBackupFileIncomplete"))
}
var oldInstall model.AppInstall
appJson, err := os.ReadFile(tmpPath + "/app.json")
if err != nil {
return err
}
if err := json.Unmarshal(appJson, &oldInstall); err != nil {
return fmt.Errorf("unmarshal app.json failed, err: %v", err)
}
if oldInstall.App.Key != install.App.Key || oldInstall.Name != install.Name {
return errors.New(i18n.GetMsgByKey("AppAttributesNotMatch"))
}
if !isRollback {
rollbackFile = path.Join(global.CONF.System.TmpDir, fmt.Sprintf("app/%s_%s.tar.gz", install.Name, time.Now().Format(constant.DateTimeSlimLayout)))
if err := handleAppBackup(install, nil, path.Dir(rollbackFile), path.Base(rollbackFile), "", "", ""); err != nil {
t.Log(fmt.Sprintf("backup app %s for rollback before recover failed, err: %v", install.Name, err))
}
}
newEnvFile := ""
resources, _ := appInstallResourceRepo.GetBy(appInstallResourceRepo.WithAppInstallId(install.ID))
for _, resource := range resources {
var database model.Database
switch resource.From {
case constant.AppResourceRemote:
database, err = databaseRepo.Get(commonRepo.WithByID(resource.LinkId))
if err != nil {
return err
}
case constant.AppResourceLocal:
resourceApp, err := appInstallRepo.GetFirst(commonRepo.WithByID(resource.LinkId))
if err != nil {
return err
}
database, err = databaseRepo.Get(databaseRepo.WithAppInstallID(resourceApp.ID), commonRepo.WithByType(resource.Key), commonRepo.WithByFrom(constant.AppResourceLocal), commonRepo.WithByName(resourceApp.Name))
if err != nil {
return err
}
}
switch database.Type {
case constant.AppPostgresql:
db, err := postgresqlRepo.Get(commonRepo.WithByID(resource.ResourceId))
if err != nil {
return err
}
taskName := task.GetTaskName(db.Name, task.TaskRecover, task.TaskScopeDatabase)
t.LogStart(taskName)
if err := handlePostgresqlRecover(dto.CommonRecover{
Name: database.Name,
DetailName: db.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name),
}, true); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
t.LogSuccess(taskName)
case constant.AppMysql, constant.AppMariaDB:
db, err := mysqlRepo.Get(commonRepo.WithByID(resource.ResourceId))
if err != nil {
return err
}
newDB, envMap, err := reCreateDB(db.ID, database, oldInstall.Env)
if err != nil {
return err
}
oldHost := fmt.Sprintf("\"PANEL_DB_HOST\":\"%v\"", envMap["PANEL_DB_HOST"].(string))
newHost := fmt.Sprintf("\"PANEL_DB_HOST\":\"%v\"", database.Address)
oldInstall.Env = strings.ReplaceAll(oldInstall.Env, oldHost, newHost)
envMap["PANEL_DB_HOST"] = database.Address
newEnvFile, err = coverEnvJsonToStr(oldInstall.Env)
if err != nil {
return err
}
_ = appInstallResourceRepo.BatchUpdateBy(map[string]interface{}{"resource_id": newDB.ID}, commonRepo.WithByID(resource.ID))
taskName := task.GetTaskName(db.Name, task.TaskRecover, task.TaskScopeDatabase)
t.LogStart(taskName)
if err := handleMysqlRecover(dto.CommonRecover{
Name: newDB.MysqlName,
DetailName: newDB.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name),
}, true); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
t.LogSuccess(taskName)
}
}
appDir := install.GetPath()
backPath := fmt.Sprintf("%s_bak", appDir)
_ = fileOp.Rename(appDir, backPath)
_ = fileOp.CreateDir(appDir, 0755)
deCompressName := i18n.GetWithName("DeCompressFile", "app.tar.gz")
t.LogStart(deCompressName)
if err := handleUnTar(tmpPath+"/app.tar.gz", install.GetAppPath(), ""); err != nil {
t.LogFailedWithErr(deCompressName, err)
_ = fileOp.DeleteDir(appDir)
_ = fileOp.Rename(backPath, appDir)
return err
}
t.LogSuccess(deCompressName)
_ = fileOp.DeleteDir(backPath)
if len(newEnvFile) != 0 {
envPath := fmt.Sprintf("%s/%s/.env", install.GetAppPath(), install.Name)
file, err := os.OpenFile(envPath, os.O_WRONLY|os.O_TRUNC, 0640)
if err != nil {
return err
}
defer file.Close()
_, _ = file.WriteString(newEnvFile)
}
oldInstall.ID = install.ID
oldInstall.Status = constant.StatusRunning
oldInstall.AppId = install.AppId
oldInstall.AppDetailId = install.AppDetailId
oldInstall.App.ID = install.AppId
if err := appInstallRepo.Save(context.Background(), &oldInstall); err != nil {
global.LOG.Errorf("save db app install failed, err: %v", err)
return err
}
isOk = true
return nil
}
rollBackApp := func(t *task.Task) {
if isRollback {
return
}
if !isOk {
t.Log(i18n.GetMsgByKey("RecoverFailedStartRollBack"))
if err := handleAppRecover(install, t, rollbackFile, true, secret, ""); err != nil {
t.LogFailedWithErr(i18n.GetMsgByKey("Rollback"), err)
return
}
t.LogSuccess(i18n.GetMsgByKey("Rollback"))
_ = os.RemoveAll(rollbackFile)
} else {
_ = os.RemoveAll(rollbackFile)
}
}
recoverTask.AddSubTask(task.GetTaskName(install.Name, task.TaskBackup, task.TaskScopeApp), recoverApp, rollBackApp)
if parentTask != nil {
return recoverApp(parentTask)
}
return nil
}

63
agent/app/service/backup_website.go
View File

@ -32,7 +32,7 @@ func (u *BackupService) WebsiteBackup(req dto.CommonBackup) error {
timeNow := time.Now().Format(constant.DateTimeSlimLayout)
itemDir := fmt.Sprintf("website/%s", req.Name)
backupDir := path.Join(global.CONF.System.Backup, itemDir)
fileName := fmt.Sprintf("%s_%s.tar.gz", website.PrimaryDomain, timeNow+common.RandStrAndNum(5))
fileName := fmt.Sprintf("%s_%s.tar.gz", website.Alias, timeNow+common.RandStrAndNum(5))
go func() {
if err = handleWebsiteBackup(&website, backupDir, fileName, "", req.Secret, req.TaskID); err != nil {
@ -41,7 +41,7 @@ func (u *BackupService) WebsiteBackup(req dto.CommonBackup) error {
}
record := &model.BackupRecord{
Type: "website",
Name: website.PrimaryDomain,
Name: website.Alias,
DetailName: req.DetailName,
SourceAccountIDs: "1",
DownloadAccountID: 1,
@ -146,7 +146,7 @@ func handleWebsiteRecover(website *model.Website, recoverFile string, isRollback
}
taskName := task.GetTaskName(app.Name, task.TaskRecover, task.TaskScopeApp)
t.LogStart(taskName)
if err := handleAppRecover(&app, fmt.Sprintf("%s/%s.app.tar.gz", tmpPath, website.Alias), true, ""); err != nil {
if err := handleAppRecover(&app, recoverTask, fmt.Sprintf("%s/%s.app.tar.gz", tmpPath, website.Alias), true, "", ""); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
@ -167,6 +167,17 @@ func handleWebsiteRecover(website *model.Website, recoverFile string, isRollback
return err
}
t.LogSuccess(taskName)
if oldWebsite.DbID > 0 {
if err := recoverWebsiteDatabase(t, oldWebsite.DbID, oldWebsite.DbType, tmpPath, website.Alias); err != nil {
return err
}
}
case constant.Static:
if oldWebsite.DbID > 0 {
if err := recoverWebsiteDatabase(t, oldWebsite.DbID, oldWebsite.DbType, tmpPath, website.Alias); err != nil {
return err
}
}
}
taskName := i18n.GetMsgByKey("TaskRecover") + i18n.GetMsgByKey("websiteDir")
t.Log(taskName)
@ -189,11 +200,11 @@ func handleWebsiteRecover(website *model.Website, recoverFile string, isRollback
}
func handleWebsiteBackup(website *model.Website, backupDir, fileName, excludes, secret, taskID string) error {
backupTask, err := task.NewTaskWithOps(website.PrimaryDomain, task.TaskBackup, task.TaskScopeWebsite, taskID, website.ID)
backupTask, err := task.NewTaskWithOps(website.Alias, task.TaskBackup, task.TaskScopeWebsite, taskID, website.ID)
if err != nil {
return err
}
backupTask.AddSubTask(task.GetTaskName(website.PrimaryDomain, task.TaskBackup, task.TaskScopeWebsite), func(t *task.Task) error {
backupTask.AddSubTask(task.GetTaskName(website.Alias, task.TaskBackup, task.TaskScopeWebsite), func(t *task.Task) error {
fileOp := files.NewFileOp()
tmpDir := fmt.Sprintf("%s/%s", backupDir, strings.ReplaceAll(fileName, ".tar.gz", ""))
if !fileOp.Stat(tmpDir) {
@ -237,13 +248,13 @@ func handleWebsiteBackup(website *model.Website, backupDir, fileName, excludes,
}
t.LogSuccess(task.GetTaskName(runtime.Name, task.TaskBackup, task.TaskScopeRuntime))
if website.DbID > 0 {
if err = backupDatabaseWithTask(t, website.DbType, tmpDir, website.PrimaryDomain, website.DbID); err != nil {
if err = backupDatabaseWithTask(t, website.DbType, tmpDir, website.Alias, website.DbID); err != nil {
return err
}
}
case constant.Static:
if website.DbID > 0 {
if err = backupDatabaseWithTask(t, website.DbType, tmpDir, website.PrimaryDomain, website.DbID); err != nil {
if err = backupDatabaseWithTask(t, website.DbType, tmpDir, website.Alias, website.DbID); err != nil {
return err
}
}
@ -285,3 +296,41 @@ func checkValidOfWebsite(oldWebsite, website *model.Website) error {
}
return nil
}
func recoverWebsiteDatabase(t *task.Task, dbID uint, dbType, tmpPath, websiteKey string) error {
switch dbType {
case constant.AppPostgresql:
db, err := postgresqlRepo.Get(commonRepo.WithByID(dbID))
if err != nil {
return err
}
taskName := task.GetTaskName(db.Name, task.TaskRecover, task.TaskScopeDatabase)
t.LogStart(taskName)
if err := handlePostgresqlRecover(dto.CommonRecover{
Name: db.PostgresqlName,
DetailName: db.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, websiteKey),
}, true); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
t.LogSuccess(taskName)
case constant.AppMysql, constant.AppMariaDB:
db, err := mysqlRepo.Get(commonRepo.WithByID(dbID))
if err != nil {
return err
}
taskName := task.GetTaskName(db.Name, task.TaskRecover, task.TaskScopeDatabase)
t.LogStart(taskName)
if err := handleMysqlRecover(dto.CommonRecover{
Name: db.MysqlName,
DetailName: db.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, websiteKey),
}, true); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
t.LogSuccess(taskName)
}
return nil
}

5
agent/i18n/lang/en.yaml
View File

@ -308,4 +308,7 @@ CompressDir: "Compress directory"
DeCompressFile: "Decompress file {{ .name }}"
ErrCheckValid: "Failed to validate backup file, {{ .name }}"
Rollback: "Rollback"
websiteDir: "Website directory"
websiteDir: "Website directory"
RecoverFailedStartRollBack: "Recovery failed, starting rollback"
AppBackupFileIncomplete: "Backup file is incomplete; missing app.json or app.tar.gz file"
AppAttributesNotMatch: "Application type or name does not match"

5
agent/i18n/lang/zh-Hant.yaml
View File

@ -311,4 +311,7 @@ CompressDir: "壓縮目錄"
DeCompressFile: "解壓檔案 {{ .name }}"
ErrCheckValid: "校驗備份檔案失敗，{{ .name }}"
Rollback: "回滾"
websiteDir: "網站目錄"
websiteDir: "網站目錄"
RecoverFailedStartRollBack: "恢復失敗，開始回滾"
AppBackupFileIncomplete: "備份文件不完整，缺少 app.json 或 app.tar.gz 文件"
AppAttributesNotMatch: "應用類型或名稱不一致"

5
agent/i18n/lang/zh.yaml
View File

@ -338,4 +338,7 @@ CompressDir: "压缩目录"
DeCompressFile: "解压文件 {{ .name }}"
ErrCheckValid: "校验备份文件失败，{{ .name }}"
Rollback: "回滚"
websiteDir: "网站目录"
websiteDir: "网站目录"
RecoverFailedStartRollBack: "恢复失败，开始回滚"
AppBackupFileIncomplete: "备份文件不完整 缺少 app.json 或者 app.tar.gz 文件"
AppAttributesNotMatch: "应用类型或者名称不一致"

2
frontend/src/components/backup/index.vue
View File

@ -70,7 +70,7 @@
<el-dialog
v-model="open"
:title="isBackup ? $t('commons.button.backup') : $t('commons.button.recover') + ' - ' + name"
width="40%"
width="30%"
:close-on-click-modal="false"
:before-close="handleBackupClose"
>