diff --git a/backend/app/dto/backup.go b/backend/app/dto/backup.go index 6dd3ce0ec..60486c9f7 100644 --- a/backend/app/dto/backup.go +++ b/backend/app/dto/backup.go @@ -52,6 +52,7 @@ type BackupRecords struct { BackupType string `json:"backupType"` FileDir string `json:"fileDir"` FileName string `json:"fileName"` + Size int64 `json:"size"` } type DownloadRecord struct { diff --git a/backend/app/service/backup.go b/backend/app/service/backup.go index 1986ed57a..a884bd657 100644 --- a/backend/app/service/backup.go +++ b/backend/app/service/backup.go @@ -81,15 +81,46 @@ func (u *BackupService) SearchRecordsWithPage(search dto.RecordSearch) (int64, [ commonRepo.WithByType(search.Type), backupRepo.WithByDetailName(search.DetailName), ) - var dtobas []dto.BackupRecords - for _, group := range records { + + var datas []dto.BackupRecords + clientMap := make(map[string]loadSizeHelper) + for i := 0; i < len(records); i++ { var item dto.BackupRecords - if err := copier.Copy(&item, &group); err != nil { + if err := copier.Copy(&item, &records[i]); err != nil { return 0, nil, errors.WithMessage(constant.ErrStructTransform, err.Error()) } - dtobas = append(dtobas, item) + itemPath := path.Join(records[i].FileDir, records[i].FileName) + if records[i].Source == "LOCAL" { + fileInfo, _ := os.Stat(itemPath) + item.Size = fileInfo.Size() + datas = append(datas, item) + continue + } + if _, ok := clientMap[records[i].Source]; !ok { + backup, err := backupRepo.Get(commonRepo.WithByType(records[i].Source)) + if err != nil { + global.LOG.Errorf("load backup model %s from db failed, err: %v", records[i].Source, err) + return total, datas, err + } + client, err := u.NewClient(&backup) + if err != nil { + global.LOG.Errorf("load backup client %s from db failed, err: %v", records[i].Source, err) + return total, datas, err + } + item.Size, _ = client.Size(path.Join(strings.TrimLeft(backup.BackupPath, "/"), itemPath)) + datas = append(datas, item) + clientMap[records[i].Source] = loadSizeHelper{backupPath: strings.TrimLeft(backup.BackupPath, "/"), client: client} + continue + } + item.Size, _ = clientMap[records[i].Source].client.Size(path.Join(clientMap[records[i].Source].backupPath, itemPath)) + datas = append(datas, item) } - return total, dtobas, err + return total, datas, err +} + +type loadSizeHelper struct { + backupPath string + client cloud_storage.CloudStorageClient } func (u *BackupService) LoadOneDriveInfo() (string, error) { @@ -139,9 +170,7 @@ func (u *BackupService) DownloadRecord(info dto.DownloadRecord) (string, error) } srcPath := fmt.Sprintf("%s/%s", info.FileDir, info.FileName) if len(backup.BackupPath) != 0 { - itemPath := strings.TrimPrefix(backup.BackupPath, "/") - itemPath = strings.TrimSuffix(itemPath, "/") + "/" - srcPath = itemPath + srcPath + srcPath = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), srcPath) } if exist, _ := backClient.Exist(srcPath); exist { isOK, err := backClient.Download(srcPath, targetPath) diff --git a/backend/app/service/cronjob_helper.go b/backend/app/service/cronjob_helper.go index 903d3b621..c29f0e9f2 100644 --- a/backend/app/service/cronjob_helper.go +++ b/backend/app/service/cronjob_helper.go @@ -157,9 +157,7 @@ func (u *CronjobService) handleBackup(cronjob *model.Cronjob, startTime time.Tim return "", err } if len(backup.BackupPath) != 0 { - itemPath := strings.TrimPrefix(backup.BackupPath, "/") - itemPath = strings.TrimSuffix(itemPath, "/") + "/" - itemFileDir = itemPath + itemFileDir + itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir) } if _, err = client.Upload(backupDir+"/"+fileName, itemFileDir+"/"+fileName); err != nil { return "", err @@ -193,9 +191,7 @@ func (u *CronjobService) HandleRmExpired(backType, backupPath, localDir string, fileItem := file if cronjob.KeepLocal { if len(backupPath) != 0 { - itemPath := strings.TrimPrefix(backupPath, "/") - itemPath = strings.TrimSuffix(itemPath, "/") + "/" - fileItem = itemPath + strings.TrimPrefix(file, localDir+"/") + fileItem = path.Join(strings.TrimPrefix(backupPath, "/") + strings.TrimPrefix(file, localDir+"/")) } else { fileItem = strings.TrimPrefix(file, localDir+"/") } @@ -332,9 +328,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, backup model.Back }() } if len(backup.BackupPath) != 0 { - itemPath := strings.TrimPrefix(backup.BackupPath, "/") - itemPath = strings.TrimSuffix(itemPath, "/") + "/" - itemFileDir = itemPath + itemFileDir + itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir) } if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil { return paths, err @@ -475,9 +469,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, backup model.BackupAcc }() } if len(backup.BackupPath) != 0 { - itemPath := strings.TrimPrefix(backup.BackupPath, "/") - itemPath = strings.TrimSuffix(itemPath, "/") + "/" - itemFileDir = itemPath + itemFileDir + itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir) } if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil { return paths, err @@ -551,9 +543,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, backup model.Backu }() } if len(backup.BackupPath) != 0 { - itemPath := strings.TrimPrefix(backup.BackupPath, "/") - itemPath = strings.TrimSuffix(itemPath, "/") + "/" - itemFileDir = itemPath + itemFileDir + itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir) } if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil { return paths, err diff --git a/backend/utils/cloud_storage/client/cos.go b/backend/utils/cloud_storage/client/cos.go index 8eef7cbe1..450cb1a16 100644 --- a/backend/utils/cloud_storage/client/cos.go +++ b/backend/utils/cloud_storage/client/cos.go @@ -7,44 +7,24 @@ import ( "net/url" "os" - "github.com/1Panel-dev/1Panel/backend/constant" cosSDK "github.com/tencentyun/cos-go-sdk-v5" ) type cosClient struct { - region string - accessKey string - secretKey string - scType string - Vars map[string]interface{} - client *cosSDK.Client + scType string + client *cosSDK.Client + clientWithBucket *cosSDK.Client } func NewCosClient(vars map[string]interface{}) (*cosClient, error) { - var accessKey string - var secretKey string - var scType string - var region string - if _, ok := vars["region"]; ok { - region = vars["region"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["accessKey"]; ok { - accessKey = vars["accessKey"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["scType"]; ok { - scType = vars["scType"].(string) - } else { + region := loadParamFromVars("region", true, vars) + accessKey := loadParamFromVars("accessKey", true, vars) + secretKey := loadParamFromVars("secretKey", true, vars) + bucket := loadParamFromVars("bucket", true, vars) + scType := loadParamFromVars("scType", true, vars) + if len(scType) == 0 { scType = "Standard" } - if _, ok := vars["secretKey"]; ok { - secretKey = vars["secretKey"].(string) - } else { - return nil, constant.ErrInvalidParams - } u, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", region)) b := &cosSDK.BaseURL{BucketURL: u} @@ -55,11 +35,23 @@ func NewCosClient(vars map[string]interface{}) (*cosClient, error) { }, }) - return &cosClient{Vars: vars, client: client, accessKey: accessKey, secretKey: secretKey, scType: scType, region: region}, nil + if len(bucket) != 0 { + u2, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", bucket, region)) + b2 := &cosSDK.BaseURL{BucketURL: u2} + clientWithBucket := cosSDK.NewClient(b2, &http.Client{ + Transport: &cosSDK.AuthorizationTransport{ + SecretID: accessKey, + SecretKey: secretKey, + }, + }) + return &cosClient{client: client, clientWithBucket: clientWithBucket, scType: scType}, nil + } + + return &cosClient{client: client, clientWithBucket: nil, scType: scType}, nil } -func (cos cosClient) ListBuckets() ([]interface{}, error) { - buckets, _, err := cos.client.Service.Get(context.Background()) +func (c cosClient) ListBuckets() ([]interface{}, error) { + buckets, _, err := c.client.Service.Get(context.Background()) if err != nil { return nil, err } @@ -70,34 +62,33 @@ func (cos cosClient) ListBuckets() ([]interface{}, error) { return datas, nil } -func (cos cosClient) Exist(path string) (bool, error) { - client, err := cos.newClientWithBucket() - if err != nil { - return false, err - } - exist, err := client.Object.IsExist(context.Background(), path) +func (c cosClient) Exist(path string) (bool, error) { + exist, err := c.clientWithBucket.Object.IsExist(context.Background(), path) if err != nil { return false, err } return exist, nil } -func (cos cosClient) Delete(path string) (bool, error) { - client, err := cos.newClientWithBucket() +func (c cosClient) Size(path string) (int64, error) { + data, _, err := c.clientWithBucket.Bucket.Get(context.Background(), &cosSDK.BucketGetOptions{Prefix: path}) if err != nil { - return false, err + return 0, err } - if _, err := client.Object.Delete(context.Background(), path); err != nil { + if len(data.Contents) == 0 { + return 0, fmt.Errorf("no such file %s", path) + } + return data.Contents[0].Size, nil +} + +func (c cosClient) Delete(path string) (bool, error) { + if _, err := c.clientWithBucket.Object.Delete(context.Background(), path); err != nil { return false, err } return true, nil } -func (cos cosClient) Upload(src, target string) (bool, error) { - client, err := cos.newClientWithBucket() - if err != nil { - return false, err - } +func (c cosClient) Upload(src, target string) (bool, error) { fileInfo, err := os.Stat(src) if err != nil { return false, err @@ -107,22 +98,22 @@ func (cos cosClient) Upload(src, target string) (bool, error) { OptIni: &cosSDK.InitiateMultipartUploadOptions{ ACLHeaderOptions: nil, ObjectPutHeaderOptions: &cosSDK.ObjectPutHeaderOptions{ - XCosStorageClass: cos.scType, + XCosStorageClass: c.scType, }, }, PartSize: 200, } - if _, _, err := client.Object.MultiUpload( + if _, _, err := c.clientWithBucket.Object.MultiUpload( context.Background(), target, src, opt, ); err != nil { return false, err } return true, nil } - if _, err := client.Object.PutFromFile(context.Background(), target, src, &cosSDK.ObjectPutOptions{ + if _, err := c.clientWithBucket.Object.PutFromFile(context.Background(), target, src, &cosSDK.ObjectPutOptions{ ACLHeaderOptions: nil, ObjectPutHeaderOptions: &cosSDK.ObjectPutHeaderOptions{ - XCosStorageClass: cos.scType, + XCosStorageClass: c.scType, }, }); err != nil { return false, err @@ -130,31 +121,15 @@ func (cos cosClient) Upload(src, target string) (bool, error) { return true, nil } -func (cos cosClient) Download(src, target string) (bool, error) { - client, err := cos.newClientWithBucket() - if err != nil { - return false, err - } - if _, err := client.Object.Download(context.Background(), src, target, &cosSDK.MultiDownloadOptions{}); err != nil { +func (c cosClient) Download(src, target string) (bool, error) { + if _, err := c.clientWithBucket.Object.Download(context.Background(), src, target, &cosSDK.MultiDownloadOptions{}); err != nil { return false, err } return true, nil } -func (cos *cosClient) GetBucket() (string, error) { - if _, ok := cos.Vars["bucket"]; ok { - return cos.Vars["bucket"].(string), nil - } else { - return "", constant.ErrInvalidParams - } -} - -func (cos cosClient) ListObjects(prefix string) ([]string, error) { - client, err := cos.newClientWithBucket() - if err != nil { - return nil, err - } - datas, _, err := client.Bucket.Get(context.Background(), &cosSDK.BucketGetOptions{Prefix: prefix}) +func (c cosClient) ListObjects(prefix string) ([]string, error) { + datas, _, err := c.clientWithBucket.Bucket.Get(context.Background(), &cosSDK.BucketGetOptions{Prefix: prefix}) if err != nil { return nil, err } @@ -165,19 +140,3 @@ func (cos cosClient) ListObjects(prefix string) ([]string, error) { } return result, nil } - -func (cos cosClient) newClientWithBucket() (*cosSDK.Client, error) { - bucket, err := cos.GetBucket() - if err != nil { - return nil, err - } - u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", bucket, cos.region)) - b := &cosSDK.BaseURL{BucketURL: u} - client := cosSDK.NewClient(b, &http.Client{ - Transport: &cosSDK.AuthorizationTransport{ - SecretID: cos.accessKey, - SecretKey: cos.secretKey, - }, - }) - return client, nil -} diff --git a/backend/utils/cloud_storage/client/helper.go b/backend/utils/cloud_storage/client/helper.go new file mode 100644 index 000000000..0c05dc5ac --- /dev/null +++ b/backend/utils/cloud_storage/client/helper.go @@ -0,0 +1,20 @@ +package client + +import ( + "fmt" + + "github.com/1Panel-dev/1Panel/backend/global" +) + +func loadParamFromVars(key string, isString bool, vars map[string]interface{}) string { + if _, ok := vars[key]; !ok { + if key != "bucket" { + global.LOG.Errorf("load param %s from vars failed, err: not exist!", key) + } + return "" + } + if isString { + return vars[key].(string) + } + return fmt.Sprintf("%v", vars[key].(float64)) +} diff --git a/backend/utils/cloud_storage/client/kodo.go b/backend/utils/cloud_storage/client/kodo.go index 872d90af1..80a4fb64e 100644 --- a/backend/utils/cloud_storage/client/kodo.go +++ b/backend/utils/cloud_storage/client/kodo.go @@ -4,45 +4,35 @@ import ( "context" "time" - "github.com/1Panel-dev/1Panel/backend/constant" "github.com/1Panel-dev/1Panel/backend/utils/files" "github.com/qiniu/go-sdk/v7/auth" - "github.com/qiniu/go-sdk/v7/auth/qbox" "github.com/qiniu/go-sdk/v7/storage" ) type kodoClient struct { - accessKey string - secretKey string - Vars map[string]interface{} - client *storage.BucketManager + bucket string + domain string + auth *auth.Credentials + client *storage.BucketManager } func NewKodoClient(vars map[string]interface{}) (*kodoClient, error) { - var accessKey string - var secretKey string - if _, ok := vars["accessKey"]; ok { - accessKey = vars["accessKey"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["secretKey"]; ok { - secretKey = vars["secretKey"].(string) - } else { - return nil, constant.ErrInvalidParams - } + accessKey := loadParamFromVars("accessKey", true, vars) + secretKey := loadParamFromVars("secretKey", true, vars) + bucket := loadParamFromVars("bucket", true, vars) + domain := loadParamFromVars("domain", true, vars) - conn := qbox.NewMac(accessKey, secretKey) + conn := auth.New(accessKey, secretKey) cfg := storage.Config{ UseHTTPS: false, } bucketManager := storage.NewBucketManager(conn, &cfg) - return &kodoClient{Vars: vars, client: bucketManager, accessKey: accessKey, secretKey: secretKey}, nil + return &kodoClient{client: bucketManager, auth: conn, bucket: bucket, domain: domain}, nil } -func (kodo kodoClient) ListBuckets() ([]interface{}, error) { - buckets, err := kodo.client.Buckets(true) +func (k kodoClient) ListBuckets() ([]interface{}, error) { + buckets, err := k.client.Buckets(true) if err != nil { return nil, err } @@ -53,38 +43,33 @@ func (kodo kodoClient) ListBuckets() ([]interface{}, error) { return datas, nil } -func (kodo kodoClient) Exist(path string) (bool, error) { - bucket, err := kodo.GetBucket() - if err != nil { - return false, err - } - if _, err := kodo.client.Stat(bucket, path); err != nil { +func (k kodoClient) Exist(path string) (bool, error) { + if _, err := k.client.Stat(k.bucket, path); err != nil { return false, err } return true, nil } -func (kodo kodoClient) Delete(path string) (bool, error) { - bucket, err := kodo.GetBucket() +func (k kodoClient) Size(path string) (int64, error) { + file, err := k.client.Stat(k.bucket, path) if err != nil { - return false, err + return 0, err } - if err := kodo.client.Delete(bucket, path); err != nil { + return file.Fsize, nil +} + +func (k kodoClient) Delete(path string) (bool, error) { + if err := k.client.Delete(k.bucket, path); err != nil { return false, err } return true, nil } -func (kodo kodoClient) Upload(src, target string) (bool, error) { - bucket, err := kodo.GetBucket() - if err != nil { - return false, err - } +func (k kodoClient) Upload(src, target string) (bool, error) { putPolicy := storage.PutPolicy{ - Scope: bucket, + Scope: k.bucket, } - mac := qbox.NewMac(kodo.accessKey, kodo.secretKey) - upToken := putPolicy.UploadToken(mac) + upToken := putPolicy.UploadToken(k.auth) cfg := storage.Config{UseHTTPS: true, UseCdnDomains: false} resumeUploader := storage.NewResumeUploaderV2(&cfg) ret := storage.PutRet{} @@ -95,14 +80,9 @@ func (kodo kodoClient) Upload(src, target string) (bool, error) { return true, nil } -func (kodo kodoClient) Download(src, target string) (bool, error) { - mac := auth.New(kodo.accessKey, kodo.secretKey) - if _, ok := kodo.Vars["domain"]; !ok { - return false, constant.ErrInvalidParams - } - domain := kodo.Vars["domain"].(string) +func (k kodoClient) Download(src, target string) (bool, error) { deadline := time.Now().Add(time.Second * 3600).Unix() - privateAccessURL := storage.MakePrivateURL(mac, domain, src, deadline) + privateAccessURL := storage.MakePrivateURL(k.auth, k.domain, src, deadline) fo := files.NewFileOp() if err := fo.DownloadFile(privateAccessURL, target); err != nil { @@ -111,24 +91,11 @@ func (kodo kodoClient) Download(src, target string) (bool, error) { return true, nil } -func (kodo *kodoClient) GetBucket() (string, error) { - if _, ok := kodo.Vars["bucket"]; ok { - return kodo.Vars["bucket"].(string), nil - } else { - return "", constant.ErrInvalidParams - } -} - -func (kodo kodoClient) ListObjects(prefix string) ([]string, error) { - bucket, err := kodo.GetBucket() - if err != nil { - return nil, constant.ErrInvalidParams - } - +func (k kodoClient) ListObjects(prefix string) ([]string, error) { var result []string marker := "" for { - entries, _, nextMarker, hashNext, err := kodo.client.ListFiles(bucket, prefix, "", marker, 1000) + entries, _, nextMarker, hashNext, err := k.client.ListFiles(k.bucket, prefix, "", marker, 1000) if err != nil { return nil, err } diff --git a/backend/utils/cloud_storage/client/minio.go b/backend/utils/cloud_storage/client/minio.go index 6264507e7..c52ae3ffb 100644 --- a/backend/utils/cloud_storage/client/minio.go +++ b/backend/utils/cloud_storage/client/minio.go @@ -14,29 +14,15 @@ import ( ) type minIoClient struct { - Vars map[string]interface{} + bucket string client *minio.Client } func NewMinIoClient(vars map[string]interface{}) (*minIoClient, error) { - var endpoint string - var accessKeyID string - var secretAccessKey string - if _, ok := vars["endpoint"]; ok { - endpoint = vars["endpoint"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["accessKey"]; ok { - accessKeyID = vars["accessKey"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["secretKey"]; ok { - secretAccessKey = vars["secretKey"].(string) - } else { - return nil, constant.ErrInvalidParams - } + endpoint := loadParamFromVars("endpoint", true, vars) + accessKeyID := loadParamFromVars("accessKey", true, vars) + secretAccessKey := loadParamFromVars("secretKey", true, vars) + bucket := loadParamFromVars("bucket", true, vars) ssl := strings.Split(endpoint, ":")[0] if len(ssl) == 0 || (ssl != "https" && ssl != "http") { return nil, constant.ErrInvalidParams @@ -59,14 +45,11 @@ func NewMinIoClient(vars map[string]interface{}) (*minIoClient, error) { if err != nil { return nil, err } - return &minIoClient{ - client: client, - Vars: vars, - }, nil + return &minIoClient{bucket: bucket, client: client}, nil } -func (minIo minIoClient) ListBuckets() ([]interface{}, error) { - buckets, err := minIo.client.ListBuckets(context.Background()) +func (m minIoClient) ListBuckets() ([]interface{}, error) { + buckets, err := m.client.ListBuckets(context.Background()) if err != nil { return nil, err } @@ -77,49 +60,44 @@ func (minIo minIoClient) ListBuckets() ([]interface{}, error) { return result, err } -func (minIo minIoClient) Exist(path string) (bool, error) { - if _, ok := minIo.Vars["bucket"]; ok { - _, err := minIo.client.GetObject(context.Background(), minIo.Vars["bucket"].(string), path, minio.GetObjectOptions{}) - if err != nil { - return false, err - } - return true, nil - } else { - return false, constant.ErrInvalidParams +func (m minIoClient) Exist(path string) (bool, error) { + if _, err := m.client.GetObject(context.Background(), m.bucket, path, minio.GetObjectOptions{}); err != nil { + return false, err } + return true, nil } -func (minIo minIoClient) Delete(path string) (bool, error) { - if _, ok := minIo.Vars["bucket"]; ok { - object, err := minIo.client.GetObject(context.Background(), minIo.Vars["bucket"].(string), path, minio.GetObjectOptions{}) - if err != nil { - return false, err - } - info, err := object.Stat() - if err != nil { - return false, err - } - err = minIo.client.RemoveObject(context.Background(), minIo.Vars["bucket"].(string), path, minio.RemoveObjectOptions{ - GovernanceBypass: true, - VersionID: info.VersionID, - }) - if err != nil { - return false, err - } - return true, nil - } else { - return false, constant.ErrInvalidParams +func (m minIoClient) Size(path string) (int64, error) { + obj, err := m.client.GetObject(context.Background(), m.bucket, path, minio.GetObjectOptions{}) + if err != nil { + return 0, err } + file, err := obj.Stat() + if err != nil { + return 0, err + } + return file.Size, nil } -func (minIo minIoClient) Upload(src, target string) (bool, error) { - var bucket string - if _, ok := minIo.Vars["bucket"]; ok { - bucket = minIo.Vars["bucket"].(string) - } else { - return false, constant.ErrInvalidParams +func (m minIoClient) Delete(path string) (bool, error) { + object, err := m.client.GetObject(context.Background(), m.bucket, path, minio.GetObjectOptions{}) + if err != nil { + return false, err } + info, err := object.Stat() + if err != nil { + return false, err + } + if err = m.client.RemoveObject(context.Background(), m.bucket, path, minio.RemoveObjectOptions{ + GovernanceBypass: true, + VersionID: info.VersionID, + }); err != nil { + return false, err + } + return true, nil +} +func (m minIoClient) Upload(src, target string) (bool, error) { file, err := os.Open(src) if err != nil { return false, err @@ -130,52 +108,36 @@ func (minIo minIoClient) Upload(src, target string) (bool, error) { if err != nil { return false, err } - _, err = minIo.client.PutObject(context.Background(), bucket, target, file, fileStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + _, err = m.client.PutObject(context.Background(), m.bucket, target, file, fileStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { return false, err } return true, nil } -func (minIo minIoClient) Download(src, target string) (bool, error) { - if _, ok := minIo.Vars["bucket"]; ok { - object, err := minIo.client.GetObject(context.Background(), minIo.Vars["bucket"].(string), src, minio.GetObjectOptions{}) - if err != nil { - return false, err - } - localFile, err := os.Create(target) - if err != nil { - return false, err - } - if _, err = io.Copy(localFile, object); err != nil { - return false, err - } - return true, nil - } else { - return false, constant.ErrInvalidParams - } -} - -func (minIo *minIoClient) GetBucket() (string, error) { - if _, ok := minIo.Vars["bucket"]; ok { - return minIo.Vars["bucket"].(string), nil - } else { - return "", constant.ErrInvalidParams - } -} - -func (minIo minIoClient) ListObjects(prefix string) ([]string, error) { - bucket, err := minIo.GetBucket() +func (m minIoClient) Download(src, target string) (bool, error) { + object, err := m.client.GetObject(context.Background(), m.bucket, src, minio.GetObjectOptions{}) if err != nil { - return nil, constant.ErrInvalidParams + return false, err } + localFile, err := os.Create(target) + if err != nil { + return false, err + } + if _, err = io.Copy(localFile, object); err != nil { + return false, err + } + return true, nil +} + +func (m minIoClient) ListObjects(prefix string) ([]string, error) { opts := minio.ListObjectsOptions{ Recursive: true, Prefix: prefix, } var result []string - for object := range minIo.client.ListObjects(context.Background(), bucket, opts) { + for object := range m.client.ListObjects(context.Background(), m.bucket, opts) { if object.Err != nil { continue } diff --git a/backend/utils/cloud_storage/client/onedrive.go b/backend/utils/cloud_storage/client/onedrive.go index f86d1e810..b349d9ff9 100644 --- a/backend/utils/cloud_storage/client/onedrive.go +++ b/backend/utils/cloud_storage/client/onedrive.go @@ -23,17 +23,11 @@ import ( ) type oneDriveClient struct { - Vars map[string]interface{} client odsdk.Client } func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) { - token := "" - if _, ok := vars["accessToken"]; ok { - token = vars["accessToken"].(string) - } else { - return nil, constant.ErrInvalidParams - } + token := loadParamFromVars("accessToken", true, vars) ctx := context.Background() newToken, err := refreshToken(token) @@ -51,13 +45,13 @@ func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) { return &oneDriveClient{client: *client}, nil } -func (onedrive oneDriveClient) ListBuckets() ([]interface{}, error) { +func (o oneDriveClient) ListBuckets() ([]interface{}, error) { return nil, nil } -func (onedrive oneDriveClient) Exist(path string) (bool, error) { +func (o oneDriveClient) Exist(path string) (bool, error) { path = "/" + strings.TrimPrefix(path, "/") - fileID, err := onedrive.loadIDByPath(path) + fileID, err := o.loadIDByPath(path) if err != nil { return false, err } @@ -65,26 +59,50 @@ func (onedrive oneDriveClient) Exist(path string) (bool, error) { return len(fileID) != 0, nil } -func (onedrive oneDriveClient) Delete(path string) (bool, error) { +func (o oneDriveClient) Size(path string) (int64, error) { path = "/" + strings.TrimPrefix(path, "/") - req, err := onedrive.client.NewRequest("DELETE", fmt.Sprintf("me/drive/root:%s", path), nil) + pathItem := "root:" + path + if path == "/" { + pathItem = "root" + } + req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil) + if err != nil { + return 0, fmt.Errorf("new request for file id failed, err: %v", err) + } + var driveItem myDriverItem + if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil { + return 0, fmt.Errorf("do request for file id failed, err: %v", err) + } + + return driveItem.Size, nil +} + +type myDriverItem struct { + Name string `json:"name"` + Id string `json:"id"` + Size int64 `json:"size"` +} + +func (o oneDriveClient) Delete(path string) (bool, error) { + path = "/" + strings.TrimPrefix(path, "/") + req, err := o.client.NewRequest("DELETE", fmt.Sprintf("me/drive/root:%s", path), nil) if err != nil { return false, fmt.Errorf("new request for delete file failed, err: %v \n", err) } - if err := onedrive.client.Do(context.Background(), req, false, nil); err != nil { + if err := o.client.Do(context.Background(), req, false, nil); err != nil { return false, fmt.Errorf("do request for delete file failed, err: %v \n", err) } return true, nil } -func (onedrive oneDriveClient) Upload(src, target string) (bool, error) { +func (o oneDriveClient) Upload(src, target string) (bool, error) { target = "/" + strings.TrimPrefix(target, "/") - if _, err := onedrive.loadIDByPath(path.Dir(target)); err != nil { + if _, err := o.loadIDByPath(path.Dir(target)); err != nil { if !strings.Contains(err.Error(), "itemNotFound") { return false, err } - if err := onedrive.createFolder(path.Dir(target)); err != nil { + if err := o.createFolder(path.Dir(target)); err != nil { return false, fmt.Errorf("create dir before upload failed, err: %v", err) } } @@ -105,7 +123,7 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) { fileName := fileInfo.Name() fileSize := fileInfo.Size() - folderID, err := onedrive.loadIDByPath(path.Dir(target)) + folderID, err := o.loadIDByPath(path.Dir(target)) if err != nil { return false, err } @@ -119,13 +137,13 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) { DeferCommit bool `json:"deferCommit"` }{sessionCreationRequestInside, false} - sessionCreationReq, err := onedrive.client.NewRequest("POST", apiURL, sessionCreationRequest) + sessionCreationReq, err := o.client.NewRequest("POST", apiURL, sessionCreationRequest) if err != nil { return false, err } var sessionCreationResp *NewUploadSessionCreationResponse - err = onedrive.client.Do(ctx, sessionCreationReq, false, &sessionCreationResp) + err = o.client.Do(ctx, sessionCreationReq, false, &sessionCreationResp) if err != nil { return false, fmt.Errorf("session creation failed %w", err) } @@ -149,11 +167,11 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) { bufferLast := buffer[:length] buffer = bufferLast } - sessionFileUploadReq, err := onedrive.NewSessionFileUploadRequest(fileSessionUploadUrl, splitNow*sizePerSplit, fileSize, bytes.NewReader(buffer)) + sessionFileUploadReq, err := o.NewSessionFileUploadRequest(fileSessionUploadUrl, splitNow*sizePerSplit, fileSize, bytes.NewReader(buffer)) if err != nil { return false, err } - if err := onedrive.client.Do(ctx, sessionFileUploadReq, false, &fileUploadResp); err != nil { + if err := o.client.Do(ctx, sessionFileUploadReq, false, &fileUploadResp); err != nil { return false, err } } @@ -164,14 +182,14 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) { return true, nil } -func (onedrive oneDriveClient) Download(src, target string) (bool, error) { +func (o oneDriveClient) Download(src, target string) (bool, error) { src = "/" + strings.TrimPrefix(src, "/") - req, err := onedrive.client.NewRequest("GET", fmt.Sprintf("me/drive/root:%s", src), nil) + req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/root:%s", src), nil) if err != nil { return false, fmt.Errorf("new request for file id failed, err: %v", err) } var driveItem *odsdk.DriveItem - if err := onedrive.client.Do(context.Background(), req, false, &driveItem); err != nil { + if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil { return false, fmt.Errorf("do request for file id failed, err: %v", err) } @@ -196,19 +214,19 @@ func (onedrive oneDriveClient) Download(src, target string) (bool, error) { return true, nil } -func (onedrive *oneDriveClient) ListObjects(prefix string) ([]string, error) { +func (o *oneDriveClient) ListObjects(prefix string) ([]string, error) { prefix = "/" + strings.TrimPrefix(prefix, "/") - folderID, err := onedrive.loadIDByPath(prefix) + folderID, err := o.loadIDByPath(prefix) if err != nil { return nil, err } - req, err := onedrive.client.NewRequest("GET", fmt.Sprintf("me/drive/items/%s/children", folderID), nil) + req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/items/%s/children", folderID), nil) if err != nil { return nil, fmt.Errorf("new request for list failed, err: %v", err) } var driveItems *odsdk.OneDriveDriveItemsResponse - if err := onedrive.client.Do(context.Background(), req, false, &driveItems); err != nil { + if err := o.client.Do(context.Background(), req, false, &driveItems); err != nil { return nil, fmt.Errorf("do request for list failed, err: %v", err) } @@ -219,17 +237,17 @@ func (onedrive *oneDriveClient) ListObjects(prefix string) ([]string, error) { return itemList, nil } -func (onedrive *oneDriveClient) loadIDByPath(path string) (string, error) { +func (o *oneDriveClient) loadIDByPath(path string) (string, error) { pathItem := "root:" + path if path == "/" { pathItem = "root" } - req, err := onedrive.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil) + req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil) if err != nil { return "", fmt.Errorf("new request for file id failed, err: %v", err) } var driveItem *odsdk.DriveItem - if err := onedrive.client.Do(context.Background(), req, false, &driveItem); err != nil { + if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil { return "", fmt.Errorf("do request for file id failed, err: %v", err) } return driveItem.Id, nil @@ -269,18 +287,18 @@ func refreshToken(oldToken string) (string, error) { return accessToken, nil } -func (onedrive *oneDriveClient) createFolder(parent string) error { - if _, err := onedrive.loadIDByPath(path.Dir(parent)); err != nil { +func (o *oneDriveClient) createFolder(parent string) error { + if _, err := o.loadIDByPath(path.Dir(parent)); err != nil { if !strings.Contains(err.Error(), "itemNotFound") { return err } - _ = onedrive.createFolder(path.Dir(parent)) + _ = o.createFolder(path.Dir(parent)) } - item2, err := onedrive.loadIDByPath(path.Dir(parent)) + item2, err := o.loadIDByPath(path.Dir(parent)) if err != nil { return err } - if _, err := onedrive.client.DriveItems.CreateNewFolder(context.Background(), "", item2, path.Base(parent)); err != nil { + if _, err := o.client.DriveItems.CreateNewFolder(context.Background(), "", item2, path.Base(parent)); err != nil { return err } return nil @@ -307,8 +325,8 @@ type DriveItem struct { WebURL string `json:"webUrl"` } -func (onedrive *oneDriveClient) NewSessionFileUploadRequest(absoluteUrl string, grandOffset, grandTotalSize int64, byteReader *bytes.Reader) (*http.Request, error) { - apiUrl, err := onedrive.client.BaseURL.Parse(absoluteUrl) +func (o *oneDriveClient) NewSessionFileUploadRequest(absoluteUrl string, grandOffset, grandTotalSize int64, byteReader *bytes.Reader) (*http.Request, error) { + apiUrl, err := o.client.BaseURL.Parse(absoluteUrl) if err != nil { return nil, err } diff --git a/backend/utils/cloud_storage/client/oss.go b/backend/utils/cloud_storage/client/oss.go index b9116e950..635039140 100644 --- a/backend/utils/cloud_storage/client/oss.go +++ b/backend/utils/cloud_storage/client/oss.go @@ -1,54 +1,36 @@ package client import ( - "github.com/1Panel-dev/1Panel/backend/constant" + "fmt" + osssdk "github.com/aliyun/aliyun-oss-go-sdk/oss" ) type ossClient struct { - scType string - Vars map[string]interface{} - client osssdk.Client + scType string + bucketStr string + client osssdk.Client } func NewOssClient(vars map[string]interface{}) (*ossClient, error) { - var endpoint string - var accessKey string - var secretKey string - var scType string - if _, ok := vars["endpoint"]; ok { - endpoint = vars["endpoint"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["accessKey"]; ok { - accessKey = vars["accessKey"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["scType"]; ok { - scType = vars["scType"].(string) - } else { + endpoint := loadParamFromVars("endpoint", true, vars) + accessKey := loadParamFromVars("accessKey", true, vars) + secretKey := loadParamFromVars("secretKey", true, vars) + bucketStr := loadParamFromVars("bucket", true, vars) + scType := loadParamFromVars("scType", true, vars) + if len(scType) == 0 { scType = "Standard" } - if _, ok := vars["secretKey"]; ok { - secretKey = vars["secretKey"].(string) - } else { - return nil, constant.ErrInvalidParams - } client, err := osssdk.New(endpoint, accessKey, secretKey) if err != nil { return nil, err } - return &ossClient{ - scType: scType, - Vars: vars, - client: *client, - }, nil + + return &ossClient{scType: scType, bucketStr: bucketStr, client: *client}, nil } -func (oss ossClient) ListBuckets() ([]interface{}, error) { - response, err := oss.client.ListBuckets() +func (o ossClient) ListBuckets() ([]interface{}, error) { + response, err := o.client.ListBuckets() if err != nil { return nil, err } @@ -59,69 +41,72 @@ func (oss ossClient) ListBuckets() ([]interface{}, error) { return result, err } -func (oss ossClient) Exist(path string) (bool, error) { - bucket, err := oss.GetBucket() +func (o ossClient) Exist(path string) (bool, error) { + bucket, err := o.client.Bucket(o.bucketStr) if err != nil { return false, err } return bucket.IsObjectExist(path) - } -func (oss ossClient) Delete(path string) (bool, error) { - bucket, err := oss.GetBucket() +func (o ossClient) Size(path string) (int64, error) { + bucket, err := o.client.Bucket(o.bucketStr) + if err != nil { + return 0, err + } + lor, err := bucket.ListObjectsV2(osssdk.Prefix(path)) + if err != nil { + return 0, err + } + if len(lor.Objects) == 0 { + return 0, fmt.Errorf("no such file %s", path) + } + return lor.Objects[0].Size, nil +} + +func (o ossClient) Delete(path string) (bool, error) { + bucket, err := o.client.Bucket(o.bucketStr) if err != nil { return false, err } - err = bucket.DeleteObject(path) - if err != nil { + if err := bucket.DeleteObject(path); err != nil { return false, err } return true, nil } -func (oss ossClient) Upload(src, target string) (bool, error) { - bucket, err := oss.GetBucket() +func (o ossClient) Upload(src, target string) (bool, error) { + bucket, err := o.client.Bucket(o.bucketStr) if err != nil { return false, err } - err = bucket.UploadFile(target, src, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, ""), osssdk.ObjectStorageClass(osssdk.StorageClassType(oss.scType))) - if err != nil { + if err := bucket.UploadFile(target, src, + 200*1024*1024, + osssdk.Routines(5), + osssdk.Checkpoint(true, ""), + osssdk.ObjectStorageClass(osssdk.StorageClassType(o.scType))); err != nil { return false, err } return true, nil } -func (oss ossClient) Download(src, target string) (bool, error) { - bucket, err := oss.GetBucket() +func (o ossClient) Download(src, target string) (bool, error) { + bucket, err := o.client.Bucket(o.bucketStr) if err != nil { return false, err } - err = bucket.DownloadFile(src, target, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, "")) - if err != nil { + if err := bucket.DownloadFile(src, target, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, "")); err != nil { return false, err } return true, nil } -func (oss *ossClient) GetBucket() (*osssdk.Bucket, error) { - if _, ok := oss.Vars["bucket"]; ok { - bucket, err := oss.client.Bucket(oss.Vars["bucket"].(string)) - if err != nil { - return nil, err - } - return bucket, nil - } else { - return nil, constant.ErrInvalidParams - } -} - -func (oss *ossClient) ListObjects(prefix string) ([]string, error) { - bucket, err := oss.GetBucket() +func (o *ossClient) ListObjects(prefix string) ([]string, error) { + bucket, err := o.client.Bucket(o.bucketStr) if err != nil { - return nil, constant.ErrInvalidParams + return nil, err } - lor, err := bucket.ListObjects(osssdk.Prefix(prefix)) + lor, err := bucket.ListObjectsV2(osssdk.Prefix(prefix)) if err != nil { return nil, err } diff --git a/backend/utils/cloud_storage/client/s3.go b/backend/utils/cloud_storage/client/s3.go index a6bce842c..0f6cb96c8 100644 --- a/backend/utils/cloud_storage/client/s3.go +++ b/backend/utils/cloud_storage/client/s3.go @@ -3,7 +3,6 @@ package client import ( "os" - "github.com/1Panel-dev/1Panel/backend/constant" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" @@ -14,41 +13,20 @@ import ( type s3Client struct { scType string - Vars map[string]interface{} + bucket string Sess session.Session } func NewS3Client(vars map[string]interface{}) (*s3Client, error) { - var accessKey string - var secretKey string - var endpoint string - var scType string - var region string - if _, ok := vars["accessKey"]; ok { - accessKey = vars["accessKey"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["secretKey"]; ok { - secretKey = vars["secretKey"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["scType"]; ok { - scType = vars["scType"].(string) - } else { + accessKey := loadParamFromVars("accessKey", true, vars) + secretKey := loadParamFromVars("secretKey", true, vars) + endpoint := loadParamFromVars("endpoint", true, vars) + region := loadParamFromVars("region", true, vars) + bucket := loadParamFromVars("bucket", true, vars) + scType := loadParamFromVars("scType", true, vars) + if len(scType) == 0 { scType = "Standard" } - if _, ok := vars["endpoint"]; ok { - endpoint = vars["endpoint"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["region"]; ok { - region = vars["region"].(string) - } else { - return nil, constant.ErrInvalidParams - } sess, err := session.NewSession(&aws.Config{ Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""), Endpoint: aws.String(endpoint), @@ -59,16 +37,12 @@ func NewS3Client(vars map[string]interface{}) (*s3Client, error) { if err != nil { return nil, err } - return &s3Client{ - scType: scType, - Vars: vars, - Sess: *sess, - }, nil + return &s3Client{scType: scType, bucket: bucket, Sess: *sess}, nil } -func (s3C s3Client) ListBuckets() ([]interface{}, error) { +func (s s3Client) ListBuckets() ([]interface{}, error) { var result []interface{} - svc := s3.New(&s3C.Sess) + svc := s3.New(&s.Sess) res, err := svc.ListBuckets(nil) if err != nil { return nil, err @@ -79,17 +53,12 @@ func (s3C s3Client) ListBuckets() ([]interface{}, error) { return result, nil } -func (s3C s3Client) Exist(path string) (bool, error) { - bucket, err := s3C.getBucket() - if err != nil { - return false, err - } - svc := s3.New(&s3C.Sess) - _, err = svc.HeadObject(&s3.HeadObjectInput{ - Bucket: &bucket, +func (s s3Client) Exist(path string) (bool, error) { + svc := s3.New(&s.Sess) + if _, err := svc.HeadObject(&s3.HeadObjectInput{ + Bucket: &s.bucket, Key: &path, - }) - if err != nil { + }); err != nil { if aerr, ok := err.(awserr.RequestFailure); ok { if aerr.StatusCode() == 404 { return false, nil @@ -101,57 +70,53 @@ func (s3C s3Client) Exist(path string) (bool, error) { return true, nil } -func (s3C s3Client) Delete(path string) (bool, error) { - bucket, err := s3C.getBucket() - if err != nil { - return false, err - } - svc := s3.New(&s3C.Sess) - _, err = svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(path)}) - if err != nil { - return false, err - } - err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(path), +func (s *s3Client) Size(path string) (int64, error) { + svc := s3.New(&s.Sess) + file, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: &s.bucket, + Key: &path, }) if err != nil { + return 0, err + } + return *file.ContentLength, nil +} + +func (s s3Client) Delete(path string) (bool, error) { + svc := s3.New(&s.Sess) + if _, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(path)}); err != nil { + return false, err + } + if err := svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(path), + }); err != nil { return false, err } return true, nil } -func (s3C s3Client) Upload(src, target string) (bool, error) { - bucket, err := s3C.getBucket() - if err != nil { - return false, err - } +func (s s3Client) Upload(src, target string) (bool, error) { file, err := os.Open(src) if err != nil { return false, err } defer file.Close() - uploader := s3manager.NewUploader(&s3C.Sess) - _, err = uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(bucket), + uploader := s3manager.NewUploader(&s.Sess) + if _, err := uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(s.bucket), Key: aws.String(target), Body: file, - StorageClass: &s3C.scType, - }) - if err != nil { + StorageClass: &s.scType, + }); err != nil { return false, err } return true, nil } -func (s3C s3Client) Download(src, target string) (bool, error) { - bucket, err := s3C.getBucket() - if err != nil { - return false, err - } - _, err = os.Stat(target) - if err != nil { +func (s s3Client) Download(src, target string) (bool, error) { + if _, err := os.Stat(target); err != nil { if os.IsNotExist(err) { os.Remove(target) } else { @@ -163,44 +128,29 @@ func (s3C s3Client) Download(src, target string) (bool, error) { return false, err } defer file.Close() - downloader := s3manager.NewDownloader(&s3C.Sess) - _, err = downloader.Download(file, - &s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(src), - }) - if err != nil { + downloader := s3manager.NewDownloader(&s.Sess) + if _, err = downloader.Download(file, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(src), + }); err != nil { os.Remove(target) return false, err } return true, nil } -func (s3C *s3Client) getBucket() (string, error) { - if _, ok := s3C.Vars["bucket"]; ok { - return s3C.Vars["bucket"].(string), nil - } else { - return "", constant.ErrInvalidParams - } -} - -func (s3C *s3Client) ListObjects(prefix string) ([]string, error) { - bucket, err := s3C.getBucket() - if err != nil { - return nil, constant.ErrInvalidParams - } - svc := s3.New(&s3C.Sess) +func (s *s3Client) ListObjects(prefix string) ([]string, error) { + svc := s3.New(&s.Sess) var result []string - if err := svc.ListObjectsPages(&s3.ListObjectsInput{ - Bucket: &bucket, + outputs, err := svc.ListObjects(&s3.ListObjectsInput{ + Bucket: &s.bucket, Prefix: &prefix, - }, func(p *s3.ListObjectsOutput, last bool) (shouldContinue bool) { - for _, obj := range p.Contents { - result = append(result, *obj.Key) - } - return true - }); err != nil { - return nil, err + }) + if err != nil { + return result, err + } + for _, item := range outputs.Contents { + result = append(result, *item.Key) } return result, nil } diff --git a/backend/utils/cloud_storage/client/sftp.go b/backend/utils/cloud_storage/client/sftp.go index d8c53f8b3..63af2744a 100644 --- a/backend/utils/cloud_storage/client/sftp.go +++ b/backend/utils/cloud_storage/client/sftp.go @@ -6,53 +6,49 @@ import ( "net" "os" "path" - "strconv" "time" - "github.com/1Panel-dev/1Panel/backend/constant" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" ) type sftpClient struct { - Bucket string - Vars map[string]interface{} - client *sftp.Client + bucket string + connInfo string + config *ssh.ClientConfig } func NewSftpClient(vars map[string]interface{}) (*sftpClient, error) { - if _, ok := vars["address"]; !ok { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["port"].(float64); !ok { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["password"]; !ok { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["username"]; !ok { - return nil, constant.ErrInvalidParams - } - var bucket string - if _, ok := vars["bucket"]; ok { - bucket = vars["bucket"].(string) - } else { - return nil, constant.ErrInvalidParams + address := loadParamFromVars("address", true, vars) + port := loadParamFromVars("port", false, vars) + password := loadParamFromVars("password", true, vars) + username := loadParamFromVars("username", true, vars) + bucket := loadParamFromVars("bucket", true, vars) + + auth := []ssh.AuthMethod{ssh.Password(password)} + clientConfig := &ssh.ClientConfig{ + User: username, + Auth: auth, + Timeout: 30 * time.Second, + HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { + return nil + }, } - port, err := strconv.Atoi(strconv.FormatFloat(vars["port"].(float64), 'G', -1, 64)) - if err != nil { - return nil, err - } - sftpC, err := connect(vars["username"].(string), vars["password"].(string), vars["address"].(string), port) - if err != nil { - return nil, err - } - return &sftpClient{Bucket: bucket, client: sftpC, Vars: vars}, nil + return &sftpClient{bucket: bucket, connInfo: fmt.Sprintf("%s:%s", address, port), config: clientConfig}, nil } func (s sftpClient) Upload(src, target string) (bool, error) { - defer s.client.Close() + sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) + if err != nil { + return false, err + } + client, err := sftp.NewClient(sshClient) + if err != nil { + return false, err + } + defer client.Close() + defer sshClient.Close() srcFile, err := os.Open(src) if err != nil { @@ -60,18 +56,18 @@ func (s sftpClient) Upload(src, target string) (bool, error) { } defer srcFile.Close() - targetFilePath := s.Bucket + "/" + target + targetFilePath := s.bucket + "/" + target targetDir, _ := path.Split(targetFilePath) - if _, err = s.client.Stat(targetDir); err != nil { + if _, err = client.Stat(targetDir); err != nil { if os.IsNotExist(err) { - if err = s.client.MkdirAll(targetDir); err != nil { + if err = client.MkdirAll(targetDir); err != nil { return false, err } } else { return false, err } } - dstFile, err := s.client.Create(targetFilePath) + dstFile, err := client.Create(targetFilePath) if err != nil { return false, err } @@ -97,8 +93,17 @@ func (s sftpClient) ListBuckets() ([]interface{}, error) { } func (s sftpClient) Download(src, target string) (bool, error) { - defer s.client.Close() - srcFile, err := s.client.Open(s.Bucket + "/" + src) + sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) + if err != nil { + return false, err + } + client, err := sftp.NewClient(sshClient) + if err != nil { + return false, err + } + defer client.Close() + defer sshClient.Close() + srcFile, err := client.Open(s.bucket + "/" + src) if err != nil { return false, err } @@ -117,8 +122,18 @@ func (s sftpClient) Download(src, target string) (bool, error) { } func (s sftpClient) Exist(path string) (bool, error) { - defer s.client.Close() - srcFile, err := s.client.Open(s.Bucket + "/" + path) + sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) + if err != nil { + return false, err + } + client, err := sftp.NewClient(sshClient) + if err != nil { + return false, err + } + defer client.Close() + defer sshClient.Close() + + srcFile, err := client.Open(s.bucket + "/" + path) if err != nil { if os.IsNotExist(err) { return false, nil @@ -130,48 +145,57 @@ func (s sftpClient) Exist(path string) (bool, error) { return true, err } +func (s sftpClient) Size(path string) (int64, error) { + sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) + if err != nil { + return 0, err + } + client, err := sftp.NewClient(sshClient) + if err != nil { + return 0, err + } + defer client.Close() + defer sshClient.Close() + + files, err := client.Stat(s.bucket + "/" + path) + if err != nil { + return 0, err + } + return files.Size(), nil +} + func (s sftpClient) Delete(filePath string) (bool, error) { - defer s.client.Close() - targetFilePath := s.Bucket + "/" + filePath - if err := s.client.Remove(targetFilePath); err != nil { + sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) + if err != nil { + return false, err + } + client, err := sftp.NewClient(sshClient) + if err != nil { + return false, err + } + defer client.Close() + defer sshClient.Close() + + targetFilePath := s.bucket + "/" + filePath + if err := client.Remove(targetFilePath); err != nil { return false, err } return true, nil } -func connect(user, password, host string, port int) (*sftp.Client, error) { - var ( - auth []ssh.AuthMethod - addr string - clientConfig *ssh.ClientConfig - sshClient *ssh.Client - sftpClient *sftp.Client - err error - ) - auth = make([]ssh.AuthMethod, 0) - auth = append(auth, ssh.Password(password)) - clientConfig = &ssh.ClientConfig{ - User: user, - Auth: auth, - Timeout: 30 * time.Second, - HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { - return nil - }, - } - addr = fmt.Sprintf("%s:%d", host, port) - - if sshClient, err = ssh.Dial("tcp", addr, clientConfig); err != nil { - return nil, err - } - if sftpClient, err = sftp.NewClient(sshClient); err != nil { - return nil, err - } - return sftpClient, nil -} - func (s sftpClient) ListObjects(prefix string) ([]string, error) { - defer s.client.Close() - files, err := s.client.ReadDir(s.Bucket + "/" + prefix) + sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) + if err != nil { + return nil, err + } + client, err := sftp.NewClient(sshClient) + if err != nil { + return nil, err + } + defer client.Close() + defer sshClient.Close() + + files, err := client.ReadDir(s.bucket + "/" + prefix) if err != nil { return nil, err } diff --git a/backend/utils/cloud_storage/client/webdav.go b/backend/utils/cloud_storage/client/webdav.go index 1034631b3..56699ceeb 100644 --- a/backend/utils/cloud_storage/client/webdav.go +++ b/backend/utils/cloud_storage/client/webdav.go @@ -5,52 +5,26 @@ import ( "io" "os" - "github.com/1Panel-dev/1Panel/backend/constant" "github.com/studio-b12/gowebdav" ) type webDAVClient struct { Bucket string client *gowebdav.Client - Vars map[string]interface{} } func NewWebDAVClient(vars map[string]interface{}) (*webDAVClient, error) { - var ( - address string - username string - password string - bucket string - ) - if _, ok := vars["address"]; ok { - address = vars["address"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["port"].(float64); !ok { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["username"]; ok { - username = vars["username"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["password"]; ok { - password = vars["password"].(string) - } else { - return nil, constant.ErrInvalidParams - } - if _, ok := vars["bucket"]; ok { - bucket = vars["bucket"].(string) - } else { - return nil, constant.ErrInvalidParams - } + address := loadParamFromVars("address", true, vars) + port := loadParamFromVars("port", false, vars) + password := loadParamFromVars("password", true, vars) + username := loadParamFromVars("username", true, vars) + bucket := loadParamFromVars("bucket", true, vars) - client := gowebdav.NewClient(fmt.Sprintf("%s:%v", address, vars["port"]), username, password) + client := gowebdav.NewClient(fmt.Sprintf("%s:%s", address, port), username, password) if err := client.Connect(); err != nil { return nil, err } - return &webDAVClient{Vars: vars, Bucket: bucket, client: client}, nil + return &webDAVClient{Bucket: bucket, client: client}, nil } func (s webDAVClient) Upload(src, target string) (bool, error) { @@ -115,6 +89,14 @@ func (s webDAVClient) Exist(path string) (bool, error) { return true, nil } +func (s webDAVClient) Size(path string) (int64, error) { + file, err := s.client.Stat(s.Bucket + "/" + path) + if err != nil { + return 0, err + } + return file.Size(), nil +} + func (s webDAVClient) Delete(filePath string) (bool, error) { if err := s.client.Remove(s.Bucket + "/" + filePath); err != nil { return false, err diff --git a/backend/utils/cloud_storage/cloud_storage_client.go b/backend/utils/cloud_storage/cloud_storage_client.go index cf5745206..dbaea36e0 100644 --- a/backend/utils/cloud_storage/cloud_storage_client.go +++ b/backend/utils/cloud_storage/cloud_storage_client.go @@ -12,6 +12,8 @@ type CloudStorageClient interface { Delete(path string) (bool, error) Upload(src, target string) (bool, error) Download(src, target string) (bool, error) + + Size(path string) (int64, error) } func NewCloudStorageClient(backupType string, vars map[string]interface{}) (CloudStorageClient, error) { diff --git a/frontend/src/components/backup/index.vue b/frontend/src/components/backup/index.vue index 1d563b1da..73deceb1f 100644 --- a/frontend/src/components/backup/index.vue +++ b/frontend/src/components/backup/index.vue @@ -27,6 +27,14 @@ + + +