mirror/1Panel
1
0
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2024-11-24 02:59:16 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: 备份列表显示文件备份文件大小 (#3134)

Browse Source 
Refs #2968 #1922
This commit is contained in:
ssongliu 2023-12-01 17:50:09 +08:00 committed by GitHub
parent 4daed1a3de
commit 64ad829e3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 491 additions and 587 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/app/dto/backup.go
View File

@ -52,6 +52,7 @@ type BackupRecords struct {
BackupType string `json:"backupType"`
FileDir string `json:"fileDir"`
FileName string `json:"fileName"`
Size int64 `json:"size"`
}
type DownloadRecord struct {

45
backend/app/service/backup.go
View File

@ -81,15 +81,46 @@ func (u *BackupService) SearchRecordsWithPage(search dto.RecordSearch) (int64, [
commonRepo.WithByType(search.Type),
backupRepo.WithByDetailName(search.DetailName),
)
var dtobas []dto.BackupRecords
for _, group := range records {
var datas []dto.BackupRecords
clientMap := make(map[string]loadSizeHelper)
for i := 0; i < len(records); i++ {
var item dto.BackupRecords
if err := copier.Copy(&item, &group); err != nil {
if err := copier.Copy(&item, &records[i]); err != nil {
return 0, nil, errors.WithMessage(constant.ErrStructTransform, err.Error())
}
dtobas = append(dtobas, item)
itemPath := path.Join(records[i].FileDir, records[i].FileName)
if records[i].Source == "LOCAL" {
fileInfo, _ := os.Stat(itemPath)
item.Size = fileInfo.Size()
datas = append(datas, item)
continue
}
if _, ok := clientMap[records[i].Source]; !ok {
backup, err := backupRepo.Get(commonRepo.WithByType(records[i].Source))
if err != nil {
global.LOG.Errorf("load backup model %s from db failed, err: %v", records[i].Source, err)
return total, datas, err
}
client, err := u.NewClient(&backup)
if err != nil {
global.LOG.Errorf("load backup client %s from db failed, err: %v", records[i].Source, err)
return total, datas, err
}
item.Size, _ = client.Size(path.Join(strings.TrimLeft(backup.BackupPath, "/"), itemPath))
datas = append(datas, item)
clientMap[records[i].Source] = loadSizeHelper{backupPath: strings.TrimLeft(backup.BackupPath, "/"), client: client}
continue
}
item.Size, _ = clientMap[records[i].Source].client.Size(path.Join(clientMap[records[i].Source].backupPath, itemPath))
datas = append(datas, item)
}
return total, dtobas, err
return total, datas, err
}
type loadSizeHelper struct {
backupPath string
client cloud_storage.CloudStorageClient
}
func (u *BackupService) LoadOneDriveInfo() (string, error) {
@ -139,9 +170,7 @@ func (u *BackupService) DownloadRecord(info dto.DownloadRecord) (string, error)
}
srcPath := fmt.Sprintf("%s/%s", info.FileDir, info.FileName)
if len(backup.BackupPath) != 0 {
itemPath := strings.TrimPrefix(backup.BackupPath, "/")
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
srcPath = itemPath + srcPath
srcPath = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), srcPath)
}
if exist, _ := backClient.Exist(srcPath); exist {
isOK, err := backClient.Download(srcPath, targetPath)

20
backend/app/service/cronjob_helper.go
View File

@ -157,9 +157,7 @@ func (u *CronjobService) handleBackup(cronjob *model.Cronjob, startTime time.Tim
return "", err
}
if len(backup.BackupPath) != 0 {
itemPath := strings.TrimPrefix(backup.BackupPath, "/")
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
itemFileDir = itemPath + itemFileDir
itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir)
}
if _, err = client.Upload(backupDir+"/"+fileName, itemFileDir+"/"+fileName); err != nil {
return "", err
@ -193,9 +191,7 @@ func (u *CronjobService) HandleRmExpired(backType, backupPath, localDir string,
fileItem := file
if cronjob.KeepLocal {
if len(backupPath) != 0 {
itemPath := strings.TrimPrefix(backupPath, "/")
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
fileItem = itemPath + strings.TrimPrefix(file, localDir+"/")
fileItem = path.Join(strings.TrimPrefix(backupPath, "/") + strings.TrimPrefix(file, localDir+"/"))
} else {
fileItem = strings.TrimPrefix(file, localDir+"/")
}
@ -332,9 +328,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, backup model.Back
}()
}
if len(backup.BackupPath) != 0 {
itemPath := strings.TrimPrefix(backup.BackupPath, "/")
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
itemFileDir = itemPath + itemFileDir
itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir)
}
if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil {
return paths, err
@ -475,9 +469,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, backup model.BackupAcc
}()
}
if len(backup.BackupPath) != 0 {
itemPath := strings.TrimPrefix(backup.BackupPath, "/")
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
itemFileDir = itemPath + itemFileDir
itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir)
}
if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil {
return paths, err
@ -551,9 +543,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, backup model.Backu
}()
}
if len(backup.BackupPath) != 0 {
itemPath := strings.TrimPrefix(backup.BackupPath, "/")
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
itemFileDir = itemPath + itemFileDir
itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir)
}
if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil {
return paths, err

133
backend/utils/cloud_storage/client/cos.go
View File

@ -7,44 +7,24 @@ import (
"net/url"
"os"
"github.com/1Panel-dev/1Panel/backend/constant"
cosSDK "github.com/tencentyun/cos-go-sdk-v5"
)
type cosClient struct {
region string
accessKey string
secretKey string
scType string
Vars map[string]interface{}
client *cosSDK.Client
scType string
client *cosSDK.Client
clientWithBucket *cosSDK.Client
}
func NewCosClient(vars map[string]interface{}) (*cosClient, error) {
var accessKey string
var secretKey string
var scType string
var region string
if _, ok := vars["region"]; ok {
region = vars["region"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["accessKey"]; ok {
accessKey = vars["accessKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["scType"]; ok {
scType = vars["scType"].(string)
} else {
region := loadParamFromVars("region", true, vars)
accessKey := loadParamFromVars("accessKey", true, vars)
secretKey := loadParamFromVars("secretKey", true, vars)
bucket := loadParamFromVars("bucket", true, vars)
scType := loadParamFromVars("scType", true, vars)
if len(scType) == 0 {
scType = "Standard"
}
if _, ok := vars["secretKey"]; ok {
secretKey = vars["secretKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
u, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", region))
b := &cosSDK.BaseURL{BucketURL: u}
@ -55,11 +35,23 @@ func NewCosClient(vars map[string]interface{}) (*cosClient, error) {
},
})
return &cosClient{Vars: vars, client: client, accessKey: accessKey, secretKey: secretKey, scType: scType, region: region}, nil
if len(bucket) != 0 {
u2, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", bucket, region))
b2 := &cosSDK.BaseURL{BucketURL: u2}
clientWithBucket := cosSDK.NewClient(b2, &http.Client{
Transport: &cosSDK.AuthorizationTransport{
SecretID: accessKey,
SecretKey: secretKey,
},
})
return &cosClient{client: client, clientWithBucket: clientWithBucket, scType: scType}, nil
}
return &cosClient{client: client, clientWithBucket: nil, scType: scType}, nil
}
func (cos cosClient) ListBuckets() ([]interface{}, error) {
buckets, _, err := cos.client.Service.Get(context.Background())
func (c cosClient) ListBuckets() ([]interface{}, error) {
buckets, _, err := c.client.Service.Get(context.Background())
if err != nil {
return nil, err
}
@ -70,34 +62,33 @@ func (cos cosClient) ListBuckets() ([]interface{}, error) {
return datas, nil
}
func (cos cosClient) Exist(path string) (bool, error) {
client, err := cos.newClientWithBucket()
if err != nil {
return false, err
}
exist, err := client.Object.IsExist(context.Background(), path)
func (c cosClient) Exist(path string) (bool, error) {
exist, err := c.clientWithBucket.Object.IsExist(context.Background(), path)
if err != nil {
return false, err
}
return exist, nil
}
func (cos cosClient) Delete(path string) (bool, error) {
client, err := cos.newClientWithBucket()
func (c cosClient) Size(path string) (int64, error) {
data, _, err := c.clientWithBucket.Bucket.Get(context.Background(), &cosSDK.BucketGetOptions{Prefix: path})
if err != nil {
return false, err
return 0, err
}
if _, err := client.Object.Delete(context.Background(), path); err != nil {
if len(data.Contents) == 0 {
return 0, fmt.Errorf("no such file %s", path)
}
return data.Contents[0].Size, nil
}
func (c cosClient) Delete(path string) (bool, error) {
if _, err := c.clientWithBucket.Object.Delete(context.Background(), path); err != nil {
return false, err
}
return true, nil
}
func (cos cosClient) Upload(src, target string) (bool, error) {
client, err := cos.newClientWithBucket()
if err != nil {
return false, err
}
func (c cosClient) Upload(src, target string) (bool, error) {
fileInfo, err := os.Stat(src)
if err != nil {
return false, err
@ -107,22 +98,22 @@ func (cos cosClient) Upload(src, target string) (bool, error) {
OptIni: &cosSDK.InitiateMultipartUploadOptions{
ACLHeaderOptions: nil,
ObjectPutHeaderOptions: &cosSDK.ObjectPutHeaderOptions{
XCosStorageClass: cos.scType,
XCosStorageClass: c.scType,
},
},
PartSize: 200,
}
if _, _, err := client.Object.MultiUpload(
if _, _, err := c.clientWithBucket.Object.MultiUpload(
context.Background(), target, src, opt,
); err != nil {
return false, err
}
return true, nil
}
if _, err := client.Object.PutFromFile(context.Background(), target, src, &cosSDK.ObjectPutOptions{
if _, err := c.clientWithBucket.Object.PutFromFile(context.Background(), target, src, &cosSDK.ObjectPutOptions{
ACLHeaderOptions: nil,
ObjectPutHeaderOptions: &cosSDK.ObjectPutHeaderOptions{
XCosStorageClass: cos.scType,
XCosStorageClass: c.scType,
},
}); err != nil {
return false, err
@ -130,31 +121,15 @@ func (cos cosClient) Upload(src, target string) (bool, error) {
return true, nil
}
func (cos cosClient) Download(src, target string) (bool, error) {
client, err := cos.newClientWithBucket()
if err != nil {
return false, err
}
if _, err := client.Object.Download(context.Background(), src, target, &cosSDK.MultiDownloadOptions{}); err != nil {
func (c cosClient) Download(src, target string) (bool, error) {
if _, err := c.clientWithBucket.Object.Download(context.Background(), src, target, &cosSDK.MultiDownloadOptions{}); err != nil {
return false, err
}
return true, nil
}
func (cos *cosClient) GetBucket() (string, error) {
if _, ok := cos.Vars["bucket"]; ok {
return cos.Vars["bucket"].(string), nil
} else {
return "", constant.ErrInvalidParams
}
}
func (cos cosClient) ListObjects(prefix string) ([]string, error) {
client, err := cos.newClientWithBucket()
if err != nil {
return nil, err
}
datas, _, err := client.Bucket.Get(context.Background(), &cosSDK.BucketGetOptions{Prefix: prefix})
func (c cosClient) ListObjects(prefix string) ([]string, error) {
datas, _, err := c.clientWithBucket.Bucket.Get(context.Background(), &cosSDK.BucketGetOptions{Prefix: prefix})
if err != nil {
return nil, err
}
@ -165,19 +140,3 @@ func (cos cosClient) ListObjects(prefix string) ([]string, error) {
}
return result, nil
}
func (cos cosClient) newClientWithBucket() (*cosSDK.Client, error) {
bucket, err := cos.GetBucket()
if err != nil {
return nil, err
}
u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", bucket, cos.region))
b := &cosSDK.BaseURL{BucketURL: u}
client := cosSDK.NewClient(b, &http.Client{
Transport: &cosSDK.AuthorizationTransport{
SecretID: cos.accessKey,
SecretKey: cos.secretKey,
},
})
return client, nil
}

20
backend/utils/cloud_storage/client/helper.go Normal file
View File

@ -0,0 +1,20 @@
package client
import (
"fmt"
"github.com/1Panel-dev/1Panel/backend/global"
)
func loadParamFromVars(key string, isString bool, vars map[string]interface{}) string {
if _, ok := vars[key]; !ok {
if key != "bucket" {
global.LOG.Errorf("load param %s from vars failed, err: not exist!", key)
}
return ""
}
if isString {
return vars[key].(string)
}
return fmt.Sprintf("%v", vars[key].(float64))
}

91
backend/utils/cloud_storage/client/kodo.go
View File

@ -4,45 +4,35 @@ import (
"context"
"time"
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/1Panel-dev/1Panel/backend/utils/files"
"github.com/qiniu/go-sdk/v7/auth"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
)
type kodoClient struct {
accessKey string
secretKey string
Vars map[string]interface{}
client *storage.BucketManager
bucket string
domain string
auth *auth.Credentials
client *storage.BucketManager
}
func NewKodoClient(vars map[string]interface{}) (*kodoClient, error) {
var accessKey string
var secretKey string
if _, ok := vars["accessKey"]; ok {
accessKey = vars["accessKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["secretKey"]; ok {
secretKey = vars["secretKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
accessKey := loadParamFromVars("accessKey", true, vars)
secretKey := loadParamFromVars("secretKey", true, vars)
bucket := loadParamFromVars("bucket", true, vars)
domain := loadParamFromVars("domain", true, vars)
conn := qbox.NewMac(accessKey, secretKey)
conn := auth.New(accessKey, secretKey)
cfg := storage.Config{
UseHTTPS: false,
}
bucketManager := storage.NewBucketManager(conn, &cfg)
return &kodoClient{Vars: vars, client: bucketManager, accessKey: accessKey, secretKey: secretKey}, nil
return &kodoClient{client: bucketManager, auth: conn, bucket: bucket, domain: domain}, nil
}
func (kodo kodoClient) ListBuckets() ([]interface{}, error) {
buckets, err := kodo.client.Buckets(true)
func (k kodoClient) ListBuckets() ([]interface{}, error) {
buckets, err := k.client.Buckets(true)
if err != nil {
return nil, err
}
@ -53,38 +43,33 @@ func (kodo kodoClient) ListBuckets() ([]interface{}, error) {
return datas, nil
}
func (kodo kodoClient) Exist(path string) (bool, error) {
bucket, err := kodo.GetBucket()
if err != nil {
return false, err
}
if _, err := kodo.client.Stat(bucket, path); err != nil {
func (k kodoClient) Exist(path string) (bool, error) {
if _, err := k.client.Stat(k.bucket, path); err != nil {
return false, err
}
return true, nil
}
func (kodo kodoClient) Delete(path string) (bool, error) {
bucket, err := kodo.GetBucket()
func (k kodoClient) Size(path string) (int64, error) {
file, err := k.client.Stat(k.bucket, path)
if err != nil {
return false, err
return 0, err
}
if err := kodo.client.Delete(bucket, path); err != nil {
return file.Fsize, nil
}
func (k kodoClient) Delete(path string) (bool, error) {
if err := k.client.Delete(k.bucket, path); err != nil {
return false, err
}
return true, nil
}
func (kodo kodoClient) Upload(src, target string) (bool, error) {
bucket, err := kodo.GetBucket()
if err != nil {
return false, err
}
func (k kodoClient) Upload(src, target string) (bool, error) {
putPolicy := storage.PutPolicy{
Scope: bucket,
Scope: k.bucket,
}
mac := qbox.NewMac(kodo.accessKey, kodo.secretKey)
upToken := putPolicy.UploadToken(mac)
upToken := putPolicy.UploadToken(k.auth)
cfg := storage.Config{UseHTTPS: true, UseCdnDomains: false}
resumeUploader := storage.NewResumeUploaderV2(&cfg)
ret := storage.PutRet{}
@ -95,14 +80,9 @@ func (kodo kodoClient) Upload(src, target string) (bool, error) {
return true, nil
}
func (kodo kodoClient) Download(src, target string) (bool, error) {
mac := auth.New(kodo.accessKey, kodo.secretKey)
if _, ok := kodo.Vars["domain"]; !ok {
return false, constant.ErrInvalidParams
}
domain := kodo.Vars["domain"].(string)
func (k kodoClient) Download(src, target string) (bool, error) {
deadline := time.Now().Add(time.Second * 3600).Unix()
privateAccessURL := storage.MakePrivateURL(mac, domain, src, deadline)
privateAccessURL := storage.MakePrivateURL(k.auth, k.domain, src, deadline)
fo := files.NewFileOp()
if err := fo.DownloadFile(privateAccessURL, target); err != nil {
@ -111,24 +91,11 @@ func (kodo kodoClient) Download(src, target string) (bool, error) {
return true, nil
}
func (kodo *kodoClient) GetBucket() (string, error) {
if _, ok := kodo.Vars["bucket"]; ok {
return kodo.Vars["bucket"].(string), nil
} else {
return "", constant.ErrInvalidParams
}
}
func (kodo kodoClient) ListObjects(prefix string) ([]string, error) {
bucket, err := kodo.GetBucket()
if err != nil {
return nil, constant.ErrInvalidParams
}
func (k kodoClient) ListObjects(prefix string) ([]string, error) {
var result []string
marker := ""
for {
entries, _, nextMarker, hashNext, err := kodo.client.ListFiles(bucket, prefix, "", marker, 1000)
entries, _, nextMarker, hashNext, err := k.client.ListFiles(k.bucket, prefix, "", marker, 1000)
if err != nil {
return nil, err
}

146
backend/utils/cloud_storage/client/minio.go
View File

@ -14,29 +14,15 @@ import (
)
type minIoClient struct {
Vars map[string]interface{}
bucket string
client *minio.Client
}
func NewMinIoClient(vars map[string]interface{}) (*minIoClient, error) {
var endpoint string
var accessKeyID string
var secretAccessKey string
if _, ok := vars["endpoint"]; ok {
endpoint = vars["endpoint"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["accessKey"]; ok {
accessKeyID = vars["accessKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["secretKey"]; ok {
secretAccessKey = vars["secretKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
endpoint := loadParamFromVars("endpoint", true, vars)
accessKeyID := loadParamFromVars("accessKey", true, vars)
secretAccessKey := loadParamFromVars("secretKey", true, vars)
bucket := loadParamFromVars("bucket", true, vars)
ssl := strings.Split(endpoint, ":")[0]
if len(ssl) == 0 || (ssl != "https" && ssl != "http") {
return nil, constant.ErrInvalidParams
@ -59,14 +45,11 @@ func NewMinIoClient(vars map[string]interface{}) (*minIoClient, error) {
if err != nil {
return nil, err
}
return &minIoClient{
client: client,
Vars: vars,
}, nil
return &minIoClient{bucket: bucket, client: client}, nil
}
func (minIo minIoClient) ListBuckets() ([]interface{}, error) {
buckets, err := minIo.client.ListBuckets(context.Background())
func (m minIoClient) ListBuckets() ([]interface{}, error) {
buckets, err := m.client.ListBuckets(context.Background())
if err != nil {
return nil, err
}
@ -77,49 +60,44 @@ func (minIo minIoClient) ListBuckets() ([]interface{}, error) {
return result, err
}
func (minIo minIoClient) Exist(path string) (bool, error) {
if _, ok := minIo.Vars["bucket"]; ok {
_, err := minIo.client.GetObject(context.Background(), minIo.Vars["bucket"].(string), path, minio.GetObjectOptions{})
if err != nil {
return false, err
}
return true, nil
} else {
return false, constant.ErrInvalidParams
func (m minIoClient) Exist(path string) (bool, error) {
if _, err := m.client.GetObject(context.Background(), m.bucket, path, minio.GetObjectOptions{}); err != nil {
return false, err
}
return true, nil
}
func (minIo minIoClient) Delete(path string) (bool, error) {
if _, ok := minIo.Vars["bucket"]; ok {
object, err := minIo.client.GetObject(context.Background(), minIo.Vars["bucket"].(string), path, minio.GetObjectOptions{})
if err != nil {
return false, err
}
info, err := object.Stat()
if err != nil {
return false, err
}
err = minIo.client.RemoveObject(context.Background(), minIo.Vars["bucket"].(string), path, minio.RemoveObjectOptions{
GovernanceBypass: true,
VersionID: info.VersionID,
})
if err != nil {
return false, err
}
return true, nil
} else {
return false, constant.ErrInvalidParams
func (m minIoClient) Size(path string) (int64, error) {
obj, err := m.client.GetObject(context.Background(), m.bucket, path, minio.GetObjectOptions{})
if err != nil {
return 0, err
}
file, err := obj.Stat()
if err != nil {
return 0, err
}
return file.Size, nil
}
func (minIo minIoClient) Upload(src, target string) (bool, error) {
var bucket string
if _, ok := minIo.Vars["bucket"]; ok {
bucket = minIo.Vars["bucket"].(string)
} else {
return false, constant.ErrInvalidParams
func (m minIoClient) Delete(path string) (bool, error) {
object, err := m.client.GetObject(context.Background(), m.bucket, path, minio.GetObjectOptions{})
if err != nil {
return false, err
}
info, err := object.Stat()
if err != nil {
return false, err
}
if err = m.client.RemoveObject(context.Background(), m.bucket, path, minio.RemoveObjectOptions{
GovernanceBypass: true,
VersionID: info.VersionID,
}); err != nil {
return false, err
}
return true, nil
}
func (m minIoClient) Upload(src, target string) (bool, error) {
file, err := os.Open(src)
if err != nil {
return false, err
@ -130,52 +108,36 @@ func (minIo minIoClient) Upload(src, target string) (bool, error) {
if err != nil {
return false, err
}
_, err = minIo.client.PutObject(context.Background(), bucket, target, file, fileStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"})
_, err = m.client.PutObject(context.Background(), m.bucket, target, file, fileStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"})
if err != nil {
return false, err
}
return true, nil
}
func (minIo minIoClient) Download(src, target string) (bool, error) {
if _, ok := minIo.Vars["bucket"]; ok {
object, err := minIo.client.GetObject(context.Background(), minIo.Vars["bucket"].(string), src, minio.GetObjectOptions{})
if err != nil {
return false, err
}
localFile, err := os.Create(target)
if err != nil {
return false, err
}
if _, err = io.Copy(localFile, object); err != nil {
return false, err
}
return true, nil
} else {
return false, constant.ErrInvalidParams
}
}
func (minIo *minIoClient) GetBucket() (string, error) {
if _, ok := minIo.Vars["bucket"]; ok {
return minIo.Vars["bucket"].(string), nil
} else {
return "", constant.ErrInvalidParams
}
}
func (minIo minIoClient) ListObjects(prefix string) ([]string, error) {
bucket, err := minIo.GetBucket()
func (m minIoClient) Download(src, target string) (bool, error) {
object, err := m.client.GetObject(context.Background(), m.bucket, src, minio.GetObjectOptions{})
if err != nil {
return nil, constant.ErrInvalidParams
return false, err
}
localFile, err := os.Create(target)
if err != nil {
return false, err
}
if _, err = io.Copy(localFile, object); err != nil {
return false, err
}
return true, nil
}
func (m minIoClient) ListObjects(prefix string) ([]string, error) {
opts := minio.ListObjectsOptions{
Recursive: true,
Prefix: prefix,
}
var result []string
for object := range minIo.client.ListObjects(context.Background(), bucket, opts) {
for object := range m.client.ListObjects(context.Background(), m.bucket, opts) {
if object.Err != nil {
continue
}

94
backend/utils/cloud_storage/client/onedrive.go
View File

@ -23,17 +23,11 @@ import (
)
type oneDriveClient struct {
Vars map[string]interface{}
client odsdk.Client
}
func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) {
token := ""
if _, ok := vars["accessToken"]; ok {
token = vars["accessToken"].(string)
} else {
return nil, constant.ErrInvalidParams
}
token := loadParamFromVars("accessToken", true, vars)
ctx := context.Background()
newToken, err := refreshToken(token)
@ -51,13 +45,13 @@ func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) {
return &oneDriveClient{client: *client}, nil
}
func (onedrive oneDriveClient) ListBuckets() ([]interface{}, error) {
func (o oneDriveClient) ListBuckets() ([]interface{}, error) {
return nil, nil
}
func (onedrive oneDriveClient) Exist(path string) (bool, error) {
func (o oneDriveClient) Exist(path string) (bool, error) {
path = "/" + strings.TrimPrefix(path, "/")
fileID, err := onedrive.loadIDByPath(path)
fileID, err := o.loadIDByPath(path)
if err != nil {
return false, err
}
@ -65,26 +59,50 @@ func (onedrive oneDriveClient) Exist(path string) (bool, error) {
return len(fileID) != 0, nil
}
func (onedrive oneDriveClient) Delete(path string) (bool, error) {
func (o oneDriveClient) Size(path string) (int64, error) {
path = "/" + strings.TrimPrefix(path, "/")
req, err := onedrive.client.NewRequest("DELETE", fmt.Sprintf("me/drive/root:%s", path), nil)
pathItem := "root:" + path
if path == "/" {
pathItem = "root"
}
req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil)
if err != nil {
return 0, fmt.Errorf("new request for file id failed, err: %v", err)
}
var driveItem myDriverItem
if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil {
return 0, fmt.Errorf("do request for file id failed, err: %v", err)
}
return driveItem.Size, nil
}
type myDriverItem struct {
Name string `json:"name"`
Id string `json:"id"`
Size int64 `json:"size"`
}
func (o oneDriveClient) Delete(path string) (bool, error) {
path = "/" + strings.TrimPrefix(path, "/")
req, err := o.client.NewRequest("DELETE", fmt.Sprintf("me/drive/root:%s", path), nil)
if err != nil {
return false, fmt.Errorf("new request for delete file failed, err: %v \n", err)
}
if err := onedrive.client.Do(context.Background(), req, false, nil); err != nil {
if err := o.client.Do(context.Background(), req, false, nil); err != nil {
return false, fmt.Errorf("do request for delete file failed, err: %v \n", err)
}
return true, nil
}
func (onedrive oneDriveClient) Upload(src, target string) (bool, error) {
func (o oneDriveClient) Upload(src, target string) (bool, error) {
target = "/" + strings.TrimPrefix(target, "/")
if _, err := onedrive.loadIDByPath(path.Dir(target)); err != nil {
if _, err := o.loadIDByPath(path.Dir(target)); err != nil {
if !strings.Contains(err.Error(), "itemNotFound") {
return false, err
}
if err := onedrive.createFolder(path.Dir(target)); err != nil {
if err := o.createFolder(path.Dir(target)); err != nil {
return false, fmt.Errorf("create dir before upload failed, err: %v", err)
}
}
@ -105,7 +123,7 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) {
fileName := fileInfo.Name()
fileSize := fileInfo.Size()
folderID, err := onedrive.loadIDByPath(path.Dir(target))
folderID, err := o.loadIDByPath(path.Dir(target))
if err != nil {
return false, err
}
@ -119,13 +137,13 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) {
DeferCommit bool `json:"deferCommit"`
}{sessionCreationRequestInside, false}
sessionCreationReq, err := onedrive.client.NewRequest("POST", apiURL, sessionCreationRequest)
sessionCreationReq, err := o.client.NewRequest("POST", apiURL, sessionCreationRequest)
if err != nil {
return false, err
}
var sessionCreationResp *NewUploadSessionCreationResponse
err = onedrive.client.Do(ctx, sessionCreationReq, false, &sessionCreationResp)
err = o.client.Do(ctx, sessionCreationReq, false, &sessionCreationResp)
if err != nil {
return false, fmt.Errorf("session creation failed %w", err)
}
@ -149,11 +167,11 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) {
bufferLast := buffer[:length]
buffer = bufferLast
}
sessionFileUploadReq, err := onedrive.NewSessionFileUploadRequest(fileSessionUploadUrl, splitNow*sizePerSplit, fileSize, bytes.NewReader(buffer))
sessionFileUploadReq, err := o.NewSessionFileUploadRequest(fileSessionUploadUrl, splitNow*sizePerSplit, fileSize, bytes.NewReader(buffer))
if err != nil {
return false, err
}
if err := onedrive.client.Do(ctx, sessionFileUploadReq, false, &fileUploadResp); err != nil {
if err := o.client.Do(ctx, sessionFileUploadReq, false, &fileUploadResp); err != nil {
return false, err
}
}
@ -164,14 +182,14 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) {
return true, nil
}
func (onedrive oneDriveClient) Download(src, target string) (bool, error) {
func (o oneDriveClient) Download(src, target string) (bool, error) {
src = "/" + strings.TrimPrefix(src, "/")
req, err := onedrive.client.NewRequest("GET", fmt.Sprintf("me/drive/root:%s", src), nil)
req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/root:%s", src), nil)
if err != nil {
return false, fmt.Errorf("new request for file id failed, err: %v", err)
}
var driveItem *odsdk.DriveItem
if err := onedrive.client.Do(context.Background(), req, false, &driveItem); err != nil {
if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil {
return false, fmt.Errorf("do request for file id failed, err: %v", err)
}
@ -196,19 +214,19 @@ func (onedrive oneDriveClient) Download(src, target string) (bool, error) {
return true, nil
}
func (onedrive *oneDriveClient) ListObjects(prefix string) ([]string, error) {
func (o *oneDriveClient) ListObjects(prefix string) ([]string, error) {
prefix = "/" + strings.TrimPrefix(prefix, "/")
folderID, err := onedrive.loadIDByPath(prefix)
folderID, err := o.loadIDByPath(prefix)
if err != nil {
return nil, err
}
req, err := onedrive.client.NewRequest("GET", fmt.Sprintf("me/drive/items/%s/children", folderID), nil)
req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/items/%s/children", folderID), nil)
if err != nil {
return nil, fmt.Errorf("new request for list failed, err: %v", err)
}
var driveItems *odsdk.OneDriveDriveItemsResponse
if err := onedrive.client.Do(context.Background(), req, false, &driveItems); err != nil {
if err := o.client.Do(context.Background(), req, false, &driveItems); err != nil {
return nil, fmt.Errorf("do request for list failed, err: %v", err)
}
@ -219,17 +237,17 @@ func (onedrive *oneDriveClient) ListObjects(prefix string) ([]string, error) {
return itemList, nil
}
func (onedrive *oneDriveClient) loadIDByPath(path string) (string, error) {
func (o *oneDriveClient) loadIDByPath(path string) (string, error) {
pathItem := "root:" + path
if path == "/" {
pathItem = "root"
}
req, err := onedrive.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil)
req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil)
if err != nil {
return "", fmt.Errorf("new request for file id failed, err: %v", err)
}
var driveItem *odsdk.DriveItem
if err := onedrive.client.Do(context.Background(), req, false, &driveItem); err != nil {
if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil {
return "", fmt.Errorf("do request for file id failed, err: %v", err)
}
return driveItem.Id, nil
@ -269,18 +287,18 @@ func refreshToken(oldToken string) (string, error) {
return accessToken, nil
}
func (onedrive *oneDriveClient) createFolder(parent string) error {
if _, err := onedrive.loadIDByPath(path.Dir(parent)); err != nil {
func (o *oneDriveClient) createFolder(parent string) error {
if _, err := o.loadIDByPath(path.Dir(parent)); err != nil {
if !strings.Contains(err.Error(), "itemNotFound") {
return err
}
_ = onedrive.createFolder(path.Dir(parent))
_ = o.createFolder(path.Dir(parent))
}
item2, err := onedrive.loadIDByPath(path.Dir(parent))
item2, err := o.loadIDByPath(path.Dir(parent))
if err != nil {
return err
}
if _, err := onedrive.client.DriveItems.CreateNewFolder(context.Background(), "", item2, path.Base(parent)); err != nil {
if _, err := o.client.DriveItems.CreateNewFolder(context.Background(), "", item2, path.Base(parent)); err != nil {
return err
}
return nil
@ -307,8 +325,8 @@ type DriveItem struct {
WebURL string `json:"webUrl"`
}
func (onedrive *oneDriveClient) NewSessionFileUploadRequest(absoluteUrl string, grandOffset, grandTotalSize int64, byteReader *bytes.Reader) (*http.Request, error) {
apiUrl, err := onedrive.client.BaseURL.Parse(absoluteUrl)
func (o *oneDriveClient) NewSessionFileUploadRequest(absoluteUrl string, grandOffset, grandTotalSize int64, byteReader *bytes.Reader) (*http.Request, error) {
apiUrl, err := o.client.BaseURL.Parse(absoluteUrl)
if err != nil {
return nil, err
}

113
backend/utils/cloud_storage/client/oss.go
View File

@ -1,54 +1,36 @@
package client
import (
"github.com/1Panel-dev/1Panel/backend/constant"
"fmt"
osssdk "github.com/aliyun/aliyun-oss-go-sdk/oss"
)
type ossClient struct {
scType string
Vars map[string]interface{}
client osssdk.Client
scType string
bucketStr string
client osssdk.Client
}
func NewOssClient(vars map[string]interface{}) (*ossClient, error) {
var endpoint string
var accessKey string
var secretKey string
var scType string
if _, ok := vars["endpoint"]; ok {
endpoint = vars["endpoint"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["accessKey"]; ok {
accessKey = vars["accessKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["scType"]; ok {
scType = vars["scType"].(string)
} else {
endpoint := loadParamFromVars("endpoint", true, vars)
accessKey := loadParamFromVars("accessKey", true, vars)
secretKey := loadParamFromVars("secretKey", true, vars)
bucketStr := loadParamFromVars("bucket", true, vars)
scType := loadParamFromVars("scType", true, vars)
if len(scType) == 0 {
scType = "Standard"
}
if _, ok := vars["secretKey"]; ok {
secretKey = vars["secretKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
client, err := osssdk.New(endpoint, accessKey, secretKey)
if err != nil {
return nil, err
}
return &ossClient{
scType: scType,
Vars: vars,
client: *client,
}, nil
return &ossClient{scType: scType, bucketStr: bucketStr, client: *client}, nil
}
func (oss ossClient) ListBuckets() ([]interface{}, error) {
response, err := oss.client.ListBuckets()
func (o ossClient) ListBuckets() ([]interface{}, error) {
response, err := o.client.ListBuckets()
if err != nil {
return nil, err
}
@ -59,69 +41,72 @@ func (oss ossClient) ListBuckets() ([]interface{}, error) {
return result, err
}
func (oss ossClient) Exist(path string) (bool, error) {
bucket, err := oss.GetBucket()
func (o ossClient) Exist(path string) (bool, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return false, err
}
return bucket.IsObjectExist(path)
}
func (oss ossClient) Delete(path string) (bool, error) {
bucket, err := oss.GetBucket()
func (o ossClient) Size(path string) (int64, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return 0, err
}
lor, err := bucket.ListObjectsV2(osssdk.Prefix(path))
if err != nil {
return 0, err
}
if len(lor.Objects) == 0 {
return 0, fmt.Errorf("no such file %s", path)
}
return lor.Objects[0].Size, nil
}
func (o ossClient) Delete(path string) (bool, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return false, err
}
err = bucket.DeleteObject(path)
if err != nil {
if err := bucket.DeleteObject(path); err != nil {
return false, err
}
return true, nil
}
func (oss ossClient) Upload(src, target string) (bool, error) {
bucket, err := oss.GetBucket()
func (o ossClient) Upload(src, target string) (bool, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return false, err
}
err = bucket.UploadFile(target, src, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, ""), osssdk.ObjectStorageClass(osssdk.StorageClassType(oss.scType)))
if err != nil {
if err := bucket.UploadFile(target, src,
200*1024*1024,
osssdk.Routines(5),
osssdk.Checkpoint(true, ""),
osssdk.ObjectStorageClass(osssdk.StorageClassType(o.scType))); err != nil {
return false, err
}
return true, nil
}
func (oss ossClient) Download(src, target string) (bool, error) {
bucket, err := oss.GetBucket()
func (o ossClient) Download(src, target string) (bool, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return false, err
}
err = bucket.DownloadFile(src, target, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, ""))
if err != nil {
if err := bucket.DownloadFile(src, target, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, "")); err != nil {
return false, err
}
return true, nil
}
func (oss *ossClient) GetBucket() (*osssdk.Bucket, error) {
if _, ok := oss.Vars["bucket"]; ok {
bucket, err := oss.client.Bucket(oss.Vars["bucket"].(string))
if err != nil {
return nil, err
}
return bucket, nil
} else {
return nil, constant.ErrInvalidParams
}
}
func (oss *ossClient) ListObjects(prefix string) ([]string, error) {
bucket, err := oss.GetBucket()
func (o *ossClient) ListObjects(prefix string) ([]string, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return nil, constant.ErrInvalidParams
return nil, err
}
lor, err := bucket.ListObjects(osssdk.Prefix(prefix))
lor, err := bucket.ListObjectsV2(osssdk.Prefix(prefix))
if err != nil {
return nil, err
}

166
backend/utils/cloud_storage/client/s3.go
View File

@ -3,7 +3,6 @@ package client
import (
"os"
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
@ -14,41 +13,20 @@ import (
type s3Client struct {
scType string
Vars map[string]interface{}
bucket string
Sess session.Session
}
func NewS3Client(vars map[string]interface{}) (*s3Client, error) {
var accessKey string
var secretKey string
var endpoint string
var scType string
var region string
if _, ok := vars["accessKey"]; ok {
accessKey = vars["accessKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["secretKey"]; ok {
secretKey = vars["secretKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["scType"]; ok {
scType = vars["scType"].(string)
} else {
accessKey := loadParamFromVars("accessKey", true, vars)
secretKey := loadParamFromVars("secretKey", true, vars)
endpoint := loadParamFromVars("endpoint", true, vars)
region := loadParamFromVars("region", true, vars)
bucket := loadParamFromVars("bucket", true, vars)
scType := loadParamFromVars("scType", true, vars)
if len(scType) == 0 {
scType = "Standard"
}
if _, ok := vars["endpoint"]; ok {
endpoint = vars["endpoint"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["region"]; ok {
region = vars["region"].(string)
} else {
return nil, constant.ErrInvalidParams
}
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
Endpoint: aws.String(endpoint),
@ -59,16 +37,12 @@ func NewS3Client(vars map[string]interface{}) (*s3Client, error) {
if err != nil {
return nil, err
}
return &s3Client{
scType: scType,
Vars: vars,
Sess: *sess,
}, nil
return &s3Client{scType: scType, bucket: bucket, Sess: *sess}, nil
}
func (s3C s3Client) ListBuckets() ([]interface{}, error) {
func (s s3Client) ListBuckets() ([]interface{}, error) {
var result []interface{}
svc := s3.New(&s3C.Sess)
svc := s3.New(&s.Sess)
res, err := svc.ListBuckets(nil)
if err != nil {
return nil, err
@ -79,17 +53,12 @@ func (s3C s3Client) ListBuckets() ([]interface{}, error) {
return result, nil
}
func (s3C s3Client) Exist(path string) (bool, error) {
bucket, err := s3C.getBucket()
if err != nil {
return false, err
}
svc := s3.New(&s3C.Sess)
_, err = svc.HeadObject(&s3.HeadObjectInput{
Bucket: &bucket,
func (s s3Client) Exist(path string) (bool, error) {
svc := s3.New(&s.Sess)
if _, err := svc.HeadObject(&s3.HeadObjectInput{
Bucket: &s.bucket,
Key: &path,
})
if err != nil {
}); err != nil {
if aerr, ok := err.(awserr.RequestFailure); ok {
if aerr.StatusCode() == 404 {
return false, nil
@ -101,57 +70,53 @@ func (s3C s3Client) Exist(path string) (bool, error) {
return true, nil
}
func (s3C s3Client) Delete(path string) (bool, error) {
bucket, err := s3C.getBucket()
if err != nil {
return false, err
}
svc := s3.New(&s3C.Sess)
_, err = svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(path)})
if err != nil {
return false, err
}
err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(path),
func (s *s3Client) Size(path string) (int64, error) {
svc := s3.New(&s.Sess)
file, err := svc.GetObject(&s3.GetObjectInput{
Bucket: &s.bucket,
Key: &path,
})
if err != nil {
return 0, err
}
return *file.ContentLength, nil
}
func (s s3Client) Delete(path string) (bool, error) {
svc := s3.New(&s.Sess)
if _, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(path)}); err != nil {
return false, err
}
if err := svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
}); err != nil {
return false, err
}
return true, nil
}
func (s3C s3Client) Upload(src, target string) (bool, error) {
bucket, err := s3C.getBucket()
if err != nil {
return false, err
}
func (s s3Client) Upload(src, target string) (bool, error) {
file, err := os.Open(src)
if err != nil {
return false, err
}
defer file.Close()
uploader := s3manager.NewUploader(&s3C.Sess)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket),
uploader := s3manager.NewUploader(&s.Sess)
if _, err := uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(target),
Body: file,
StorageClass: &s3C.scType,
})
if err != nil {
StorageClass: &s.scType,
}); err != nil {
return false, err
}
return true, nil
}
func (s3C s3Client) Download(src, target string) (bool, error) {
bucket, err := s3C.getBucket()
if err != nil {
return false, err
}
_, err = os.Stat(target)
if err != nil {
func (s s3Client) Download(src, target string) (bool, error) {
if _, err := os.Stat(target); err != nil {
if os.IsNotExist(err) {
os.Remove(target)
} else {
@ -163,44 +128,29 @@ func (s3C s3Client) Download(src, target string) (bool, error) {
return false, err
}
defer file.Close()
downloader := s3manager.NewDownloader(&s3C.Sess)
_, err = downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(src),
})
if err != nil {
downloader := s3manager.NewDownloader(&s.Sess)
if _, err = downloader.Download(file, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(src),
}); err != nil {
os.Remove(target)
return false, err
}
return true, nil
}
func (s3C *s3Client) getBucket() (string, error) {
if _, ok := s3C.Vars["bucket"]; ok {
return s3C.Vars["bucket"].(string), nil
} else {
return "", constant.ErrInvalidParams
}
}
func (s3C *s3Client) ListObjects(prefix string) ([]string, error) {
bucket, err := s3C.getBucket()
if err != nil {
return nil, constant.ErrInvalidParams
}
svc := s3.New(&s3C.Sess)
func (s *s3Client) ListObjects(prefix string) ([]string, error) {
svc := s3.New(&s.Sess)
var result []string
if err := svc.ListObjectsPages(&s3.ListObjectsInput{
Bucket: &bucket,
outputs, err := svc.ListObjects(&s3.ListObjectsInput{
Bucket: &s.bucket,
Prefix: &prefix,
}, func(p *s3.ListObjectsOutput, last bool) (shouldContinue bool) {
for _, obj := range p.Contents {
result = append(result, *obj.Key)
}
return true
}); err != nil {
return nil, err
})
if err != nil {
return result, err
}
for _, item := range outputs.Contents {
result = append(result, *item.Key)
}
return result, nil
}

174
backend/utils/cloud_storage/client/sftp.go
View File

@ -6,53 +6,49 @@ import (
"net"
"os"
"path"
"strconv"
"time"
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
)
type sftpClient struct {
Bucket string
Vars map[string]interface{}
client *sftp.Client
bucket string
connInfo string
config *ssh.ClientConfig
}
func NewSftpClient(vars map[string]interface{}) (*sftpClient, error) {
if _, ok := vars["address"]; !ok {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["port"].(float64); !ok {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["password"]; !ok {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["username"]; !ok {
return nil, constant.ErrInvalidParams
}
var bucket string
if _, ok := vars["bucket"]; ok {
bucket = vars["bucket"].(string)
} else {
return nil, constant.ErrInvalidParams
address := loadParamFromVars("address", true, vars)
port := loadParamFromVars("port", false, vars)
password := loadParamFromVars("password", true, vars)
username := loadParamFromVars("username", true, vars)
bucket := loadParamFromVars("bucket", true, vars)
auth := []ssh.AuthMethod{ssh.Password(password)}
clientConfig := &ssh.ClientConfig{
User: username,
Auth: auth,
Timeout: 30 * time.Second,
HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return nil
},
}
port, err := strconv.Atoi(strconv.FormatFloat(vars["port"].(float64), 'G', -1, 64))
if err != nil {
return nil, err
}
sftpC, err := connect(vars["username"].(string), vars["password"].(string), vars["address"].(string), port)
if err != nil {
return nil, err
}
return &sftpClient{Bucket: bucket, client: sftpC, Vars: vars}, nil
return &sftpClient{bucket: bucket, connInfo: fmt.Sprintf("%s:%s", address, port), config: clientConfig}, nil
}
func (s sftpClient) Upload(src, target string) (bool, error) {
defer s.client.Close()
sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
if err != nil {
return false, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return false, err
}
defer client.Close()
defer sshClient.Close()
srcFile, err := os.Open(src)
if err != nil {
@ -60,18 +56,18 @@ func (s sftpClient) Upload(src, target string) (bool, error) {
}
defer srcFile.Close()
targetFilePath := s.Bucket + "/" + target
targetFilePath := s.bucket + "/" + target
targetDir, _ := path.Split(targetFilePath)
if _, err = s.client.Stat(targetDir); err != nil {
if _, err = client.Stat(targetDir); err != nil {
if os.IsNotExist(err) {
if err = s.client.MkdirAll(targetDir); err != nil {
if err = client.MkdirAll(targetDir); err != nil {
return false, err
}
} else {
return false, err
}
}
dstFile, err := s.client.Create(targetFilePath)
dstFile, err := client.Create(targetFilePath)
if err != nil {
return false, err
}
@ -97,8 +93,17 @@ func (s sftpClient) ListBuckets() ([]interface{}, error) {
}
func (s sftpClient) Download(src, target string) (bool, error) {
defer s.client.Close()
srcFile, err := s.client.Open(s.Bucket + "/" + src)
sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
if err != nil {
return false, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return false, err
}
defer client.Close()
defer sshClient.Close()
srcFile, err := client.Open(s.bucket + "/" + src)
if err != nil {
return false, err
}
@ -117,8 +122,18 @@ func (s sftpClient) Download(src, target string) (bool, error) {
}
func (s sftpClient) Exist(path string) (bool, error) {
defer s.client.Close()
srcFile, err := s.client.Open(s.Bucket + "/" + path)
sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
if err != nil {
return false, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return false, err
}
defer client.Close()
defer sshClient.Close()
srcFile, err := client.Open(s.bucket + "/" + path)
if err != nil {
if os.IsNotExist(err) {
return false, nil
@ -130,48 +145,57 @@ func (s sftpClient) Exist(path string) (bool, error) {
return true, err
}
func (s sftpClient) Size(path string) (int64, error) {
sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
if err != nil {
return 0, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return 0, err
}
defer client.Close()
defer sshClient.Close()
files, err := client.Stat(s.bucket + "/" + path)
if err != nil {
return 0, err
}
return files.Size(), nil
}
func (s sftpClient) Delete(filePath string) (bool, error) {
defer s.client.Close()
targetFilePath := s.Bucket + "/" + filePath
if err := s.client.Remove(targetFilePath); err != nil {
sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
if err != nil {
return false, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return false, err
}
defer client.Close()
defer sshClient.Close()
targetFilePath := s.bucket + "/" + filePath
if err := client.Remove(targetFilePath); err != nil {
return false, err
}
return true, nil
}
func connect(user, password, host string, port int) (*sftp.Client, error) {
var (
auth []ssh.AuthMethod
addr string
clientConfig *ssh.ClientConfig
sshClient *ssh.Client
sftpClient *sftp.Client
err error
)
auth = make([]ssh.AuthMethod, 0)
auth = append(auth, ssh.Password(password))
clientConfig = &ssh.ClientConfig{
User: user,
Auth: auth,
Timeout: 30 * time.Second,
HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return nil
},
}
addr = fmt.Sprintf("%s:%d", host, port)
if sshClient, err = ssh.Dial("tcp", addr, clientConfig); err != nil {
return nil, err
}
if sftpClient, err = sftp.NewClient(sshClient); err != nil {
return nil, err
}
return sftpClient, nil
}
func (s sftpClient) ListObjects(prefix string) ([]string, error) {
defer s.client.Close()
files, err := s.client.ReadDir(s.Bucket + "/" + prefix)
sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
if err != nil {
return nil, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return nil, err
}
defer client.Close()
defer sshClient.Close()
files, err := client.ReadDir(s.bucket + "/" + prefix)
if err != nil {
return nil, err
}

48
backend/utils/cloud_storage/client/webdav.go
View File

@ -5,52 +5,26 @@ import (
"io"
"os"
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/studio-b12/gowebdav"
)
type webDAVClient struct {
Bucket string
client *gowebdav.Client
Vars map[string]interface{}
}
func NewWebDAVClient(vars map[string]interface{}) (*webDAVClient, error) {
var (
address string
username string
password string
bucket string
)
if _, ok := vars["address"]; ok {
address = vars["address"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["port"].(float64); !ok {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["username"]; ok {
username = vars["username"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["password"]; ok {
password = vars["password"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["bucket"]; ok {
bucket = vars["bucket"].(string)
} else {
return nil, constant.ErrInvalidParams
}
address := loadParamFromVars("address", true, vars)
port := loadParamFromVars("port", false, vars)
password := loadParamFromVars("password", true, vars)
username := loadParamFromVars("username", true, vars)
bucket := loadParamFromVars("bucket", true, vars)
client := gowebdav.NewClient(fmt.Sprintf("%s:%v", address, vars["port"]), username, password)
client := gowebdav.NewClient(fmt.Sprintf("%s:%s", address, port), username, password)
if err := client.Connect(); err != nil {
return nil, err
}
return &webDAVClient{Vars: vars, Bucket: bucket, client: client}, nil
return &webDAVClient{Bucket: bucket, client: client}, nil
}
func (s webDAVClient) Upload(src, target string) (bool, error) {
@ -115,6 +89,14 @@ func (s webDAVClient) Exist(path string) (bool, error) {
return true, nil
}
func (s webDAVClient) Size(path string) (int64, error) {
file, err := s.client.Stat(s.Bucket + "/" + path)
if err != nil {
return 0, err
}
return file.Size(), nil
}
func (s webDAVClient) Delete(filePath string) (bool, error) {
if err := s.client.Remove(s.Bucket + "/" + filePath); err != nil {
return false, err

2
backend/utils/cloud_storage/cloud_storage_client.go
View File

@ -12,6 +12,8 @@ type CloudStorageClient interface {
Delete(path string) (bool, error)
Upload(src, target string) (bool, error)
Download(src, target string) (bool, error)
Size(path string) (int64, error)
}
func NewCloudStorageClient(backupType string, vars map[string]interface{}) (CloudStorageClient, error) {

23
frontend/src/components/backup/index.vue
View File

@ -27,6 +27,14 @@
</template>
<el-table-column type="selection" fix />
<el-table-column :label="$t('commons.table.name')" prop="fileName" show-overflow-tooltip />
<el-table-column :label="$t('file.size')" prop="size" show-overflow-tooltip>
<template #default="{ row }">
<span v-if="row.size">
{{ computeSize(row.size) }}
</span>
<span v-else>-</span>
</template>
</el-table-column>
<el-table-column :label="$t('database.source')" prop="backupType">
<template #default="{ row }">
<span v-if="row.source">
@ -52,7 +60,7 @@
<script lang="ts" setup>
import { reactive, ref } from 'vue';
import OpDialog from '@/components/del-dialog/index.vue';
import { dateFormat, downloadFile } from '@/utils/util';
import { computeSize, dateFormat, downloadFile } from '@/utils/util';
import { handleBackup, handleRecover } from '@/api/modules/setting';
import i18n from '@/lang';
import DrawerHeader from '@/components/drawer-header/index.vue';
@ -101,9 +109,16 @@ const search = async () => {
name: name.value,
detailName: detailName.value,
};
const res = await searchBackupRecords(params);
data.value = res.data.items || [];
paginationConfig.total = res.data.total;
loading.value = true;
await searchBackupRecords(params)
.then((res) => {
loading.value = false;
data.value = res.data.items || [];
paginationConfig.total = res.data.total;
})
.catch(() => {
loading.value = false;
});
};
const onBackup = async () => {

2
frontend/src/views/setting/backup-account/operate/index.vue
View File

@ -211,7 +211,7 @@
:rules="Rules.requiredInput"
>
<el-input v-model.trim="dialogData.rowData!.varsJson['address']" />
<span class="input-help">https://172.16.10.111</span>
<span class="input-help">http://172.16.10.111</span>
</el-form-item>
<el-form-item :label="$t('commons.table.port')" prop="varsJson.port" :rules="[Rules.port]">
<el-input-number