mirror/1Panel
1
0
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2024-11-24 02:59:16 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: 备份账号增加 Google Drive

Browse Source
This commit is contained in:
ssongliu 2024-11-11 18:28:27 +08:00
parent 9e82e2837d
commit e5e9653e62
16 changed files with 766 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
agent/constant/backup.go
View File

@ -15,6 +15,7 @@ const (
Local = "LOCAL"
UPYUN = "UPYUN"
ALIYUN = "ALIYUN"
GoogleDrive = "GoogleDrive"
OneDriveRedirectURI = "http://localhost/login/authorized"
)

5
agent/go.mod
View File

@ -50,12 +50,14 @@ require (
golang.org/x/oauth2 v0.21.0
golang.org/x/sys v0.22.0
golang.org/x/text v0.16.0
google.golang.org/api v0.172.0
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/gorm v1.25.11
)
require (
cloud.google.com/go/compute/metadata v0.3.0 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
@ -123,7 +125,10 @@ require (
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.3 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
github.com/h2non/filetype v1.1.1 // indirect

10
agent/go.sum
View File

@ -19,6 +19,8 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
@ -381,14 +383,20 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg=
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA=
github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4=
github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
@ -1088,6 +1096,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk=
google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

409
agent/utils/cloud_storage/client/google_drive.go Normal file
View File

@ -0,0 +1,409 @@
package client
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
"github.com/go-resty/resty/v2"
)
type googleDriveClient struct {
accessToken string
}
func NewGoogleDriveClient(vars map[string]interface{}) (*googleDriveClient, error) {
accessToken, err := RefreshGoogleToken("refresh_token", "accessToken", vars)
if err != nil {
return nil, err
}
return &googleDriveClient{accessToken: accessToken}, nil
}
func (g *googleDriveClient) ListBuckets() ([]interface{}, error) {
return nil, nil
}
func (g *googleDriveClient) Exist(pathItem string) (bool, error) {
pathItem = path.Join("root", pathItem)
if _, err := g.loadFileWithName(pathItem); err != nil {
return false, err
}
return true, nil
}
func (g *googleDriveClient) Size(pathItem string) (int64, error) {
pathItem = path.Join("root", pathItem)
fileInfo, err := g.loadFileWithName(pathItem)
if err != nil {
return 0, err
}
size, _ := strconv.ParseInt(fileInfo.Size, 10, 64)
return size, nil
}
func (g *googleDriveClient) Delete(pathItem string) (bool, error) {
pathItem = path.Join("root", pathItem)
fileInfo, err := g.loadFileWithName(pathItem)
if err != nil {
return false, err
}
if len(fileInfo.ID) == 0 {
return false, fmt.Errorf("no such file %s", pathItem)
}
res, err := g.googleRequest("https://www.googleapis.com/drive/v3/files/"+fileInfo.ID, http.MethodDelete, nil, nil)
if err != nil {
return false, err
}
fmt.Println(string(res))
return true, nil
}
func (g *googleDriveClient) Upload(src, target string) (bool, error) {
target = path.Join("/root", target)
parentID := "root"
var err error
if path.Dir(target) != "/root" {
parentID, err = g.mkdirWithPath(path.Dir(target))
if err != nil {
return false, err
}
}
file, err := os.Open(src)
if err != nil {
return false, err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return false, err
}
data := map[string]interface{}{
"name": fileInfo.Name(),
"parents": []string{parentID},
}
urlItem := "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true"
client := resty.New()
client.SetProxy("http://127.0.0.1:7890")
resp, err := client.R().
SetHeader("Authorization", "Bearer "+g.accessToken).
SetBody(data).
Post(urlItem)
if err != nil {
return false, err
}
uploadUrl := resp.Header().Get("location")
if _, err := g.googleRequest(uploadUrl, http.MethodPut, func(req *resty.Request) {
req.SetHeader("Content-Length", strconv.FormatInt(fileInfo.Size(), 10)).SetBody(file)
}, nil); err != nil {
return false, err
}
return true, nil
}
func (g *googleDriveClient) Download(src, target string) (bool, error) {
src = path.Join("/root", src)
fileInfo, err := g.loadFileWithName(src)
if err != nil {
return false, err
}
url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s?alt=media&acknowledgeAbuse=true", fileInfo.ID)
if err := g.handleDownload(url, target); err != nil {
return false, err
}
return true, nil
}
func (g *googleDriveClient) ListObjects(src string) ([]string, error) {
if len(src) == 0 || src == "root" || src == "/root" {
src = "root"
} else {
src = path.Join("/root", src)
}
fileInfos, err := g.loadDirWithPath(src)
if err != nil {
return nil, err
}
var names []string
for _, item := range fileInfos {
names = append(names, item.Name)
}
return names, nil
}
type googleFileResp struct {
Files []googleFile `json:"files"`
}
type googleFile struct {
ID string `json:"id"`
Name string `json:"name"`
Size string `json:"size"`
}
func (g *googleDriveClient) mkdirWithPath(target string) (string, error) {
pathItems := strings.Split(target, "/")
var (
fileInfos []googleFile
err error
)
parentID := "root"
for i := 0; i < len(pathItems); i++ {
if len(pathItems[i]) == 0 {
continue
}
fileInfos, err = g.loadFileWithParentID(parentID)
if err != nil {
return "", err
}
isEnd := false
if i == len(pathItems)-2 {
isEnd = true
}
exist := false
for _, item := range fileInfos {
if item.Name == pathItems[i+1] {
parentID = item.ID
if isEnd {
return item.ID, nil
} else {
exist = true
}
}
}
if !exist {
parentID, err = g.mkdir(parentID, pathItems[i+1])
if err != nil {
return parentID, err
}
if isEnd {
return parentID, nil
}
}
}
return "", errors.New("mkdir failed.")
}
type googleMkdirRes struct {
ID string `json:"id"`
}
func (g *googleDriveClient) mkdir(parentID, name string) (string, error) {
data := map[string]interface{}{
"name": name,
"parents": []string{parentID},
"mimeType": "application/vnd.google-apps.folder",
}
res, err := g.googleRequest("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) {
req.SetBody(data)
}, nil)
if err != nil {
return "", err
}
var mkdirResp googleMkdirRes
if err := json.Unmarshal(res, &mkdirResp); err != nil {
return "", err
}
return mkdirResp.ID, nil
}
func (g *googleDriveClient) handleDownload(urlItem string, target string) error {
req, err := http.NewRequest(http.MethodGet, urlItem, nil)
if err != nil {
return err
}
req.Header.Add("Authorization", "Bearer "+g.accessToken)
proxyURL, _ := url.Parse("http://127.0.0.1:7890")
client := &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyURL(proxyURL),
},
}
response, err := client.Do(req)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("handle download with url failed, code: %v", response.StatusCode)
}
if _, err := os.Stat(path.Dir(target)); err != nil {
_ = os.MkdirAll(path.Dir(target), os.ModePerm)
}
out, err := os.Create(target)
if err != nil {
return err
}
defer out.Close()
if _, err = io.Copy(out, response.Body); err != nil {
return err
}
return nil
}
func (g *googleDriveClient) loadFileWithName(pathItem string) (googleFile, error) {
pathItems := strings.Split(pathItem, "/")
var (
fileInfos []googleFile
err error
)
parentID := "root"
for i := 0; i < len(pathItems); i++ {
if len(pathItems[i]) == 0 {
continue
}
fileInfos, err = g.loadFileWithParentID(parentID)
if err != nil {
return googleFile{}, err
}
isEnd := false
if i == len(pathItems)-2 {
isEnd = true
}
exist := false
for _, item := range fileInfos {
if item.Name == pathItems[i+1] {
if isEnd {
return item, nil
} else {
parentID = item.ID
exist = true
}
}
}
if !exist {
return googleFile{}, errors.New("no such file or dir")
}
}
return googleFile{}, errors.New("no such file or dir")
}
func (g *googleDriveClient) loadFileWithParentID(parentID string) ([]googleFile, error) {
query := map[string]string{
"fields": "files(id,name,mimeType,size)",
"q": fmt.Sprintf("'%s' in parents and trashed = false", parentID),
}
res, err := g.googleRequest("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) {
req.SetQueryParams(query)
}, nil)
if err != nil {
return nil, err
}
var fileResp googleFileResp
if err := json.Unmarshal(res, &fileResp); err != nil {
return nil, err
}
return fileResp.Files, nil
}
func (g googleDriveClient) loadDirWithPath(path string) ([]googleFile, error) {
pathItems := strings.Split(path, "/")
var (
fileInfos []googleFile
err error
)
parentID := "root"
for i := 0; i < len(pathItems); i++ {
if len(pathItems[i]) == 0 {
continue
}
fileInfos, err = g.loadFileWithParentID(parentID)
if err != nil {
return fileInfos, err
}
if i == len(pathItems)-1 {
return fileInfos, nil
}
exist := false
for _, item := range fileInfos {
if item.Name == pathItems[i+1] {
parentID = item.ID
exist = true
}
}
if !exist {
return nil, errors.New("no such file or dir")
}
}
return fileInfos, errors.New("no such file or dir")
}
type reqCallback func(req *resty.Request)
func (g *googleDriveClient) googleRequest(urlItem, method string, callback reqCallback, resp interface{}) ([]byte, error) {
client := resty.New()
client.SetProxy("http://127.0.0.1:7890")
req := client.R()
req.SetHeader("Authorization", "Bearer "+g.accessToken)
if callback != nil {
callback(req)
}
if resp != nil {
req.SetResult(req)
}
res, err := req.Execute(method, urlItem)
if err != nil {
return nil, err
}
if res.StatusCode() == 401 {
// refresh and retry
return nil, fmt.Errorf("request for %s failed, code: %v", urlItem, res.StatusCode())
}
if res.StatusCode() > 300 {
return nil, fmt.Errorf("request for %s failed, err: %v", urlItem, res.StatusCode())
}
return res.Body(), nil
}
type googleTokenRes struct {
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
}
func RefreshGoogleToken(grantType string, tokenType string, varMap map[string]interface{}) (string, error) {
client := resty.New()
client.SetProxy("http://127.0.0.1:7890")
data := map[string]interface{}{
"client_id": loadParamFromVars("client_id", varMap),
"client_secret": loadParamFromVars("client_secret", varMap),
"redirect_uri": loadParamFromVars("redirect_uri", varMap),
}
if grantType == "refresh_token" {
data["grant_type"] = "refresh_token"
data["refresh_token"] = loadParamFromVars("refresh_token", varMap)
} else {
data["grant_type"] = "authorization_code"
data["code"] = loadParamFromVars("code", varMap)
}
urlItem := "https://www.googleapis.com/oauth2/v4/token"
resp, err := client.R().
SetBody(data).
Post(urlItem)
if err != nil {
return "", fmt.Errorf("load account token failed, err: %v", err)
}
if resp.StatusCode() != 200 {
return "", fmt.Errorf("load account token failed, code: %v", resp.StatusCode())
}
var respItem googleTokenRes
if err := json.Unmarshal(resp.Body(), &respItem); err != nil {
return "", err
}
fmt.Println(respItem)
if tokenType == "accessToken" {
return respItem.AccessToken, nil
}
return respItem.RefreshToken, nil
}

11
core/app/api/v2/backup.go
View File

@ -1,6 +1,8 @@
package v2
import (
"fmt"
"github.com/1Panel-dev/1Panel/core/app/api/v2/helper"
"github.com/1Panel-dev/1Panel/core/app/dto"
"github.com/gin-gonic/gin"
@ -67,9 +69,14 @@ func (b *BaseApi) ListBuckets(c *gin.Context) {
// @Accept json
// @Success 200 {object} dto.OneDriveInfo
// @Security ApiKeyAuth
// @Router /core/backup/onedrive [get]
// @Router /core/backup/client/:clientType [get]
func (b *BaseApi) LoadOneDriveInfo(c *gin.Context) {
data, err := backupService.LoadOneDriveInfo()
clientType, ok := c.Params.Get("clientType")
if !ok {
helper.BadRequest(c, fmt.Errorf("error %s in path", "clientType"))
return
}
data, err := backupService.LoadBackupClientInfo(clientType)
if err != nil {
helper.InternalServer(c, err)
return

2
core/app/dto/backup.go
View File

@ -35,7 +35,7 @@ type BackupInfo struct {
RememberAuth bool `json:"rememberAuth"`
}
type OneDriveInfo struct {
type BackupClientInfo struct {
ClientID string `json:"client_id"`
ClientSecret string `json:"client_secret"`
RedirectUri string `json:"redirect_uri"`

6
core/app/model/backup.go
View File

@ -12,8 +12,6 @@ type BackupAccount struct {
BackupPath string `json:"backupPath"`
Vars string `json:"vars"`
RememberAuth bool `json:"rememberAuth"`
EntryID uint `json:"entryID"`
DeletedAt time.Time `json:"deletedAt"`
RememberAuth bool `json:"rememberAuth"`
DeletedAt time.Time `json:"deletedAt"`
}

139
core/app/service/backup.go
View File

@ -24,7 +24,6 @@ import (
"github.com/1Panel-dev/1Panel/core/utils/xpack"
"github.com/jinzhu/copier"
"github.com/pkg/errors"
"github.com/robfig/cron/v3"
)
type BackupService struct{}
@ -36,7 +35,7 @@ type IBackupService interface {
GetLocalDir() (string, error)
LoadBackupOptions() ([]dto.BackupOption, error)
SearchWithPage(search dto.SearchPageWithType) (int64, interface{}, error)
LoadOneDriveInfo() (dto.OneDriveInfo, error)
LoadBackupClientInfo(clientType string) (dto.BackupClientInfo, error)
Create(backupDto dto.BackupOperate) error
GetBuckets(backupDto dto.ForBuckets) ([]interface{}, error)
Update(req dto.BackupOperate) error
@ -156,7 +155,7 @@ func (u *BackupService) SearchWithPage(req dto.SearchPageWithType) (int64, inter
item.Credential = base64.StdEncoding.EncodeToString([]byte(item.Credential))
}
if account.Type == constant.OneDrive || account.Type == constant.ALIYUN {
if account.Type == constant.OneDrive || account.Type == constant.ALIYUN || account.Type == constant.GoogleDrive {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(item.Vars), &varMap); err != nil {
continue
@ -171,10 +170,18 @@ func (u *BackupService) SearchWithPage(req dto.SearchPageWithType) (int64, inter
return count, data, nil
}
func (u *BackupService) LoadOneDriveInfo() (dto.OneDriveInfo, error) {
var data dto.OneDriveInfo
data.RedirectUri = constant.OneDriveRedirectURI
clientID, err := settingRepo.Get(commonRepo.WithByKey("OneDriveID"))
func (u *BackupService) LoadBackupClientInfo(clientType string) (dto.BackupClientInfo, error) {
var data dto.BackupClientInfo
clientIDKey := "OneDriveID"
clientIDSc := "OneDriveSc"
if clientType == constant.GoogleDrive {
clientIDKey = "GoogleID"
clientIDSc = "GoogleSc"
data.RedirectUri = constant.GoogleRedirectURI
} else {
data.RedirectUri = constant.OneDriveRedirectURI
}
clientID, err := settingRepo.Get(commonRepo.WithByKey(clientIDKey))
if err != nil {
return data, err
}
@ -183,7 +190,7 @@ func (u *BackupService) LoadOneDriveInfo() (dto.OneDriveInfo, error) {
return data, err
}
data.ClientID = string(idItem)
clientSecret, err := settingRepo.Get(commonRepo.WithByKey("OneDriveSc"))
clientSecret, err := settingRepo.Get(commonRepo.WithByKey(clientIDSc))
if err != nil {
return data, err
}
@ -215,8 +222,8 @@ func (u *BackupService) Create(req dto.BackupOperate) error {
}
backup.Credential = string(itemCredential)
if req.Type == constant.OneDrive {
if err := u.loadAccessToken(&backup); err != nil {
if req.Type == constant.OneDrive || req.Type == constant.GoogleDrive {
if err := u.loadRefreshTokenByCode(&backup); err != nil {
return err
}
}
@ -225,11 +232,6 @@ func (u *BackupService) Create(req dto.BackupOperate) error {
return buserr.WithMap("ErrBackupCheck", map[string]interface{}{"err": err.Error()}, err)
}
}
if backup.Type == constant.OneDrive {
if err := StartRefreshOneDriveToken(&backup); err != nil {
return err
}
}
backup.AccessKey, err = encrypt.StringEncrypt(backup.AccessKey)
if err != nil {
@ -284,9 +286,6 @@ func (u *BackupService) Delete(id uint) error {
if backup.Type == constant.Local {
return buserr.New(constant.ErrBackupLocalDelete)
}
if backup.Type == constant.OneDrive {
global.Cron.Remove(cron.EntryID(backup.EntryID))
}
if _, err := httpUtils.NewLocalClient(fmt.Sprintf("/api/v2/backups/check/%v", id), http.MethodGet, nil); err != nil {
global.LOG.Errorf("check used of local cronjob failed, err: %v", err)
return buserr.New(constant.ErrBackupInUsed)
@ -338,9 +337,8 @@ func (u *BackupService) Update(req dto.BackupOperate) error {
}
}
if newBackup.Type == constant.OneDrive {
global.Cron.Remove(cron.EntryID(backup.EntryID))
if err := u.loadAccessToken(&backup); err != nil {
if newBackup.Type == constant.OneDrive || newBackup.Type == constant.GoogleDrive {
if err := u.loadRefreshTokenByCode(&backup); err != nil {
return err
}
}
@ -392,14 +390,23 @@ func (u *BackupService) NewClient(backup *model.BackupAccount) (cloud_storage.Cl
return backClient, nil
}
func (u *BackupService) loadAccessToken(backup *model.BackupAccount) error {
func (u *BackupService) loadRefreshTokenByCode(backup *model.BackupAccount) error {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
return fmt.Errorf("unmarshal backup vars failed, err: %v", err)
}
refreshToken, err := client.RefreshToken("authorization_code", "refreshToken", varMap)
if err != nil {
return err
refreshToken := ""
var err error
if backup.Type == constant.GoogleDrive {
refreshToken, err = client.RefreshGoogleToken("authorization_code", "refreshToken", varMap)
if err != nil {
return err
}
} else {
refreshToken, err = client.RefreshToken("authorization_code", "refreshToken", varMap)
if err != nil {
return err
}
}
delete(varMap, "code")
varMap["refresh_status"] = constant.StatusSuccess
@ -489,87 +496,59 @@ func (u *BackupService) checkBackupConn(backup *model.BackupAccount) (bool, erro
return client.Upload(fileItem, targetPath)
}
func StartRefreshOneDriveToken(backup *model.BackupAccount) error {
func StartRefreshForToken() error {
service := NewIBackupService()
oneDriveCronID, err := global.Cron.AddJob("0 3 */31 * *", service)
refreshID, err := global.Cron.AddJob("0 3 */31 * *", service)
if err != nil {
global.LOG.Errorf("can not add OneDrive corn job: %s", err.Error())
global.LOG.Errorf("add cron job of refresh backup account token failed, err: %s", err.Error())
return err
}
backup.EntryID = uint(oneDriveCronID)
global.BackupAccountTokenEntryID = refreshID
return nil
}
func (u *BackupService) Run() {
refreshOneDrive()
refreshALIYUN()
refreshToken()
}
func refreshOneDrive() {
func refreshToken() {
var backups []model.BackupAccount
_ = global.DB.Where("`type` = ?", "OneDrive").Find(&backups)
_ = global.DB.Where("`type` in (?)", []string{constant.OneDrive, constant.ALIYUN, constant.GoogleDrive}).Find(&backups)
if len(backups) == 0 {
return
}
for _, backupItem := range backups {
if backupItem.ID == 0 {
return
continue
}
global.LOG.Infof("start to refresh token of OneDrive %s ...", backupItem.Name)
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil {
global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err)
return
global.LOG.Errorf("Failed to refresh %s - %s token, please retry, err: %v", backupItem.Type, backupItem.Name, err)
continue
}
var (
refreshToken string
err error
)
switch backupItem.Type {
case constant.OneDrive:
refreshToken, err = client.RefreshToken("refresh_token", "refreshToken", varMap)
case constant.GoogleDrive:
refreshToken, err = client.RefreshGoogleToken("refresh_token", "refreshToken", varMap)
case constant.ALIYUN:
refreshToken, err = client.RefreshALIToken(varMap)
}
refreshToken, err := client.RefreshToken("refresh_token", "refreshToken", varMap)
varMap["refresh_status"] = constant.StatusSuccess
varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout)
if err != nil {
varMap["refresh_status"] = constant.StatusFailed
varMap["refresh_msg"] = err.Error()
global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err)
return
continue
}
varMap["refresh_token"] = refreshToken
varsItem, _ := json.Marshal(varMap)
_ = global.DB.Model(&model.BackupAccount{}).
Where("id = ?", backupItem.ID).
Updates(map[string]interface{}{
"vars": varsItem,
}).Error
global.LOG.Info("Successfully refreshed OneDrive token.")
}
}
func refreshALIYUN() {
var backups []model.BackupAccount
_ = global.DB.Where("`type` = ?", "ALIYUN").Find(&backups)
for _, backupItem := range backups {
global.LOG.Infof("start to refresh token of ALIYUN %s ...", backupItem.Name)
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil {
global.LOG.Errorf("Failed to refresh ALIYUN token, please retry, err: %v", err)
return
}
if _, ok := varMap["refresh_token"]; !ok {
global.LOG.Error("no such refresh token find in db")
return
}
refreshToken, err := client.RefreshALIToken(fmt.Sprintf("%v", varMap["refresh_token"]))
varMap["refresh_status"] = constant.StatusSuccess
varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout)
if err != nil {
varMap["refresh_status"] = constant.StatusFailed
varMap["refresh_msg"] = err.Error()
global.LOG.Errorf("Failed to refresh ALIYUN token, please retry, err: %v", err)
return
}
varMap["refresh_token"] = refreshToken
varsItem, _ := json.Marshal(varMap)
_ = global.DB.Model(&model.BackupAccount{}).
Where("id = ?", backupItem.ID).
Updates(map[string]interface{}{
"vars": varsItem,
}).Error
global.LOG.Info("Successfully refreshed ALIYUN token.")
_ = global.DB.Model(&model.BackupAccount{}).Where("id = ?", backupItem.ID).Updates(map[string]interface{}{"vars": varsItem}).Error
}
}

25
core/constant/common.go
View File

@ -19,16 +19,19 @@ const (
StatusDisable = "Disable"
// backup
S3 = "S3"
OSS = "OSS"
Sftp = "SFTP"
OneDrive = "OneDrive"
MinIo = "MINIO"
Cos = "COS"
Kodo = "KODO"
WebDAV = "WebDAV"
Local = "LOCAL"
UPYUN = "UPYUN"
ALIYUN = "ALIYUN"
S3 = "S3"
OSS = "OSS"
Sftp = "SFTP"
OneDrive = "OneDrive"
MinIo = "MINIO"
Cos = "COS"
Kodo = "KODO"
WebDAV = "WebDAV"
Local = "LOCAL"
UPYUN = "UPYUN"
ALIYUN = "ALIYUN"
GoogleDrive = "GoogleDrive"
OneDriveRedirectURI = "http://localhost/login/authorized"
GoogleRedirectURI = "http://localhost:8080"
)

2
core/global/global.go
View File

@ -26,4 +26,6 @@ var (
I18n *i18n.Localizer
Cron *cron.Cron
BackupAccountTokenEntryID cron.EntryID
)

7
core/init/cron/cron.go
View File

@ -3,7 +3,6 @@ package cron
import (
"time"
"github.com/1Panel-dev/1Panel/core/app/model"
"github.com/1Panel-dev/1Panel/core/app/service"
"github.com/1Panel-dev/1Panel/core/global"
"github.com/1Panel-dev/1Panel/core/utils/common"
@ -14,10 +13,6 @@ func Init() {
nyc, _ := time.LoadLocation(common.LoadTimeZone())
global.Cron = cron.New(cron.WithLocation(nyc), cron.WithChain(cron.Recover(cron.DefaultLogger)), cron.WithChain(cron.DelayIfStillRunning(cron.DefaultLogger)))
var accounts []model.BackupAccount
_ = global.DB.Where("type = ?", "OneDrive").Find(&accounts).Error
for i := 0; i < len(accounts); i++ {
_ = service.StartRefreshOneDriveToken(&accounts[i])
}
_ = service.StartRefreshForToken()
global.Cron.Start()
}

1
core/init/migration/migrate.go
View File

@ -17,6 +17,7 @@ func Init() {
migrations.InitTerminalSetting,
migrations.InitAppLauncher,
migrations.InitBackup,
migrations.InitGoogle,
})
if err := m.Migrate(); err != nil {
global.LOG.Error(err)

13
core/init/migration/migrations/init.go
View File

@ -247,3 +247,16 @@ var InitBackup = &gormigrate.Migration{
return tx.AutoMigrate(&model.BackupAccount{})
},
}
var InitGoogle = &gormigrate.Migration{
ID: "20241111-init-google",
Migrate: func(tx *gorm.DB) error {
if err := tx.Create(&model.Setting{Key: "GoogleID", Value: "NTU2NTQ3NDYwMTQtY2Q0bGR0dDk2aGNsNWcxYWtwdmJhZTFmcjJlZ2Y0MXAuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20K"}).Error; err != nil {
return err
}
if err := tx.Create(&model.Setting{Key: "GoogleSc", Value: "R09DU1BYLXRibXg0QVdVZ3d3Ykc2QW1XTHQ3YUdaZElVeE4K"}).Error; err != nil {
return err
}
return nil
},
}

9
core/utils/cloud_storage/client/ali.go
View File

@ -19,10 +19,9 @@ type aliClient struct {
}
func NewALIClient(vars map[string]interface{}) (*aliClient, error) {
refresh_token := loadParamFromVars("refresh_token", vars)
drive_id := loadParamFromVars("drive_id", vars)
token, err := RefreshALIToken(refresh_token)
token, err := RefreshALIToken(vars)
if err != nil {
return nil, err
}
@ -278,7 +277,11 @@ type tokenResp struct {
AccessToken string `json:"access_token"`
}
func RefreshALIToken(refresh_token string) (string, error) {
func RefreshALIToken(varMap map[string]interface{}) (string, error) {
refresh_token := loadParamFromVars("refresh_token", varMap)
if len(refresh_token) == 0 {
return "", errors.New("no such refresh token find in db")
}
client := resty.New()
data := map[string]interface{}{
"grant_type": "refresh_token",

231
core/utils/cloud_storage/client/google_drive.go Normal file
View File

@ -0,0 +1,231 @@
package client
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"path"
"strconv"
"strings"
"github.com/go-resty/resty/v2"
)
type googleDriveClient struct {
accessToken string
}
func NewGoogleDriveClient(vars map[string]interface{}) (*googleDriveClient, error) {
accessToken, err := RefreshGoogleToken("refresh_token", "accessToken", vars)
if err != nil {
return nil, err
}
return &googleDriveClient{accessToken: accessToken}, nil
}
func (g *googleDriveClient) ListBuckets() ([]interface{}, error) {
return nil, nil
}
func (g *googleDriveClient) Upload(src, target string) (bool, error) {
target = path.Join("/root", target)
parentID := "root"
var err error
if path.Dir(target) != "/root" {
parentID, err = g.mkdirWithPath(path.Dir(target))
if err != nil {
return false, err
}
}
file, err := os.Open(src)
if err != nil {
return false, err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return false, err
}
data := map[string]interface{}{
"name": fileInfo.Name(),
"parents": []string{parentID},
}
urlItem := "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true"
client := resty.New()
client.SetProxy("http://127.0.0.1:7890")
resp, err := client.R().
SetHeader("Authorization", "Bearer "+g.accessToken).
SetBody(data).
Post(urlItem)
if err != nil {
return false, err
}
uploadUrl := resp.Header().Get("location")
if _, err := g.googleRequest(uploadUrl, http.MethodPut, func(req *resty.Request) {
req.SetHeader("Content-Length", strconv.FormatInt(fileInfo.Size(), 10)).SetBody(file)
}, nil); err != nil {
return false, err
}
return true, nil
}
type googleFileResp struct {
Files []googleFile `json:"files"`
}
type googleFile struct {
ID string `json:"id"`
Name string `json:"name"`
Size string `json:"size"`
}
func (g *googleDriveClient) mkdirWithPath(target string) (string, error) {
pathItems := strings.Split(target, "/")
var (
fileInfos []googleFile
err error
)
parentID := "root"
for i := 0; i < len(pathItems); i++ {
if len(pathItems[i]) == 0 {
continue
}
fileInfos, err = g.loadFileWithParentID(parentID)
if err != nil {
return "", err
}
isEnd := false
if i == len(pathItems)-2 {
isEnd = true
}
exist := false
for _, item := range fileInfos {
if item.Name == pathItems[i+1] {
parentID = item.ID
if isEnd {
return item.ID, nil
} else {
exist = true
}
}
}
if !exist {
parentID, err = g.mkdir(parentID, pathItems[i+1])
if err != nil {
return parentID, err
}
if isEnd {
return parentID, nil
}
}
}
return "", errors.New("mkdir failed.")
}
type googleMkdirRes struct {
ID string `json:"id"`
}
func (g *googleDriveClient) mkdir(parentID, name string) (string, error) {
data := map[string]interface{}{
"name": name,
"parents": []string{parentID},
"mimeType": "application/vnd.google-apps.folder",
}
res, err := g.googleRequest("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) {
req.SetBody(data)
}, nil)
if err != nil {
return "", err
}
var mkdirResp googleMkdirRes
if err := json.Unmarshal(res, &mkdirResp); err != nil {
return "", err
}
return mkdirResp.ID, nil
}
func (g *googleDriveClient) loadFileWithParentID(parentID string) ([]googleFile, error) {
query := map[string]string{
"fields": "files(id,name,mimeType,size)",
"q": fmt.Sprintf("'%s' in parents and trashed = false", parentID),
}
res, err := g.googleRequest("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) {
req.SetQueryParams(query)
}, nil)
if err != nil {
return nil, err
}
var fileResp googleFileResp
if err := json.Unmarshal(res, &fileResp); err != nil {
return nil, err
}
return fileResp.Files, nil
}
type reqCallback func(req *resty.Request)
func (g *googleDriveClient) googleRequest(urlItem, method string, callback reqCallback, resp interface{}) ([]byte, error) {
client := resty.New()
client.SetProxy("http://127.0.0.1:7890")
req := client.R()
req.SetHeader("Authorization", "Bearer "+g.accessToken)
if callback != nil {
callback(req)
}
if resp != nil {
req.SetResult(req)
}
res, err := req.Execute(method, urlItem)
if err != nil {
return nil, err
}
if res.StatusCode() > 300 {
return nil, fmt.Errorf("request for %s failed, err: %v", urlItem, res.StatusCode())
}
return res.Body(), nil
}
type googleTokenRes struct {
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
}
func RefreshGoogleToken(grantType string, tokenType string, varMap map[string]interface{}) (string, error) {
client := resty.New()
client.SetProxy("http://127.0.0.1:7890")
data := map[string]interface{}{
"client_id": loadParamFromVars("client_id", varMap),
"client_secret": loadParamFromVars("client_secret", varMap),
"redirect_uri": loadParamFromVars("redirect_uri", varMap),
}
if grantType == "refresh_token" {
data["grant_type"] = "refresh_token"
data["refresh_token"] = loadParamFromVars("refresh_token", varMap)
} else {
data["grant_type"] = "authorization_code"
data["code"] = loadParamFromVars("code", varMap)
}
urlItem := "https://www.googleapis.com/oauth2/v4/token"
resp, err := client.R().
SetBody(data).
Post(urlItem)
if err != nil {
return "", fmt.Errorf("load account token failed, err: %v", err)
}
if resp.StatusCode() != 200 {
return "", fmt.Errorf("load account token failed, code: %v", resp.StatusCode())
}
var respItem googleTokenRes
if err := json.Unmarshal(resp.Body(), &respItem); err != nil {
return "", err
}
if tokenType == "accessToken" {
return respItem.AccessToken, nil
}
return respItem.RefreshToken, nil
}

2
core/utils/cloud_storage/cloud_storage_client.go
View File

@ -34,6 +34,8 @@ func NewCloudStorageClient(backupType string, vars map[string]interface{}) (Clou
return client.NewUpClient(vars)
case constant.ALIYUN:
return client.NewALIClient(vars)
case constant.GoogleDrive:
return client.NewGoogleDriveClient(vars)
default:
return nil, constant.ErrNotSupportType
}