From 2dfbe210e02adf05227d8782c32ebd8bc2b38437 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 11 Apr 2022 11:16:20 +0500 Subject: [PATCH 1/2] avoid breaking loop in cleanupUploads if error is empty --- weed/shell/command_s3_clean_uploads.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/weed/shell/command_s3_clean_uploads.go b/weed/shell/command_s3_clean_uploads.go index 4f893df7a..b7cf9e426 100644 --- a/weed/shell/command_s3_clean_uploads.go +++ b/weed/shell/command_s3_clean_uploads.go @@ -55,7 +55,9 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer } for _, bucket := range buckets { - c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo) + if err := c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo); err != nil { + fmt.Fprintf(writer, fmt.Sprintf("failed cleanup uploads for backet %s: %v", bucket, err)) + } } return err @@ -82,7 +84,7 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io fmt.Fprintf(writer, "purge %s\n", deleteUrl) err = util.Delete(deleteUrl, "") - if err != nil { + if err != nil && err.Error() != "" { return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err) } } From 76b1c5ce67bb1b680b2257bcb2996bed06691be8 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 12 Apr 2022 13:46:11 +0500 Subject: [PATCH 2/2] cleanupUploads use jwt token --- weed/shell/command_s3_clean_uploads.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/weed/shell/command_s3_clean_uploads.go b/weed/shell/command_s3_clean_uploads.go index b7cf9e426..a6dc8f574 100644 --- a/weed/shell/command_s3_clean_uploads.go +++ b/weed/shell/command_s3_clean_uploads.go @@ -3,6 +3,7 @@ package shell import ( "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util" "io" "math" @@ -39,6 +40,8 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer return nil } + signingKey := util.GetViper().GetString("jwt.signing.key") + var filerBucketsPath string filerBucketsPath, err = readFilerBucketsPath(commandEnv) if err != nil { @@ -55,7 +58,7 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer } for _, bucket := range buckets { - if err := c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo); err != nil { + if err := c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo, signingKey); err != nil { fmt.Fprintf(writer, fmt.Sprintf("failed cleanup uploads for backet %s: %v", bucket, err)) } } @@ -64,7 +67,7 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer } -func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration) error { +func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration, signingKey string) error { uploadsDir := filerBucketsPath + "/" + bucket + "/.uploads" var staleUploads []string now := time.Now() @@ -79,11 +82,16 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io return fmt.Errorf("list uploads under %v: %v", uploadsDir, err) } + var encodedJwt security.EncodedJwt + if signingKey != "" { + encodedJwt = security.GenJwtForFilerServer(security.SigningKey(signingKey), 15*60) + } + for _, staleUpload := range staleUploads { deleteUrl := fmt.Sprintf("http://%s%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerAddress.ToHttpAddress(), uploadsDir, staleUpload) fmt.Fprintf(writer, "purge %s\n", deleteUrl) - err = util.Delete(deleteUrl, "") + err = util.Delete(deleteUrl, string(encodedJwt)) if err != nil && err.Error() != "" { return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err) }