From 2232cfb5b72382e9e5ac1b986c745c0112d4e91a Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Mon, 11 Apr 2022 18:59:36 +0800 Subject: [PATCH 1/3] Check object name and uploadID when processing multipart uploading --- weed/s3api/s3api_object_multipart_handlers.go | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 35bc174c8..1c42b6105 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -70,6 +70,11 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r // Get upload id. uploadID, _, _, _ := getObjectResources(r.URL.Query()) + err := s3a.chkUploadID(bucket, object, uploadID) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) + return + } response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{ Bucket: aws.String(bucket), @@ -94,6 +99,11 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht // Get upload id. uploadID, _, _, _ := getObjectResources(r.URL.Query()) + err := s3a.chkUploadID(bucket, object, uploadID) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) + return + } response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{ Bucket: aws.String(bucket), @@ -165,6 +175,12 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re return } + err := s3a.chkUploadID(bucket, object, uploadID) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) + return + } + response, errCode := s3a.listObjectParts(&s3.ListPartsInput{ Bucket: aws.String(bucket), Key: objectKey(aws.String(object)), @@ -186,11 +202,11 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re // PutObjectPartHandler - Put an object part in a multipart upload. func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { - bucket, _ := xhttp.GetBucketAndObject(r) + bucket, object := xhttp.GetBucketAndObject(r) uploadID := r.URL.Query().Get("uploadId") - exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true) - if !exists { + err := s3a.chkUploadID(bucket, object, uploadID) + if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return } @@ -250,6 +266,22 @@ func (s3a *S3ApiServer) genUploadsFolder(bucket string) string { return fmt.Sprintf("%s/%s/.uploads", s3a.option.BucketsPath, bucket) } +//Check object name and uploadID when processing multipart uploading +func (s3a *S3ApiServer) chkUploadID(bucket string, object string, id string) error { + entry, err := s3a.getEntry(s3a.genUploadsFolder(bucket), id) + if err != nil { + glog.Errorf("getEntry %v/%v : %v", s3a.genUploadsFolder(bucket), id, err) + return err + } + + key := "/" + string(entry.Extended["key"]) + if key != object { + glog.Errorf("object %s and uploadID %s are not matched", object, id) + return fmt.Errorf("object %s and uploadID %s are not matched", object, id) + } + return nil +} + // Parse bucket url queries for ?uploads func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) { prefix = values.Get("prefix") From 180aa88a92a930add1b3199183ae929b6f89b76b Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Tue, 12 Apr 2022 11:04:38 +0800 Subject: [PATCH 2/3] check uploadid using object name hash string --- weed/s3api/filer_multipart.go | 4 +-- weed/s3api/s3api_object_multipart_handlers.go | 36 ++++++++++++------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 64ce16b45..f133767f9 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -13,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" - "github.com/google/uuid" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -29,8 +28,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp glog.V(2).Infof("createMultipartUpload input %v", input) - uploadId, _ := uuid.NewRandom() - uploadIdString := uploadId.String() + uploadIdString := s3a.generateUploadID(*input.Key) if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) { if entry.Extended == nil { diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 1c42b6105..09ecad759 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -2,6 +2,7 @@ package s3api import ( "encoding/xml" + "crypto/sha1" "fmt" "github.com/chrislusf/seaweedfs/weed/glog" xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" @@ -70,7 +71,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r // Get upload id. uploadID, _, _, _ := getObjectResources(r.URL.Query()) - err := s3a.chkUploadID(bucket, object, uploadID) + err := s3a.chkUploadID(object, uploadID) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return @@ -99,7 +100,7 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht // Get upload id. uploadID, _, _, _ := getObjectResources(r.URL.Query()) - err := s3a.chkUploadID(bucket, object, uploadID) + err := s3a.chkUploadID(object, uploadID) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return @@ -175,7 +176,7 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re return } - err := s3a.chkUploadID(bucket, object, uploadID) + err := s3a.chkUploadID(object, uploadID) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return @@ -205,7 +206,13 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ bucket, object := xhttp.GetBucketAndObject(r) uploadID := r.URL.Query().Get("uploadId") - err := s3a.chkUploadID(bucket, object, uploadID) + exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true) + if !exists { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) + return + } + + err = s3a.chkUploadID(object, uploadID) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return @@ -266,16 +273,21 @@ func (s3a *S3ApiServer) genUploadsFolder(bucket string) string { return fmt.Sprintf("%s/%s/.uploads", s3a.option.BucketsPath, bucket) } -//Check object name and uploadID when processing multipart uploading -func (s3a *S3ApiServer) chkUploadID(bucket string, object string, id string) error { - entry, err := s3a.getEntry(s3a.genUploadsFolder(bucket), id) - if err != nil { - glog.Errorf("getEntry %v/%v : %v", s3a.genUploadsFolder(bucket), id, err) - return err +// Generate uploadID hash string from object +func (s3a *S3ApiServer) generateUploadID(object string) string { + if strings.HasPrefix(object, "/") { + object = object[1:] } + h := sha1.New() + h.Write([]byte(object)) + return fmt.Sprintf("%x", h.Sum(nil)) +} - key := "/" + string(entry.Extended["key"]) - if key != object { +//Check object name and uploadID when processing multipart uploading +func (s3a *S3ApiServer) chkUploadID(object string, id string) error { + + hash := s3a.generateUploadID(object) + if hash != id { glog.Errorf("object %s and uploadID %s are not matched", object, id) return fmt.Errorf("object %s and uploadID %s are not matched", object, id) } From 44d810d163af8eb417a0a79f09e80eef8aff9e25 Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Tue, 12 Apr 2022 13:29:50 +0800 Subject: [PATCH 3/3] rename functions and remove uncessary check --- weed/s3api/s3api_object_multipart_handlers.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 09ecad759..d2fa21c2e 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -71,7 +71,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r // Get upload id. uploadID, _, _, _ := getObjectResources(r.URL.Query()) - err := s3a.chkUploadID(object, uploadID) + err := s3a.checkUploadId(object, uploadID) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return @@ -100,7 +100,7 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht // Get upload id. uploadID, _, _, _ := getObjectResources(r.URL.Query()) - err := s3a.chkUploadID(object, uploadID) + err := s3a.checkUploadId(object, uploadID) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return @@ -176,7 +176,7 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re return } - err := s3a.chkUploadID(object, uploadID) + err := s3a.checkUploadId(object, uploadID) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return @@ -206,13 +206,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ bucket, object := xhttp.GetBucketAndObject(r) uploadID := r.URL.Query().Get("uploadId") - exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true) - if !exists { - s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) - return - } - - err = s3a.chkUploadID(object, uploadID) + err := s3a.checkUploadId(object, uploadID) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return @@ -284,7 +278,7 @@ func (s3a *S3ApiServer) generateUploadID(object string) string { } //Check object name and uploadID when processing multipart uploading -func (s3a *S3ApiServer) chkUploadID(object string, id string) error { +func (s3a *S3ApiServer) checkUploadId(object string, id string) error { hash := s3a.generateUploadID(object) if hash != id {