From 4a0dc883cfee63832fb2f73d1e4478e918eb0f58 Mon Sep 17 00:00:00 2001 From: Kevin Bulteel Date: Fri, 15 Nov 2024 20:25:18 +0100 Subject: [PATCH] Fix/parse upload filename (#6241) * fix: parse filename in PUT + refactor * fix: master iu public url with http * fix: better parsing and handle disposition header * fix: take mime type from file extension if not set --- weed/operation/needle_parse_test.go | 2 +- weed/server/master_ui/master.html | 8 +- weed/server/master_ui/templates.go | 15 +- weed/storage/needle/needle_parse_upload.go | 264 ++++++++++----------- 4 files changed, 139 insertions(+), 150 deletions(-) diff --git a/weed/operation/needle_parse_test.go b/weed/operation/needle_parse_test.go index b4bac5976..7526a6e79 100644 --- a/weed/operation/needle_parse_test.go +++ b/weed/operation/needle_parse_test.go @@ -44,7 +44,7 @@ func TestCreateNeedleFromRequest(t *testing.T) { { mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) { assert.Equal(t, nil, err, "upload: %v", err) - assert.Equal(t, "", string(n.Mime), "mime detection failed: %v", string(n.Mime)) + assert.Equal(t, "text/plain; charset=utf-8", string(n.Mime), "mime detection failed: %v", string(n.Mime)) assert.Equal(t, true, n.IsCompressed(), "this should be compressed") assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip") fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize) diff --git a/weed/server/master_ui/master.html b/weed/server/master_ui/master.html index 2e37c3a62..40d49991b 100644 --- a/weed/server/master_ui/master.html +++ b/weed/server/master_ui/master.html @@ -33,14 +33,14 @@ {{ with .RaftServer }} Leader - {{ .Leader }} + {{ .Leader }} Other Masters @@ -88,9 +88,9 @@ {{ $dc.Id }} {{ $rack.Id }} - {{ $dn.Url }} + {{ $dn.Url }} {{ if ne $dn.PublicUrl $dn.Url }} - / {{ $dn.PublicUrl }} + / {{ $dn.PublicUrl }} {{ end }} {{ $dn.Volumes }} diff --git a/weed/server/master_ui/templates.go b/weed/server/master_ui/templates.go index a6dcc57d7..d32c5efdf 100644 --- a/weed/server/master_ui/templates.go +++ b/weed/server/master_ui/templates.go @@ -3,6 +3,7 @@ package master_ui import ( _ "embed" "html/template" + "strings" ) //go:embed master.html @@ -11,5 +12,17 @@ var masterHtml string //go:embed masterNewRaft.html var masterNewRaftHtml string -var StatusTpl = template.Must(template.New("status").Parse(masterHtml)) +var templateFunctions = template.FuncMap{ + "url": func(input string) string { + + if !strings.HasPrefix(input, "http://") && !strings.HasPrefix(input, "https://") { + return "http://" + input + } + + return input + }, +} + +var StatusTpl = template.Must(template.New("status").Funcs(templateFunctions).Parse(masterHtml)) + var StatusNewRaftTpl = template.Must(template.New("status").Parse(masterNewRaftHtml)) diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go index 28de27910..89708303d 100644 --- a/weed/storage/needle/needle_parse_upload.go +++ b/weed/storage/needle/needle_parse_upload.go @@ -43,19 +43,8 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p } } - if r.Method == http.MethodPost { - contentType := r.Header.Get("Content-Type") + e = parseUpload(r, sizeLimit, pu) - // If content-type is explicitly set, upload the file without parsing form-data - if contentType != "" && !strings.Contains(contentType, "form-data") { - e = parseRawPost(r, sizeLimit, pu) - } else { - e = parseMultipart(r, sizeLimit, pu) - } - - } else { - e = parsePut(r, sizeLimit, pu) - } if e != nil { return } @@ -108,87 +97,136 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p return } -func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) error { - pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip" - // pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd" - pu.MimeType = r.Header.Get("Content-Type") - pu.FileName = "" - dataSize, err := pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1)) - if err == io.EOF || dataSize == sizeLimit+1 { - io.Copy(io.Discard, r.Body) - } - pu.Data = pu.bytesBuffer.Bytes() - r.Body.Close() - return nil -} +func parseUpload(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) { -func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) { defer func() { if e != nil && r.Body != nil { io.Copy(io.Discard, r.Body) r.Body.Close() } }() - form, fe := r.MultipartReader() - if fe != nil { - glog.V(0).Infoln("MultipartReader [ERROR]", fe) - e = fe - return - } - - // first multi-part item - part, fe := form.NextPart() - if fe != nil { - glog.V(0).Infoln("Reading Multi part [ERROR]", fe) - e = fe - return - } - - pu.FileName = part.FileName() - if pu.FileName != "" { - pu.FileName = path.Base(pu.FileName) - } + contentType := r.Header.Get("Content-Type") var dataSize int64 - dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1)) - if e != nil { - glog.V(0).Infoln("Reading Content [ERROR]", e) - return - } - if dataSize == sizeLimit+1 { - e = fmt.Errorf("file over the limited %d bytes", sizeLimit) - return - } - pu.Data = pu.bytesBuffer.Bytes() - // if the filename is empty string, do a search on the other multi-part items - for pu.FileName == "" { - part2, fe := form.NextPart() + if r.Method == http.MethodPost && (contentType == "" || strings.Contains(contentType, "form-data")) { + form, fe := r.MultipartReader() + if fe != nil { - break // no more or on error, just safely break + glog.V(0).Infoln("MultipartReader [ERROR]", fe) + e = fe + return } - fName := part2.FileName() - - // found the first multi-part has filename - if fName != "" { - pu.bytesBuffer.Reset() - dataSize2, fe2 := pu.bytesBuffer.ReadFrom(io.LimitReader(part2, sizeLimit+1)) - if fe2 != nil { - glog.V(0).Infoln("Reading Content [ERROR]", fe2) - e = fe2 - return - } - if dataSize2 == sizeLimit+1 { - e = fmt.Errorf("file over the limited %d bytes", sizeLimit) - return - } - - // update - pu.Data = pu.bytesBuffer.Bytes() - pu.FileName = path.Base(fName) - break + // first multi-part item + part, fe := form.NextPart() + if fe != nil { + glog.V(0).Infoln("Reading Multi part [ERROR]", fe) + e = fe + return } + + pu.FileName = part.FileName() + if pu.FileName != "" { + pu.FileName = path.Base(pu.FileName) + } + + dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1)) + if e != nil { + glog.V(0).Infoln("Reading Content [ERROR]", e) + return + } + if dataSize == sizeLimit+1 { + e = fmt.Errorf("file over the limited %d bytes", sizeLimit) + return + } + pu.Data = pu.bytesBuffer.Bytes() + + contentType = part.Header.Get("Content-Type") + + // if the filename is empty string, do a search on the other multi-part items + for pu.FileName == "" { + part2, fe := form.NextPart() + if fe != nil { + break // no more or on error, just safely break + } + + fName := part2.FileName() + + // found the first multi-part has filename + if fName != "" { + pu.bytesBuffer.Reset() + dataSize2, fe2 := pu.bytesBuffer.ReadFrom(io.LimitReader(part2, sizeLimit+1)) + if fe2 != nil { + glog.V(0).Infoln("Reading Content [ERROR]", fe2) + e = fe2 + return + } + if dataSize2 == sizeLimit+1 { + e = fmt.Errorf("file over the limited %d bytes", sizeLimit) + return + } + + // update + pu.Data = pu.bytesBuffer.Bytes() + pu.FileName = path.Base(fName) + contentType = part.Header.Get("Content-Type") + part = part2 + break + } + } + + pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip" + // pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd" + + } else { + disposition := r.Header.Get("Content-Disposition") + + if strings.Contains(disposition, "name=") { + + if !strings.HasPrefix(disposition, "inline") && !strings.HasPrefix(disposition, "attachment") { + disposition = "attachment; " + disposition + } + + _, mediaTypeParams, err := mime.ParseMediaType(disposition) + + if err == nil { + dpFilename, hasFilename := mediaTypeParams["filename"] + dpName, hasName := mediaTypeParams["name"] + + if hasFilename { + pu.FileName = dpFilename + } else if hasName { + pu.FileName = dpName + } + + } + + } else { + pu.FileName = "" + } + + if pu.FileName != "" { + pu.FileName = path.Base(pu.FileName) + } else { + pu.FileName = path.Base(r.URL.Path) + } + + dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1)) + + if e != nil { + glog.V(0).Infoln("Reading Content [ERROR]", e) + return + } + if dataSize == sizeLimit+1 { + e = fmt.Errorf("file over the limited %d bytes", sizeLimit) + return + } + + pu.Data = pu.bytesBuffer.Bytes() + pu.MimeType = contentType + pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip" + // pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd" } pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm")) @@ -197,81 +235,19 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error dotIndex := strings.LastIndex(pu.FileName, ".") ext, mtype := "", "" + if dotIndex > 0 { ext = strings.ToLower(pu.FileName[dotIndex:]) mtype = mime.TypeByExtension(ext) } - contentType := part.Header.Get("Content-Type") - if contentType != "" && contentType != "application/octet-stream" && mtype != contentType { - pu.MimeType = contentType // only return mime type if not deducible - mtype = contentType - } - - } - pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip" - // pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd" - - return -} - -func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) { - - defer func() { - if e != nil && r.Body != nil { - io.Copy(io.Discard, r.Body) - r.Body.Close() - } - }() - - pu.FileName = r.Header.Get("Content-Disposition") - - if pu.FileName != "" && strings.Contains(pu.FileName, "filename=") { - parts := strings.Split(pu.FileName, "filename=") - parts = strings.Split(parts[1], "\"") - - pu.FileName = parts[1] - } else { - pu.FileName = "" - } - - if pu.FileName != "" { - pu.FileName = path.Base(pu.FileName) - } else { - pu.FileName = path.Base(r.URL.Path) - } - - var dataSize int64 - dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1)) - - if e != nil { - glog.V(0).Infoln("Reading Content [ERROR]", e) - return - } - if dataSize == sizeLimit+1 { - e = fmt.Errorf("file over the limited %d bytes", sizeLimit) - return - } - pu.Data = pu.bytesBuffer.Bytes() - pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm")) - - if !pu.IsChunkedFile { - - dotIndex := strings.LastIndex(pu.FileName, ".") - ext, mtype := "", "" - if dotIndex > 0 { - ext = strings.ToLower(pu.FileName[dotIndex:]) - mtype = mime.TypeByExtension(ext) - } - contentType := r.Header.Get("Content-Type") if contentType != "" && contentType != "application/octet-stream" && mtype != contentType { pu.MimeType = contentType // only return mime type if not deducible - mtype = contentType + } else if mtype != "" && pu.MimeType == "" && mtype != "application/octet-stream" { + pu.MimeType = mtype } } - pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip" - // pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd" return }