mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-11-23 18:49:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix/parse upload filename (#6241)

Browse Source 
* fix: parse filename in PUT + refactor

* fix: master iu public url with http

* fix: better parsing and handle disposition header

* fix: take mime type from file extension if not set
This commit is contained in:
Kevin Bulteel 2024-11-15 20:25:18 +01:00 committed by GitHub
parent 8ebbf37e8e
commit 4a0dc883cf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 139 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
weed/operation/needle_parse_test.go
View File

@ -44,7 +44,7 @@ func TestCreateNeedleFromRequest(t *testing.T) {
{
mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
assert.Equal(t, nil, err, "upload: %v", err)
assert.Equal(t, "", string(n.Mime), "mime detection failed: %v", string(n.Mime))
assert.Equal(t, "text/plain; charset=utf-8", string(n.Mime), "mime detection failed: %v", string(n.Mime))
assert.Equal(t, true, n.IsCompressed(), "this should be compressed")
assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip")
fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize)

8
weed/server/master_ui/master.html
View File

@ -33,14 +33,14 @@
{{ with .RaftServer }}
<tr>
<th>Leader</th>
<td><a href="http://{{ .Leader }}">{{ .Leader }}</a></td>
<td><a href="{{ url .Leader }}">{{ .Leader }}</a></td>
</tr>
<tr>
<th>Other Masters</th>
<td class="col-sm-5">
<ul class="list-unstyled">
{{ range $k, $p := .Peers }}
<li><a href="http://{{ $p.Name }}/ui/index.html">{{ $p.Name }}</a></li>
<li><a href="{{ url $p.Name }}/ui/index.html">{{ $p.Name }}</a></li>
{{ end }}
</ul>
</td>
@ -88,9 +88,9 @@
<tr>
<td><code>{{ $dc.Id }}</code></td>
<td>{{ $rack.Id }}</td>
<td><a href="http://{{ $dn.Url }}/ui/index.html">{{ $dn.Url }}</a>
<td><a href="{{ url $dn.Url }}/ui/index.html">{{ $dn.Url }}</a>
{{ if ne $dn.PublicUrl $dn.Url }}
/ <a href="http://{{ $dn.PublicUrl }}/ui/index.html">{{ $dn.PublicUrl }}</a>
/ <a href="{{ url $dn.PublicUrl }}/ui/index.html">{{ $dn.PublicUrl }}</a>
{{ end }}
</td>
<td>{{ $dn.Volumes }}</td>

15
weed/server/master_ui/templates.go
View File

@ -3,6 +3,7 @@ package master_ui
import (
_ "embed"
"html/template"
"strings"
)
//go:embed master.html
@ -11,5 +12,17 @@ var masterHtml string
//go:embed masterNewRaft.html
var masterNewRaftHtml string
var StatusTpl = template.Must(template.New("status").Parse(masterHtml))
var templateFunctions = template.FuncMap{
"url": func(input string) string {
if !strings.HasPrefix(input, "http://") && !strings.HasPrefix(input, "https://") {
return "http://" + input
}
return input
},
}
var StatusTpl = template.Must(template.New("status").Funcs(templateFunctions).Parse(masterHtml))
var StatusNewRaftTpl = template.Must(template.New("status").Parse(masterNewRaftHtml))

100
weed/storage/needle/needle_parse_upload.go
View File

@ -43,19 +43,8 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
}
}
if r.Method == http.MethodPost {
contentType := r.Header.Get("Content-Type")
e = parseUpload(r, sizeLimit, pu)
// If content-type is explicitly set, upload the file without parsing form-data
if contentType != "" && !strings.Contains(contentType, "form-data") {
e = parseRawPost(r, sizeLimit, pu)
} else {
e = parseMultipart(r, sizeLimit, pu)
}
} else {
e = parsePut(r, sizeLimit, pu)
}
if e != nil {
return
}
@ -108,28 +97,21 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
return
}
func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) error {
pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
pu.MimeType = r.Header.Get("Content-Type")
pu.FileName = ""
dataSize, err := pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
if err == io.EOF || dataSize == sizeLimit+1 {
io.Copy(io.Discard, r.Body)
}
pu.Data = pu.bytesBuffer.Bytes()
r.Body.Close()
return nil
}
func parseUpload(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
defer func() {
if e != nil && r.Body != nil {
io.Copy(io.Discard, r.Body)
r.Body.Close()
}
}()
contentType := r.Header.Get("Content-Type")
var dataSize int64
if r.Method == http.MethodPost && (contentType == "" || strings.Contains(contentType, "form-data")) {
form, fe := r.MultipartReader()
if fe != nil {
glog.V(0).Infoln("MultipartReader [ERROR]", fe)
e = fe
@ -149,7 +131,6 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
pu.FileName = path.Base(pu.FileName)
}
var dataSize int64
dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1))
if e != nil {
glog.V(0).Infoln("Reading Content [ERROR]", e)
@ -161,6 +142,8 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
}
pu.Data = pu.bytesBuffer.Bytes()
contentType = part.Header.Get("Content-Type")
// if the filename is empty string, do a search on the other multi-part items
for pu.FileName == "" {
part2, fe := form.NextPart()
@ -187,49 +170,38 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
// update
pu.Data = pu.bytesBuffer.Bytes()
pu.FileName = path.Base(fName)
contentType = part.Header.Get("Content-Type")
part = part2
break
}
}
pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
if !pu.IsChunkedFile {
dotIndex := strings.LastIndex(pu.FileName, ".")
ext, mtype := "", ""
if dotIndex > 0 {
ext = strings.ToLower(pu.FileName[dotIndex:])
mtype = mime.TypeByExtension(ext)
}
contentType := part.Header.Get("Content-Type")
if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
pu.MimeType = contentType // only return mime type if not deducible
mtype = contentType
}
}
pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
return
} else {
disposition := r.Header.Get("Content-Disposition")
if strings.Contains(disposition, "name=") {
if !strings.HasPrefix(disposition, "inline") && !strings.HasPrefix(disposition, "attachment") {
disposition = "attachment; " + disposition
}
func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
_, mediaTypeParams, err := mime.ParseMediaType(disposition)
defer func() {
if e != nil && r.Body != nil {
io.Copy(io.Discard, r.Body)
r.Body.Close()
if err == nil {
dpFilename, hasFilename := mediaTypeParams["filename"]
dpName, hasName := mediaTypeParams["name"]
if hasFilename {
pu.FileName = dpFilename
} else if hasName {
pu.FileName = dpName
}
}()
pu.FileName = r.Header.Get("Content-Disposition")
}
if pu.FileName != "" && strings.Contains(pu.FileName, "filename=") {
parts := strings.Split(pu.FileName, "filename=")
parts = strings.Split(parts[1], "\"")
pu.FileName = parts[1]
} else {
pu.FileName = ""
}
@ -240,7 +212,6 @@ func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error)
pu.FileName = path.Base(r.URL.Path)
}
var dataSize int64
dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
if e != nil {
@ -251,27 +222,32 @@ func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error)
e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
return
}
pu.Data = pu.bytesBuffer.Bytes()
pu.MimeType = contentType
pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
}
pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
if !pu.IsChunkedFile {
dotIndex := strings.LastIndex(pu.FileName, ".")
ext, mtype := "", ""
if dotIndex > 0 {
ext = strings.ToLower(pu.FileName[dotIndex:])
mtype = mime.TypeByExtension(ext)
}
contentType := r.Header.Get("Content-Type")
if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
pu.MimeType = contentType // only return mime type if not deducible
mtype = contentType
} else if mtype != "" && pu.MimeType == "" && mtype != "application/octet-stream" {
pu.MimeType = mtype
}
}
pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
return
}