mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-11-28 05:15:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

CheckAllChunkViews() for HEAD requests only

Browse Source
This commit is contained in:
Konstantin Lebedev 2021-03-16 14:15:17 +05:00
parent 3a3699867b
commit 06da02739d
6 changed files with 20 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
weed/command/filer_cat.go
View File

@ -110,7 +110,7 @@ func runFilerCat(cmd *Command, args []string) bool {
filerCat.filerClient = client filerCat.filerClient = client
return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
}) })

2
weed/filer/filer_on_meta_event.go
View File

@ -52,7 +52,7 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR
func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
var buf bytes.Buffer var buf bytes.Buffer
err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64) err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }

2
weed/filer/read_write.go
View File

@ -27,7 +27,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
return err return err
} }
return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
} }

23
weed/filer/stream.go
View File

@ -14,7 +14,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/chrislusf/seaweedfs/weed/wdclient"
) )
func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error { func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64, isCheck bool) error {
glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks) glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size) chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
@ -34,15 +34,20 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
fileId2Url[chunkView.FileId] = urlStrings fileId2Url[chunkView.FileId] = urlStrings
} }
for idx, chunkView := range chunkViews { if isCheck {
urlStrings := fileId2Url[chunkView.FileId]
// Pre-check all chunkViews urls // Pre-check all chunkViews urls
gErr := new(errgroup.Group) gErr := new(errgroup.Group)
if len(chunkViews) > 1 && idx == 0 { CheckAllChunkViews(chunkViews, &fileId2Url, gErr)
CheckAllChunkViews(chunkViews[1:], &fileId2Url, gErr) if err := gErr.Wait(); err != nil {
glog.Errorf("check all chunks: %v", err)
return fmt.Errorf("check all chunks: %v", err)
} }
return nil
}
for _, chunkView := range chunkViews {
urlStrings := fileId2Url[chunkView.FileId]
data, err := retriedFetchChunkData( data, err := retriedFetchChunkData(
urlStrings, urlStrings,
chunkView.CipherKey, chunkView.CipherKey,
@ -56,10 +61,6 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
glog.Errorf("read chunk: %v", err) glog.Errorf("read chunk: %v", err)
return fmt.Errorf("read chunk: %v", err) return fmt.Errorf("read chunk: %v", err)
} }
if err := gErr.Wait(); err != nil {
glog.Errorf("check all chunks: %v", err)
return fmt.Errorf("check all chunks: %v", err)
}
_, err = w.Write(data) _, err = w.Write(data)
if err != nil { if err != nil {
glog.Errorf("write chunk: %v", err) glog.Errorf("write chunk: %v", err)

5
weed/server/filer_server_handlers_read.go
View File

@ -131,6 +131,9 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
if r.Method == "HEAD" { if r.Method == "HEAD" {
w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, true)
})
return return
} }
@ -158,7 +161,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
} }
return err return err
} }
return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size) return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, false)
}) })
} }

2
weed/shell/command_fs_cat.go
View File

@ -52,7 +52,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write
return err return err
} }
return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
}) })