mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-12-19 21:47:50 +08:00
86d92a42b4
* Added global http client * Added Do func for global http client * Changed the code to use the global http client * Fix http client in volume uploader * Fixed pkg name * Fixed http util funcs * Fixed http client for bench_filer_upload * Fixed http client for stress_filer_upload * Fixed http client for filer_server_handlers_proxy * Fixed http client for command_fs_merge_volumes * Fixed http client for command_fs_merge_volumes and command_volume_fsck * Fixed http client for s3api_server * Added init global client for main funcs * Rename global_client to client * Changed: - fixed NewHttpClient; - added CheckIsHttpsClientEnabled func - updated security.toml in scaffold * Reduce the visibility of some functions in the util/http/client pkg * Added the loadSecurityConfig function * Use util.LoadSecurityConfiguration() in NewHttpClient func
89 lines
2.3 KiB
Go
89 lines
2.3 KiB
Go
package broker
|
|
|
|
import (
|
|
"fmt"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/operation"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"os"
|
|
"time"
|
|
)
|
|
|
|
func (b *MessageQueueBroker) appendToFile(targetFile string, data []byte) error {
|
|
|
|
fileId, uploadResult, err2 := b.assignAndUpload(targetFile, data)
|
|
if err2 != nil {
|
|
return err2
|
|
}
|
|
|
|
// find out existing entry
|
|
fullpath := util.FullPath(targetFile)
|
|
dir, name := fullpath.DirAndName()
|
|
entry, err := filer_pb.GetEntry(b, fullpath)
|
|
var offset int64 = 0
|
|
if err == filer_pb.ErrNotFound {
|
|
entry = &filer_pb.Entry{
|
|
Name: name,
|
|
IsDirectory: false,
|
|
Attributes: &filer_pb.FuseAttributes{
|
|
Crtime: time.Now().Unix(),
|
|
Mtime: time.Now().Unix(),
|
|
FileMode: uint32(os.FileMode(0644)),
|
|
Uid: uint32(os.Getuid()),
|
|
Gid: uint32(os.Getgid()),
|
|
},
|
|
}
|
|
} else if err != nil {
|
|
return fmt.Errorf("find %s: %v", fullpath, err)
|
|
} else {
|
|
offset = int64(filer.TotalSize(entry.GetChunks()))
|
|
}
|
|
|
|
// append to existing chunks
|
|
entry.Chunks = append(entry.GetChunks(), uploadResult.ToPbFileChunk(fileId, offset, time.Now().UnixNano()))
|
|
|
|
// update the entry
|
|
return b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
|
return filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
|
|
Directory: dir,
|
|
Entry: entry,
|
|
})
|
|
})
|
|
}
|
|
|
|
func (b *MessageQueueBroker) assignAndUpload(targetFile string, data []byte) (fileId string, uploadResult *operation.UploadResult, err error) {
|
|
|
|
reader := util.NewBytesReader(data)
|
|
|
|
uploader, err := operation.NewUploader()
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
fileId, uploadResult, err, _ = uploader.UploadWithRetry(
|
|
b,
|
|
&filer_pb.AssignVolumeRequest{
|
|
Count: 1,
|
|
Replication: b.option.DefaultReplication,
|
|
Collection: "topics",
|
|
// TtlSec: wfs.option.TtlSec,
|
|
// DiskType: string(wfs.option.DiskType),
|
|
DataCenter: b.option.DataCenter,
|
|
Path: targetFile,
|
|
},
|
|
&operation.UploadOption{
|
|
Cipher: b.option.Cipher,
|
|
},
|
|
func(host, fileId string) string {
|
|
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
|
|
if b.option.VolumeServerAccess == "filerProxy" {
|
|
fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", b.currentFiler, fileId)
|
|
}
|
|
return fileUrl
|
|
},
|
|
reader,
|
|
)
|
|
return
|
|
}
|