mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-11-30 15:19:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

remove zstd

Browse Source 
fix https://github.com/chrislusf/seaweedfs/issues/1629
This commit is contained in:
Chris Lu 2020-11-21 13:06:35 -08:00
parent 025bd8d447
commit 92f906b6fc
6 changed files with 28 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -40,7 +40,7 @@ require (
github.com/json-iterator/go v1.1.10
github.com/karlseguin/ccache v2.0.3+incompatible
github.com/karlseguin/expect v1.0.1 // indirect
github.com/klauspost/compress v1.10.9
github.com/klauspost/compress v1.10.9 // indirect
github.com/klauspost/cpuid v1.2.1 // indirect
github.com/klauspost/crc32 v1.2.0
github.com/klauspost/reedsolomon v1.9.2

2
weed/operation/needle_parse_test.go
View File

@ -75,6 +75,7 @@ func TestCreateNeedleFromRequest(t *testing.T) {
Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(gzippedData), true, "text/plain", nil, "")
}
/*
{
mc.needleHandling = func(n *needle.Needle, originalSize int, err error) {
assert.Equal(t, nil, err, "upload: %v", err)
@ -98,6 +99,7 @@ func TestCreateNeedleFromRequest(t *testing.T) {
zstdData, _ := util.ZstdData([]byte(textContent))
Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(zstdData), false, "application/zstd", nil, "")
}
*/
}

4
weed/server/volume_server_handlers_read.go
View File

@ -159,8 +159,8 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
if n.Data, err = util.DecompressData(n.Data); err != nil {
glog.V(0).Infoln("ungzip error:", err, r.URL.Path)
}
} else if strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") && util.IsZstdContent(n.Data) {
w.Header().Set("Content-Encoding", "zstd")
// } else if strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") && util.IsZstdContent(n.Data) {
// w.Header().Set("Content-Encoding", "zstd")
} else if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && util.IsGzippedContent(n.Data) {
w.Header().Set("Content-Encoding", "gzip")
} else {

5
weed/shell/command_fs_meta_cat.go
View File

@ -72,8 +72,9 @@ func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.W
bytes, _ := proto.Marshal(respLookupEntry.Entry)
gzippedBytes, _ := util.GzipData(bytes)
zstdBytes, _ := util.ZstdData(bytes)
fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d zstd:%d\n", len(respLookupEntry.Entry.Chunks), len(bytes), len(gzippedBytes), len(zstdBytes))
// zstdBytes, _ := util.ZstdData(bytes)
// fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d zstd:%d\n", len(respLookupEntry.Entry.Chunks), len(bytes), len(gzippedBytes), len(zstdBytes))
fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d\n", len(respLookupEntry.Entry.Chunks), len(bytes), len(gzippedBytes))
return nil

6
weed/storage/needle/needle_parse_upload.go
View File

@ -23,7 +23,7 @@ type ParsedUpload struct {
MimeType string
PairMap map[string]string
IsGzipped bool
IsZstd bool
// IsZstd bool
OriginalDataSize int
ModifiedTime uint64
Ttl *TTL
@ -100,7 +100,7 @@ func ParseUpload(r *http.Request, sizeLimit int64) (pu *ParsedUpload, e error) {
func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
pu.MimeType = r.Header.Get("Content-Type")
pu.FileName = ""
pu.Data, e = ioutil.ReadAll(io.LimitReader(r.Body, sizeLimit+1))
@ -194,7 +194,7 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
}
pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
}
return

29
weed/util/compression.go
View File

@ -9,7 +9,7 @@ import (
"strings"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/klauspost/compress/zstd"
// "github.com/klauspost/compress/zstd"
)
var (
@ -55,19 +55,16 @@ func GzipData(input []byte) ([]byte, error) {
return buf.Bytes(), nil
}
var zstdEncoder, _ = zstd.NewWriter(nil)
func ZstdData(input []byte) ([]byte, error) {
return zstdEncoder.EncodeAll(input, nil), nil
}
func DecompressData(input []byte) ([]byte, error) {
if IsGzippedContent(input) {
return ungzipData(input)
}
/*
if IsZstdContent(input) {
return unzstdData(input)
}
*/
return input, UnsupportedCompression
}
@ -82,12 +79,6 @@ func ungzipData(input []byte) ([]byte, error) {
return output, err
}
var decoder, _ = zstd.NewReader(nil)
func unzstdData(input []byte) ([]byte, error) {
return decoder.DecodeAll(input, nil)
}
func IsGzippedContent(data []byte) bool {
if len(data) < 2 {
return false
@ -95,12 +86,26 @@ func IsGzippedContent(data []byte) bool {
return data[0] == 31 && data[1] == 139
}
/*
var zstdEncoder, _ = zstd.NewWriter(nil)
func ZstdData(input []byte) ([]byte, error) {
return zstdEncoder.EncodeAll(input, nil), nil
}
var decoder, _ = zstd.NewReader(nil)
func unzstdData(input []byte) ([]byte, error) {
return decoder.DecodeAll(input, nil)
}
func IsZstdContent(data []byte) bool {
if len(data) < 4 {
return false
}
return data[3] == 0xFD && data[2] == 0x2F && data[1] == 0xB5 && data[0] == 0x28
}
*/
/*
* Default not to compressed since compression can be done on client side.