2019-04-19 12:43:36 +08:00
|
|
|
package needle
|
2011-12-16 22:51:26 +08:00
|
|
|
|
|
|
|
import (
|
2017-01-08 22:34:26 +08:00
|
|
|
"encoding/json"
|
2014-12-26 13:29:44 +08:00
|
|
|
"fmt"
|
2020-01-03 16:37:24 +08:00
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
2012-06-29 15:53:47 +08:00
|
|
|
"net/http"
|
2012-07-17 01:15:16 +08:00
|
|
|
"strconv"
|
2011-12-16 22:51:26 +08:00
|
|
|
"strings"
|
2013-07-09 14:38:38 +08:00
|
|
|
"time"
|
2014-10-27 02:34:55 +08:00
|
|
|
|
2016-06-03 09:09:14 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/images"
|
2018-07-08 17:28:04 +08:00
|
|
|
. "github.com/chrislusf/seaweedfs/weed/storage/types"
|
2020-03-02 15:37:02 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/util"
|
2011-12-16 22:51:26 +08:00
|
|
|
)
|
|
|
|
|
2012-12-21 16:36:55 +08:00
|
|
|
const (
|
2018-07-08 17:28:04 +08:00
|
|
|
NeedleChecksumSize = 4
|
|
|
|
PairNamePrefix = "Seaweed-"
|
2012-12-21 16:36:55 +08:00
|
|
|
)
|
|
|
|
|
2013-09-20 02:06:14 +08:00
|
|
|
/*
|
2014-12-26 15:36:33 +08:00
|
|
|
* A Needle means a uploaded and stored file.
|
2013-09-20 02:06:14 +08:00
|
|
|
* Needle file size is limited to 4GB for now.
|
|
|
|
*/
|
2011-12-19 13:59:37 +08:00
|
|
|
type Needle struct {
|
2018-07-08 17:28:04 +08:00
|
|
|
Cookie Cookie `comment:"random number to mitigate brute force lookups"`
|
|
|
|
Id NeedleId `comment:"needle id"`
|
|
|
|
Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
|
2012-12-21 18:13:02 +08:00
|
|
|
|
2013-07-09 14:38:38 +08:00
|
|
|
DataSize uint32 `comment:"Data size"` //version2
|
|
|
|
Data []byte `comment:"The actual file data"`
|
2019-01-17 09:17:19 +08:00
|
|
|
Flags byte `comment:"boolean flags"` //version2
|
|
|
|
NameSize uint8 //version2
|
2013-07-09 14:38:38 +08:00
|
|
|
Name []byte `comment:"maximum 256 characters"` //version2
|
2019-01-17 09:17:19 +08:00
|
|
|
MimeSize uint8 //version2
|
2013-07-09 14:38:38 +08:00
|
|
|
Mime []byte `comment:"maximum 256 characters"` //version2
|
2019-01-17 09:17:19 +08:00
|
|
|
PairsSize uint16 //version2
|
2017-01-08 09:16:29 +08:00
|
|
|
Pairs []byte `comment:"additional name value pairs, json format, maximum 64kB"`
|
2013-07-09 14:38:38 +08:00
|
|
|
LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
|
2014-09-21 03:38:59 +08:00
|
|
|
Ttl *TTL
|
2012-12-21 18:13:02 +08:00
|
|
|
|
2018-07-24 16:36:04 +08:00
|
|
|
Checksum CRC `comment:"CRC32 to check integrity"`
|
|
|
|
AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
|
|
|
|
Padding []byte `comment:"Aligned to 8 bytes"`
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
|
|
|
|
2014-12-26 13:29:44 +08:00
|
|
|
func (n *Needle) String() (str string) {
|
2018-07-22 08:41:21 +08:00
|
|
|
str = fmt.Sprintf("%s Size:%d, DataSize:%d, Name:%s, Mime:%s", formatNeedleIdCookie(n.Id, n.Cookie), n.Size, n.DataSize, n.Name, n.Mime)
|
2014-12-26 13:29:44 +08:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-01-03 16:37:24 +08:00
|
|
|
func ParseUpload(r *http.Request, sizeLimit int64) (
|
2018-12-23 05:11:07 +08:00
|
|
|
fileName string, data []byte, mimeType string, pairMap map[string]string, isGzipped bool, originalDataSize int,
|
2015-12-15 14:38:58 +08:00
|
|
|
modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) {
|
2017-01-08 22:34:26 +08:00
|
|
|
pairMap = make(map[string]string)
|
2017-01-08 09:16:29 +08:00
|
|
|
for k, v := range r.Header {
|
|
|
|
if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) {
|
|
|
|
pairMap[k] = v[0]
|
|
|
|
}
|
|
|
|
}
|
2017-01-08 22:34:26 +08:00
|
|
|
|
2018-07-22 06:58:48 +08:00
|
|
|
if r.Method == "POST" {
|
2020-01-03 16:37:24 +08:00
|
|
|
fileName, data, mimeType, isGzipped, originalDataSize, isChunkedFile, e = parseMultipart(r, sizeLimit)
|
2018-07-22 06:58:48 +08:00
|
|
|
} else {
|
2020-03-02 15:37:02 +08:00
|
|
|
isGzipped = r.Header.Get("Content-Encoding") == "gzip"
|
2018-07-22 06:58:48 +08:00
|
|
|
mimeType = r.Header.Get("Content-Type")
|
|
|
|
fileName = ""
|
2020-01-03 16:37:24 +08:00
|
|
|
data, e = ioutil.ReadAll(io.LimitReader(r.Body, sizeLimit+1))
|
2019-01-03 04:57:33 +08:00
|
|
|
originalDataSize = len(data)
|
2020-01-03 16:37:24 +08:00
|
|
|
if e == io.EOF || int64(originalDataSize) == sizeLimit+1 {
|
|
|
|
io.Copy(ioutil.Discard, r.Body)
|
|
|
|
}
|
|
|
|
r.Body.Close()
|
2020-03-02 15:37:02 +08:00
|
|
|
if isGzipped {
|
|
|
|
if unzipped, e := util.UnGzipData(data); e == nil {
|
|
|
|
originalDataSize = len(unzipped)
|
|
|
|
}
|
|
|
|
} else if shouldGzip, _ := util.IsGzippableFileType("", mimeType); shouldGzip {
|
|
|
|
if compressedData, err := util.GzipData(data); err == nil {
|
|
|
|
data = compressedData
|
|
|
|
isGzipped = true
|
|
|
|
}
|
|
|
|
}
|
2018-07-22 06:58:48 +08:00
|
|
|
}
|
2013-08-07 02:23:24 +08:00
|
|
|
if e != nil {
|
2013-08-06 04:37:41 +08:00
|
|
|
return
|
|
|
|
}
|
2015-01-16 17:30:23 +08:00
|
|
|
|
2013-08-07 02:23:24 +08:00
|
|
|
modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
|
2014-09-21 03:38:59 +08:00
|
|
|
ttl, _ = ReadTTL(r.FormValue("ttl"))
|
2017-01-04 11:23:40 +08:00
|
|
|
|
2013-09-02 14:58:21 +08:00
|
|
|
return
|
2013-08-07 02:23:24 +08:00
|
|
|
}
|
2020-01-03 16:37:24 +08:00
|
|
|
func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit int64) (n *Needle, originalSize int, e error) {
|
2017-01-08 22:34:26 +08:00
|
|
|
var pairMap map[string]string
|
2015-12-01 20:23:50 +08:00
|
|
|
fname, mimeType, isGzipped, isChunkedFile := "", "", false, false
|
2013-08-07 02:23:24 +08:00
|
|
|
n = new(Needle)
|
2020-01-03 16:37:24 +08:00
|
|
|
fname, n.Data, mimeType, pairMap, isGzipped, originalSize, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r, sizeLimit)
|
2013-08-07 02:23:24 +08:00
|
|
|
if e != nil {
|
2013-09-02 14:58:21 +08:00
|
|
|
return
|
|
|
|
}
|
|
|
|
if len(fname) < 256 {
|
|
|
|
n.Name = []byte(fname)
|
|
|
|
n.SetHasName()
|
2013-08-07 02:23:24 +08:00
|
|
|
}
|
|
|
|
if len(mimeType) < 256 {
|
|
|
|
n.Mime = []byte(mimeType)
|
|
|
|
n.SetHasMime()
|
|
|
|
}
|
2017-01-08 22:34:26 +08:00
|
|
|
if len(pairMap) != 0 {
|
|
|
|
trimmedPairMap := make(map[string]string)
|
|
|
|
for k, v := range pairMap {
|
|
|
|
trimmedPairMap[k[len(PairNamePrefix):]] = v
|
|
|
|
}
|
|
|
|
|
|
|
|
pairs, _ := json.Marshal(trimmedPairMap)
|
|
|
|
if len(pairs) < 65536 {
|
|
|
|
n.Pairs = pairs
|
|
|
|
n.PairsSize = uint16(len(pairs))
|
|
|
|
n.SetHasPairs()
|
|
|
|
}
|
2017-01-08 09:16:29 +08:00
|
|
|
}
|
2013-08-07 02:23:24 +08:00
|
|
|
if isGzipped {
|
|
|
|
n.SetGzipped()
|
2012-07-30 16:37:10 +08:00
|
|
|
}
|
2013-08-07 02:23:24 +08:00
|
|
|
if n.LastModified == 0 {
|
2013-07-10 15:25:14 +08:00
|
|
|
n.LastModified = uint64(time.Now().Unix())
|
|
|
|
}
|
2013-10-16 23:39:09 +08:00
|
|
|
n.SetHasLastModifiedDate()
|
2014-09-21 11:51:24 +08:00
|
|
|
if n.Ttl != EMPTY_TTL {
|
2014-09-21 03:38:59 +08:00
|
|
|
n.SetHasTtl()
|
|
|
|
}
|
2012-12-22 18:10:45 +08:00
|
|
|
|
2015-12-01 20:23:50 +08:00
|
|
|
if isChunkedFile {
|
2015-12-15 14:38:58 +08:00
|
|
|
n.SetIsChunkManifest()
|
2015-12-01 20:23:50 +08:00
|
|
|
}
|
|
|
|
|
2014-05-16 16:10:46 +08:00
|
|
|
if fixJpgOrientation {
|
|
|
|
loweredName := strings.ToLower(fname)
|
|
|
|
if mimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
|
|
|
|
n.Data = images.FixJpgOrientation(n.Data)
|
|
|
|
}
|
2014-05-15 16:56:08 +08:00
|
|
|
}
|
|
|
|
|
2013-08-07 02:23:24 +08:00
|
|
|
n.Checksum = NewCRC(n.Data)
|
2011-12-16 22:51:26 +08:00
|
|
|
|
2012-06-29 15:53:47 +08:00
|
|
|
commaSep := strings.LastIndex(r.URL.Path, ",")
|
2011-12-22 12:04:47 +08:00
|
|
|
dotSep := strings.LastIndex(r.URL.Path, ".")
|
|
|
|
fid := r.URL.Path[commaSep+1:]
|
|
|
|
if dotSep > 0 {
|
2018-07-24 16:38:08 +08:00
|
|
|
fid = r.URL.Path[commaSep+1 : dotSep]
|
2011-12-22 12:04:47 +08:00
|
|
|
}
|
|
|
|
|
2013-12-10 05:53:24 +08:00
|
|
|
e = n.ParsePath(fid)
|
2011-12-19 13:59:37 +08:00
|
|
|
|
|
|
|
return
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
2013-12-10 05:53:24 +08:00
|
|
|
func (n *Needle) ParsePath(fid string) (err error) {
|
2011-12-22 12:04:47 +08:00
|
|
|
length := len(fid)
|
2018-07-08 17:28:04 +08:00
|
|
|
if length <= CookieSize*2 {
|
2016-04-10 16:50:58 +08:00
|
|
|
return fmt.Errorf("Invalid fid: %s", fid)
|
2011-12-20 17:00:01 +08:00
|
|
|
}
|
2012-07-17 01:15:16 +08:00
|
|
|
delta := ""
|
|
|
|
deltaIndex := strings.LastIndex(fid, "_")
|
|
|
|
if deltaIndex > 0 {
|
|
|
|
fid, delta = fid[0:deltaIndex], fid[deltaIndex+1:]
|
|
|
|
}
|
2018-07-08 17:28:04 +08:00
|
|
|
n.Id, n.Cookie, err = ParseNeedleIdCookie(fid)
|
2013-12-10 05:53:24 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2012-07-17 01:15:16 +08:00
|
|
|
if delta != "" {
|
2013-12-10 05:53:24 +08:00
|
|
|
if d, e := strconv.ParseUint(delta, 10, 64); e == nil {
|
2019-06-21 16:14:10 +08:00
|
|
|
n.Id += Uint64ToNeedleId(d)
|
2013-12-10 05:53:24 +08:00
|
|
|
} else {
|
|
|
|
return e
|
2012-07-17 01:15:16 +08:00
|
|
|
}
|
|
|
|
}
|
2013-12-10 05:53:24 +08:00
|
|
|
return err
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
2013-01-21 11:44:23 +08:00
|
|
|
|
2018-07-08 17:28:04 +08:00
|
|
|
func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error) {
|
|
|
|
if len(key_hash_string) <= CookieSize*2 {
|
2018-08-01 14:25:26 +08:00
|
|
|
return NeedleIdEmpty, 0, fmt.Errorf("KeyHash is too short.")
|
2012-12-21 16:36:55 +08:00
|
|
|
}
|
2018-07-08 17:28:04 +08:00
|
|
|
if len(key_hash_string) > (NeedleIdSize+CookieSize)*2 {
|
2018-08-01 14:25:26 +08:00
|
|
|
return NeedleIdEmpty, 0, fmt.Errorf("KeyHash is too long.")
|
2016-04-10 15:54:40 +08:00
|
|
|
}
|
2018-07-08 17:28:04 +08:00
|
|
|
split := len(key_hash_string) - CookieSize*2
|
|
|
|
needleId, err := ParseNeedleId(key_hash_string[:split])
|
2016-04-10 15:54:40 +08:00
|
|
|
if err != nil {
|
2018-08-01 14:25:26 +08:00
|
|
|
return NeedleIdEmpty, 0, fmt.Errorf("Parse needleId error: %v", err)
|
2016-04-10 15:54:40 +08:00
|
|
|
}
|
2018-07-08 17:28:04 +08:00
|
|
|
cookie, err := ParseCookie(key_hash_string[split:])
|
2016-04-10 15:54:40 +08:00
|
|
|
if err != nil {
|
2018-08-01 14:25:26 +08:00
|
|
|
return NeedleIdEmpty, 0, fmt.Errorf("Parse cookie error: %v", err)
|
2016-04-10 15:54:40 +08:00
|
|
|
}
|
2018-07-08 17:28:04 +08:00
|
|
|
return needleId, cookie, nil
|
2016-04-10 15:54:40 +08:00
|
|
|
}
|
2018-07-15 11:51:17 +08:00
|
|
|
|
|
|
|
func (n *Needle) LastModifiedString() string {
|
|
|
|
return time.Unix(int64(n.LastModified), 0).Format("2006-01-02T15:04:05")
|
|
|
|
}
|