more efficient client side gzip compression

weed/operation/compress.go

@@ -16,40 +16,9 @@ import (
  */
 func IsGzippable(ext, mtype string, data []byte) bool {
 
-	// text
-	if strings.HasPrefix(mtype, "text/") {
-		return true
-	}
-
-	// images
-	switch ext {
-	case ".svg", ".bmp":
-		return true
-	}
-	if strings.HasPrefix(mtype, "image/") {
-		return false
-	}
-
-	// by file name extension
-	switch ext {
-	case ".zip", ".rar", ".gz", ".bz2", ".xz":
-		return false
-	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
-		return true
-	case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
-		return true
-	case ".png", ".jpg", ".jpeg":
-		return false
-	}
-
-	// by mime type
-	if strings.HasPrefix(mtype, "application/") {
-		if strings.HasSuffix(mtype, "xml") {
-			return true
-		}
-		if strings.HasSuffix(mtype, "script") {
-			return true
-		}
+	shouldBeZipped, iAmSure := IsGzippableFileType(ext, mtype)
+	if iAmSure {
+		return shouldBeZipped
 	}
 
 	isMostlyText := util.IsText(data)
@@ -57,6 +26,50 @@ func IsGzippable(ext, mtype string, data []byte) bool {
 	return isMostlyText
 }
 
+/*
+* Default more not to gzip since gzip can be done on client side.
+ */
+func IsGzippableFileType(ext, mtype string) (shouldBeZipped, iAmSure bool) {
+
+	// text
+	if strings.HasPrefix(mtype, "text/") {
+		return true, true
+	}
+
+	// images
+	switch ext {
+	case ".svg", ".bmp":
+		return true, true
+	}
+	if strings.HasPrefix(mtype, "image/") {
+		return false, true
+	}
+
+	// by file name extension
+	switch ext {
+	case ".zip", ".rar", ".gz", ".bz2", ".xz":
+		return false, true
+	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
+		return true, true
+	case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
+		return true, true
+	case ".png", ".jpg", ".jpeg":
+		return false, true
+	}
+
+	// by mime type
+	if strings.HasPrefix(mtype, "application/") {
+		if strings.HasSuffix(mtype, "xml") {
+			return true, true
+		}
+		if strings.HasSuffix(mtype, "script") {
+			return true, true
+		}
+	}
+
+	return false, false
+}
+
 func GzipData(input []byte) ([]byte, error) {
 	buf := new(bytes.Buffer)
 	w, _ := gzip.NewWriterLevel(buf, flate.BestCompression)

weed/operation/upload_content.go

@@ -2,6 +2,7 @@ package operation
 
 import (
 	"bytes"
+	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -39,10 +40,24 @@ var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
 // Upload sends a POST request to a volume server to upload the content
 func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
+	contentIsGzipped := isGzipped
+	shouldGzipNow := false
+	if !isGzipped {
+		if shouldBeZipped, iAmSure := IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
+			shouldGzipNow = true
+			contentIsGzipped = true
+		}
+	}
 	return upload_content(uploadUrl, func(w io.Writer) (err error) {
-		_, err = io.Copy(w, reader)
+		if shouldGzipNow {
+			gzWriter := gzip.NewWriter(w)
+			_, err = io.Copy(gzWriter, reader)
+			gzWriter.Close()
+		} else {
+			_, err = io.Copy(w, reader)
+		}
 		return
-	}, filename, isGzipped, mtype, pairMap, jwt)
+	}, filename, contentIsGzipped, mtype, pairMap, jwt)
 }
 func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
 	body_buf := bytes.NewBufferString("")