From 8d1f6cd567c6bddb2fac668363b5f6cbc740fab7 Mon Sep 17 00:00:00 2001
From: Kimbsen <kim@jotta.no>
Date: Tue, 23 Jun 2020 16:24:11 +0200
Subject: [PATCH 1/5] added optional md5 verification of uploaded data

---
 weed/storage/needle/needle_parse_upload.go | 41 +++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go
index 67b798821..7df0069e9 100644
--- a/weed/storage/needle/needle_parse_upload.go
+++ b/weed/storage/needle/needle_parse_upload.go
@@ -1,7 +1,10 @@
 package needle
 
 import (
+	"compress/gzip"
+	"crypto/md5"
 	"fmt"
+	"hash"
 	"io"
 	"io/ioutil"
 	"mime"
@@ -93,6 +96,23 @@ func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
 	return nil
 }
 
+type ChecksumReader struct {
+	h hash.Hash
+	r io.Reader
+}
+
+func (cr *ChecksumReader) Read(p []byte) (int, error) {
+	n, err := cr.r.Read(p)
+	if err == nil {
+		cr.h.Write(p[:n])
+	}
+	return n, err
+}
+
+func (cr *ChecksumReader) Checksum() string {
+	return fmt.Sprintf("%x", cr.h.Sum(nil))
+}
+
 func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
 	defer func() {
 		if e != nil && r.Body != nil {
@@ -120,7 +140,26 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
 		pu.FileName = path.Base(pu.FileName)
 	}
 
-	pu.Data, e = ioutil.ReadAll(io.LimitReader(part, sizeLimit+1))
+	reader := io.LimitReader(part, sizeLimit+1)
+	if expectedChecksum := r.Header.Get("Content-MD5"); expectedChecksum != "" {
+		if part.Header.Get("Content-Encoding") == "gzip" {
+			gr, err := gzip.NewReader(reader)
+			if err != nil {
+				e = fmt.Errorf("Content-Encoding == gzip but content was not gzipped: %s", err)
+				return
+			}
+			reader = gr
+		}
+		cr := &ChecksumReader{md5.New(), reader}
+		pu.Data, e = ioutil.ReadAll(cr)
+		if expectedChecksum != cr.Checksum() {
+			e = fmt.Errorf("Content-MD5 did not match md5 of file data [%s] != [%s]", expectedChecksum, cr.Checksum())
+			return
+		}
+	} else {
+		pu.Data, e = ioutil.ReadAll(reader)
+	}
+
 	if e != nil {
 		glog.V(0).Infoln("Reading Content [ERROR]", e)
 		return

From 38626cb584819857b61a86518ab23bc0113ef93b Mon Sep 17 00:00:00 2001
From: Kimbsen <kim@jotta.no>
Date: Tue, 23 Jun 2020 16:53:49 +0200
Subject: [PATCH 2/5] don't forget the last few bytes

---
 weed/storage/needle/needle_parse_upload.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go
index 7df0069e9..6dd61821f 100644
--- a/weed/storage/needle/needle_parse_upload.go
+++ b/weed/storage/needle/needle_parse_upload.go
@@ -103,9 +103,7 @@ type ChecksumReader struct {
 
 func (cr *ChecksumReader) Read(p []byte) (int, error) {
 	n, err := cr.r.Read(p)
-	if err == nil {
-		cr.h.Write(p[:n])
-	}
+	cr.h.Write(p[:n])
 	return n, err
 }
 

From 6b1e93ba0b8a74a0391da4e1431d8d3d8e5fb82e Mon Sep 17 00:00:00 2001
From: Kimbsen <kim@jotta.no>
Date: Tue, 23 Jun 2020 17:04:24 +0200
Subject: [PATCH 3/5] check request header not part header for content-encoding

---
 weed/storage/needle/needle_parse_upload.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go
index 6dd61821f..3fb1c0968 100644
--- a/weed/storage/needle/needle_parse_upload.go
+++ b/weed/storage/needle/needle_parse_upload.go
@@ -140,7 +140,7 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
 
 	reader := io.LimitReader(part, sizeLimit+1)
 	if expectedChecksum := r.Header.Get("Content-MD5"); expectedChecksum != "" {
-		if part.Header.Get("Content-Encoding") == "gzip" {
+		if r.Header.Get("Content-Encoding") == "gzip" {
 			gr, err := gzip.NewReader(reader)
 			if err != nil {
 				e = fmt.Errorf("Content-Encoding == gzip but content was not gzipped: %s", err)

From ffddecebef0e91d1fbe12854168db64f66c02a8d Mon Sep 17 00:00:00 2001
From: Kimbsen <kim@jotta.no>
Date: Wed, 24 Jun 2020 13:35:13 +0200
Subject: [PATCH 4/5] do md5 validation AFTER decompression

---
 weed/storage/needle/needle_parse_upload.go | 46 ++++++----------------
 1 file changed, 11 insertions(+), 35 deletions(-)

diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go
index 3fb1c0968..8705cfdb3 100644
--- a/weed/storage/needle/needle_parse_upload.go
+++ b/weed/storage/needle/needle_parse_upload.go
@@ -81,6 +81,16 @@ func ParseUpload(r *http.Request, sizeLimit int64) (pu *ParsedUpload, e error) {
 			}
 		}
 	}
+
+	if expectedChecksum := r.Header.Get("Content-MD5"); expectedChecksum != "" {
+		h := md5.New()
+		h.Write(pu.UncompressedData)
+		if receivedChecksum := fmt.Sprintf("%x", h.Sum(nil)); expectedChecksum != receivedChecksum {
+			e = fmt.Errorf("Content-MD5 did not match md5 of file data [%s] != [%s]", expectedChecksum, receivedChecksum)
+			return
+		}
+	}
+
 	return
 }
 
@@ -96,21 +106,6 @@ func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
 	return nil
 }
 
-type ChecksumReader struct {
-	h hash.Hash
-	r io.Reader
-}
-
-func (cr *ChecksumReader) Read(p []byte) (int, error) {
-	n, err := cr.r.Read(p)
-	cr.h.Write(p[:n])
-	return n, err
-}
-
-func (cr *ChecksumReader) Checksum() string {
-	return fmt.Sprintf("%x", cr.h.Sum(nil))
-}
-
 func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
 	defer func() {
 		if e != nil && r.Body != nil {
@@ -138,26 +133,7 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
 		pu.FileName = path.Base(pu.FileName)
 	}
 
-	reader := io.LimitReader(part, sizeLimit+1)
-	if expectedChecksum := r.Header.Get("Content-MD5"); expectedChecksum != "" {
-		if r.Header.Get("Content-Encoding") == "gzip" {
-			gr, err := gzip.NewReader(reader)
-			if err != nil {
-				e = fmt.Errorf("Content-Encoding == gzip but content was not gzipped: %s", err)
-				return
-			}
-			reader = gr
-		}
-		cr := &ChecksumReader{md5.New(), reader}
-		pu.Data, e = ioutil.ReadAll(cr)
-		if expectedChecksum != cr.Checksum() {
-			e = fmt.Errorf("Content-MD5 did not match md5 of file data [%s] != [%s]", expectedChecksum, cr.Checksum())
-			return
-		}
-	} else {
-		pu.Data, e = ioutil.ReadAll(reader)
-	}
-
+	pu.Data, e = ioutil.ReadAll(io.LimitReader(part, sizeLimit+1))
 	if e != nil {
 		glog.V(0).Infoln("Reading Content [ERROR]", e)
 		return

From 4eeab2a3793103027767968ca73ae03b2fe51c01 Mon Sep 17 00:00:00 2001
From: Kimbsen <kim@jotta.no>
Date: Wed, 24 Jun 2020 13:37:00 +0200
Subject: [PATCH 5/5] remove unused imports

---
 weed/storage/needle/needle_parse_upload.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go
index 8705cfdb3..e339c65a4 100644
--- a/weed/storage/needle/needle_parse_upload.go
+++ b/weed/storage/needle/needle_parse_upload.go
@@ -1,10 +1,8 @@
 package needle
 
 import (
-	"compress/gzip"
 	"crypto/md5"
 	"fmt"
-	"hash"
 	"io"
 	"io/ioutil"
 	"mime"