mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/chrislusf/seaweedfs synced 2024-05-30 23:30:18 +02:00
Code Issues Projects Releases Wiki Activity

refactoring

Browse source
This commit is contained in:
Chris Lu 2021-09-06 16:20:49 -07:00
parent 64f6532fbe
commit 6923af7280
15 changed files with 228 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
unmaintained/repeated_vacuum/repeated_vacuum.go
View file

@ -65,7 +65,16 @@ func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, st
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
_, err = operation.UploadData(targetUrl, fmt.Sprintf("test%d", i), false, data, false, "bench/test", nil, assignResult.Auth)
uploadOption := &operation.UploadOption{
UploadUrl: targetUrl,
Filename: fmt.Sprintf("test%d", i),
Cipher: false,
IsInputCompressed: true,
MimeType: "bench/test",
PairMap: nil,
Jwt: assignResult.Auth,
}
_, err = operation.UploadData(data, uploadOption)
if err != nil {
log.Fatalf("upload: %v", err)
}

35
weed/command/filer_copy.go
View file

@ -392,8 +392,16 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
}
targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
uploadResult, err := operation.UploadData(targetUrl, fileName, worker.options.cipher, data, false, mimeType, nil, security.EncodedJwt(assignResult.Auth))
uploadOption := &operation.UploadOption{
UploadUrl: targetUrl,
Filename: fileName,
Cipher: worker.options.cipher,
IsInputCompressed: false,
MimeType: mimeType,
PairMap: nil,
Jwt: security.EncodedJwt(assignResult.Auth),
}
uploadResult, err := operation.UploadData(data, uploadOption)
if err != nil {
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
}
@ -498,7 +506,16 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
replication = assignResult.Replication
}
uploadResult, err, _ := operation.Upload(targetUrl, fileName+"-"+strconv.FormatInt(i+1, 10), worker.options.cipher, io.NewSectionReader(f, i*chunkSize, chunkSize), false, "", nil, security.EncodedJwt(assignResult.Auth))
uploadOption := &operation.UploadOption{
UploadUrl: targetUrl,
Filename: fileName+"-"+strconv.FormatInt(i+1, 10),
Cipher: worker.options.cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: security.EncodedJwt(assignResult.Auth),
}
uploadResult, err, _ := operation.Upload(io.NewSectionReader(f, i*chunkSize, chunkSize), uploadOption)
if err != nil {
uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
return
@ -630,8 +647,16 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
return nil, collection, replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr)
}
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
uploadResult, flushErr, _ := operation.Upload(fileUrl, name, worker.options.cipher, reader, false, "", nil, auth)
uploadOption := &operation.UploadOption{
UploadUrl: fmt.Sprintf("http://%s/%s", host, fileId),
Filename: name,
Cipher: worker.options.cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: auth,
}
uploadResult, flushErr, _ := operation.Upload(reader, uploadOption)
if flushErr != nil {
return nil, collection, replication, fmt.Errorf("upload data: %v", flushErr)
}

11
weed/filer/filer_notify_append.go
View file

@ -66,7 +66,16 @@ func (f *Filer) assignAndUpload(targetFile string, data []byte) (*operation.Assi
// upload data
targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
uploadResult, err := operation.UploadData(targetUrl, "", f.Cipher, data, false, "", nil, assignResult.Auth)
uploadOption := &operation.UploadOption{
UploadUrl: targetUrl,
Filename: "",
Cipher: f.Cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: assignResult.Auth,
}
uploadResult, err := operation.UploadData(data, uploadOption)
if err != nil {
return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
}

11
weed/filesys/wfs_write.go
View file

@ -58,7 +58,16 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.Sa
if wfs.option.VolumeServerAccess == "filerProxy" {
fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.getCurrentFiler(), fileId)
}
uploadResult, err, data := operation.Upload(fileUrl, filename, wfs.option.Cipher, reader, false, "", nil, auth)
uploadOption := &operation.UploadOption{
UploadUrl: fileUrl,
Filename: filename,
Cipher: wfs.option.Cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: auth,
}
uploadResult, err, data := operation.Upload(reader, uploadOption)
if err != nil {
glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err)
return nil, "", "", fmt.Errorf("upload data: %v", err)

11
weed/messaging/broker/broker_append.go
View file

@ -88,7 +88,16 @@ func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConf
// upload data
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
uploadResult, err := operation.UploadData(targetUrl, "", broker.option.Cipher, data, false, "", nil, assignResult.Auth)
uploadOption := &operation.UploadOption{
UploadUrl: targetUrl,
Filename: "",
Cipher: broker.option.Cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: assignResult.Auth,
}
uploadResult, err := operation.UploadData(data, uploadOption)
if err != nil {
return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
}

22
weed/operation/needle_parse_test.go
View file

@ -53,7 +53,16 @@ func TestCreateNeedleFromRequest(t *testing.T) {
assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip")
fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize)
}
uploadResult, err, data := Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader([]byte(textContent)), false, "", nil, "")
uploadOption := &UploadOption{
UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
Filename: "t.txt",
Cipher: false,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: "",
}
uploadResult, err, data := Upload(bytes.NewReader([]byte(textContent)), uploadOption)
if len(data) != len(textContent) {
t.Errorf("data actual %d expected %d", len(data), len(textContent))
}
@ -72,7 +81,16 @@ func TestCreateNeedleFromRequest(t *testing.T) {
fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
}
gzippedData, _ := util.GzipData([]byte(textContent))
Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(gzippedData), true, "text/plain", nil, "")
uploadOption := &UploadOption{
UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
Filename: "t.txt",
Cipher: false,
IsInputCompressed: true,
MimeType: "text/plain",
PairMap: nil,
Jwt: "",
}
Upload(bytes.NewReader(gzippedData), uploadOption)
}
/*

33
weed/operation/submit.go
View file

@ -206,7 +206,16 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
}
} else {
ret, e, _ := Upload(fileUrl, baseName, false, fi.Reader, false, fi.MimeType, nil, jwt)
uploadOption := &UploadOption{
UploadUrl: fileUrl,
Filename: baseName,
Cipher: false,
IsInputCompressed: false,
MimeType: fi.MimeType,
PairMap: nil,
Jwt: jwt,
}
ret, e, _ := Upload(fi.Reader, uploadOption)
if e != nil {
return 0, e
}
@ -219,7 +228,16 @@ func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn,
fileUrl string, jwt security.EncodedJwt,
) (size uint32, e error) {
glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
uploadResult, uploadError, _ := Upload(fileUrl, filename, false, reader, false, "", nil, jwt)
uploadOption := &UploadOption{
UploadUrl: fileUrl,
Filename: filename,
Cipher: false,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: jwt,
}
uploadResult, uploadError, _ := Upload(reader, uploadOption)
if uploadError != nil {
return 0, uploadError
}
@ -236,6 +254,15 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s
q := u.Query()
q.Set("cm", "true")
u.RawQuery = q.Encode()
_, e = UploadData(u.String(), manifest.Name, false, buf, false, "application/json", nil, jwt)
uploadOption := &UploadOption{
UploadUrl: u.String(),
Filename: manifest.Name,
Cipher: false,
IsInputCompressed: false,
MimeType: "application/json",
PairMap: nil,
Jwt: jwt,
}
_, e = UploadData(buf, uploadOption)
return e
}

66
weed/operation/upload_content.go
View file

@ -20,6 +20,16 @@ import (
"github.com/chrislusf/seaweedfs/weed/util"
)
type UploadOption struct {
UploadUrl string
Filename string
Cipher bool
IsInputCompressed bool
MimeType string
PairMap map[string]string
Jwt security.EncodedJwt
}
type UploadResult struct {
Name string `json:"name,omitempty"`
Size uint32 `json:"size,omitempty"`
@ -65,18 +75,18 @@ func init() {
var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
uploadResult, err = retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
func UploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
uploadResult, err = retriedUploadData(data, option)
return
}
// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
uploadResult, err, data = doUpload(uploadUrl, filename, cipher, reader, isInputCompressed, mtype, pairMap, jwt)
func Upload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) {
uploadResult, err, data = doUpload(reader, option)
return
}
func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
func doUpload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) {
bytesReader, ok := reader.(*util.BytesReader)
if ok {
data = bytesReader.Bytes
@ -87,38 +97,38 @@ func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader,
return
}
}
uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
uploadResult, uploadErr := retriedUploadData(data, option)
return uploadResult, uploadErr, data
}
func retriedUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
func retriedUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
for i := 0; i < 3; i++ {
uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
uploadResult, err = doUploadData(data, option)
if err == nil {
uploadResult.RetryCount = i
return
} else {
glog.Warningf("uploading to %s: %v", uploadUrl, err)
glog.Warningf("uploading to %s: %v", option.UploadUrl, err)
}
time.Sleep(time.Millisecond * time.Duration(237*(i+1)))
}
return
}
func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
contentIsGzipped := isInputCompressed
func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
contentIsGzipped := option.IsInputCompressed
shouldGzipNow := false
if !isInputCompressed {
if mtype == "" {
mtype = http.DetectContentType(data)
// println("detect1 mimetype to", mtype)
if mtype == "application/octet-stream" {
mtype = ""
if !option.IsInputCompressed {
if option.MimeType == "" {
option.MimeType = http.DetectContentType(data)
// println("detect1 mimetype to", MimeType)
if option.MimeType == "application/octet-stream" {
option.MimeType = ""
}
}
if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed {
if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
shouldGzipNow = true
} else if !iAmSure && mtype == "" && len(data) > 16*1024 {
} else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 {
var compressed []byte
compressed, err = util.GzipData(data[0:128])
shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
@ -131,14 +141,14 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
// this could be double copying
clearDataLen = len(data)
clearData := data
if shouldGzipNow && !cipher {
if shouldGzipNow && !option.Cipher {
compressed, compressErr := util.GzipData(data)
// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
if compressErr == nil {
data = compressed
contentIsGzipped = true
}
} else if isInputCompressed {
} else if option.IsInputCompressed {
// just to get the clear data length
clearData, err = util.DecompressData(data)
if err == nil {
@ -146,7 +156,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
}
}
if cipher {
if option.Cipher {
// encrypt(gzip(data))
// encrypt
@ -158,23 +168,23 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
}
// upload data
uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
uploadResult, err = upload_content(option.UploadUrl, func(w io.Writer) (err error) {
_, err = w.Write(encryptedData)
return
}, "", false, len(encryptedData), "", nil, jwt)
}, "", false, len(encryptedData), "", nil, option.Jwt)
if uploadResult == nil {
return
}
uploadResult.Name = filename
uploadResult.Mime = mtype
uploadResult.Name = option.Filename
uploadResult.Mime = option.MimeType
uploadResult.CipherKey = cipherKey
uploadResult.Size = uint32(clearDataLen)
} else {
// upload data
uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
uploadResult, err = upload_content(option.UploadUrl, func(w io.Writer) (err error) {
_, err = w.Write(data)
return
}, filename, contentIsGzipped, len(data), mtype, pairMap, jwt)
}, option.Filename, contentIsGzipped, len(data), option.MimeType, option.PairMap, option.Jwt)
if uploadResult == nil {
return
}

11
weed/replication/sink/filersink/fetch_write.go
View file

@ -107,7 +107,16 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string)
glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header)
// fetch data as is, regardless whether it is encrypted or not
uploadResult, err, _ := operation.Upload(fileUrl, filename, false, resp.Body, "gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth)
uploadOption := &operation.UploadOption{
UploadUrl: fileUrl,
Filename: filename,
Cipher: false,
IsInputCompressed: "gzip" == header.Get("Content-Encoding"),
MimeType: header.Get("Content-Type"),
PairMap: nil,
Jwt: auth,
}
uploadResult, err, _ := operation.Upload(resp.Body, uploadOption)
if err != nil {
glog.V(0).Infof("upload source data %v to %s: %v", sourceChunk.GetFileIdString(), fileUrl, err)
return "", fmt.Errorf("upload data: %v", err)

11
weed/server/common.go
View file

@ -161,7 +161,16 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn ope
}
debug("upload file to store", url)
uploadResult, err := operation.UploadData(url, pu.FileName, false, pu.Data, pu.IsGzipped, pu.MimeType, pu.PairMap, assignResult.Auth)
uploadOption := &operation.UploadOption{
UploadUrl: url,
Filename: pu.FileName,
Cipher: false,
IsInputCompressed: pu.IsGzipped,
MimeType: pu.MimeType,
PairMap: pu.PairMap,
Jwt: assignResult.Auth,
}
uploadResult, err := operation.UploadData(pu.Data, uploadOption)
if err != nil {
writeJsonError(w, r, http.StatusInternalServerError, err)
return

11
weed/server/filer_server_handlers_write_autochunk.go
View file

@ -241,7 +241,16 @@ func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAs
}
// upload the chunk to the volume server
uploadResult, uploadErr, _ := operation.Upload(urlLocation, name, fs.option.Cipher, reader, false, "", nil, auth)
uploadOption := &operation.UploadOption{
UploadUrl: urlLocation,
Filename: name,
Cipher: fs.option.Cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: auth,
}
uploadResult, uploadErr, _ := operation.Upload(reader, uploadOption)
if uploadErr != nil {
return nil, "", "", uploadErr
}

11
weed/server/filer_server_handlers_write_cipher.go
View file

@ -44,7 +44,16 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht
// println("detect2 mimetype to", pu.MimeType)
}
uploadResult, uploadError := operation.UploadData(urlLocation, pu.FileName, true, uncompressedData, false, pu.MimeType, pu.PairMap, auth)
uploadOption := &operation.UploadOption{
UploadUrl: urlLocation,
Filename: pu.FileName,
Cipher: true,
IsInputCompressed: false,
MimeType: pu.MimeType,
PairMap: pu.PairMap,
Jwt: auth,
}
uploadResult, uploadError := operation.UploadData(uncompressedData, uploadOption)
if uploadError != nil {
return nil, fmt.Errorf("upload to volume server: %v", uploadError)
}

11
weed/server/filer_server_handlers_write_upload.go
View file

@ -127,7 +127,16 @@ func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fil
stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
}()
uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
uploadOption := &operation.UploadOption{
UploadUrl: urlLocation,
Filename: fileName,
Cipher: fs.option.Cipher,
IsInputCompressed: false,
MimeType: contentType,
PairMap: pairMap,
Jwt: auth,
}
uploadResult, err, data := operation.Upload(limitedReader, uploadOption)
if uploadResult != nil && uploadResult.RetryCount > 0 {
stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
}

11
weed/server/webdav_server.go
View file

@ -413,7 +413,16 @@ func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64
}
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
uploadResult, flushErr, _ := operation.Upload(fileUrl, f.name, f.fs.option.Cipher, reader, false, "", nil, auth)
uploadOption := &operation.UploadOption{
UploadUrl: fileUrl,
Filename: f.name,
Cipher: f.fs.option.Cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: auth,
}
uploadResult, flushErr, _ := operation.Upload(reader, uploadOption)
if flushErr != nil {
glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, flushErr)
return nil, f.collection, f.replication, fmt.Errorf("upload data: %v", flushErr)

11
weed/topology/store_replicate.go
View file

@ -83,7 +83,16 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
// volume server do not know about encryption
// TODO optimize here to compress data only once
_, err := operation.UploadData(u.String(), string(n.Name), false, n.Data, n.IsCompressed(), string(n.Mime), pairMap, jwt)
uploadOption := &operation.UploadOption{
UploadUrl: u.String(),
Filename: string(n.Name),
Cipher: false,
IsInputCompressed: n.IsCompressed(),
MimeType: string(n.Mime),
PairMap: pairMap,
Jwt: jwt,
}
_, err := operation.UploadData(n.Data, uploadOption)
return err
}); err != nil {
err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err)