diff --git a/weed/filer2/stream.go b/weed/filer2/stream.go index 3cb69f72b..bf1b4c24a 100644 --- a/weed/filer2/stream.go +++ b/weed/filer2/stream.go @@ -44,8 +44,37 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f } +// ---------------- ReadAllReader ---------------------------------- + +func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) { + + buffer := bytes.Buffer{} + + chunkViews := ViewFromChunks(chunks, 0, math.MaxInt32) + + lookupFileId := func(fileId string) (targetUrl string, err error) { + return masterClient.LookupFileId(fileId) + } + + for _, chunkView := range chunkViews { + urlString, err := lookupFileId(chunkView.FileId) + if err != nil { + glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err) + return nil, err + } + err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) { + buffer.Write(data) + }) + if err != nil { + glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err) + return nil, err + } + } + return buffer.Bytes(), nil +} + +// ---------------- ChunkStreamReader ---------------------------------- type ChunkStreamReader struct { - masterClient *wdclient.MasterClient chunkViews []*ChunkView logicOffset int64 buffer []byte @@ -69,6 +98,7 @@ func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks [ } } + func (c *ChunkStreamReader) Read(p []byte) (n int, err error) { if c.isBufferEmpty() { if c.chunkIndex >= len(c.chunkViews) { @@ -144,6 +174,10 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error { return nil } +func (c *ChunkStreamReader) Close() { + // TODO try to release and reuse buffer +} + func VolumeId(fileId string) string { lastCommaIndex := strings.LastIndex(fileId, ",") if lastCommaIndex > 0 { diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 96fc38bf5..76c924df1 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -1,6 +1,7 @@ package weed_server import ( + "bytes" "context" "io" "mime" @@ -114,8 +115,13 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, ext := filepath.Ext(filename) width, height, mode, shouldResize := shouldResizeImages(ext, r) if shouldResize { - chunkedFileReader := filer2.NewChunkStreamReaderFromFiler(fs.filer.MasterClient, entry.Chunks) - rs, _, _ := images.Resized(ext, chunkedFileReader, width, height, mode) + data, err := filer2.ReadAll(fs.filer.MasterClient, entry.Chunks) + if err != nil { + glog.Errorf("failed to read %s: %v", path, err) + w.WriteHeader(http.StatusNotModified) + return + } + rs, _, _ := images.Resized(ext, bytes.NewReader(data), width, height, mode) io.Copy(w, rs) return }