mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/chrislusf/seaweedfs synced 2024-06-02 16:50:25 +02:00
Code Issues Projects Releases Wiki Activity

remove fixJpgOrientation

Browse source
This commit is contained in:
Chris Lu 2020-06-20 12:50:40 -07:00
parent 3080c197e3
commit de5ca9b258
12 changed files with 13 additions and 264 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
k8s/seaweedfs/templates/volume-statefulset.yaml
View file

@ -88,9 +88,6 @@ spec:
{{- if .Values.volume.whiteList }}
-whiteList={{ .Values.volume.whiteList }} \
{{- end }}
{{- if .Values.volume.imagesFixOrientation }}
-images.fix.orientation \
{{- end }}
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \
-compactionMBps={{ .Values.volume.compactionMBps }} \
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}

3
k8s/seaweedfs/values.yaml
View file

@ -123,9 +123,6 @@ volume:
# Comma separated Ip addresses having write permission. No limit if empty.
whiteList: null
# Adjust jpg orientation when uploading.
imagesFixOrientation: false
extraVolumes: ""
extraVolumeMounts: ""

1
weed/command/server.go
View file

@ -92,7 +92,6 @@ func init() {
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")

4
weed/command/volume.go
View file

@ -47,7 +47,6 @@ type VolumeServerOptions struct {
rack *string
whiteList []string
indexType *string
fixJpgOrientation *bool
readRedirect *bool
cpuProfile *string
memProfile *string
@ -71,7 +70,6 @@ func init() {
v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
@ -192,7 +190,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
volumeNeedleMapKind,
strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
v.whiteList,
*v.fixJpgOrientation, *v.readRedirect,
*v.readRedirect,
*v.compactionMBPerSecond,
*v.fileSizeLimitMB,
)

182
weed/images/orientation.go
View file

@ -1,182 +0,0 @@
package images
import (
"bytes"
"image"
"image/draw"
"image/jpeg"
"log"
"github.com/seaweedfs/goexif/exif"
)
//many code is copied from http://camlistore.org/pkg/images/images.go
func FixJpgOrientation(data []byte) (oriented []byte) {
ex, err := exif.Decode(bytes.NewReader(data))
if err != nil {
return data
}
tag, err := ex.Get(exif.Orientation)
if err != nil {
return data
}
angle := 0
flipMode := FlipDirection(0)
orient, err := tag.Int(0)
if err != nil {
return data
}
switch orient {
case topLeftSide:
// do nothing
return data
case topRightSide:
flipMode = 2
case bottomRightSide:
angle = 180
case bottomLeftSide:
angle = 180
flipMode = 2
case leftSideTop:
angle = -90
flipMode = 2
case rightSideTop:
angle = -90
case rightSideBottom:
angle = 90
flipMode = 2
case leftSideBottom:
angle = 90
}
if srcImage, _, err := image.Decode(bytes.NewReader(data)); err == nil {
dstImage := flip(rotate(srcImage, angle), flipMode)
var buf bytes.Buffer
jpeg.Encode(&buf, dstImage, nil)
return buf.Bytes()
}
return data
}
// Exif Orientation Tag values
// http://sylvana.net/jpegcrop/exif_orientation.html
const (
topLeftSide = 1
topRightSide = 2
bottomRightSide = 3
bottomLeftSide = 4
leftSideTop = 5
rightSideTop = 6
rightSideBottom = 7
leftSideBottom = 8
)
// The FlipDirection type is used by the Flip option in DecodeOpts
// to indicate in which direction to flip an image.
type FlipDirection int
// FlipVertical and FlipHorizontal are two possible FlipDirections
// values to indicate in which direction an image will be flipped.
const (
FlipVertical FlipDirection = 1 << iota
FlipHorizontal
)
type DecodeOpts struct {
// Rotate specifies how to rotate the image.
// If nil, the image is rotated automatically based on EXIF metadata.
// If an int, Rotate is the number of degrees to rotate
// counter clockwise and must be one of 0, 90, -90, 180, or
// -180.
Rotate interface{}
// Flip specifies how to flip the image.
// If nil, the image is flipped automatically based on EXIF metadata.
// Otherwise, Flip is a FlipDirection bitfield indicating how to flip.
Flip interface{}
}
func rotate(im image.Image, angle int) image.Image {
var rotated *image.NRGBA
// trigonometric (i.e counter clock-wise)
switch angle {
case 90:
newH, newW := im.Bounds().Dx(), im.Bounds().Dy()
rotated = image.NewNRGBA(image.Rect(0, 0, newW, newH))
for y := 0; y < newH; y++ {
for x := 0; x < newW; x++ {
rotated.Set(x, y, im.At(newH-1-y, x))
}
}
case -90:
newH, newW := im.Bounds().Dx(), im.Bounds().Dy()
rotated = image.NewNRGBA(image.Rect(0, 0, newW, newH))
for y := 0; y < newH; y++ {
for x := 0; x < newW; x++ {
rotated.Set(x, y, im.At(y, newW-1-x))
}
}
case 180, -180:
newW, newH := im.Bounds().Dx(), im.Bounds().Dy()
rotated = image.NewNRGBA(image.Rect(0, 0, newW, newH))
for y := 0; y < newH; y++ {
for x := 0; x < newW; x++ {
rotated.Set(x, y, im.At(newW-1-x, newH-1-y))
}
}
default:
return im
}
return rotated
}
// flip returns a flipped version of the image im, according to
// the direction(s) in dir.
// It may flip the imput im in place and return it, or it may allocate a
// new NRGBA (if im is an *image.YCbCr).
func flip(im image.Image, dir FlipDirection) image.Image {
if dir == 0 {
return im
}
ycbcr := false
var nrgba image.Image
dx, dy := im.Bounds().Dx(), im.Bounds().Dy()
di, ok := im.(draw.Image)
if !ok {
if _, ok := im.(*image.YCbCr); !ok {
log.Printf("failed to flip image: input does not satisfy draw.Image")
return im
}
// because YCbCr does not implement Set, we replace it with a new NRGBA
ycbcr = true
nrgba = image.NewNRGBA(image.Rect(0, 0, dx, dy))
di, ok = nrgba.(draw.Image)
if !ok {
log.Print("failed to flip image: could not cast an NRGBA to a draw.Image")
return im
}
}
if dir&FlipHorizontal != 0 {
for y := 0; y < dy; y++ {
for x := 0; x < dx/2; x++ {
old := im.At(x, y)
di.Set(x, y, im.At(dx-1-x, y))
di.Set(dx-1-x, y, old)
}
}
}
if dir&FlipVertical != 0 {
for y := 0; y < dy/2; y++ {
for x := 0; x < dx; x++ {
old := im.At(x, y)
di.Set(x, y, im.At(x, dy-1-y))
di.Set(x, dy-1-y, old)
}
}
}
if ycbcr {
return nrgba
}
return im
}

20
weed/images/orientation_test.go
View file

@ -1,20 +0,0 @@
package images
import (
"io/ioutil"
"os"
"testing"
)
func TestXYZ(t *testing.T) {
fname := "sample1.jpg"
dat, _ := ioutil.ReadFile(fname)
fixed_data := FixJpgOrientation(dat)
ioutil.WriteFile("fixed1.jpg", fixed_data, 0644)
os.Remove("fixed1.jpg")
}

29
weed/images/preprocess.go
View file

@ -1,29 +0,0 @@
package images
import (
"bytes"
"io"
"path/filepath"
"strings"
)
/*
* Preprocess image files on client side.
* 1. possibly adjust the orientation
* 2. resize the image to a width or height limit
* 3. remove the exif data
* Call this function on any file uploaded to SeaweedFS
*
*/
func MaybePreprocessImage(filename string, data []byte, width, height int) (resized io.ReadSeeker, w int, h int) {
ext := filepath.Ext(filename)
ext = strings.ToLower(ext)
switch ext {
case ".png", ".gif":
return Resized(ext, bytes.NewReader(data), width, height, "")
case ".jpg", ".jpeg":
data = FixJpgOrientation(data)
return Resized(ext, bytes.NewReader(data), width, height, "")
}
return bytes.NewReader(data), 0, 0
}

2
weed/pb/shared_values.go
View file

@ -1,5 +1,5 @@
package pb
const (
AdminShellClient = "shell"
AdminShellClient = "adminShell"
)

3
weed/server/volume_server.go
View file

@ -25,7 +25,6 @@ type VolumeServer struct {
grpcDialOption grpc.DialOption
needleMapKind storage.NeedleMapType
FixJpgOrientation bool
ReadRedirect bool
compactionBytePerSecond int64
MetricsAddress string
@ -40,7 +39,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
masterNodes []string, pulseSeconds int,
dataCenter string, rack string,
whiteList []string,
fixJpgOrientation bool,
readRedirect bool,
compactionMBPerSecond int,
fileSizeLimitMB int,
@ -61,7 +59,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
dataCenter: dataCenter,
rack: rack,
needleMapKind: needleMapKind,
FixJpgOrientation: fixJpgOrientation,
ReadRedirect: readRedirect,
grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.volume"),
compactionBytePerSecond: int64(compactionMBPerSecond) * 1024 * 1024,

2
weed/server/volume_server_handlers_write.go
View file

@ -42,7 +42,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
return
}
reqNeedle, originalSize, ne := needle.CreateNeedleFromRequest(r, vs.FixJpgOrientation, vs.fileSizeLimitBytes)
reqNeedle, originalSize, ne := needle.CreateNeedleFromRequest(r, vs.fileSizeLimitBytes)
if ne != nil {
writeJsonError(w, r, http.StatusBadRequest, ne)
return

10
weed/storage/needle/needle.go
View file

@ -8,7 +8,6 @@ import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/images"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
)
@ -48,7 +47,7 @@ func (n *Needle) String() (str string) {
return
}
func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit int64) (n *Needle, originalSize int, e error) {
func CreateNeedleFromRequest(r *http.Request, sizeLimit int64) (n *Needle, originalSize int, e error) {
n = new(Needle)
pu, e := ParseUpload(r, sizeLimit)
if e != nil {
@ -95,13 +94,6 @@ func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit
n.SetIsChunkManifest()
}
if fixJpgOrientation {
loweredName := strings.ToLower(pu.FileName)
if pu.MimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
n.Data = images.FixJpgOrientation(n.Data)
}
}
n.Checksum = NewCRC(n.Data)
commaSep := strings.LastIndex(r.URL.Path, ",")

18
weed/wdclient/masterclient.go
View file

@ -45,7 +45,7 @@ func (mc *MasterClient) WaitUntilConnected() {
}
func (mc *MasterClient) KeepConnectedToMaster() {
glog.V(1).Infof("%s bootstraps with masters %v", mc.clientType, mc.masters)
glog.V(1).Infof("%s masterClient bootstraps with masters %v", mc.clientType, mc.masters)
for {
mc.tryAllMasters()
time.Sleep(time.Second)
@ -67,27 +67,27 @@ func (mc *MasterClient) tryAllMasters() {
}
func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader string) {
glog.V(1).Infof("%s Connecting to master %v", mc.clientType, master)
glog.V(1).Infof("%s masterClient Connecting to master %v", mc.clientType, master)
gprcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
stream, err := client.KeepConnected(context.Background())
if err != nil {
glog.V(0).Infof("%s failed to keep connected to %s: %v", mc.clientType, master, err)
glog.V(0).Infof("%s masterClient failed to keep connected to %s: %v", mc.clientType, master, err)
return err
}
if err = stream.Send(&master_pb.KeepConnectedRequest{Name: mc.clientType, GrpcPort: mc.grpcPort}); err != nil {
glog.V(0).Infof("%s failed to send to %s: %v", mc.clientType, master, err)
glog.V(0).Infof("%s masterClient failed to send to %s: %v", mc.clientType, master, err)
return err
}
glog.V(1).Infof("%s Connected to %v", mc.clientType, master)
glog.V(1).Infof("%s masterClient Connected to %v", mc.clientType, master)
mc.currentMaster = master
for {
volumeLocation, err := stream.Recv()
if err != nil {
glog.V(0).Infof("%s failed to receive from %s: %v", mc.clientType, master, err)
glog.V(0).Infof("%s masterClient failed to receive from %s: %v", mc.clientType, master, err)
return err
}
@ -104,18 +104,18 @@ func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader stri
PublicUrl: volumeLocation.PublicUrl,
}
for _, newVid := range volumeLocation.NewVids {
glog.V(1).Infof("%s: %s adds volume %d", mc.clientType, loc.Url, newVid)
glog.V(1).Infof("%s: %s masterClient adds volume %d", mc.clientType, loc.Url, newVid)
mc.addLocation(newVid, loc)
}
for _, deletedVid := range volumeLocation.DeletedVids {
glog.V(1).Infof("%s: %s removes volume %d", mc.clientType, loc.Url, deletedVid)
glog.V(1).Infof("%s: %s masterClient removes volume %d", mc.clientType, loc.Url, deletedVid)
mc.deleteLocation(deletedVid, loc)
}
}
})
if gprcErr != nil {
glog.V(0).Infof("%s failed to connect with master %v: %v", mc.clientType, master, gprcErr)
glog.V(0).Infof("%s masterClient failed to connect with master %v: %v", mc.clientType, master, gprcErr)
}
return
}