mirror of
https://github.com/chrislusf/seaweedfs
synced 2024-05-04 10:31:06 +02:00
Merge pull request #1137 from BruceWangNo1/s3-select
followed @chrislusf and @iliul 's suggestions
This commit is contained in:
Chris Lu
GitHub
commit
8d068dc5d6
|
|
@ -1,8 +1,6 @@
|
|||
sudo: false
|
||||
language: go
|
||||
go:
|
||||
- 1.11.x
|
||||
- 1.12.x
|
||||
- 1.13.x
|
||||
|
||||
before_install:
|
||||
|
|
|
|||
22
go.mod
22
go.mod
|
|
@ -10,10 +10,9 @@ require (
|
|||
contrib.go.opencensus.io/resource v0.1.2 // indirect
|
||||
github.com/Azure/azure-amqp-common-go v1.1.4 // indirect
|
||||
github.com/Azure/azure-pipeline-go v0.2.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go v33.0.0+incompatible // indirect
|
||||
github.com/Azure/azure-storage-blob-go v0.8.0
|
||||
github.com/Azure/go-autorest v13.0.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/tracing v0.5.0 // indirect
|
||||
github.com/Azure/go-autorest/autorest v0.9.2 // indirect
|
||||
github.com/DataDog/zstd v1.4.1 // indirect
|
||||
github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190828224159-d93c53a4824c // indirect
|
||||
github.com/Shopify/sarama v1.23.1
|
||||
|
|
@ -22,12 +21,10 @@ require (
|
|||
github.com/aws/aws-sdk-go v1.23.13
|
||||
github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92
|
||||
github.com/coreos/etcd v3.3.15+incompatible // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible
|
||||
github.com/disintegration/imaging v1.6.1
|
||||
github.com/dustin/go-humanize v1.0.0
|
||||
github.com/eapache/go-resiliency v1.2.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v0.3.17
|
||||
github.com/go-kit/kit v0.9.0 // indirect
|
||||
github.com/go-redis/redis v6.15.2+incompatible
|
||||
|
|
@ -39,28 +36,23 @@ require (
|
|||
github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70 // indirect
|
||||
github.com/google/uuid v1.1.1
|
||||
github.com/gorilla/mux v1.7.3
|
||||
github.com/gorilla/websocket v1.4.1 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
|
||||
github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f
|
||||
github.com/jcmturner/gofork v1.0.0 // indirect
|
||||
github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 // indirect
|
||||
github.com/karlseguin/ccache v2.0.3+incompatible
|
||||
github.com/karlseguin/expect v1.0.1 // indirect
|
||||
github.com/klauspost/cpuid v1.2.1 // indirect
|
||||
github.com/klauspost/crc32 v1.2.0
|
||||
github.com/klauspost/reedsolomon v1.9.2
|
||||
github.com/klauspost/reedsolomon v1.9.3
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
|
||||
github.com/kr/pty v1.1.8 // indirect
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3
|
||||
github.com/lib/pq v1.2.0
|
||||
github.com/magiconair/properties v1.8.1 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
|
||||
github.com/mattn/go-isatty v0.0.9 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.4 // indirect
|
||||
github.com/minio/minio v0.0.0-20191122214605-0a56e33ce1b5
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
|
||||
github.com/nats-io/gnatsd v1.4.1 // indirect
|
||||
github.com/nats-io/go-nats v1.7.2 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.0.4 // indirect
|
||||
github.com/onsi/ginkgo v1.10.1 // indirect
|
||||
github.com/onsi/gomega v1.7.0 // indirect
|
||||
github.com/opentracing/opentracing-go v1.1.0 // indirect
|
||||
|
|
@ -81,7 +73,6 @@ require (
|
|||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff
|
||||
github.com/sirupsen/logrus v1.4.2 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/afero v1.2.2 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
|
|
@ -107,13 +98,12 @@ require (
|
|||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979 // indirect
|
||||
golang.org/x/image v0.0.0-20190829233526-b3c06291d021 // indirect
|
||||
golang.org/x/mobile v0.0.0-20190830201351-c6da95954960 // indirect
|
||||
golang.org/x/net v0.0.0-20190909003024-a7b16738d86b
|
||||
golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478
|
||||
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69
|
||||
golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110
|
||||
google.golang.org/api v0.9.0
|
||||
google.golang.org/appengine v1.6.2 // indirect
|
||||
google.golang.org/grpc v1.23.0
|
||||
gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
|
||||
gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
|
||||
gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect
|
||||
honnef.co/go/tools v0.0.1-2019.2.2 // indirect
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ import (
|
|||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/minio/minio/cmd"
|
||||
"github.com/minio/minio/pkg/s3select"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
|
@ -12,6 +14,7 @@ import (
|
|||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/server"
|
||||
"github.com/gorilla/mux"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -115,10 +118,15 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
|||
}
|
||||
|
||||
func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResonse *http.Response, w http.ResponseWriter)) {
|
||||
var method string
|
||||
method = r.Method
|
||||
|
||||
glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl)
|
||||
if r.URL.RawQuery == "select&select-type=2" {
|
||||
method = "GET"
|
||||
}
|
||||
glog.V(2).Infof("s3 proxying %s to %s", method, destUrl)
|
||||
|
||||
proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body)
|
||||
proxyReq, err := http.NewRequest(method, destUrl, r.Body)
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("NewRequest %s: %v", destUrl, err)
|
||||
|
|
@ -224,3 +232,63 @@ func getObject(vars map[string]string) string {
|
|||
}
|
||||
return object
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) SelectObjectContent(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
object := getObject(vars)
|
||||
|
||||
s3Select, err := s3select.NewS3Select(r.Body)
|
||||
if err != nil {
|
||||
if serr, ok := err.(s3select.SelectError); ok {
|
||||
encodedErrorResponse := encodeResponse(cmd.APIErrorResponse{
|
||||
Code: serr.ErrorCode(),
|
||||
Message: serr.ErrorMessage(),
|
||||
BucketName: bucket,
|
||||
Key: object,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: "",
|
||||
})
|
||||
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, "application/xml")
|
||||
} else {
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if strings.HasSuffix(r.URL.Path, "/") {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
destUrl := fmt.Sprintf("http://%s%s/%s%s",
|
||||
s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
|
||||
passThroughResponseSelectObjectContent := func(proxyResonse *http.Response,
|
||||
w http.ResponseWriter) {
|
||||
getObject := func(offset, length int64) (io.ReadCloser, error) {
|
||||
return proxyResonse.Body, nil
|
||||
}
|
||||
if err = s3Select.Open(getObject); err != nil {
|
||||
if serr, ok := err.(s3select.SelectError); ok {
|
||||
encodedErrorResponse := encodeResponse(cmd.APIErrorResponse{
|
||||
Code: serr.ErrorCode(),
|
||||
Message: serr.ErrorMessage(),
|
||||
BucketName: bucket,
|
||||
Key: object,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: "",
|
||||
})
|
||||
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
|
||||
} else {
|
||||
writeResponse(w, http.StatusInternalServerError, encodeResponse("not s3select.SelectError"), mimeXML)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
s3Select.Evaluate(w)
|
||||
s3Select.Close()
|
||||
}
|
||||
s3a.proxyToFiler(w, r, destUrl, passThroughResponseSelectObjectContent)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,6 +79,8 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
|
|||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectHandler)
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV1Handler)
|
||||
// SelectObjectContent
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(s3a.SelectObjectContent).Queries("select", "", "select-type", "2")
|
||||
|
||||
// DeleteMultipleObjects
|
||||
bucket.Methods("POST").HandlerFunc(s3a.DeleteMultipleObjectsHandler).Queries("delete", "")
|
||||
|
|
|
|||
Loading…
Reference in a new issue