mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/chrislusf/seaweedfs synced 2024-06-25 03:48:10 +02:00
Code Issues Projects Releases Wiki Activity

new pkg s3iam

Browse source
This commit is contained in:
Konstantin Lebedev 2020-11-26 01:30:11 +05:00
parent 6206737df2
commit a26f1b2040
4 changed files with 164 additions and 193 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
weed/s3api/auth_credentials.go
View file

@ -3,11 +3,15 @@ package s3api
import (
"bytes"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"google.golang.org/grpc"
"io/ioutil"
"net/http"
xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"github.com/chrislusf/seaweedfs/weed/s3iam"
"github.com/golang/protobuf/jsonpb"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -48,19 +52,33 @@ func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManag
iam := &IdentityAccessManagement{
domain: option.DomainName,
}
if err := loadS3config(iam, option); err != nil {
if err := iam.loadS3ApiConfigurationFromFiler(option); err != nil {
glog.Warningf("fail to load config %v", err)
}
if len(iam.identities) == 0 && option.Config != "" {
if err := iam.loadS3ApiConfiguration(option.Config); err != nil {
if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil {
glog.Fatalf("fail to load config file %s: %v", option.Config, err)
}
}
return iam
}
func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) error {
func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) error {
s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}
return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
store := s3iam.NewIAMFilerStore(&client)
if err := store.LoadIAMConfig(s3ApiConfiguration); err != nil {
return nil
}
if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
return err
}
return nil
}, option.FilerGrpcAddress, option.GrpcDialOption)
}
func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error {
s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}
rawData, readErr := ioutil.ReadFile(fileName)
if readErr != nil {
@ -73,8 +91,14 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) err
glog.Warningf("unmarshal error: %v", err)
return fmt.Errorf("unmarshal %s error: %v", fileName, err)
}
if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
return err
}
return nil
}
for _, ident := range s3ApiConfiguration.Identities {
func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
for _, ident := range config.Identities {
t := &Identity{
Name: ident.Name,
Credentials: nil,
@ -91,7 +115,6 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) err
}
iam.identities = append(iam.identities, t)
}
return nil
}

137
weed/s3api/filer_util.go
View file

@ -3,20 +3,12 @@ package s3api
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"strings"
proto "github.com/golang/protobuf/proto"
"google.golang.org/grpc"
)
const S3ConfName = "s3.conf"
func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
return filer_pb.Mkdir(s3a, parentDirectoryPath, dirName, fn)
@ -88,135 +80,6 @@ func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *
return filer_pb.GetEntry(s3a, fullPath)
}
func LoadS3configFromEntryExtended(extended *map[string][]byte, identities *[]*Identity) (err error) {
for name, ident := range *extended {
t := &Identity{
Name: name,
Credentials: nil,
Actions: nil,
}
identity := &iam_pb.Identity{}
if err := proto.Unmarshal(ident, identity); err != nil {
return err
}
for _, action := range identity.Actions {
t.Actions = append(t.Actions, Action(action))
}
for _, cred := range identity.Credentials {
t.Credentials = append(t.Credentials, &Credential{
AccessKey: cred.AccessKey,
SecretKey: cred.SecretKey,
})
}
*identities = append(*identities, t)
}
return nil
}
func SaveS3configToEntryExtended(extended *map[string][]byte, identities *[]*Identity) (err error) {
for _, identity := range *identities {
i := &iam_pb.Identity{
Name: identity.Name,
Credentials: []*iam_pb.Credential{},
Actions: []string{},
}
for _, cred := range identity.Credentials {
i.Credentials = append(i.Credentials, &iam_pb.Credential{
AccessKey: cred.AccessKey,
SecretKey: cred.SecretKey,
})
}
for _, action := range identity.Actions {
i.Actions = append(i.Actions, string(action))
}
ident, err := proto.Marshal(i)
if err != nil {
return err
}
(*extended)[identity.Name] = ident
}
return nil
}
func loadS3config(iam *IdentityAccessManagement, option *S3ApiServerOption) error {
return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{
Directory: filer.DirectoryEtc,
Name: S3ConfName,
})
if err != nil {
return err
}
if err = LoadS3configFromEntryExtended(&resp.Entry.Extended, &iam.identities); err != nil {
return err
}
return nil
}, option.FilerGrpcAddress, option.GrpcDialOption)
}
/* testing save
func saveS3config(iam *IdentityAccessManagement, option *S3ApiServerOption) (error) {
return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
entry := &filer_pb.Entry{
Name: "s3identities",
IsDirectory: false,
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
Crtime: time.Now().Unix(),
FileMode: uint32(0644),
Collection: "",
Replication: "",
},
Extended: make(map[string][]byte),
}
for _, identity := range iam.identities {
glog.V(0).Infof("get iam identities %s", identity.Name)
i := &iam_pb.Identity{
Name: identity.Name,
Credentials: []*iam_pb.Credential{},
Actions: []string{},
}
for _, cred := range identity.Credentials {
i.Credentials = append(i.Credentials, &iam_pb.Credential{
AccessKey: cred.AccessKey,
SecretKey: cred.SecretKey,
})
}
for _, action := range identity.Actions {
i.Actions = append(i.Actions, string(action))
}
ident, err := proto.Marshal(i)
if err != nil {
return err
}
entry.Extended[identity.Name] = ident
}
_, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{
Directory: "/.configs",
Name: "s3identities",
})
if err == filer_pb.ErrNotFound {
err = filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
Directory: "/.configs",
Entry: entry,
IsFromOtherCluster: false,
Signatures: nil,
})
} else {
err = filer_pb.UpdateEntry(client, &filer_pb.UpdateEntryRequest{
Directory: "/.configs",
Entry: entry,
IsFromOtherCluster: false,
Signatures: nil,
})
}
return err
},option.FilerGrpcAddress, option.GrpcDialOption)
}
*/
func objectKey(key *string) *string {
if strings.HasPrefix(*key, "/") {
t := (*key)[1:]

104
weed/s3iam/s3iam_filer_store.go Normal file
View file

@ -0,0 +1,104 @@
package s3iam
import (
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
"time"
proto "github.com/golang/protobuf/proto"
)
const (
iamConfigPrefix = "/etc/iam"
iamIdentityFile = "identity.json"
)
type IAMFilerStore struct {
client *filer_pb.SeaweedFilerClient
}
func NewIAMFilerStore(client *filer_pb.SeaweedFilerClient) *IAMFilerStore {
return &IAMFilerStore{client: client}
}
func (ifs *IAMFilerStore) getIAMConfigRequest() *filer_pb.LookupDirectoryEntryRequest {
return &filer_pb.LookupDirectoryEntryRequest{
Directory: iamConfigPrefix,
Name: iamIdentityFile,
}
}
func (ifs *IAMFilerStore) LoadIAMConfig(config *iam_pb.S3ApiConfiguration) error {
resp, err := filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest())
if err != nil {
return err
}
err = ifs.loadIAMConfigFromEntryExtended(&resp.Entry.Extended, config)
if err != nil {
return err
}
return nil
}
func (ifs *IAMFilerStore) SaveIAMConfig(config *iam_pb.S3ApiConfiguration) error {
entry := &filer_pb.Entry{
Name: iamIdentityFile,
IsDirectory: false,
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
Crtime: time.Now().Unix(),
FileMode: uint32(0644),
Collection: "",
Replication: "",
},
Extended: make(map[string][]byte),
}
err := ifs.saveIAMConfigToEntryExtended(&entry.Extended, config)
if err != nil {
return err
}
_, err = filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest())
if err == filer_pb.ErrNotFound {
err = filer_pb.CreateEntry(*ifs.client, &filer_pb.CreateEntryRequest{
Directory: iamConfigPrefix,
Entry: entry,
IsFromOtherCluster: false,
Signatures: nil,
})
} else {
err = filer_pb.UpdateEntry(*ifs.client, &filer_pb.UpdateEntryRequest{
Directory: iamConfigPrefix,
Entry: entry,
IsFromOtherCluster: false,
Signatures: nil,
})
}
if err != nil {
return err
}
return nil
}
func (ifs *IAMFilerStore) loadIAMConfigFromEntryExtended(extended *map[string][]byte, config *iam_pb.S3ApiConfiguration) error {
for _, ident := range *extended {
identity := &iam_pb.Identity{}
if err := proto.Unmarshal(ident, identity); err != nil {
return err
}
config.Identities = append(config.Identities, identity)
}
return nil
}
func (ifs *IAMFilerStore) saveIAMConfigToEntryExtended(extended *map[string][]byte, config *iam_pb.S3ApiConfiguration) error {
for _, identity := range config.Identities {
ident, err := proto.Marshal(identity)
if err != nil {
return err
}
(*extended)[identity.Name] = ident
}
return nil
}

83
weed/shell/command_s3_configure.go
View file

@ -3,13 +3,13 @@ package shell
import (
"flag"
"fmt"
"github.com/chrislusf/seaweedfs/weed/s3api"
"io"
"sort"
"strings"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
"github.com/chrislusf/seaweedfs/weed/s3iam"
)
func init() {
@ -44,18 +44,12 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
return nil
}
var identities []*s3api.Identity
s3cfg := &iam_pb.S3ApiConfiguration{}
ifs := &s3iam.IAMFilerStore{}
if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.LookupDirectoryEntryRequest{
Directory: filer.DirectoryEtc,
Name: s3api.S3ConfName,
}
respLookupEntry, err := filer_pb.LookupEntry(client, request)
if err != nil {
return err
}
if err = s3api.LoadS3configFromEntryExtended(&respLookupEntry.Entry.Extended, &identities); err != nil {
return err
ifs = s3iam.NewIAMFilerStore(&client)
if err := ifs.LoadIAMConfig(s3cfg); err != nil {
return nil
}
return nil
}); err != nil {
@ -65,7 +59,7 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
idx := 0
changed := false
if *user != "" && *buckets != "" {
for i, identity := range identities {
for i, identity := range s3cfg.Identities {
if *user == identity.Name {
idx = i
changed = true
@ -73,22 +67,17 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
}
}
}
cmdActions := []s3api.Action{}
var cmdActions []string
for _, bucket := range strings.Split(*buckets, ",") {
for _, action := range strings.Split(*actions, ",") {
cmdActions = append(cmdActions, s3api.Action(fmt.Sprintf("%s:%s", action, bucket)))
cmdActions = append(cmdActions, fmt.Sprintf("%s:%s", action, bucket))
}
}
cmdCredential := &s3api.Credential{
AccessKey: *accessKey,
SecretKey: *secretKey,
}
if changed {
if *isDelete {
exists := []int{}
var exists []int
for _, cmdAction := range cmdActions {
for i, currentAction := range identities[idx].Actions {
for i, currentAction := range s3cfg.Identities[idx].Actions {
if cmdAction == currentAction {
exists = append(exists, i)
}
@ -96,65 +85,57 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
}
sort.Sort(sort.Reverse(sort.IntSlice(exists)))
for _, i := range exists {
identities[idx].Actions = append(identities[idx].Actions[:i], identities[idx].Actions[i+1:]...)
s3cfg.Identities[idx].Actions = append(
s3cfg.Identities[idx].Actions[:i],
s3cfg.Identities[idx].Actions[i+1:]...,
)
}
if *accessKey != "" {
exists = []int{}
for i, credential := range identities[idx].Credentials {
for i, credential := range s3cfg.Identities[idx].Credentials {
if credential.AccessKey == *accessKey {
exists = append(exists, i)
}
}
sort.Sort(sort.Reverse(sort.IntSlice(exists)))
for _, i := range exists {
identities[idx].Credentials = append(identities[idx].Credentials[:i], identities[idx].Credentials[:i+1]...)
s3cfg.Identities[idx].Credentials = append(
s3cfg.Identities[idx].Credentials[:i],
s3cfg.Identities[idx].Credentials[:i+1]...,
)
}
}
if *actions == "" && *accessKey == "" {
identities = append(identities[:idx], identities[idx+1:]...)
s3cfg.Identities = append(s3cfg.Identities[:idx], s3cfg.Identities[idx+1:]...)
}
} else {
identities[idx].Actions = append(identities[idx].Actions, cmdActions...)
identities[idx].Credentials = append(identities[idx].Credentials, &s3api.Credential{
s3cfg.Identities[idx].Actions = append(s3cfg.Identities[idx].Actions, cmdActions...)
s3cfg.Identities[idx].Credentials = append(s3cfg.Identities[idx].Credentials, &iam_pb.Credential{
AccessKey: *accessKey,
SecretKey: *secretKey,
})
}
} else {
identity := s3api.Identity{
identity := iam_pb.Identity{
Name: *user,
Actions: cmdActions,
}
identity.Credentials = append(identity.Credentials, &s3api.Credential{
identity.Credentials = append(identity.Credentials, &iam_pb.Credential{
AccessKey: *accessKey,
SecretKey: *secretKey,
})
identities = append(identities, &identity)
s3cfg.Identities = append(s3cfg.Identities, &identity)
}
fmt.Fprintf(writer, fmt.Sprintf("%+v\n", identities))
fmt.Fprintf(writer, fmt.Sprintf("%+v\n", s3cfg.Identities))
fmt.Fprintln(writer)
if !*apply {
return nil
if *apply {
if err := ifs.SaveIAMConfig(s3cfg); err != nil {
return err
}
}
if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.LookupDirectoryEntryRequest{
Directory: filer.DirectoryEtc,
Name: s3api.S3ConfName,
}
respLookupEntry, err := filer_pb.LookupEntry(client, request)
if err != nil {
return err
}
if err = s3api.SaveS3configToEntryExtended(&respLookupEntry.Entry.Extended, &identities); err != nil {
return err
}
return nil
}); err != nil {
return err
}
return nil
}