From a26f1b2040762903c4b700a0508c0b4c66d4cad5 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Thu, 26 Nov 2020 01:30:11 +0500 Subject: [PATCH] new pkg s3iam --- weed/s3api/auth_credentials.go | 33 +++++-- weed/s3api/filer_util.go | 137 ----------------------------- weed/s3iam/s3iam_filer_store.go | 104 ++++++++++++++++++++++ weed/shell/command_s3_configure.go | 83 +++++++---------- 4 files changed, 164 insertions(+), 193 deletions(-) create mode 100644 weed/s3iam/s3iam_filer_store.go diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index cc259645d..de1a0e3a1 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -3,11 +3,15 @@ package s3api import ( "bytes" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "google.golang.org/grpc" "io/ioutil" "net/http" xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/chrislusf/seaweedfs/weed/s3iam" "github.com/golang/protobuf/jsonpb" "github.com/chrislusf/seaweedfs/weed/glog" @@ -48,19 +52,33 @@ func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManag iam := &IdentityAccessManagement{ domain: option.DomainName, } - if err := loadS3config(iam, option); err != nil { + if err := iam.loadS3ApiConfigurationFromFiler(option); err != nil { glog.Warningf("fail to load config %v", err) } if len(iam.identities) == 0 && option.Config != "" { - if err := iam.loadS3ApiConfiguration(option.Config); err != nil { + if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil { glog.Fatalf("fail to load config file %s: %v", option.Config, err) } } return iam } -func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) error { +func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) error { + s3ApiConfiguration := &iam_pb.S3ApiConfiguration{} + return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + client := filer_pb.NewSeaweedFilerClient(grpcConnection) + store := s3iam.NewIAMFilerStore(&client) + if err := store.LoadIAMConfig(s3ApiConfiguration); err != nil { + return nil + } + if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil { + return err + } + return nil + }, option.FilerGrpcAddress, option.GrpcDialOption) +} +func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error { s3ApiConfiguration := &iam_pb.S3ApiConfiguration{} rawData, readErr := ioutil.ReadFile(fileName) if readErr != nil { @@ -73,8 +91,14 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) err glog.Warningf("unmarshal error: %v", err) return fmt.Errorf("unmarshal %s error: %v", fileName, err) } + if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil { + return err + } + return nil +} - for _, ident := range s3ApiConfiguration.Identities { +func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error { + for _, ident := range config.Identities { t := &Identity{ Name: ident.Name, Credentials: nil, @@ -91,7 +115,6 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) err } iam.identities = append(iam.identities, t) } - return nil } diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go index 7e61aa46d..3626ece98 100644 --- a/weed/s3api/filer_util.go +++ b/weed/s3api/filer_util.go @@ -3,20 +3,12 @@ package s3api import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" "github.com/chrislusf/seaweedfs/weed/util" "strings" - - proto "github.com/golang/protobuf/proto" - "google.golang.org/grpc" ) -const S3ConfName = "s3.conf" - func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error { return filer_pb.Mkdir(s3a, parentDirectoryPath, dirName, fn) @@ -88,135 +80,6 @@ func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry * return filer_pb.GetEntry(s3a, fullPath) } -func LoadS3configFromEntryExtended(extended *map[string][]byte, identities *[]*Identity) (err error) { - for name, ident := range *extended { - t := &Identity{ - Name: name, - Credentials: nil, - Actions: nil, - } - identity := &iam_pb.Identity{} - if err := proto.Unmarshal(ident, identity); err != nil { - return err - } - for _, action := range identity.Actions { - t.Actions = append(t.Actions, Action(action)) - } - for _, cred := range identity.Credentials { - t.Credentials = append(t.Credentials, &Credential{ - AccessKey: cred.AccessKey, - SecretKey: cred.SecretKey, - }) - } - *identities = append(*identities, t) - } - return nil -} - -func SaveS3configToEntryExtended(extended *map[string][]byte, identities *[]*Identity) (err error) { - for _, identity := range *identities { - i := &iam_pb.Identity{ - Name: identity.Name, - Credentials: []*iam_pb.Credential{}, - Actions: []string{}, - } - for _, cred := range identity.Credentials { - i.Credentials = append(i.Credentials, &iam_pb.Credential{ - AccessKey: cred.AccessKey, - SecretKey: cred.SecretKey, - }) - } - for _, action := range identity.Actions { - i.Actions = append(i.Actions, string(action)) - } - ident, err := proto.Marshal(i) - if err != nil { - return err - } - (*extended)[identity.Name] = ident - } - return nil -} - -func loadS3config(iam *IdentityAccessManagement, option *S3ApiServerOption) error { - return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { - client := filer_pb.NewSeaweedFilerClient(grpcConnection) - resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ - Directory: filer.DirectoryEtc, - Name: S3ConfName, - }) - if err != nil { - return err - } - if err = LoadS3configFromEntryExtended(&resp.Entry.Extended, &iam.identities); err != nil { - return err - } - return nil - }, option.FilerGrpcAddress, option.GrpcDialOption) -} - -/* testing save -func saveS3config(iam *IdentityAccessManagement, option *S3ApiServerOption) (error) { - return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { - client := filer_pb.NewSeaweedFilerClient(grpcConnection) - entry := &filer_pb.Entry{ - Name: "s3identities", - IsDirectory: false, - Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), - Crtime: time.Now().Unix(), - FileMode: uint32(0644), - Collection: "", - Replication: "", - }, - Extended: make(map[string][]byte), - } - for _, identity := range iam.identities { - glog.V(0).Infof("get iam identities %s", identity.Name) - i := &iam_pb.Identity{ - Name: identity.Name, - Credentials: []*iam_pb.Credential{}, - Actions: []string{}, - } - for _, cred := range identity.Credentials { - i.Credentials = append(i.Credentials, &iam_pb.Credential{ - AccessKey: cred.AccessKey, - SecretKey: cred.SecretKey, - }) - } - for _, action := range identity.Actions { - i.Actions = append(i.Actions, string(action)) - } - ident, err := proto.Marshal(i) - if err != nil { - return err - } - entry.Extended[identity.Name] = ident - } - _, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ - Directory: "/.configs", - Name: "s3identities", - }) - if err == filer_pb.ErrNotFound { - err = filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{ - Directory: "/.configs", - Entry: entry, - IsFromOtherCluster: false, - Signatures: nil, - }) - } else { - err = filer_pb.UpdateEntry(client, &filer_pb.UpdateEntryRequest{ - Directory: "/.configs", - Entry: entry, - IsFromOtherCluster: false, - Signatures: nil, - }) - } - return err - },option.FilerGrpcAddress, option.GrpcDialOption) -} -*/ - func objectKey(key *string) *string { if strings.HasPrefix(*key, "/") { t := (*key)[1:] diff --git a/weed/s3iam/s3iam_filer_store.go b/weed/s3iam/s3iam_filer_store.go new file mode 100644 index 000000000..22bce2bc4 --- /dev/null +++ b/weed/s3iam/s3iam_filer_store.go @@ -0,0 +1,104 @@ +package s3iam + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" + "time" + + proto "github.com/golang/protobuf/proto" +) + +const ( + iamConfigPrefix = "/etc/iam" + iamIdentityFile = "identity.json" +) + +type IAMFilerStore struct { + client *filer_pb.SeaweedFilerClient +} + +func NewIAMFilerStore(client *filer_pb.SeaweedFilerClient) *IAMFilerStore { + return &IAMFilerStore{client: client} +} + +func (ifs *IAMFilerStore) getIAMConfigRequest() *filer_pb.LookupDirectoryEntryRequest { + return &filer_pb.LookupDirectoryEntryRequest{ + Directory: iamConfigPrefix, + Name: iamIdentityFile, + } +} + +func (ifs *IAMFilerStore) LoadIAMConfig(config *iam_pb.S3ApiConfiguration) error { + resp, err := filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest()) + if err != nil { + return err + } + err = ifs.loadIAMConfigFromEntryExtended(&resp.Entry.Extended, config) + if err != nil { + return err + } + return nil +} + +func (ifs *IAMFilerStore) SaveIAMConfig(config *iam_pb.S3ApiConfiguration) error { + entry := &filer_pb.Entry{ + Name: iamIdentityFile, + IsDirectory: false, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(0644), + Collection: "", + Replication: "", + }, + Extended: make(map[string][]byte), + } + + err := ifs.saveIAMConfigToEntryExtended(&entry.Extended, config) + if err != nil { + return err + } + + _, err = filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest()) + if err == filer_pb.ErrNotFound { + err = filer_pb.CreateEntry(*ifs.client, &filer_pb.CreateEntryRequest{ + Directory: iamConfigPrefix, + Entry: entry, + IsFromOtherCluster: false, + Signatures: nil, + }) + } else { + err = filer_pb.UpdateEntry(*ifs.client, &filer_pb.UpdateEntryRequest{ + Directory: iamConfigPrefix, + Entry: entry, + IsFromOtherCluster: false, + Signatures: nil, + }) + } + if err != nil { + return err + } + return nil +} + +func (ifs *IAMFilerStore) loadIAMConfigFromEntryExtended(extended *map[string][]byte, config *iam_pb.S3ApiConfiguration) error { + for _, ident := range *extended { + identity := &iam_pb.Identity{} + if err := proto.Unmarshal(ident, identity); err != nil { + return err + } + config.Identities = append(config.Identities, identity) + } + return nil +} + +func (ifs *IAMFilerStore) saveIAMConfigToEntryExtended(extended *map[string][]byte, config *iam_pb.S3ApiConfiguration) error { + for _, identity := range config.Identities { + ident, err := proto.Marshal(identity) + if err != nil { + return err + } + (*extended)[identity.Name] = ident + } + return nil +} diff --git a/weed/shell/command_s3_configure.go b/weed/shell/command_s3_configure.go index a4d45a4f9..340d20894 100644 --- a/weed/shell/command_s3_configure.go +++ b/weed/shell/command_s3_configure.go @@ -3,13 +3,13 @@ package shell import ( "flag" "fmt" - "github.com/chrislusf/seaweedfs/weed/s3api" "io" "sort" "strings" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" + "github.com/chrislusf/seaweedfs/weed/s3iam" ) func init() { @@ -44,18 +44,12 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io return nil } - var identities []*s3api.Identity + s3cfg := &iam_pb.S3ApiConfiguration{} + ifs := &s3iam.IAMFilerStore{} if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - request := &filer_pb.LookupDirectoryEntryRequest{ - Directory: filer.DirectoryEtc, - Name: s3api.S3ConfName, - } - respLookupEntry, err := filer_pb.LookupEntry(client, request) - if err != nil { - return err - } - if err = s3api.LoadS3configFromEntryExtended(&respLookupEntry.Entry.Extended, &identities); err != nil { - return err + ifs = s3iam.NewIAMFilerStore(&client) + if err := ifs.LoadIAMConfig(s3cfg); err != nil { + return nil } return nil }); err != nil { @@ -65,7 +59,7 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io idx := 0 changed := false if *user != "" && *buckets != "" { - for i, identity := range identities { + for i, identity := range s3cfg.Identities { if *user == identity.Name { idx = i changed = true @@ -73,22 +67,17 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io } } } - cmdActions := []s3api.Action{} + var cmdActions []string for _, bucket := range strings.Split(*buckets, ",") { for _, action := range strings.Split(*actions, ",") { - cmdActions = append(cmdActions, s3api.Action(fmt.Sprintf("%s:%s", action, bucket))) + cmdActions = append(cmdActions, fmt.Sprintf("%s:%s", action, bucket)) } } - cmdCredential := &s3api.Credential{ - AccessKey: *accessKey, - SecretKey: *secretKey, - } - if changed { if *isDelete { - exists := []int{} + var exists []int for _, cmdAction := range cmdActions { - for i, currentAction := range identities[idx].Actions { + for i, currentAction := range s3cfg.Identities[idx].Actions { if cmdAction == currentAction { exists = append(exists, i) } @@ -96,65 +85,57 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io } sort.Sort(sort.Reverse(sort.IntSlice(exists))) for _, i := range exists { - identities[idx].Actions = append(identities[idx].Actions[:i], identities[idx].Actions[i+1:]...) + s3cfg.Identities[idx].Actions = append( + s3cfg.Identities[idx].Actions[:i], + s3cfg.Identities[idx].Actions[i+1:]..., + ) } if *accessKey != "" { exists = []int{} - for i, credential := range identities[idx].Credentials { + for i, credential := range s3cfg.Identities[idx].Credentials { if credential.AccessKey == *accessKey { exists = append(exists, i) } } sort.Sort(sort.Reverse(sort.IntSlice(exists))) for _, i := range exists { - identities[idx].Credentials = append(identities[idx].Credentials[:i], identities[idx].Credentials[:i+1]...) + s3cfg.Identities[idx].Credentials = append( + s3cfg.Identities[idx].Credentials[:i], + s3cfg.Identities[idx].Credentials[:i+1]..., + ) } } if *actions == "" && *accessKey == "" { - identities = append(identities[:idx], identities[idx+1:]...) + s3cfg.Identities = append(s3cfg.Identities[:idx], s3cfg.Identities[idx+1:]...) } } else { - identities[idx].Actions = append(identities[idx].Actions, cmdActions...) - identities[idx].Credentials = append(identities[idx].Credentials, &s3api.Credential{ + s3cfg.Identities[idx].Actions = append(s3cfg.Identities[idx].Actions, cmdActions...) + s3cfg.Identities[idx].Credentials = append(s3cfg.Identities[idx].Credentials, &iam_pb.Credential{ AccessKey: *accessKey, SecretKey: *secretKey, }) } } else { - identity := s3api.Identity{ + identity := iam_pb.Identity{ Name: *user, Actions: cmdActions, } - identity.Credentials = append(identity.Credentials, &s3api.Credential{ + identity.Credentials = append(identity.Credentials, &iam_pb.Credential{ AccessKey: *accessKey, SecretKey: *secretKey, }) - identities = append(identities, &identity) + s3cfg.Identities = append(s3cfg.Identities, &identity) } - fmt.Fprintf(writer, fmt.Sprintf("%+v\n", identities)) + fmt.Fprintf(writer, fmt.Sprintf("%+v\n", s3cfg.Identities)) fmt.Fprintln(writer) - if !*apply { - return nil + if *apply { + if err := ifs.SaveIAMConfig(s3cfg); err != nil { + return err + } } - if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - request := &filer_pb.LookupDirectoryEntryRequest{ - Directory: filer.DirectoryEtc, - Name: s3api.S3ConfName, - } - respLookupEntry, err := filer_pb.LookupEntry(client, request) - if err != nil { - return err - } - if err = s3api.SaveS3configToEntryExtended(&respLookupEntry.Entry.Extended, &identities); err != nil { - return err - } - return nil - }); err != nil { - return err - } return nil }