mirror of
https://github.com/chrislusf/seaweedfs
synced 2024-06-18 08:31:22 +02:00
audit log SignatureVersion
This commit is contained in:
kmlebedev
parent
98251fe16a
commit
4f98553ba9
|
|
@ -46,6 +46,9 @@ dev_registry: build
|
|||
dev_replicate: build
|
||||
docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up
|
||||
|
||||
dev_auditlog: build
|
||||
docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
|
||||
|
||||
cluster: build
|
||||
docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
version: '2'
|
||||
|
||||
services:
|
||||
server:
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8333:8333
|
||||
|
|
@ -11,10 +11,26 @@ services:
|
|||
- 18084:18080
|
||||
- 8888:8888
|
||||
- 18888:18888
|
||||
command: "server -ip=server -filer -s3 -s3.auditLogConfig=/etc/seaweedfs/fluent.json -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1"
|
||||
- 8000:8000
|
||||
command: "server -ip=s3 -filer -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.auditLogConfig=/etc/seaweedfs/fluent.json -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1"
|
||||
volumes:
|
||||
- ./fluent.json:/etc/seaweedfs/fluent.json
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
depends_on:
|
||||
- fluent
|
||||
fluent:
|
||||
image: fluent/fluentd:v1.14
|
||||
ports:
|
||||
- 24224:24224
|
||||
- 24224:24224
|
||||
#s3tests:
|
||||
# image: chrislusf/ceph-s3-tests:local
|
||||
# volumes:
|
||||
# - ./s3tests.conf:/opt/s3-tests/s3tests.conf
|
||||
# environment:
|
||||
# S3TEST_CONF: "s3tests.conf"
|
||||
# NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
|
||||
# NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
|
||||
# NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
|
||||
# depends_on:
|
||||
# - s3
|
||||
# - fluent
|
||||
|
|
@ -38,7 +38,7 @@ services:
|
|||
S3TEST_CONF: "s3tests.conf"
|
||||
NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
|
||||
NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
|
||||
NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
|
||||
NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
|
|
|
|||
|
|
@ -197,8 +197,10 @@ func (s3opt *S3Options) startS3Server() bool {
|
|||
|
||||
if len(*s3opt.auditLogConfig) > 0 {
|
||||
s3err.InitAuditLog(*s3opt.auditLogConfig)
|
||||
if s3err.Logger != nil {
|
||||
defer s3err.Logger.Close()
|
||||
}
|
||||
}
|
||||
defer s3err.Logger.Close()
|
||||
|
||||
if *s3opt.tlsPrivateKey != "" {
|
||||
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
|
||||
|
|
|
|||
|
|
@ -203,33 +203,44 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
|
|||
var identity *Identity
|
||||
var s3Err s3err.ErrorCode
|
||||
var found bool
|
||||
var authType string
|
||||
switch getRequestAuthType(r) {
|
||||
case authTypeStreamingSigned:
|
||||
return identity, s3err.ErrNone
|
||||
case authTypeUnknown:
|
||||
glog.V(3).Infof("unknown auth type")
|
||||
r.Header.Set(xhttp.AmzAuthType, "Unknown")
|
||||
return identity, s3err.ErrAccessDenied
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
glog.V(3).Infof("v2 auth type")
|
||||
identity, s3Err = iam.isReqAuthenticatedV2(r)
|
||||
authType = "SigV2"
|
||||
case authTypeSigned, authTypePresigned:
|
||||
glog.V(3).Infof("v4 auth type")
|
||||
identity, s3Err = iam.reqSignatureV4Verify(r)
|
||||
authType = "SigV4"
|
||||
case authTypePostPolicy:
|
||||
glog.V(3).Infof("post policy auth type")
|
||||
r.Header.Set(xhttp.AmzAuthType, "PostPolicy")
|
||||
return identity, s3err.ErrNone
|
||||
case authTypeJWT:
|
||||
glog.V(3).Infof("jwt auth type")
|
||||
r.Header.Set(xhttp.AmzAuthType, "Jwt")
|
||||
return identity, s3err.ErrNotImplemented
|
||||
case authTypeAnonymous:
|
||||
authType = "Anonymous"
|
||||
identity, found = iam.lookupAnonymous()
|
||||
if !found {
|
||||
r.Header.Set(xhttp.AmzAuthType, authType)
|
||||
return identity, s3err.ErrAccessDenied
|
||||
}
|
||||
default:
|
||||
return identity, s3err.ErrNotImplemented
|
||||
}
|
||||
|
||||
if len(authType) > 0 {
|
||||
r.Header.Set(xhttp.AmzAuthType, authType)
|
||||
}
|
||||
if s3Err != s3err.ErrNone {
|
||||
return identity, s3Err
|
||||
}
|
||||
|
|
@ -250,33 +261,45 @@ func (iam *IdentityAccessManagement) authUser(r *http.Request) (*Identity, s3err
|
|||
var identity *Identity
|
||||
var s3Err s3err.ErrorCode
|
||||
var found bool
|
||||
var authType string
|
||||
switch getRequestAuthType(r) {
|
||||
case authTypeStreamingSigned:
|
||||
return identity, s3err.ErrNone
|
||||
case authTypeUnknown:
|
||||
glog.V(3).Infof("unknown auth type")
|
||||
r.Header.Set(xhttp.AmzAuthType, "Unknown")
|
||||
return identity, s3err.ErrAccessDenied
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
glog.V(3).Infof("v2 auth type")
|
||||
identity, s3Err = iam.isReqAuthenticatedV2(r)
|
||||
authType = "SigV2"
|
||||
case authTypeSigned, authTypePresigned:
|
||||
glog.V(3).Infof("v4 auth type")
|
||||
identity, s3Err = iam.reqSignatureV4Verify(r)
|
||||
authType = "SigV4"
|
||||
case authTypePostPolicy:
|
||||
glog.V(3).Infof("post policy auth type")
|
||||
r.Header.Set(xhttp.AmzAuthType, "PostPolicy")
|
||||
return identity, s3err.ErrNone
|
||||
case authTypeJWT:
|
||||
glog.V(3).Infof("jwt auth type")
|
||||
r.Header.Set(xhttp.AmzAuthType, "Jwt")
|
||||
return identity, s3err.ErrNotImplemented
|
||||
case authTypeAnonymous:
|
||||
authType = "Anonymous"
|
||||
identity, found = iam.lookupAnonymous()
|
||||
if !found {
|
||||
r.Header.Set(xhttp.AmzAuthType, authType)
|
||||
return identity, s3err.ErrAccessDenied
|
||||
}
|
||||
default:
|
||||
return identity, s3err.ErrNotImplemented
|
||||
}
|
||||
|
||||
if len(authType) > 0 {
|
||||
r.Header.Set(xhttp.AmzAuthType, authType)
|
||||
}
|
||||
|
||||
glog.V(3).Infof("auth error: %v", s3Err)
|
||||
if s3Err != s3err.ErrNone {
|
||||
return identity, s3Err
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ const (
|
|||
// Non-Standard S3 HTTP request constants
|
||||
const (
|
||||
AmzIdentityId = "s3-identity-id"
|
||||
AmzAuthType = "s3-auth-type"
|
||||
AmzIsAdmin = "s3-is-admin" // only set to http request header as a context
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -48,10 +48,9 @@ type AccessLogHTTP struct {
|
|||
const tag = "s3.access"
|
||||
|
||||
var (
|
||||
Logger *fluent.Fluent
|
||||
hostname = os.Getenv("HOSTNAME")
|
||||
environment = os.Getenv("ENVIRONMENT")
|
||||
fluentConfig *fluent.Config
|
||||
Logger *fluent.Fluent
|
||||
hostname = os.Getenv("HOSTNAME")
|
||||
environment = os.Getenv("ENVIRONMENT")
|
||||
)
|
||||
|
||||
func InitAuditLog(config string) {
|
||||
|
|
@ -60,8 +59,9 @@ func InitAuditLog(config string) {
|
|||
glog.Errorf("fail to read fluent config %s : %v", config, readErr)
|
||||
return
|
||||
}
|
||||
fluentConfig := &fluent.Config{}
|
||||
if err := json.Unmarshal(configContent, fluentConfig); err != nil {
|
||||
glog.Errorf("fail to parse fluent config %s : %v", config, err)
|
||||
glog.Errorf("fail to parse fluent config %s : %v", string(configContent), err)
|
||||
return
|
||||
}
|
||||
if len(fluentConfig.TagPrefix) == 0 && len(environment) > 0 {
|
||||
|
|
@ -142,18 +142,19 @@ func GetAccessLog(r *http.Request, HTTPStatusCode int, s3errCode ErrorCode) *Acc
|
|||
hostHeader = r.Host
|
||||
}
|
||||
return &AccessLog{
|
||||
HostHeader: hostHeader,
|
||||
RequestID: r.Header.Get("X-Request-ID"),
|
||||
RemoteIP: remoteIP,
|
||||
Requester: r.Header.Get(xhttp.AmzIdentityId),
|
||||
UserAgent: r.Header.Get("user-agent"),
|
||||
HostId: hostname,
|
||||
Bucket: bucket,
|
||||
HTTPStatus: HTTPStatusCode,
|
||||
Time: time.Now().Unix(),
|
||||
Key: key,
|
||||
Operation: getOperation(key, r),
|
||||
ErrorCode: errorCode,
|
||||
HostHeader: hostHeader,
|
||||
RequestID: r.Header.Get("X-Request-ID"),
|
||||
RemoteIP: remoteIP,
|
||||
Requester: r.Header.Get(xhttp.AmzIdentityId),
|
||||
SignatureVersion: r.Header.Get(xhttp.AmzAuthType),
|
||||
UserAgent: r.Header.Get("user-agent"),
|
||||
HostId: hostname,
|
||||
Bucket: bucket,
|
||||
HTTPStatus: HTTPStatusCode,
|
||||
Time: time.Now().Unix(),
|
||||
Key: key,
|
||||
Operation: getOperation(key, r),
|
||||
ErrorCode: errorCode,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue