From 3dd60529c517d6cabfb5ff513013680678d8d3ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9F=B3=E6=98=8C=E6=9E=97?= Date: Fri, 17 Jun 2022 19:07:39 +0800 Subject: [PATCH] some code optimizations --- weed/pb/s3.proto | 6 +- weed/pb/s3_pb/s3.pb.go | 128 ++++++++-------- weed/s3api/auth_credentials_subscribe.go | 4 +- .../{s3_config => s3_constants}/s3_config.go | 5 +- weed/s3api/s3api_circuit_breaker.go | 16 +- weed/s3api/s3api_circuit_breaker_test.go | 24 +-- weed/shell/command_s3_circuitbreaker.go | 56 ++++--- weed/shell/command_s3_circuitbreaker_test.go | 145 ++++++++++++++++-- 8 files changed, 256 insertions(+), 128 deletions(-) rename weed/s3api/{s3_config => s3_constants}/s3_config.go (55%) diff --git a/weed/pb/s3.proto b/weed/pb/s3.proto index b08631ae7..45a877fac 100644 --- a/weed/pb/s3.proto +++ b/weed/pb/s3.proto @@ -25,11 +25,11 @@ message S3ConfigureResponse { } message S3CircuitBreakerConfig { - CbOptions global=1; - map buckets= 2; + S3CircuitBreakerOptions global=1; + map buckets= 2; } -message CbOptions { +message S3CircuitBreakerOptions { bool enabled=1; map actions = 2; } diff --git a/weed/pb/s3_pb/s3.pb.go b/weed/pb/s3_pb/s3.pb.go index 0cac7d828..c1bd23556 100644 --- a/weed/pb/s3_pb/s3.pb.go +++ b/weed/pb/s3_pb/s3.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc v3.21.1 // source: s3.proto package s3_pb @@ -110,8 +110,8 @@ type S3CircuitBreakerConfig struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Global *CbOptions `protobuf:"bytes,1,opt,name=global,proto3" json:"global,omitempty"` - Buckets map[string]*CbOptions `protobuf:"bytes,2,rep,name=buckets,proto3" json:"buckets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Global *S3CircuitBreakerOptions `protobuf:"bytes,1,opt,name=global,proto3" json:"global,omitempty"` + Buckets map[string]*S3CircuitBreakerOptions `protobuf:"bytes,2,rep,name=buckets,proto3" json:"buckets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *S3CircuitBreakerConfig) Reset() { @@ -146,21 +146,21 @@ func (*S3CircuitBreakerConfig) Descriptor() ([]byte, []int) { return file_s3_proto_rawDescGZIP(), []int{2} } -func (x *S3CircuitBreakerConfig) GetGlobal() *CbOptions { +func (x *S3CircuitBreakerConfig) GetGlobal() *S3CircuitBreakerOptions { if x != nil { return x.Global } return nil } -func (x *S3CircuitBreakerConfig) GetBuckets() map[string]*CbOptions { +func (x *S3CircuitBreakerConfig) GetBuckets() map[string]*S3CircuitBreakerOptions { if x != nil { return x.Buckets } return nil } -type CbOptions struct { +type S3CircuitBreakerOptions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -169,8 +169,8 @@ type CbOptions struct { Actions map[string]int64 `protobuf:"bytes,2,rep,name=actions,proto3" json:"actions,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } -func (x *CbOptions) Reset() { - *x = CbOptions{} +func (x *S3CircuitBreakerOptions) Reset() { + *x = S3CircuitBreakerOptions{} if protoimpl.UnsafeEnabled { mi := &file_s3_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -178,13 +178,13 @@ func (x *CbOptions) Reset() { } } -func (x *CbOptions) String() string { +func (x *S3CircuitBreakerOptions) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CbOptions) ProtoMessage() {} +func (*S3CircuitBreakerOptions) ProtoMessage() {} -func (x *CbOptions) ProtoReflect() protoreflect.Message { +func (x *S3CircuitBreakerOptions) ProtoReflect() protoreflect.Message { mi := &file_s3_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -196,19 +196,19 @@ func (x *CbOptions) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CbOptions.ProtoReflect.Descriptor instead. -func (*CbOptions) Descriptor() ([]byte, []int) { +// Deprecated: Use S3CircuitBreakerOptions.ProtoReflect.Descriptor instead. +func (*S3CircuitBreakerOptions) Descriptor() ([]byte, []int) { return file_s3_proto_rawDescGZIP(), []int{3} } -func (x *CbOptions) GetEnabled() bool { +func (x *S3CircuitBreakerOptions) GetEnabled() bool { if x != nil { return x.Enabled } return false } -func (x *CbOptions) GetActions() map[string]int64 { +func (x *S3CircuitBreakerOptions) GetActions() map[string]int64 { if x != nil { return x.Actions } @@ -226,43 +226,47 @@ var file_s3_proto_rawDesc = []byte{ 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x1a, 0x73, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xeb, 0x01, 0x0a, 0x16, 0x53, 0x33, 0x43, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x87, 0x02, 0x0a, 0x16, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x2f, 0x0a, 0x06, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, - 0x70, 0x62, 0x2e, 0x43, 0x62, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x06, 0x67, 0x6c, - 0x6f, 0x62, 0x61, 0x6c, 0x12, 0x4b, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, - 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, - 0x65, 0x61, 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x42, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x73, 0x1a, 0x53, 0x0a, 0x0c, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, - 0x62, 0x2e, 0x43, 0x62, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa1, 0x01, 0x0a, 0x09, 0x43, 0x62, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x3e, - 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x24, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x43, - 0x62, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3a, - 0x0a, 0x0c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0x5f, 0x0a, 0x09, 0x53, 0x65, - 0x61, 0x77, 0x65, 0x65, 0x64, 0x53, 0x33, 0x12, 0x52, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x75, 0x72, 0x65, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, - 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, - 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x49, 0x0a, 0x10, 0x73, - 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, - 0x07, 0x53, 0x33, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, - 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, - 0x2f, 0x73, 0x33, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x66, 0x69, 0x67, 0x12, 0x3d, 0x0a, 0x06, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, + 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, + 0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x06, 0x67, 0x6c, 0x6f, 0x62, + 0x61, 0x6c, 0x12, 0x4b, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, + 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, + 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x1a, + 0x61, 0x0a, 0x0c, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x3b, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, + 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0xbd, 0x01, 0x0a, 0x17, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, + 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, + 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x61, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x32, 0x5f, 0x0a, 0x09, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x53, 0x33, 0x12, + 0x52, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x20, 0x2e, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, + 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x42, 0x49, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, + 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x07, 0x53, 0x33, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, + 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, + 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x33, 0x5f, 0x70, 0x62, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -279,18 +283,18 @@ func file_s3_proto_rawDescGZIP() []byte { var file_s3_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_s3_proto_goTypes = []interface{}{ - (*S3ConfigureRequest)(nil), // 0: messaging_pb.S3ConfigureRequest - (*S3ConfigureResponse)(nil), // 1: messaging_pb.S3ConfigureResponse - (*S3CircuitBreakerConfig)(nil), // 2: messaging_pb.S3CircuitBreakerConfig - (*CbOptions)(nil), // 3: messaging_pb.CbOptions - nil, // 4: messaging_pb.S3CircuitBreakerConfig.BucketsEntry - nil, // 5: messaging_pb.CbOptions.ActionsEntry + (*S3ConfigureRequest)(nil), // 0: messaging_pb.S3ConfigureRequest + (*S3ConfigureResponse)(nil), // 1: messaging_pb.S3ConfigureResponse + (*S3CircuitBreakerConfig)(nil), // 2: messaging_pb.S3CircuitBreakerConfig + (*S3CircuitBreakerOptions)(nil), // 3: messaging_pb.S3CircuitBreakerOptions + nil, // 4: messaging_pb.S3CircuitBreakerConfig.BucketsEntry + nil, // 5: messaging_pb.S3CircuitBreakerOptions.ActionsEntry } var file_s3_proto_depIdxs = []int32{ - 3, // 0: messaging_pb.S3CircuitBreakerConfig.global:type_name -> messaging_pb.CbOptions + 3, // 0: messaging_pb.S3CircuitBreakerConfig.global:type_name -> messaging_pb.S3CircuitBreakerOptions 4, // 1: messaging_pb.S3CircuitBreakerConfig.buckets:type_name -> messaging_pb.S3CircuitBreakerConfig.BucketsEntry - 5, // 2: messaging_pb.CbOptions.actions:type_name -> messaging_pb.CbOptions.ActionsEntry - 3, // 3: messaging_pb.S3CircuitBreakerConfig.BucketsEntry.value:type_name -> messaging_pb.CbOptions + 5, // 2: messaging_pb.S3CircuitBreakerOptions.actions:type_name -> messaging_pb.S3CircuitBreakerOptions.ActionsEntry + 3, // 3: messaging_pb.S3CircuitBreakerConfig.BucketsEntry.value:type_name -> messaging_pb.S3CircuitBreakerOptions 0, // 4: messaging_pb.SeaweedS3.Configure:input_type -> messaging_pb.S3ConfigureRequest 1, // 5: messaging_pb.SeaweedS3.Configure:output_type -> messaging_pb.S3ConfigureResponse 5, // [5:6] is the sub-list for method output_type @@ -343,7 +347,7 @@ func file_s3_proto_init() { } } file_s3_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CbOptions); i { + switch v := v.(*S3CircuitBreakerOptions); i { case 0: return &v.state case 1: diff --git a/weed/s3api/auth_credentials_subscribe.go b/weed/s3api/auth_credentials_subscribe.go index d64b38b86..f2bd94f56 100644 --- a/weed/s3api/auth_credentials_subscribe.go +++ b/weed/s3api/auth_credentials_subscribe.go @@ -5,7 +5,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/s3api/s3_config" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -53,7 +53,7 @@ func (s3a *S3ApiServer) onIamConfigUpdate(dir, filename string, content []byte) //reload circuit breaker config func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, content []byte) error { - if dir == s3_config.CircuitBreakerConfigDir && filename == s3_config.CircuitBreakerConfigFile { + if dir == s3_constants.CircuitBreakerConfigDir && filename == s3_constants.CircuitBreakerConfigFile { if err := s3a.cb.LoadS3ApiConfigurationFromBytes(content); err != nil { return err } diff --git a/weed/s3api/s3_config/s3_config.go b/weed/s3api/s3_constants/s3_config.go similarity index 55% rename from weed/s3api/s3_config/s3_config.go rename to weed/s3api/s3_constants/s3_config.go index c9ae7c1f6..950124f05 100644 --- a/weed/s3api/s3_config/s3_config.go +++ b/weed/s3api/s3_constants/s3_config.go @@ -1,14 +1,13 @@ -package s3_config +package s3_constants import ( - "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "strings" ) var ( CircuitBreakerConfigDir = "/etc/s3" CircuitBreakerConfigFile = "circuit_breaker.json" - AllowedActions = []string{s3_constants.ACTION_READ, s3_constants.ACTION_WRITE, s3_constants.ACTION_LIST, s3_constants.ACTION_TAGGING, s3_constants.ACTION_ADMIN} + AllowedActions = []string{ACTION_READ, ACTION_WRITE, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN} LimitTypeCount = "count" LimitTypeBytes = "bytes" Separator = ":" diff --git a/weed/s3api/s3api_circuit_breaker.go b/weed/s3api/s3api_circuit_breaker.go index 73f0062bb..f5b0be94e 100644 --- a/weed/s3api/s3api_circuit_breaker.go +++ b/weed/s3api/s3api_circuit_breaker.go @@ -7,7 +7,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/s3_pb" - "github.com/chrislusf/seaweedfs/weed/s3api/s3_config" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "github.com/gorilla/mux" "go.uber.org/atomic" @@ -29,7 +29,7 @@ func NewCircuitBreaker(option *S3ApiServerOption) *CircuitBreaker { } err := pb.WithFilerClient(false, option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { - content, err := filer.ReadInsideFiler(client, s3_config.CircuitBreakerConfigDir, s3_config.CircuitBreakerConfigFile) + content, err := filer.ReadInsideFiler(client, s3_constants.CircuitBreakerConfigDir, s3_constants.CircuitBreakerConfigFile) if err != nil { return fmt.Errorf("read S3 circuit breaker config: %v", err) } @@ -73,7 +73,7 @@ func (cb *CircuitBreaker) loadCircuitBreakerConfig(cfg *s3_pb.S3CircuitBreakerCo for bucket, cbOptions := range cfg.Buckets { if cbOptions.Enabled { for action, limit := range cbOptions.Actions { - limitations[s3_config.Concat(bucket, action)] = limit + limitations[s3_constants.Concat(bucket, action)] = limit } } } @@ -110,7 +110,7 @@ func (cb *CircuitBreaker) Limit(f func(w http.ResponseWriter, r *http.Request), func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (rollback []func(), errCode s3err.ErrorCode) { //bucket simultaneous request count - bucketCountRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_config.LimitTypeCount, 1, s3err.ErrTooManyRequest) + bucketCountRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_constants.LimitTypeCount, 1, s3err.ErrTooManyRequest) if bucketCountRollBack != nil { rollback = append(rollback, bucketCountRollBack) } @@ -119,7 +119,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) ( } //bucket simultaneous request content bytes - bucketContentLengthRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_config.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed) + bucketContentLengthRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_constants.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed) if bucketContentLengthRollBack != nil { rollback = append(rollback, bucketContentLengthRollBack) } @@ -128,7 +128,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) ( } //global simultaneous request count - globalCountRollBack, errCode := cb.loadCounterAndCompare("", action, s3_config.LimitTypeCount, 1, s3err.ErrTooManyRequest) + globalCountRollBack, errCode := cb.loadCounterAndCompare("", action, s3_constants.LimitTypeCount, 1, s3err.ErrTooManyRequest) if globalCountRollBack != nil { rollback = append(rollback, globalCountRollBack) } @@ -137,7 +137,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) ( } //global simultaneous request content bytes - globalContentLengthRollBack, errCode := cb.loadCounterAndCompare("", action, s3_config.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed) + globalContentLengthRollBack, errCode := cb.loadCounterAndCompare("", action, s3_constants.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed) if globalContentLengthRollBack != nil { rollback = append(rollback, globalContentLengthRollBack) } @@ -148,7 +148,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) ( } func (cb *CircuitBreaker) loadCounterAndCompare(bucket, action, limitType string, inc int64, errCode s3err.ErrorCode) (f func(), e s3err.ErrorCode) { - key := s3_config.Concat(bucket, action, limitType) + key := s3_constants.Concat(bucket, action, limitType) e = s3err.ErrNone if max, ok := cb.limitations[key]; ok { counter, exists := cb.counters[key] diff --git a/weed/s3api/s3api_circuit_breaker_test.go b/weed/s3api/s3api_circuit_breaker_test.go index 57654c744..63d2dc250 100644 --- a/weed/s3api/s3api_circuit_breaker_test.go +++ b/weed/s3api/s3api_circuit_breaker_test.go @@ -2,7 +2,6 @@ package s3api import ( "github.com/chrislusf/seaweedfs/weed/pb/s3_pb" - "github.com/chrislusf/seaweedfs/weed/s3api/s3_config" "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "go.uber.org/atomic" @@ -27,30 +26,31 @@ var ( bucket = "/test" action = s3_constants.ACTION_READ TestLimitCases = []*TestLimitCase{ - {action, s3_config.LimitTypeCount, 5, 5, 6, 1024, 5}, - {action, s3_config.LimitTypeCount, 6, 6, 6, 1024, 6}, - {action, s3_config.LimitTypeCount, 5, 6, 6, 1024, 5}, - {action, s3_config.LimitTypeBytes, 1024, 1024, 6, 200, 5}, - {action, s3_config.LimitTypeBytes, 1200, 1200, 6, 200, 6}, - {action, s3_config.LimitTypeBytes, 11990, 11990, 60, 200, 59}, - {action, s3_config.LimitTypeBytes, 11790, 11990, 60, 200, 58}, + {action, s3_constants.LimitTypeCount, 5, 5, 6, 1024, 5}, + {action, s3_constants.LimitTypeCount, 6, 6, 6, 1024, 6}, + {action, s3_constants.LimitTypeCount, 5, 6, 6, 1024, 5}, + {action, s3_constants.LimitTypeBytes, 1024, 1024, 6, 200, 5}, + {action, s3_constants.LimitTypeBytes, 1200, 1200, 6, 200, 6}, + {action, s3_constants.LimitTypeBytes, 11990, 11990, 60, 200, 59}, + {action, s3_constants.LimitTypeBytes, 11790, 11990, 60, 200, 58}, } ) func TestLimit(t *testing.T) { for _, tc := range TestLimitCases { circuitBreakerConfig := &s3_pb.S3CircuitBreakerConfig{ - Global: &s3_pb.CbOptions{ + Global: &s3_pb.S3CircuitBreakerOptions{ Enabled: true, Actions: map[string]int64{ - s3_config.Concat(tc.actionName, tc.limitType): tc.globalLimitValue, + s3_constants.Concat(tc.actionName, tc.limitType): tc.globalLimitValue, + s3_constants.Concat(tc.actionName, tc.limitType): tc.globalLimitValue, }, }, - Buckets: map[string]*s3_pb.CbOptions{ + Buckets: map[string]*s3_pb.S3CircuitBreakerOptions{ bucket: { Enabled: true, Actions: map[string]int64{ - s3_config.Concat(tc.actionName, tc.limitType): tc.bucketLimitValue, + s3_constants.Concat(tc.actionName, tc.limitType): tc.bucketLimitValue, }, }, }, diff --git a/weed/shell/command_s3_circuitbreaker.go b/weed/shell/command_s3_circuitbreaker.go index f1a8b08bb..7d6eebf34 100644 --- a/weed/shell/command_s3_circuitbreaker.go +++ b/weed/shell/command_s3_circuitbreaker.go @@ -7,7 +7,7 @@ import ( "github.com/alecthomas/units" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/s3_pb" - "github.com/chrislusf/seaweedfs/weed/s3api/s3_config" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "io" "strconv" "strings" @@ -32,21 +32,29 @@ func (c *commandS3CircuitBreaker) Help() string { return `configure and apply s3 circuit breaker options for each bucket # examples - # add - s3.circuitBreaker -actions Read,Write -values 500,200 -global -enable -apply -type count - s3.circuitBreaker -actions Write -values 200MiB -global -enable -apply -type bytes - s3.circuitBreaker -actions Write -values 200MiB -bucket x,y,z -enable -apply -type bytes + # add circuit breaker config for global + s3.circuitBreaker -global -type count -actions Read,Write -values 500,200 -apply - #delete - s3.circuitBreaker -actions Write -bucket x,y,z -delete -apply -type bytes - s3.circuitBreaker -actions Write -bucket x,y,z -delete -apply - s3.circuitBreaker -actions Write -delete -apply + # disable global config + s3.circuitBreaker -global -disable -apply + + # add circuit breaker config for buckets x,y,z + s3.circuitBreaker -buckets x,y,z -type count -actions Read,Write -values 200,100 -apply + + # disable circuit breaker config of x + s3.circuitBreaker -buckets x -disable -apply + + # delete circuit breaker config of x + s3.circuitBreaker -buckets x -delete -apply + + # clear all circuit breaker config + s3.circuitBreaker -delete -apply ` } func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - dir := s3_config.CircuitBreakerConfigDir - file := s3_config.CircuitBreakerConfigFile + dir := s3_constants.CircuitBreakerConfigDir + file := s3_constants.CircuitBreakerConfigFile s3CircuitBreakerCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) buckets := s3CircuitBreakerCommand.String("buckets", "", "the bucket name(s) to configure, eg: -buckets x,y,z") @@ -73,7 +81,7 @@ func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writ } cbCfg := &s3_pb.S3CircuitBreakerConfig{ - Buckets: make(map[string]*s3_pb.CbOptions), + Buckets: make(map[string]*s3_pb.S3CircuitBreakerOptions), } if buf.Len() > 0 { if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), cbCfg); err != nil { @@ -121,10 +129,10 @@ func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writ if len(*buckets) > 0 { for _, bucket := range cmdBuckets { - var cbOptions *s3_pb.CbOptions + var cbOptions *s3_pb.S3CircuitBreakerOptions var exists bool if cbOptions, exists = cbCfg.Buckets[bucket]; !exists { - cbOptions = &s3_pb.CbOptions{} + cbOptions = &s3_pb.S3CircuitBreakerOptions{} cbCfg.Buckets[bucket] = cbOptions } cbOptions.Enabled = !*disabled @@ -145,7 +153,7 @@ func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writ if *global { globalOptions := cbCfg.Global if globalOptions == nil { - globalOptions = &s3_pb.CbOptions{Actions: make(map[string]int64, len(cmdActions))} + globalOptions = &s3_pb.S3CircuitBreakerOptions{Actions: make(map[string]int64, len(cmdActions))} cbCfg.Global = globalOptions } globalOptions.Enabled = !*disabled @@ -192,7 +200,7 @@ func loadConfig(commandEnv *CommandEnv, dir string, file string, buf *bytes.Buff return nil } -func insertOrUpdateValues(cbOptions *s3_pb.CbOptions, cmdActions []string, cmdValues []int64, limitType *string) error { +func insertOrUpdateValues(cbOptions *s3_pb.S3CircuitBreakerOptions, cmdActions []string, cmdValues []int64, limitType *string) error { if len(*limitType) == 0 { return fmt.Errorf("type not valid, only 'count' and 'bytes' are allowed") } @@ -203,7 +211,7 @@ func insertOrUpdateValues(cbOptions *s3_pb.CbOptions, cmdActions []string, cmdVa if len(cmdValues) > 0 { for i, action := range cmdActions { - cbOptions.Actions[s3_config.Concat(action, *limitType)] = cmdValues[i] + cbOptions.Actions[s3_constants.Concat(action, *limitType)] = cmdValues[i] } } return nil @@ -223,7 +231,7 @@ func deleteBucketsActions(cmdBuckets []string, cbCfg *s3_pb.S3CircuitBreakerConf if cbOption, ok := cbCfg.Buckets[bucket]; ok { if len(cmdActions) > 0 && cbOption.Actions != nil { for _, action := range cmdActions { - delete(cbOption.Actions, s3_config.Concat(action, *limitType)) + delete(cbOption.Actions, s3_constants.Concat(action, *limitType)) } } @@ -250,7 +258,7 @@ func deleteGlobalActions(cbCfg *s3_pb.S3CircuitBreakerConfig, cmdActions []strin return } else { for _, action := range cmdActions { - delete(globalOptions.Actions, s3_config.Concat(action, *limitType)) + delete(globalOptions.Actions, s3_constants.Concat(action, *limitType)) } } @@ -270,27 +278,27 @@ func (c *commandS3CircuitBreaker) initActionsAndValues(buckets, actions, limitTy //check action valid for _, action := range cmdActions { var found bool - for _, allowedAction := range s3_config.AllowedActions { + for _, allowedAction := range s3_constants.AllowedActions { if allowedAction == action { found = true } } if !found { - return nil, nil, nil, fmt.Errorf("value(%s) of flag[-action] not valid, allowed actions: %v", *actions, s3_config.AllowedActions) + return nil, nil, nil, fmt.Errorf("value(%s) of flag[-action] not valid, allowed actions: %v", *actions, s3_constants.AllowedActions) } } } if !parseValues { if len(cmdActions) < 0 { - for _, action := range s3_config.AllowedActions { + for _, action := range s3_constants.AllowedActions { cmdActions = append(cmdActions, action) } } if len(*limitType) > 0 { switch *limitType { - case s3_config.LimitTypeCount: + case s3_constants.LimitTypeCount: elements := strings.Split(*values, ",") if len(cmdActions) != len(elements) { if len(elements) != 1 || len(elements) == 0 { @@ -312,7 +320,7 @@ func (c *commandS3CircuitBreaker) initActionsAndValues(buckets, actions, limitTy cmdValues = append(cmdValues, int64(v)) } } - case s3_config.LimitTypeBytes: + case s3_constants.LimitTypeBytes: elements := strings.Split(*values, ",") if len(cmdActions) != len(elements) { if len(elements) != 1 || len(elements) == 0 { diff --git a/weed/shell/command_s3_circuitbreaker_test.go b/weed/shell/command_s3_circuitbreaker_test.go index 214256b5c..191a62e6a 100644 --- a/weed/shell/command_s3_circuitbreaker_test.go +++ b/weed/shell/command_s3_circuitbreaker_test.go @@ -2,6 +2,8 @@ package shell import ( "bytes" + "encoding/json" + "reflect" "strings" "testing" ) @@ -15,33 +17,137 @@ var ( TestCases = []*Case{ //add circuit breaker config for global { - args: strings.Split("-global -type count -actions Read,Write -values 500,200", " "), - result: "{\n \"global\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"500\",\n \"Write:count\": \"200\"\n }\n }\n}\n", + args: strings.Split("-global -type count -actions Read,Write -values 500,200", " "), + result: `{ + "global": { + "enabled": true, + "actions": { + "Read:count": "500", + "Write:count": "200" + } + } + }`, }, + //disable global config { - args: strings.Split("-global -disable", " "), - result: "{\n \"global\": {\n \"actions\": {\n \"Read:count\": \"500\",\n \"Write:count\": \"200\"\n }\n }\n}\n", + args: strings.Split("-global -disable", " "), + result: `{ + "global": { + "actions": { + "Read:count": "500", + "Write:count": "200" + } + } + }`, }, + //add circuit breaker config for buckets x,y,z { - args: strings.Split("-buckets x,y,z -type count -actions Read,Write -values 200,100", " "), - result: "{\n \"global\": {\n \"actions\": {\n \"Read:count\": \"500\",\n \"Write:count\": \"200\"\n }\n },\n \"buckets\": {\n \"x\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n },\n \"y\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n },\n \"z\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n }\n }\n}\n", + args: strings.Split("-buckets x,y,z -type count -actions Read,Write -values 200,100", " "), + result: `{ + "global": { + "actions": { + "Read:count": "500", + "Write:count": "200" + } + }, + "buckets": { + "x": { + "enabled": true, + "actions": { + "Read:count": "200", + "Write:count": "100" + } + }, + "y": { + "enabled": true, + "actions": { + "Read:count": "200", + "Write:count": "100" + } + }, + "z": { + "enabled": true, + "actions": { + "Read:count": "200", + "Write:count": "100" + } + } + } + }`, }, + //disable circuit breaker config of x { - args: strings.Split("-buckets x -disable", " "), - result: "{\n \"global\": {\n \"actions\": {\n \"Read:count\": \"500\",\n \"Write:count\": \"200\"\n }\n },\n \"buckets\": {\n \"x\": {\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n },\n \"y\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n },\n \"z\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n }\n }\n}\n", + args: strings.Split("-buckets x -disable", " "), + result: `{ + "global": { + "actions": { + "Read:count": "500", + "Write:count": "200" + } + }, + "buckets": { + "x": { + "actions": { + "Read:count": "200", + "Write:count": "100" + } + }, + "y": { + "enabled": true, + "actions": { + "Read:count": "200", + "Write:count": "100" + } + }, + "z": { + "enabled": true, + "actions": { + "Read:count": "200", + "Write:count": "100" + } + } + } + }`, }, + //delete circuit breaker config of x { - args: strings.Split("-buckets x -delete", " "), - result: "{\n \"global\": {\n \"actions\": {\n \"Read:count\": \"500\",\n \"Write:count\": \"200\"\n }\n },\n \"buckets\": {\n \"y\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n },\n \"z\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n }\n }\n}\n", + args: strings.Split("-buckets x -delete", " "), + result: `{ + "global": { + "actions": { + "Read:count": "500", + "Write:count": "200" + } + }, + "buckets": { + "y": { + "enabled": true, + "actions": { + "Read:count": "200", + "Write:count": "100" + } + }, + "z": { + "enabled": true, + "actions": { + "Read:count": "200", + "Write:count": "100" + } + } + } + }`, }, + //clear all circuit breaker config { - args: strings.Split("-delete", " "), - result: "{\n\n}\n", + args: strings.Split("-delete", " "), + result: `{ + + }`, }, } ) @@ -65,10 +171,21 @@ func TestCircuitBreakerShell(t *testing.T) { } if i != 0 { result := writeBuf.String() - if result != tc.result { - t.Fatal("result of s3 circuit breaker shell command is unexpect!") + + actual := make(map[string]interface{}) + err := json.Unmarshal([]byte(result), &actual) + if err != nil { + t.Error(err) } + expect := make(map[string]interface{}) + err = json.Unmarshal([]byte(result), &expect) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(actual, expect) { + t.Fatal("result of s3 circuit breaker shell command is unexpect!") + } } } }