mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-01-18 06:30:07 +08:00
hilimd
GitHub
commit
8693cdacae
@ -7,6 +7,7 @@
|
||||
[](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
|
||||
[](https://github.com/chrislusf/seaweedfs/wiki)
|
||||
[](https://hub.docker.com/r/chrislusf/seaweedfs/)
|
||||
[](https://search.maven.org/search?q=g:com.github.chrislusf)
|
||||
|
||||
|
||||
|
||||
@ -42,6 +43,7 @@ Your support will be really appreciated by me and other supporters!
|
||||
- [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
|
||||
- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
|
||||
- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
|
||||
- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
|
||||
- [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
|
||||
|
||||
Table of Contents
|
||||
@ -70,7 +72,7 @@ Table of Contents
|
||||
* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
|
||||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thoudsands of machines. That is it!
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
|
||||
## Introduction ##
|
||||
|
||||
|
||||
@ -28,4 +28,4 @@ ENV \
|
||||
S3TEST_CONF="/s3test.conf"
|
||||
|
||||
ENTRYPOINT ["/bin/bash", "-c"]
|
||||
CMD ["exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]
|
||||
CMD ["sleep 10 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]
|
||||
@ -4,8 +4,10 @@ all: gen
|
||||
|
||||
gen: dev
|
||||
|
||||
build:
|
||||
binary:
|
||||
cd ../weed; CGO_ENABLED=0 GOOS=linux go build -ldflags "-extldflags -static"; mv weed ../docker/
|
||||
|
||||
build: binary
|
||||
docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
|
||||
rm ./weed
|
||||
|
||||
@ -21,6 +23,9 @@ dev_tls: build certstrap
|
||||
dev_mount: build
|
||||
docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
|
||||
|
||||
profile_mount: build
|
||||
docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
|
||||
|
||||
k8s: build
|
||||
docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
|
||||
|
||||
|
||||
50
docker/compose/local-minio-gateway-compose.yml
Normal file
50
docker/compose/local-minio-gateway-compose.yml
Normal file
@ -0,0 +1,50 @@
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
command: "master -ip=master -volumeSizeLimitMB=1024"
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8888:8888
|
||||
- 18888:18888
|
||||
- 8333:8333
|
||||
command: '-v 1 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
minio-gateway-s3:
|
||||
image: minio/minio
|
||||
ports:
|
||||
- 9000:9000
|
||||
command: 'minio gateway s3 http://s3:8333'
|
||||
restart: on-failure
|
||||
environment:
|
||||
MINIO_ACCESS_KEY: "some_access_key1"
|
||||
MINIO_SECRET_KEY: "some_secret_key1"
|
||||
depends_on:
|
||||
- s3
|
||||
minio-warp:
|
||||
image: minio/warp
|
||||
command: 'mixed --duration=5m --obj.size=3mb --autoterm'
|
||||
restart: on-failure
|
||||
environment:
|
||||
WARP_HOST: "minio-gateway-s3:9000"
|
||||
WARP_ACCESS_KEY: "some_access_key1"
|
||||
WARP_SECRET_KEY: "some_secret_key1"
|
||||
depends_on:
|
||||
- minio-gateway-s3
|
||||
47
docker/compose/local-mount-profile-compose.yml
Normal file
47
docker/compose/local-mount-profile-compose.yml
Normal file
@ -0,0 +1,47 @@
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
command: "master -ip=master"
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 7455:8080
|
||||
- 9325:9325
|
||||
volumes:
|
||||
- /Volumes/mobile_disk/99:/data
|
||||
command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8888:8888
|
||||
- 18888:18888
|
||||
- 9326:9326
|
||||
volumes:
|
||||
- /Volumes/mobile_disk/99:/data
|
||||
command: 'filer -master="master:9333" -metricsPort=9326'
|
||||
tty: true
|
||||
stdin_open: true
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
mount:
|
||||
image: chrislusf/seaweedfs:local
|
||||
privileged: true
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
devices:
|
||||
- fuse
|
||||
volumes:
|
||||
- /Volumes/mobile_disk/99:/data
|
||||
entrypoint: '/bin/sh -c "mkdir -p t1 && weed mount -filer=filer:8888 -dir=./t1 -cacheCapacityMB=0 -memprofile=/data/mount.mem.pprof"'
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
- filer
|
||||
@ -15,24 +15,18 @@ services:
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8888:8888
|
||||
- 18888:18888
|
||||
command: 'filer -master="master:9333"'
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8333:8333
|
||||
command: '-v 9 s3 -filer="filer:8888"'
|
||||
command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
- filer
|
||||
minio:
|
||||
image: minio/minio
|
||||
ports:
|
||||
|
||||
@ -38,7 +38,7 @@ services:
|
||||
S3TEST_CONF: "s3tests.conf"
|
||||
NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
|
||||
NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
|
||||
NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_delimiter_not_skip_special|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_head_zero_bytes|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_authenticated|object_raw_response_headers|object_raw_authenticated_bucket_acl|object_raw_authenticated_object_acl|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_not_expired|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_create_exists|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|bucket_create_special_key_names|object_copy_zero_size|object_copy_verify_contenttype|object_copy_to_itself|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
|
||||
NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
|
||||
7
go.mod
7
go.mod
@ -13,6 +13,7 @@ require (
|
||||
github.com/Shopify/sarama v1.23.1
|
||||
github.com/aws/aws-sdk-go v1.34.30
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/chrislusf/raft v1.0.4
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
@ -38,7 +39,7 @@ require (
|
||||
github.com/google/uuid v1.1.1
|
||||
github.com/gorilla/mux v1.7.4
|
||||
github.com/gorilla/websocket v1.4.1 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
|
||||
github.com/jcmturner/gofork v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.10
|
||||
@ -60,7 +61,7 @@ require (
|
||||
github.com/prometheus/client_golang v1.3.0
|
||||
github.com/rakyll/statik v0.1.7
|
||||
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
|
||||
github.com/seaweedfs/fuse v1.1.1
|
||||
github.com/seaweedfs/fuse v1.1.3
|
||||
github.com/seaweedfs/goexif v1.0.2
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
@ -88,7 +89,7 @@ require (
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.20.0
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
|
||||
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
|
||||
golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd
|
||||
golang.org/x/tools v0.0.0-20200608174601-1b747fd94509
|
||||
google.golang.org/api v0.26.0
|
||||
|
||||
6
go.sum
6
go.sum
@ -141,6 +141,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA=
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg=
|
||||
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
|
||||
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
|
||||
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
|
||||
@ -683,6 +685,8 @@ github.com/seaweedfs/fuse v1.1.0 h1:cL1qPHFNtFv0UuJTLjKKgWDzfJ4iZzTa4Y7ipc2acGw=
|
||||
github.com/seaweedfs/fuse v1.1.0/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
|
||||
github.com/seaweedfs/fuse v1.1.1 h1:WD51YFJcBViOx8I89jeqPD+vAKl4EowzBy9GUw0plb0=
|
||||
github.com/seaweedfs/fuse v1.1.1/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
|
||||
github.com/seaweedfs/fuse v1.1.3 h1:0DddotXwSRGbYG2kynoJyr8GHCy30Z2SpdhP3vdyijY=
|
||||
github.com/seaweedfs/fuse v1.1.3/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
|
||||
github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E=
|
||||
github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk=
|
||||
github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw=
|
||||
@ -957,6 +961,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6cnrWpidecD1UuSYbhKm0=
|
||||
golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "2.31"
|
||||
version: 2.31
|
||||
appVersion: "2.38"
|
||||
version: 2.38
|
||||
|
||||
@ -4,7 +4,7 @@ global:
|
||||
registry: ""
|
||||
repository: ""
|
||||
imageName: chrislusf/seaweedfs
|
||||
# imageTag: "2.31" - started using {.Chart.appVersion}
|
||||
# imageTag: "2.38" - started using {.Chart.appVersion}
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: imagepullsecret
|
||||
restartPolicy: Always
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>1.6.2</version>
|
||||
<version>1.6.4</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
@ -17,7 +17,7 @@
|
||||
<protobuf.version>3.9.1</protobuf.version>
|
||||
<!-- follow https://github.com/grpc/grpc-java -->
|
||||
<grpc.version>1.23.0</grpc.version>
|
||||
<guava.version>28.0-jre</guava.version>
|
||||
<guava.version>30.0-jre</guava.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>1.6.2</version>
|
||||
<version>1.6.4</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>1.6.2</version>
|
||||
<version>1.6.4</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
|
||||
@ -3,6 +3,7 @@ package seaweedfs.client;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.nio.Buffer;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
@ -34,7 +35,7 @@ public class ByteBufferPool {
|
||||
}
|
||||
|
||||
public static synchronized void release(ByteBuffer obj) {
|
||||
obj.clear();
|
||||
((Buffer)obj).clear();
|
||||
bufferList.add(0, obj);
|
||||
}
|
||||
|
||||
|
||||
@ -94,7 +94,7 @@ public class FilerClient extends FilerGrpcClient {
|
||||
return true;
|
||||
}
|
||||
File pathFile = new File(path);
|
||||
String parent = pathFile.getParent();
|
||||
String parent = pathFile.getParent().replace('\\','/');
|
||||
String name = pathFile.getName();
|
||||
|
||||
mkdirs(parent, mode, uid, gid, userName, groupNames);
|
||||
@ -115,11 +115,11 @@ public class FilerClient extends FilerGrpcClient {
|
||||
public boolean mv(String oldPath, String newPath) {
|
||||
|
||||
File oldPathFile = new File(oldPath);
|
||||
String oldParent = oldPathFile.getParent();
|
||||
String oldParent = oldPathFile.getParent().replace('\\','/');
|
||||
String oldName = oldPathFile.getName();
|
||||
|
||||
File newPathFile = new File(newPath);
|
||||
String newParent = newPathFile.getParent();
|
||||
String newParent = newPathFile.getParent().replace('\\','/');
|
||||
String newName = newPathFile.getName();
|
||||
|
||||
return atomicRenameEntry(oldParent, oldName, newParent, newName);
|
||||
@ -129,7 +129,7 @@ public class FilerClient extends FilerGrpcClient {
|
||||
public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) {
|
||||
|
||||
File pathFile = new File(path);
|
||||
String parent = pathFile.getParent();
|
||||
String parent = pathFile.getParent().replace('\\','/');
|
||||
String name = pathFile.getName();
|
||||
|
||||
return deleteEntry(
|
||||
@ -148,7 +148,7 @@ public class FilerClient extends FilerGrpcClient {
|
||||
public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
|
||||
|
||||
File pathFile = new File(path);
|
||||
String parent = pathFile.getParent();
|
||||
String parent = pathFile.getParent().replace('\\','/');
|
||||
String name = pathFile.getName();
|
||||
|
||||
FilerProto.Entry entry = lookupEntry(parent, name);
|
||||
|
||||
@ -11,13 +11,13 @@
|
||||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>1.6.2</version>
|
||||
<version>1.6.4</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-hadoop2-client</artifactId>
|
||||
<version>1.6.2</version>
|
||||
<version>1.6.4</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
||||
@ -301,7 +301,7 @@
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
<properties>
|
||||
<seaweedfs.client.version>1.6.2</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>1.6.4</seaweedfs.client.version>
|
||||
<hadoop.version>2.9.2</hadoop.version>
|
||||
</properties>
|
||||
</project>
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<seaweedfs.client.version>1.6.2</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>1.6.4</seaweedfs.client.version>
|
||||
<hadoop.version>2.9.2</hadoop.version>
|
||||
</properties>
|
||||
|
||||
|
||||
@ -309,7 +309,7 @@
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
<properties>
|
||||
<seaweedfs.client.version>1.6.2</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>1.6.4</seaweedfs.client.version>
|
||||
<hadoop.version>3.1.1</hadoop.version>
|
||||
</properties>
|
||||
</project>
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<seaweedfs.client.version>1.6.2</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>1.6.4</seaweedfs.client.version>
|
||||
<hadoop.version>3.1.1</hadoop.version>
|
||||
</properties>
|
||||
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
|
||||
<properties>
|
||||
<guava.version>28.0-jre</guava.version>
|
||||
<guava.version>30.0-jre</guava.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
@ -305,7 +305,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
|
||||
}
|
||||
var bytes []byte
|
||||
for _, url := range urls {
|
||||
bytes, _, err = util.FastGet(url)
|
||||
bytes, _, err = util.Get(url)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
@ -47,6 +47,7 @@ type FilerOptions struct {
|
||||
metricsHttpPort *int
|
||||
saveToFilerLimit *int
|
||||
defaultLevelDbDirectory *string
|
||||
concurrentUploadLimitMB *int
|
||||
}
|
||||
|
||||
func init() {
|
||||
@ -54,12 +55,12 @@ func init() {
|
||||
f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
|
||||
f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
|
||||
f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
|
||||
f.bindIp = cmdFiler.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
|
||||
f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to")
|
||||
f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
|
||||
f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
|
||||
f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
|
||||
f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
|
||||
f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
|
||||
f.maxMB = cmdFiler.Flag.Int("maxMB", 4, "split files larger than the limit")
|
||||
f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
|
||||
f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
|
||||
f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
|
||||
@ -69,6 +70,7 @@ func init() {
|
||||
f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
|
||||
f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
|
||||
f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
|
||||
|
||||
// start s3 on filer
|
||||
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
|
||||
@ -159,21 +161,22 @@ func (fo *FilerOptions) startFiler() {
|
||||
}
|
||||
|
||||
fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
|
||||
Masters: strings.Split(*fo.masters, ","),
|
||||
Collection: *fo.collection,
|
||||
DefaultReplication: *fo.defaultReplicaPlacement,
|
||||
DisableDirListing: *fo.disableDirListing,
|
||||
MaxMB: *fo.maxMB,
|
||||
DirListingLimit: *fo.dirListingLimit,
|
||||
DataCenter: *fo.dataCenter,
|
||||
Rack: *fo.rack,
|
||||
DefaultLevelDbDir: defaultLevelDbDirectory,
|
||||
DisableHttp: *fo.disableHttp,
|
||||
Host: *fo.ip,
|
||||
Port: uint32(*fo.port),
|
||||
Cipher: *fo.cipher,
|
||||
SaveToFilerLimit: *fo.saveToFilerLimit,
|
||||
Filers: peers,
|
||||
Masters: strings.Split(*fo.masters, ","),
|
||||
Collection: *fo.collection,
|
||||
DefaultReplication: *fo.defaultReplicaPlacement,
|
||||
DisableDirListing: *fo.disableDirListing,
|
||||
MaxMB: *fo.maxMB,
|
||||
DirListingLimit: *fo.dirListingLimit,
|
||||
DataCenter: *fo.dataCenter,
|
||||
Rack: *fo.rack,
|
||||
DefaultLevelDbDir: defaultLevelDbDirectory,
|
||||
DisableHttp: *fo.disableHttp,
|
||||
Host: *fo.ip,
|
||||
Port: uint32(*fo.port),
|
||||
Cipher: *fo.cipher,
|
||||
SaveToFilerLimit: int64(*fo.saveToFilerLimit),
|
||||
Filers: peers,
|
||||
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
|
||||
})
|
||||
if nfs_err != nil {
|
||||
glog.Fatalf("Filer startup error: %v", nfs_err)
|
||||
|
||||
@ -110,7 +110,7 @@ func runFilerCat(cmd *Command, args []string) bool {
|
||||
|
||||
filerCat.filerClient = client
|
||||
|
||||
return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
|
||||
return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
|
||||
|
||||
})
|
||||
|
||||
|
||||
@ -56,7 +56,7 @@ func init() {
|
||||
copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
|
||||
copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
|
||||
copy.diskType = cmdCopy.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
|
||||
copy.maxMB = cmdCopy.Flag.Int("maxMB", 4, "split files larger than the limit")
|
||||
copy.concurrenctFiles = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
|
||||
copy.concurrenctChunks = cmdCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
|
||||
}
|
||||
|
||||
@ -6,7 +6,6 @@ import (
|
||||
"google.golang.org/grpc/reflection"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@ -48,8 +47,8 @@ type MasterOptions struct {
|
||||
func init() {
|
||||
cmdMaster.Run = runMaster // break init cycle
|
||||
m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
|
||||
m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address")
|
||||
m.ipBind = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
|
||||
m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address, also used as identifier")
|
||||
m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to")
|
||||
m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
|
||||
m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
|
||||
m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
|
||||
@ -86,7 +85,6 @@ func runMaster(cmd *Command, args []string) bool {
|
||||
util.LoadConfiguration("security", false)
|
||||
util.LoadConfiguration("master", false)
|
||||
|
||||
runtime.GOMAXPROCS(runtime.NumCPU())
|
||||
grace.SetupProfiling(*masterCpuProfile, *masterMemProfile)
|
||||
|
||||
parent, _ := util.FullPath(*m.metaFolder).DirAndName()
|
||||
|
||||
@ -25,6 +25,7 @@ type MountOptions struct {
|
||||
volumeServerAccess *string
|
||||
uidMap *string
|
||||
gidMap *string
|
||||
readOnly *bool
|
||||
}
|
||||
|
||||
var (
|
||||
@ -45,7 +46,7 @@ func init() {
|
||||
mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
|
||||
mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
|
||||
mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 128, "limit concurrent goroutine writers if not 0")
|
||||
mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
|
||||
mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
|
||||
mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)")
|
||||
mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
|
||||
@ -55,6 +56,7 @@ func init() {
|
||||
mountOptions.volumeServerAccess = cmdMount.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]")
|
||||
mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
|
||||
mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
|
||||
mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
|
||||
|
||||
mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
|
||||
|
||||
@ -75,7 +75,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
if err != nil {
|
||||
glog.V(0).Infof("failed to talk to filer %s: %v", filerGrpcAddress, err)
|
||||
glog.V(0).Infof("wait for %d seconds ...", i+1)
|
||||
time.Sleep(time.Duration(i+1)*time.Second)
|
||||
time.Sleep(time.Duration(i+1) * time.Second)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
@ -203,6 +203,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
VolumeServerAccess: *mountOptions.volumeServerAccess,
|
||||
Cipher: cipher,
|
||||
UidGidMapper: uidGidMapper,
|
||||
ReadOnly: *option.readOnly,
|
||||
})
|
||||
|
||||
// mount
|
||||
|
||||
@ -103,9 +103,9 @@ dir = "./filerrdb" # directory to store rocksdb files
|
||||
|
||||
[mysql] # or memsql, tidb
|
||||
# CREATE TABLE IF NOT EXISTS filemeta (
|
||||
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
# name VARCHAR(1000) COMMENT 'directory or file name',
|
||||
# directory TEXT COMMENT 'full path to parent directory',
|
||||
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
# name VARCHAR(1000) BINARY COMMENT 'directory or file name',
|
||||
# directory TEXT COMMENT 'full path to parent directory',
|
||||
# meta LONGBLOB,
|
||||
# PRIMARY KEY (dirhash, name)
|
||||
# ) DEFAULT CHARSET=utf8;
|
||||
@ -120,13 +120,16 @@ connection_max_idle = 2
|
||||
connection_max_open = 100
|
||||
connection_max_lifetime_seconds = 0
|
||||
interpolateParams = false
|
||||
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
|
||||
enableUpsert = true
|
||||
upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
|
||||
|
||||
[mysql2] # or memsql, tidb
|
||||
enabled = false
|
||||
createTable = """
|
||||
CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
|
||||
dirhash BIGINT,
|
||||
name VARCHAR(1000),
|
||||
name VARCHAR(1000) BINARY,
|
||||
directory TEXT,
|
||||
meta LONGBLOB,
|
||||
PRIMARY KEY (dirhash, name)
|
||||
@ -141,6 +144,9 @@ connection_max_idle = 2
|
||||
connection_max_open = 100
|
||||
connection_max_lifetime_seconds = 0
|
||||
interpolateParams = false
|
||||
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
|
||||
enableUpsert = true
|
||||
upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
|
||||
|
||||
[postgres] # or cockroachdb, YugabyteDB
|
||||
# CREATE TABLE IF NOT EXISTS filemeta (
|
||||
@ -161,6 +167,9 @@ sslmode = "disable"
|
||||
connection_max_idle = 100
|
||||
connection_max_open = 100
|
||||
connection_max_lifetime_seconds = 0
|
||||
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
|
||||
enableUpsert = true
|
||||
upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
|
||||
|
||||
[postgres2]
|
||||
enabled = false
|
||||
@ -183,6 +192,9 @@ sslmode = "disable"
|
||||
connection_max_idle = 100
|
||||
connection_max_open = 100
|
||||
connection_max_lifetime_seconds = 0
|
||||
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
|
||||
enableUpsert = true
|
||||
upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
|
||||
|
||||
[cassandra]
|
||||
# CREATE TABLE filemeta (
|
||||
@ -506,7 +518,7 @@ default = "localhost:8888" # used by maintenance scripts if the scripts needs
|
||||
|
||||
|
||||
[master.sequencer]
|
||||
type = "raft" # Choose [raft|etcd] type for storing the file id sequence
|
||||
type = "raft" # Choose [raft|etcd|snowflake] type for storing the file id sequence
|
||||
# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
|
||||
# example : http://127.0.0.1:2379,http://127.0.0.1:2389
|
||||
sequencer_etcd_urls = "http://127.0.0.1:2379"
|
||||
|
||||
@ -2,9 +2,8 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
"os"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -16,6 +15,7 @@ import (
|
||||
|
||||
type ServerOptions struct {
|
||||
cpuprofile *string
|
||||
memprofile *string
|
||||
v VolumeServerOptions
|
||||
}
|
||||
|
||||
@ -49,8 +49,8 @@ var cmdServer = &Command{
|
||||
}
|
||||
|
||||
var (
|
||||
serverIp = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
|
||||
serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
|
||||
serverIp = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
|
||||
serverBindIp = cmdServer.Flag.String("ip.bind", "", "ip address to bind to")
|
||||
serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
|
||||
serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
|
||||
serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
|
||||
@ -76,6 +76,7 @@ var (
|
||||
|
||||
func init() {
|
||||
serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
serverOptions.memprofile = cmdServer.Flag.String("memprofile", "", "memory profile output file")
|
||||
|
||||
masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
|
||||
masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
|
||||
@ -93,11 +94,12 @@ func init() {
|
||||
filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
|
||||
filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
|
||||
filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
|
||||
filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
|
||||
filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit")
|
||||
filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
|
||||
filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
|
||||
filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")
|
||||
filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
|
||||
filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
|
||||
|
||||
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
|
||||
serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
|
||||
@ -107,6 +109,7 @@ func init() {
|
||||
serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
|
||||
serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
|
||||
serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
|
||||
serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
|
||||
serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
|
||||
serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
|
||||
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
|
||||
@ -138,14 +141,7 @@ func runServer(cmd *Command, args []string) bool {
|
||||
util.LoadConfiguration("security", false)
|
||||
util.LoadConfiguration("master", false)
|
||||
|
||||
if *serverOptions.cpuprofile != "" {
|
||||
f, err := os.Create(*serverOptions.cpuprofile)
|
||||
if err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
pprof.StartCPUProfile(f)
|
||||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
grace.SetupProfiling(*serverOptions.cpuprofile, *serverOptions.memprofile)
|
||||
|
||||
if *isStartingS3 {
|
||||
*isStartingFiler = true
|
||||
@ -192,7 +188,6 @@ func runServer(cmd *Command, args []string) bool {
|
||||
webdavOptions.filer = &filerAddress
|
||||
msgBrokerOptions.filer = &filerAddress
|
||||
|
||||
runtime.GOMAXPROCS(runtime.NumCPU())
|
||||
go stats_collect.StartMetricsServer(*serverMetricsHttpPort)
|
||||
|
||||
folders := strings.Split(*volumeDataFolders, ",")
|
||||
|
||||
@ -43,7 +43,7 @@ func init() {
|
||||
upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
|
||||
upload.diskType = cmdUpload.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
|
||||
upload.maxMB = cmdUpload.Flag.Int("maxMB", 32, "split files larger than the limit")
|
||||
upload.maxMB = cmdUpload.Flag.Int("maxMB", 4, "split files larger than the limit")
|
||||
upload.usePublicUrl = cmdUpload.Flag.Bool("usePublicUrl", false, "upload to public url from volume server")
|
||||
}
|
||||
|
||||
|
||||
@ -6,7 +6,6 @@ import (
|
||||
"net/http"
|
||||
httppprof "net/http/pprof"
|
||||
"os"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"strconv"
|
||||
"strings"
|
||||
@ -36,31 +35,32 @@ var (
|
||||
)
|
||||
|
||||
type VolumeServerOptions struct {
|
||||
port *int
|
||||
publicPort *int
|
||||
folders []string
|
||||
folderMaxLimits []int
|
||||
idxFolder *string
|
||||
ip *string
|
||||
publicUrl *string
|
||||
bindIp *string
|
||||
masters *string
|
||||
idleConnectionTimeout *int
|
||||
dataCenter *string
|
||||
rack *string
|
||||
whiteList []string
|
||||
indexType *string
|
||||
diskType *string
|
||||
fixJpgOrientation *bool
|
||||
readRedirect *bool
|
||||
cpuProfile *string
|
||||
memProfile *string
|
||||
compactionMBPerSecond *int
|
||||
fileSizeLimitMB *int
|
||||
minFreeSpacePercents []float32
|
||||
pprof *bool
|
||||
preStopSeconds *int
|
||||
metricsHttpPort *int
|
||||
port *int
|
||||
publicPort *int
|
||||
folders []string
|
||||
folderMaxLimits []int
|
||||
idxFolder *string
|
||||
ip *string
|
||||
publicUrl *string
|
||||
bindIp *string
|
||||
masters *string
|
||||
idleConnectionTimeout *int
|
||||
dataCenter *string
|
||||
rack *string
|
||||
whiteList []string
|
||||
indexType *string
|
||||
diskType *string
|
||||
fixJpgOrientation *bool
|
||||
readRedirect *bool
|
||||
cpuProfile *string
|
||||
memProfile *string
|
||||
compactionMBPerSecond *int
|
||||
fileSizeLimitMB *int
|
||||
concurrentUploadLimitMB *int
|
||||
minFreeSpacePercents []float32
|
||||
pprof *bool
|
||||
preStopSeconds *int
|
||||
metricsHttpPort *int
|
||||
// pulseSeconds *int
|
||||
enableTcp *bool
|
||||
}
|
||||
@ -69,9 +69,9 @@ func init() {
|
||||
cmdVolume.Run = runVolume // break init cycle
|
||||
v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
|
||||
v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
|
||||
v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
|
||||
v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
|
||||
v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
|
||||
v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
|
||||
v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to")
|
||||
v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
|
||||
v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
|
||||
// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
|
||||
@ -86,6 +86,7 @@ func init() {
|
||||
v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
|
||||
v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
|
||||
v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
|
||||
v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
|
||||
v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
|
||||
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
|
||||
@ -111,8 +112,6 @@ func runVolume(cmd *Command, args []string) bool {
|
||||
|
||||
util.LoadConfiguration("security", false)
|
||||
|
||||
runtime.GOMAXPROCS(runtime.NumCPU())
|
||||
|
||||
// If --pprof is set we assume the caller wants to be able to collect
|
||||
// cpu and memory profiles via go tool pprof
|
||||
if !*v.pprof {
|
||||
@ -240,6 +239,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
||||
*v.fixJpgOrientation, *v.readRedirect,
|
||||
*v.compactionMBPerSecond,
|
||||
*v.fileSizeLimitMB,
|
||||
int64(*v.concurrentUploadLimitMB)*1024*1024,
|
||||
)
|
||||
// starting grpc server
|
||||
grpcS := v.startGrpcService(volumeServer)
|
||||
|
||||
@ -13,15 +13,15 @@ import (
|
||||
)
|
||||
|
||||
type SqlGenerator interface {
|
||||
GetSqlInsert(bucket string) string
|
||||
GetSqlUpdate(bucket string) string
|
||||
GetSqlFind(bucket string) string
|
||||
GetSqlDelete(bucket string) string
|
||||
GetSqlDeleteFolderChildren(bucket string) string
|
||||
GetSqlListExclusive(bucket string) string
|
||||
GetSqlListInclusive(bucket string) string
|
||||
GetSqlCreateTable(bucket string) string
|
||||
GetSqlDropTable(bucket string) string
|
||||
GetSqlInsert(tableName string) string
|
||||
GetSqlUpdate(tableName string) string
|
||||
GetSqlFind(tableName string) string
|
||||
GetSqlDelete(tableName string) string
|
||||
GetSqlDeleteFolderChildren(tableName string) string
|
||||
GetSqlListExclusive(tableName string) string
|
||||
GetSqlListInclusive(tableName string) string
|
||||
GetSqlCreateTable(tableName string) string
|
||||
GetSqlDropTable(tableName string) string
|
||||
}
|
||||
|
||||
type AbstractSqlStore struct {
|
||||
@ -32,6 +32,29 @@ type AbstractSqlStore struct {
|
||||
dbsLock sync.Mutex
|
||||
}
|
||||
|
||||
func (store *AbstractSqlStore) OnBucketCreation(bucket string) {
|
||||
store.dbsLock.Lock()
|
||||
defer store.dbsLock.Unlock()
|
||||
|
||||
store.CreateTable(context.Background(), bucket)
|
||||
|
||||
if store.dbs == nil {
|
||||
return
|
||||
}
|
||||
store.dbs[bucket] = true
|
||||
}
|
||||
func (store *AbstractSqlStore) OnBucketDeletion(bucket string) {
|
||||
store.dbsLock.Lock()
|
||||
defer store.dbsLock.Unlock()
|
||||
|
||||
store.deleteTable(context.Background(), bucket)
|
||||
|
||||
if store.dbs == nil {
|
||||
return
|
||||
}
|
||||
delete(store.dbs, bucket)
|
||||
}
|
||||
|
||||
const (
|
||||
DEFAULT_TABLE = "filemeta"
|
||||
)
|
||||
|
||||
@ -97,20 +97,20 @@ func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string,
|
||||
func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {
|
||||
|
||||
var err error
|
||||
var buffer bytes.Buffer
|
||||
var shouldRetry bool
|
||||
receivedData := make([]byte, 0, size)
|
||||
|
||||
for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
|
||||
for _, urlString := range urlStrings {
|
||||
shouldRetry, err = util.FastReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
|
||||
buffer.Write(data)
|
||||
receivedData = receivedData[:0]
|
||||
shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
|
||||
receivedData = append(receivedData, data...)
|
||||
})
|
||||
if !shouldRetry {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
glog.V(0).Infof("read %s failed, err: %v", urlString, err)
|
||||
buffer.Reset()
|
||||
} else {
|
||||
break
|
||||
}
|
||||
@ -123,7 +123,8 @@ func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool
|
||||
}
|
||||
}
|
||||
|
||||
return buffer.Bytes(), err
|
||||
return receivedData, err
|
||||
|
||||
}
|
||||
|
||||
func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) {
|
||||
|
||||
@ -241,12 +241,12 @@ func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err er
|
||||
if oldEntry != nil {
|
||||
entry.Attr.Crtime = oldEntry.Attr.Crtime
|
||||
if oldEntry.IsDirectory() && !entry.IsDirectory() {
|
||||
glog.Errorf("existing %s is a directory", entry.FullPath)
|
||||
return fmt.Errorf("existing %s is a directory", entry.FullPath)
|
||||
glog.Errorf("existing %s is a directory", oldEntry.FullPath)
|
||||
return fmt.Errorf("existing %s is a directory", oldEntry.FullPath)
|
||||
}
|
||||
if !oldEntry.IsDirectory() && entry.IsDirectory() {
|
||||
glog.Errorf("existing %s is a file", entry.FullPath)
|
||||
return fmt.Errorf("existing %s is a file", entry.FullPath)
|
||||
glog.Errorf("existing %s is a file", oldEntry.FullPath)
|
||||
return fmt.Errorf("existing %s is a file", oldEntry.FullPath)
|
||||
}
|
||||
}
|
||||
return f.Store.UpdateEntry(ctx, entry)
|
||||
|
||||
@ -12,6 +12,24 @@ import (
|
||||
// onMetadataChangeEvent is triggered after filer processed change events from local or remote filers
|
||||
func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) {
|
||||
f.maybeReloadFilerConfiguration(event)
|
||||
f.onBucketEvents(event)
|
||||
}
|
||||
|
||||
func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
|
||||
message := event.EventNotification
|
||||
for _, sig := range message.Signatures {
|
||||
if sig == f.Signature {
|
||||
return
|
||||
}
|
||||
}
|
||||
if f.DirBucketsPath == event.Directory {
|
||||
if message.OldEntry == nil && message.NewEntry != nil {
|
||||
f.Store.OnBucketCreation(message.NewEntry.Name)
|
||||
}
|
||||
if message.OldEntry != nil && message.NewEntry == nil {
|
||||
f.Store.OnBucketDeletion(message.OldEntry.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
|
||||
@ -34,7 +52,7 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR
|
||||
|
||||
func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64)
|
||||
err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -39,3 +39,8 @@ type FilerStore interface {
|
||||
|
||||
Shutdown()
|
||||
}
|
||||
|
||||
type BucketAware interface {
|
||||
OnBucketCreation(bucket string)
|
||||
OnBucketDeletion(bucket string)
|
||||
}
|
||||
|
||||
@ -21,6 +21,8 @@ type VirtualFilerStore interface {
|
||||
DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
|
||||
DeleteOneEntry(ctx context.Context, entry *Entry) error
|
||||
AddPathSpecificStore(path string, storeId string, store FilerStore)
|
||||
OnBucketCreation(bucket string)
|
||||
OnBucketDeletion(bucket string)
|
||||
}
|
||||
|
||||
type FilerStoreWrapper struct {
|
||||
@ -40,6 +42,27 @@ func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
func (fsw *FilerStoreWrapper) OnBucketCreation(bucket string) {
|
||||
for _, store := range fsw.storeIdToStore {
|
||||
if ba, ok := store.(BucketAware); ok {
|
||||
ba.OnBucketCreation(bucket)
|
||||
}
|
||||
}
|
||||
if ba, ok := fsw.defaultStore.(BucketAware); ok {
|
||||
ba.OnBucketCreation(bucket)
|
||||
}
|
||||
}
|
||||
func (fsw *FilerStoreWrapper) OnBucketDeletion(bucket string) {
|
||||
for _, store := range fsw.storeIdToStore {
|
||||
if ba, ok := store.(BucketAware); ok {
|
||||
ba.OnBucketDeletion(bucket)
|
||||
}
|
||||
}
|
||||
if ba, ok := fsw.defaultStore.(BucketAware); ok {
|
||||
ba.OnBucketDeletion(bucket)
|
||||
}
|
||||
}
|
||||
|
||||
func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, store FilerStore) {
|
||||
fsw.storeIdToStore[storeId] = NewFilerStorePathTranlator(path, store)
|
||||
err := fsw.pathToStore.Put([]byte(path), storeId)
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
|
||||
"os"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
@ -38,6 +39,7 @@ func (store *LevelDBStore) Initialize(configuration weed_util.Configuration, pre
|
||||
|
||||
func (store *LevelDBStore) initialize(dir string) (err error) {
|
||||
glog.Infof("filer store dir: %s", dir)
|
||||
os.MkdirAll(dir, 0755)
|
||||
if err := weed_util.TestFolderWritable(dir); err != nil {
|
||||
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
|
||||
}
|
||||
|
||||
@ -38,6 +38,7 @@ func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, pr
|
||||
|
||||
func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
|
||||
glog.Infof("filer store leveldb2 dir: %s", dir)
|
||||
os.MkdirAll(dir, 0755)
|
||||
if err := weed_util.TestFolderWritable(dir); err != nil {
|
||||
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
|
||||
}
|
||||
|
||||
@ -45,6 +45,7 @@ func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, pr
|
||||
|
||||
func (store *LevelDB3Store) initialize(dir string) (err error) {
|
||||
glog.Infof("filer store leveldb3 dir: %s", dir)
|
||||
os.MkdirAll(dir, 0755)
|
||||
if err := weed_util.TestFolderWritable(dir); err != nil {
|
||||
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@ package mysql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
@ -9,44 +10,49 @@ import (
|
||||
type SqlGenMysql struct {
|
||||
CreateTableSqlTemplate string
|
||||
DropTableSqlTemplate string
|
||||
UpsertQueryTemplate string
|
||||
}
|
||||
|
||||
var (
|
||||
_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
|
||||
)
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlInsert(bucket string) string {
|
||||
return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", bucket)
|
||||
func (gen *SqlGenMysql) GetSqlInsert(tableName string) string {
|
||||
if gen.UpsertQueryTemplate != "" {
|
||||
return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
|
||||
} else {
|
||||
return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
|
||||
}
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string {
|
||||
return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", bucket)
|
||||
func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string {
|
||||
return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlFind(bucket string) string {
|
||||
return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", bucket)
|
||||
func (gen *SqlGenMysql) GetSqlFind(tableName string) string {
|
||||
return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlDelete(bucket string) string {
|
||||
return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", bucket)
|
||||
func (gen *SqlGenMysql) GetSqlDelete(tableName string) string {
|
||||
return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string {
|
||||
return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", bucket)
|
||||
func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(tableName string) string {
|
||||
return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string {
|
||||
return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
|
||||
func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
|
||||
return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string {
|
||||
return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
|
||||
func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
|
||||
return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string {
|
||||
return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket)
|
||||
func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {
|
||||
return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string {
|
||||
return fmt.Sprintf(gen.DropTableSqlTemplate, bucket)
|
||||
func (gen *SqlGenMysql) GetSqlDropTable(tableName string) string {
|
||||
return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
|
||||
}
|
||||
|
||||
@ -3,9 +3,10 @@ package mysql
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
@ -29,6 +30,8 @@ func (store *MysqlStore) GetName() string {
|
||||
|
||||
func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
|
||||
return store.initialize(
|
||||
configuration.GetString(prefix+"upsertQuery"),
|
||||
configuration.GetBool(prefix+"enableUpsert"),
|
||||
configuration.GetString(prefix+"username"),
|
||||
configuration.GetString(prefix+"password"),
|
||||
configuration.GetString(prefix+"hostname"),
|
||||
@ -41,13 +44,17 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str
|
||||
)
|
||||
}
|
||||
|
||||
func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen,
|
||||
func (store *MysqlStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
|
||||
maxLifetimeSeconds int, interpolateParams bool) (err error) {
|
||||
|
||||
store.SupportBucketTable = false
|
||||
if !enableUpsert {
|
||||
upsertQuery = ""
|
||||
}
|
||||
store.SqlGenerator = &SqlGenMysql{
|
||||
CreateTableSqlTemplate: "",
|
||||
DropTableSqlTemplate: "drop table `%s`",
|
||||
UpsertQueryTemplate: upsertQuery,
|
||||
}
|
||||
|
||||
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
|
||||
|
||||
@ -32,6 +32,8 @@ func (store *MysqlStore2) GetName() string {
|
||||
func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
|
||||
return store.initialize(
|
||||
configuration.GetString(prefix+"createTable"),
|
||||
configuration.GetString(prefix+"upsertQuery"),
|
||||
configuration.GetBool(prefix+"enableUpsert"),
|
||||
configuration.GetString(prefix+"username"),
|
||||
configuration.GetString(prefix+"password"),
|
||||
configuration.GetString(prefix+"hostname"),
|
||||
@ -44,13 +46,17 @@ func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix st
|
||||
)
|
||||
}
|
||||
|
||||
func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen,
|
||||
func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
|
||||
maxLifetimeSeconds int, interpolateParams bool) (err error) {
|
||||
|
||||
store.SupportBucketTable = true
|
||||
if !enableUpsert {
|
||||
upsertQuery = ""
|
||||
}
|
||||
store.SqlGenerator = &mysql.SqlGenMysql{
|
||||
CreateTableSqlTemplate: createTable,
|
||||
DropTableSqlTemplate: "drop table `%s`",
|
||||
UpsertQueryTemplate: upsertQuery,
|
||||
}
|
||||
|
||||
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
|
||||
|
||||
@ -10,44 +10,49 @@ import (
|
||||
type SqlGenPostgres struct {
|
||||
CreateTableSqlTemplate string
|
||||
DropTableSqlTemplate string
|
||||
UpsertQueryTemplate string
|
||||
}
|
||||
|
||||
var (
|
||||
_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
|
||||
)
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlInsert(bucket string) string {
|
||||
return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, bucket)
|
||||
func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
|
||||
if gen.UpsertQueryTemplate != "" {
|
||||
return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
|
||||
} else {
|
||||
return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
|
||||
}
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlUpdate(bucket string) string {
|
||||
return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, bucket)
|
||||
func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string {
|
||||
return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlFind(bucket string) string {
|
||||
return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, bucket)
|
||||
func (gen *SqlGenPostgres) GetSqlFind(tableName string) string {
|
||||
return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlDelete(bucket string) string {
|
||||
return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, bucket)
|
||||
func (gen *SqlGenPostgres) GetSqlDelete(tableName string) string {
|
||||
return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(bucket string) string {
|
||||
return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, bucket)
|
||||
func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(tableName string) string {
|
||||
return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlListExclusive(bucket string) string {
|
||||
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, bucket)
|
||||
func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
|
||||
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlListInclusive(bucket string) string {
|
||||
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, bucket)
|
||||
func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {
|
||||
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlCreateTable(bucket string) string {
|
||||
return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket)
|
||||
func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {
|
||||
return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlDropTable(bucket string) string {
|
||||
return fmt.Sprintf(gen.DropTableSqlTemplate, bucket)
|
||||
func (gen *SqlGenPostgres) GetSqlDropTable(tableName string) string {
|
||||
return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
|
||||
}
|
||||
|
||||
@ -29,6 +29,8 @@ func (store *PostgresStore) GetName() string {
|
||||
|
||||
func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
|
||||
return store.initialize(
|
||||
configuration.GetString(prefix+"upsertQuery"),
|
||||
configuration.GetBool(prefix+"enableUpsert"),
|
||||
configuration.GetString(prefix+"username"),
|
||||
configuration.GetString(prefix+"password"),
|
||||
configuration.GetString(prefix+"hostname"),
|
||||
@ -42,12 +44,16 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
|
||||
)
|
||||
}
|
||||
|
||||
func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
|
||||
func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
|
||||
|
||||
store.SupportBucketTable = false
|
||||
if !enableUpsert {
|
||||
upsertQuery = ""
|
||||
}
|
||||
store.SqlGenerator = &SqlGenPostgres{
|
||||
CreateTableSqlTemplate: "",
|
||||
DropTableSqlTemplate: `drop table "%s"`,
|
||||
UpsertQueryTemplate: upsertQuery,
|
||||
}
|
||||
|
||||
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)
|
||||
|
||||
@ -32,6 +32,8 @@ func (store *PostgresStore2) GetName() string {
|
||||
func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
|
||||
return store.initialize(
|
||||
configuration.GetString(prefix+"createTable"),
|
||||
configuration.GetString(prefix+"upsertQuery"),
|
||||
configuration.GetBool(prefix+"enableUpsert"),
|
||||
configuration.GetString(prefix+"username"),
|
||||
configuration.GetString(prefix+"password"),
|
||||
configuration.GetString(prefix+"hostname"),
|
||||
@ -45,12 +47,16 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
|
||||
)
|
||||
}
|
||||
|
||||
func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
|
||||
func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
|
||||
|
||||
store.SupportBucketTable = true
|
||||
if !enableUpsert {
|
||||
upsertQuery = ""
|
||||
}
|
||||
store.SqlGenerator = &postgres.SqlGenPostgres{
|
||||
CreateTableSqlTemplate: createTable,
|
||||
DropTableSqlTemplate: `drop table "%s"`,
|
||||
UpsertQueryTemplate: upsertQuery,
|
||||
}
|
||||
|
||||
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)
|
||||
|
||||
@ -27,7 +27,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
|
||||
return err
|
||||
}
|
||||
|
||||
return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
|
||||
return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
|
||||
|
||||
}
|
||||
|
||||
@ -35,7 +35,7 @@ func ReadContent(filerAddress string, dir, name string) ([]byte, error) {
|
||||
|
||||
target := fmt.Sprintf("http://%s%s/%s", filerAddress, dir, name)
|
||||
|
||||
data, _, err := util.FastGet(target)
|
||||
data, _, err := util.Get(target)
|
||||
|
||||
return data, err
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/tecbot/gorocksdb"
|
||||
|
||||
@ -56,6 +57,7 @@ func (store *RocksDBStore) Initialize(configuration weed_util.Configuration, pre
|
||||
|
||||
func (store *RocksDBStore) initialize(dir string) (err error) {
|
||||
glog.Infof("filer store rocksdb dir: %s", dir)
|
||||
os.MkdirAll(dir, 0755)
|
||||
if err := weed_util.TestFolderWritable(dir); err != nil {
|
||||
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ package filer
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"io"
|
||||
"math"
|
||||
"strings"
|
||||
@ -13,7 +14,7 @@ import (
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
|
||||
func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64, isCheck bool) error {
|
||||
|
||||
glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
|
||||
chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
|
||||
@ -33,10 +34,20 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
|
||||
fileId2Url[chunkView.FileId] = urlStrings
|
||||
}
|
||||
|
||||
if isCheck {
|
||||
// Pre-check all chunkViews urls
|
||||
gErr := new(errgroup.Group)
|
||||
CheckAllChunkViews(chunkViews, &fileId2Url, gErr)
|
||||
if err := gErr.Wait(); err != nil {
|
||||
glog.Errorf("check all chunks: %v", err)
|
||||
return fmt.Errorf("check all chunks: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, chunkView := range chunkViews {
|
||||
|
||||
urlStrings := fileId2Url[chunkView.FileId]
|
||||
|
||||
data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
|
||||
if err != nil {
|
||||
glog.Errorf("read chunk: %v", err)
|
||||
@ -54,6 +65,17 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
|
||||
|
||||
}
|
||||
|
||||
func CheckAllChunkViews(chunkViews []*ChunkView, fileId2Url *map[string][]string, gErr *errgroup.Group) {
|
||||
for _, chunkView := range chunkViews {
|
||||
urlStrings := (*fileId2Url)[chunkView.FileId]
|
||||
glog.V(9).Infof("Check chunk: %+v\n url: %v", chunkView, urlStrings)
|
||||
gErr.Go(func() error {
|
||||
_, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
|
||||
return err
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------- ReadAllReader ----------------------------------
|
||||
|
||||
func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) {
|
||||
@ -185,7 +207,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
|
||||
var buffer bytes.Buffer
|
||||
var shouldRetry bool
|
||||
for _, urlString := range urlStrings {
|
||||
shouldRetry, err = util.FastReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
|
||||
shouldRetry, err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
|
||||
buffer.Write(data)
|
||||
})
|
||||
if !shouldRetry {
|
||||
|
||||
@ -128,6 +128,10 @@ func (dir *Dir) newDirectory(fullpath util.FullPath, entry *filer_pb.Entry) fs.N
|
||||
func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
|
||||
resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
|
||||
|
||||
if dir.wfs.option.ReadOnly {
|
||||
return nil, nil, fuse.EPERM
|
||||
}
|
||||
|
||||
request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, req.Flags&fuse.OpenExclusive != 0)
|
||||
|
||||
if err != nil {
|
||||
@ -148,6 +152,10 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
|
||||
|
||||
func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) {
|
||||
|
||||
if dir.wfs.option.ReadOnly {
|
||||
return nil, fuse.EPERM
|
||||
}
|
||||
|
||||
request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, false)
|
||||
|
||||
if err != nil {
|
||||
@ -202,6 +210,10 @@ func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, ex
|
||||
|
||||
func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
|
||||
|
||||
if dir.wfs.option.ReadOnly {
|
||||
return nil, fuse.EPERM
|
||||
}
|
||||
|
||||
glog.V(4).Infof("mkdir %s: %s", dir.FullPath(), req.Name)
|
||||
|
||||
newEntry := &filer_pb.Entry{
|
||||
@ -356,6 +368,11 @@ func findFileType(mode uint16) fuse.DirentType {
|
||||
|
||||
func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
|
||||
|
||||
if dir.wfs.option.ReadOnly {
|
||||
return fuse.EPERM
|
||||
}
|
||||
|
||||
|
||||
if !req.Dir {
|
||||
return dir.removeOneFile(req)
|
||||
}
|
||||
@ -429,6 +446,10 @@ func (dir *Dir) removeFolder(req *fuse.RemoveRequest) error {
|
||||
|
||||
func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
|
||||
|
||||
if dir.wfs.option.ReadOnly {
|
||||
return fuse.EPERM
|
||||
}
|
||||
|
||||
glog.V(4).Infof("%v dir setattr %+v", dir.FullPath(), req)
|
||||
|
||||
if err := dir.maybeLoadEntry(); err != nil {
|
||||
@ -457,6 +478,10 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
|
||||
|
||||
func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
|
||||
|
||||
if dir.wfs.option.ReadOnly {
|
||||
return fuse.EPERM
|
||||
}
|
||||
|
||||
glog.V(4).Infof("dir Setxattr %s: %s", dir.FullPath(), req.Name)
|
||||
|
||||
if err := dir.maybeLoadEntry(); err != nil {
|
||||
@ -473,6 +498,10 @@ func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
|
||||
|
||||
func (dir *Dir) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {
|
||||
|
||||
if dir.wfs.option.ReadOnly {
|
||||
return fuse.EPERM
|
||||
}
|
||||
|
||||
glog.V(4).Infof("dir Removexattr %s: %s", dir.FullPath(), req.Name)
|
||||
|
||||
if err := dir.maybeLoadEntry(); err != nil {
|
||||
|
||||
@ -24,6 +24,10 @@ const (
|
||||
|
||||
func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (fs.Node, error) {
|
||||
|
||||
if dir.wfs.option.ReadOnly {
|
||||
return nil, fuse.EPERM
|
||||
}
|
||||
|
||||
oldFile, ok := old.(*File)
|
||||
if !ok {
|
||||
glog.Errorf("old node is not a file: %+v", old)
|
||||
@ -105,6 +109,10 @@ func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (f
|
||||
|
||||
func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, error) {
|
||||
|
||||
if dir.wfs.option.ReadOnly {
|
||||
return nil, fuse.EPERM
|
||||
}
|
||||
|
||||
glog.V(4).Infof("Symlink: %v/%v to %v", dir.FullPath(), req.NewName, req.Target)
|
||||
|
||||
request := &filer_pb.CreateEntryRequest{
|
||||
|
||||
@ -13,6 +13,10 @@ import (
|
||||
|
||||
func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
|
||||
|
||||
if dir.wfs.option.ReadOnly {
|
||||
return fuse.EPERM
|
||||
}
|
||||
|
||||
newDir := newDirectory.(*Dir)
|
||||
|
||||
newPath := util.NewFullPath(newDir.FullPath(), req.NewName)
|
||||
|
||||
@ -1,34 +0,0 @@
|
||||
package filesys
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDirPath(t *testing.T) {
|
||||
|
||||
p := &Dir{name: "/some"}
|
||||
p = &Dir{name: "path", parent: p}
|
||||
p = &Dir{name: "to", parent: p}
|
||||
p = &Dir{name: "a", parent: p}
|
||||
p = &Dir{name: "file", parent: p}
|
||||
|
||||
assert.Equal(t, "/some/path/to/a/file", p.FullPath())
|
||||
|
||||
p = &Dir{name: "/some"}
|
||||
assert.Equal(t, "/some", p.FullPath())
|
||||
|
||||
p = &Dir{name: "/"}
|
||||
assert.Equal(t, "/", p.FullPath())
|
||||
|
||||
p = &Dir{name: "/"}
|
||||
p = &Dir{name: "path", parent: p}
|
||||
assert.Equal(t, "/path", p.FullPath())
|
||||
|
||||
p = &Dir{name: "/"}
|
||||
p = &Dir{name: "path", parent: p}
|
||||
p = &Dir{name: "to", parent: p}
|
||||
assert.Equal(t, "/path/to", p.FullPath())
|
||||
|
||||
}
|
||||
@ -56,6 +56,10 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) {
|
||||
}
|
||||
}
|
||||
|
||||
if entry == nil {
|
||||
return fuse.ENOENT
|
||||
}
|
||||
|
||||
// attr.Inode = file.fullpath().AsInode()
|
||||
attr.Valid = time.Second
|
||||
attr.Mode = os.FileMode(entry.Attributes.FileMode)
|
||||
@ -106,6 +110,10 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op
|
||||
|
||||
func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
|
||||
|
||||
if file.wfs.option.ReadOnly {
|
||||
return fuse.EPERM
|
||||
}
|
||||
|
||||
glog.V(4).Infof("%v file setattr %+v", file.fullpath(), req)
|
||||
|
||||
entry, err := file.maybeLoadEntry(ctx)
|
||||
@ -147,7 +155,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
|
||||
}
|
||||
entry.Chunks = chunks
|
||||
file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), chunks)
|
||||
file.reader = nil
|
||||
file.setReader(nil)
|
||||
}
|
||||
entry.Attributes.FileSize = req.Size
|
||||
file.dirtyMetadata = true
|
||||
@ -196,6 +204,10 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
|
||||
|
||||
func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
|
||||
|
||||
if file.wfs.option.ReadOnly {
|
||||
return fuse.EPERM
|
||||
}
|
||||
|
||||
glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name)
|
||||
|
||||
entry, err := file.maybeLoadEntry(ctx)
|
||||
@ -213,6 +225,10 @@ func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error
|
||||
|
||||
func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {
|
||||
|
||||
if file.wfs.option.ReadOnly {
|
||||
return fuse.EPERM
|
||||
}
|
||||
|
||||
glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name)
|
||||
|
||||
entry, err := file.maybeLoadEntry(ctx)
|
||||
@ -257,6 +273,8 @@ func (file *File) Forget() {
|
||||
t := util.NewFullPath(file.dir.FullPath(), file.Name)
|
||||
glog.V(4).Infof("Forget file %s", t)
|
||||
file.wfs.fsNodeCache.DeleteFsNode(t)
|
||||
file.wfs.ReleaseHandle(t, 0)
|
||||
file.setReader(nil)
|
||||
}
|
||||
|
||||
func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, err error) {
|
||||
@ -323,19 +341,29 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
|
||||
file.entryViewCache = filer.MergeIntoVisibles(file.entryViewCache, chunk)
|
||||
}
|
||||
|
||||
file.reader = nil
|
||||
file.setReader(nil)
|
||||
|
||||
glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(entry.Chunks), len(chunks))
|
||||
|
||||
entry.Chunks = append(entry.Chunks, newChunks...)
|
||||
}
|
||||
|
||||
func (file *File) setReader(reader io.ReaderAt) {
|
||||
r := file.reader
|
||||
if r != nil {
|
||||
if closer, ok := r.(io.Closer); ok {
|
||||
closer.Close()
|
||||
}
|
||||
}
|
||||
file.reader = reader
|
||||
}
|
||||
|
||||
func (file *File) setEntry(entry *filer_pb.Entry) {
|
||||
file.entryLock.Lock()
|
||||
defer file.entryLock.Unlock()
|
||||
file.entry = entry
|
||||
file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), entry.Chunks)
|
||||
file.reader = nil
|
||||
file.setReader(nil)
|
||||
}
|
||||
|
||||
func (file *File) clearEntry() {
|
||||
@ -343,7 +371,7 @@ func (file *File) clearEntry() {
|
||||
defer file.entryLock.Unlock()
|
||||
file.entry = nil
|
||||
file.entryViewCache = nil
|
||||
file.reader = nil
|
||||
file.setReader(nil)
|
||||
}
|
||||
|
||||
func (file *File) saveEntry(entry *filer_pb.Entry) error {
|
||||
|
||||
@ -23,7 +23,7 @@ type FileHandle struct {
|
||||
dirtyPages *ContinuousDirtyPages
|
||||
contentType string
|
||||
handle uint64
|
||||
sync.RWMutex
|
||||
sync.Mutex
|
||||
|
||||
f *File
|
||||
RequestId fuse.RequestID // unique ID for request
|
||||
@ -59,8 +59,8 @@ var _ = fs.HandleReleaser(&FileHandle{})
|
||||
func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
|
||||
|
||||
glog.V(4).Infof("%s read fh %d: [%d,%d) size %d resp.Data cap=%d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, cap(resp.Data))
|
||||
fh.RLock()
|
||||
defer fh.RUnlock()
|
||||
fh.Lock()
|
||||
defer fh.Unlock()
|
||||
|
||||
if req.Size <= 0 {
|
||||
return nil
|
||||
@ -130,7 +130,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
|
||||
if chunkResolveErr != nil {
|
||||
return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr)
|
||||
}
|
||||
fh.f.reader = nil
|
||||
fh.f.setReader(nil)
|
||||
}
|
||||
|
||||
reader := fh.f.reader
|
||||
@ -138,7 +138,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
|
||||
chunkViews := filer.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64)
|
||||
reader = filer.NewChunkReaderAtFromClient(fh.f.wfs.LookupFn(), chunkViews, fh.f.wfs.chunkCache, fileSize)
|
||||
}
|
||||
fh.f.reader = reader
|
||||
fh.f.setReader(reader)
|
||||
|
||||
totalRead, err := reader.ReadAt(buff, offset)
|
||||
|
||||
@ -154,6 +154,10 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
|
||||
// Write to the file handle
|
||||
func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
|
||||
|
||||
if fh.f.wfs.option.ReadOnly {
|
||||
return fuse.EPERM
|
||||
}
|
||||
|
||||
fh.Lock()
|
||||
defer fh.Unlock()
|
||||
|
||||
@ -207,12 +211,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
|
||||
fh.f.isOpen--
|
||||
|
||||
fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
|
||||
if closer, ok := fh.f.reader.(io.Closer); ok {
|
||||
if closer != nil {
|
||||
closer.Close()
|
||||
}
|
||||
}
|
||||
fh.f.reader = nil
|
||||
fh.f.setReader(nil)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@ -43,6 +43,7 @@ type Option struct {
|
||||
DataCenter string
|
||||
EntryCacheTtl time.Duration
|
||||
Umask os.FileMode
|
||||
ReadOnly bool
|
||||
|
||||
MountUid uint32
|
||||
MountGid uint32
|
||||
|
||||
24
weed/operation/buffer_pool.go
Normal file
24
weed/operation/buffer_pool.go
Normal file
@ -0,0 +1,24 @@
|
||||
package operation
|
||||
|
||||
import (
|
||||
"github.com/valyala/bytebufferpool"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
var bufferCounter int64
|
||||
|
||||
func GetBuffer() *bytebufferpool.ByteBuffer {
|
||||
defer func() {
|
||||
atomic.AddInt64(&bufferCounter, 1)
|
||||
// println("+", bufferCounter)
|
||||
}()
|
||||
return bytebufferpool.Get()
|
||||
}
|
||||
|
||||
func PutBuffer(buf *bytebufferpool.ByteBuffer) {
|
||||
defer func() {
|
||||
atomic.AddInt64(&bufferCounter, -1)
|
||||
// println("-", bufferCounter)
|
||||
}()
|
||||
bytebufferpool.Put(buf)
|
||||
}
|
||||
@ -19,7 +19,6 @@ import (
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/valyala/bytebufferpool"
|
||||
)
|
||||
|
||||
type UploadResult struct {
|
||||
@ -190,8 +189,8 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
|
||||
}
|
||||
|
||||
func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
|
||||
buf := bytebufferpool.Get()
|
||||
defer bytebufferpool.Put(buf)
|
||||
buf := GetBuffer()
|
||||
defer PutBuffer(buf)
|
||||
body_writer := multipart.NewWriter(buf)
|
||||
h := make(textproto.MIMEHeader)
|
||||
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.24.0
|
||||
// protoc-gen-go v1.25.0
|
||||
// protoc v3.12.3
|
||||
// source: filer.proto
|
||||
|
||||
|
||||
@ -104,7 +104,7 @@ func SeaweedList(client SeaweedFilerClient, parentDirectoryPath, prefix string,
|
||||
func doSeaweedList(client SeaweedFilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunciton, startFrom string, inclusive bool, limit uint32) (err error) {
|
||||
// Redundancy limit to make it correctly judge whether it is the last file.
|
||||
redLimit := limit
|
||||
if limit != math.MaxInt32 && limit != 0{
|
||||
if limit != math.MaxInt32 && limit != 0 {
|
||||
redLimit = limit + 1
|
||||
}
|
||||
request := &ListEntriesRequest{
|
||||
@ -182,6 +182,26 @@ func Exists(filerClient FilerClient, parentDirectoryPath string, entryName strin
|
||||
return
|
||||
}
|
||||
|
||||
func Touch(filerClient FilerClient, parentDirectoryPath string, entryName string, entry *Entry) (err error) {
|
||||
|
||||
return filerClient.WithFilerClient(func(client SeaweedFilerClient) error {
|
||||
|
||||
request := &UpdateEntryRequest{
|
||||
Directory: parentDirectoryPath,
|
||||
Entry: entry,
|
||||
}
|
||||
|
||||
glog.V(4).Infof("touch entry %v/%v: %v", parentDirectoryPath, entryName, request)
|
||||
if err := UpdateEntry(client, request); err != nil {
|
||||
glog.V(0).Infof("touch exists entry %v: %v", request, err)
|
||||
return fmt.Errorf("touch exists entry %s/%s: %v", parentDirectoryPath, entryName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func Mkdir(filerClient FilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error {
|
||||
return filerClient.WithFilerClient(func(client SeaweedFilerClient) error {
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.24.0
|
||||
// protoc-gen-go v1.25.0
|
||||
// protoc v3.12.3
|
||||
// source: iam.proto
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.24.0
|
||||
// protoc-gen-go v1.25.0
|
||||
// protoc v3.12.3
|
||||
// source: master.proto
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.24.0
|
||||
// protoc-gen-go v1.25.0
|
||||
// protoc v3.12.3
|
||||
// source: messaging.proto
|
||||
|
||||
|
||||
@ -52,6 +52,11 @@ service VolumeServer {
|
||||
rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
|
||||
}
|
||||
|
||||
rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
|
||||
}
|
||||
rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
|
||||
}
|
||||
|
||||
rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
|
||||
}
|
||||
rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
|
||||
@ -253,6 +258,25 @@ message CopyFileResponse {
|
||||
bytes file_content = 1;
|
||||
}
|
||||
|
||||
message ReadNeedleBlobRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 needle_id = 2;
|
||||
int64 offset = 3; // actual offset
|
||||
int32 size = 4;
|
||||
}
|
||||
message ReadNeedleBlobResponse {
|
||||
bytes needle_blob = 1;
|
||||
}
|
||||
|
||||
message WriteNeedleBlobRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 needle_id = 2;
|
||||
int32 size = 3;
|
||||
bytes needle_blob = 4;
|
||||
}
|
||||
message WriteNeedleBlobResponse {
|
||||
}
|
||||
|
||||
message VolumeTailSenderRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 since_ns = 2;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -20,7 +20,7 @@ func CopyFromChunkViews(chunkViews []*filer.ChunkView, filerSource *source.Filer
|
||||
var shouldRetry bool
|
||||
|
||||
for _, fileUrl := range fileUrls {
|
||||
shouldRetry, err = util.FastReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
|
||||
shouldRetry, err = util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
|
||||
writeErr = writeFunc(data)
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@ -79,6 +79,12 @@ func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isD
|
||||
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) touch(parentDirectoryPath string, entryName string, entry *filer_pb.Entry) (err error) {
|
||||
|
||||
return filer_pb.Touch(s3a, parentDirectoryPath, entryName, entry)
|
||||
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *filer_pb.Entry, err error) {
|
||||
fullPath := util.NewFullPath(parentDirectoryPath, entryName)
|
||||
return filer_pb.GetEntry(s3a, fullPath)
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
|
||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
@ -25,6 +26,26 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
|
||||
srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
|
||||
|
||||
if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && isReplace(r) {
|
||||
fullPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
|
||||
dir, name := fullPath.DirAndName()
|
||||
entry, err := s3a.getEntry(dir, name)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
|
||||
}
|
||||
entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r))
|
||||
err = s3a.touch(dir, name, entry)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
|
||||
}
|
||||
writeSuccessResponseXML(w, encodeResponse(CopyObjectResult{
|
||||
ETag: fmt.Sprintf("%x", entry.Attributes.Md5),
|
||||
LastModified: time.Now().UTC(),
|
||||
}))
|
||||
return
|
||||
}
|
||||
|
||||
// If source object is empty or bucket is empty, reply back invalid copy source.
|
||||
if srcObject == "" || srcBucket == "" {
|
||||
writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
|
||||
@ -32,7 +53,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
|
||||
if srcBucket == dstBucket && srcObject == dstObject {
|
||||
writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
|
||||
writeErrorResponse(w, s3err.ErrInvalidCopyDest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@ -147,3 +168,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
|
||||
writeSuccessResponseXML(w, encodeResponse(response))
|
||||
|
||||
}
|
||||
|
||||
func isReplace(r *http.Request) bool {
|
||||
return r.Header.Get("X-Amz-Metadata-Directive") == "REPLACE"
|
||||
}
|
||||
|
||||
@ -311,7 +311,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
|
||||
}
|
||||
defer util.CloseResponse(resp)
|
||||
|
||||
if (resp.ContentLength == -1 || resp.StatusCode == 404) && !strings.HasSuffix(destUrl, "/") {
|
||||
if resp.ContentLength == -1 || resp.StatusCode == 404 {
|
||||
if r.Method != "DELETE" {
|
||||
writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
|
||||
return
|
||||
@ -326,7 +326,11 @@ func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
|
||||
for k, v := range proxyResponse.Header {
|
||||
w.Header()[k] = v
|
||||
}
|
||||
w.WriteHeader(proxyResponse.StatusCode)
|
||||
if proxyResponse.Header.Get("Content-Range") != "" && proxyResponse.StatusCode == 200 {
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
} else {
|
||||
w.WriteHeader(proxyResponse.StatusCode)
|
||||
}
|
||||
io.Copy(w, proxyResponse.Body)
|
||||
}
|
||||
|
||||
|
||||
@ -206,7 +206,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
||||
isTruncated = isTruncated || subIsTruncated
|
||||
maxKeys -= subCounter
|
||||
nextMarker = subDir + "/" + subNextMarker
|
||||
counter += subCounter
|
||||
// finished processing this sub directory
|
||||
marker = subDir
|
||||
}
|
||||
|
||||
46
weed/sequence/snowflake_sequencer.go
Normal file
46
weed/sequence/snowflake_sequencer.go
Normal file
@ -0,0 +1,46 @@
|
||||
package sequence
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
|
||||
"github.com/bwmarrin/snowflake"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
// a simple snowflake Sequencer
|
||||
type SnowflakeSequencer struct {
|
||||
node *snowflake.Node
|
||||
}
|
||||
|
||||
func NewSnowflakeSequencer(nodeid string) (*SnowflakeSequencer, error) {
|
||||
nodeid_hash := hash(nodeid) & 0x3ff
|
||||
glog.V(0).Infof("use snowflake seq id generator, nodeid:%s hex_of_nodeid: %x", nodeid, nodeid_hash)
|
||||
node, err := snowflake.NewNode(int64(nodeid_hash))
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sequencer := &SnowflakeSequencer{node: node}
|
||||
return sequencer, nil
|
||||
}
|
||||
|
||||
func hash(s string) uint32 {
|
||||
h := fnv.New32a()
|
||||
h.Write([]byte(s))
|
||||
return h.Sum32()
|
||||
}
|
||||
|
||||
func (m *SnowflakeSequencer) NextFileId(count uint64) uint64 {
|
||||
return uint64(m.node.Generate().Int64())
|
||||
}
|
||||
|
||||
// ignore setmax as we are snowflake
|
||||
func (m *SnowflakeSequencer) SetMax(seenValue uint64) {
|
||||
}
|
||||
|
||||
// return a new id as no Peek is stored
|
||||
func (m *SnowflakeSequencer) Peek() uint64 {
|
||||
return uint64(m.node.Generate().Int64())
|
||||
}
|
||||
@ -234,12 +234,12 @@ func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, file
|
||||
}
|
||||
}
|
||||
|
||||
func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64, httpStatusCode int) error) {
|
||||
func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
|
||||
rangeReq := r.Header.Get("Range")
|
||||
|
||||
if rangeReq == "" {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
|
||||
if err := writeFn(w, 0, totalSize, 0); err != nil {
|
||||
if err := writeFn(w, 0, totalSize); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
@ -279,7 +279,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
|
||||
w.Header().Set("Content-Range", ra.contentRange(totalSize))
|
||||
|
||||
err = writeFn(w, ra.start, ra.length, http.StatusPartialContent)
|
||||
err = writeFn(w, ra.start, ra.length)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
@ -307,7 +307,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
|
||||
pw.CloseWithError(e)
|
||||
return
|
||||
}
|
||||
if e = writeFn(part, ra.start, ra.length, 0); e != nil {
|
||||
if e = writeFn(part, ra.start, ra.length); e != nil {
|
||||
pw.CloseWithError(e)
|
||||
return
|
||||
}
|
||||
|
||||
@ -33,8 +33,7 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom
|
||||
return nil, fmt.Errorf("%s/%s not found: %v", req.OldDirectory, req.OldName, err)
|
||||
}
|
||||
|
||||
var events MoveEvents
|
||||
moveErr := fs.moveEntry(ctx, oldParent, oldEntry, newParent, req.NewName, &events)
|
||||
moveErr := fs.moveEntry(ctx, oldParent, oldEntry, newParent, req.NewName)
|
||||
if moveErr != nil {
|
||||
fs.filer.RollbackTransaction(ctx)
|
||||
return nil, fmt.Errorf("%s/%s move error: %v", req.OldDirectory, req.OldName, moveErr)
|
||||
@ -48,11 +47,11 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom
|
||||
return &filer_pb.AtomicRenameEntryResponse{}, nil
|
||||
}
|
||||
|
||||
func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents) error {
|
||||
func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string) error {
|
||||
|
||||
if err := fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, events, func() error {
|
||||
if err := fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, func() error {
|
||||
if entry.IsDirectory() {
|
||||
if err := fs.moveFolderSubEntries(ctx, oldParent, entry, newParent, newName, events); err != nil {
|
||||
if err := fs.moveFolderSubEntries(ctx, oldParent, entry, newParent, newName); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@ -64,7 +63,7 @@ func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, e
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents) error {
|
||||
func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string) error {
|
||||
|
||||
currentDirPath := oldParent.Child(entry.Name())
|
||||
newDirPath := newParent.Child(newName)
|
||||
@ -85,7 +84,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.
|
||||
for _, item := range entries {
|
||||
lastFileName = item.Name()
|
||||
// println("processing", lastFileName)
|
||||
err := fs.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name(), events)
|
||||
err := fs.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -97,8 +96,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents,
|
||||
moveFolderSubEntries func() error) error {
|
||||
func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, moveFolderSubEntries func() error) error {
|
||||
|
||||
oldPath, newPath := oldParent.Child(entry.Name()), newParent.Child(newName)
|
||||
|
||||
@ -122,8 +120,6 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat
|
||||
return createErr
|
||||
}
|
||||
|
||||
events.newEntries = append(events.newEntries, newEntry)
|
||||
|
||||
if moveFolderSubEntries != nil {
|
||||
if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil {
|
||||
return moveChildrenErr
|
||||
@ -136,13 +132,6 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat
|
||||
return deleteErr
|
||||
}
|
||||
|
||||
events.oldEntries = append(events.oldEntries, entry)
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
type MoveEvents struct {
|
||||
oldEntries []*filer.Entry
|
||||
newEntries []*filer.Entry
|
||||
}
|
||||
|
||||
@ -45,22 +45,23 @@ import (
|
||||
)
|
||||
|
||||
type FilerOption struct {
|
||||
Masters []string
|
||||
Collection string
|
||||
DefaultReplication string
|
||||
DisableDirListing bool
|
||||
MaxMB int
|
||||
DirListingLimit int
|
||||
DataCenter string
|
||||
Rack string
|
||||
DefaultLevelDbDir string
|
||||
DisableHttp bool
|
||||
Host string
|
||||
Port uint32
|
||||
recursiveDelete bool
|
||||
Cipher bool
|
||||
SaveToFilerLimit int
|
||||
Filers []string
|
||||
Masters []string
|
||||
Collection string
|
||||
DefaultReplication string
|
||||
DisableDirListing bool
|
||||
MaxMB int
|
||||
DirListingLimit int
|
||||
DataCenter string
|
||||
Rack string
|
||||
DefaultLevelDbDir string
|
||||
DisableHttp bool
|
||||
Host string
|
||||
Port uint32
|
||||
recursiveDelete bool
|
||||
Cipher bool
|
||||
SaveToFilerLimit int64
|
||||
Filers []string
|
||||
ConcurrentUploadLimit int64
|
||||
}
|
||||
|
||||
type FilerServer struct {
|
||||
@ -79,14 +80,18 @@ type FilerServer struct {
|
||||
|
||||
brokers map[string]map[string]bool
|
||||
brokersLock sync.Mutex
|
||||
|
||||
inFlightDataSize int64
|
||||
inFlightDataLimitCond *sync.Cond
|
||||
}
|
||||
|
||||
func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) {
|
||||
|
||||
fs = &FilerServer{
|
||||
option: option,
|
||||
grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
|
||||
brokers: make(map[string]map[string]bool),
|
||||
option: option,
|
||||
grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
|
||||
brokers: make(map[string]map[string]bool),
|
||||
inFlightDataLimitCond: sync.NewCond(new(sync.Mutex)),
|
||||
}
|
||||
fs.listenersCond = sync.NewCond(&fs.listenersLock)
|
||||
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/stats"
|
||||
@ -47,18 +48,34 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
|
||||
fs.DeleteHandler(w, r)
|
||||
}
|
||||
stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds())
|
||||
case "PUT":
|
||||
stats.FilerRequestCounter.WithLabelValues("put").Inc()
|
||||
if _, ok := r.URL.Query()["tagging"]; ok {
|
||||
fs.PutTaggingHandler(w, r)
|
||||
} else {
|
||||
fs.PostHandler(w, r)
|
||||
case "POST", "PUT":
|
||||
|
||||
// wait until in flight data is less than the limit
|
||||
contentLength := getContentLength(r)
|
||||
fs.inFlightDataLimitCond.L.Lock()
|
||||
for atomic.LoadInt64(&fs.inFlightDataSize) > fs.option.ConcurrentUploadLimit {
|
||||
fs.inFlightDataLimitCond.Wait()
|
||||
}
|
||||
atomic.AddInt64(&fs.inFlightDataSize, contentLength)
|
||||
fs.inFlightDataLimitCond.L.Unlock()
|
||||
defer func() {
|
||||
atomic.AddInt64(&fs.inFlightDataSize, -contentLength)
|
||||
fs.inFlightDataLimitCond.Signal()
|
||||
}()
|
||||
|
||||
if r.Method == "PUT" {
|
||||
stats.FilerRequestCounter.WithLabelValues("put").Inc()
|
||||
if _, ok := r.URL.Query()["tagging"]; ok {
|
||||
fs.PutTaggingHandler(w, r)
|
||||
} else {
|
||||
fs.PostHandler(w, r, contentLength)
|
||||
}
|
||||
stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
|
||||
} else { // method == "POST"
|
||||
stats.FilerRequestCounter.WithLabelValues("post").Inc()
|
||||
fs.PostHandler(w, r, contentLength)
|
||||
stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
|
||||
}
|
||||
stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
|
||||
case "POST":
|
||||
stats.FilerRequestCounter.WithLabelValues("post").Inc()
|
||||
fs.PostHandler(w, r)
|
||||
stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
|
||||
case "OPTIONS":
|
||||
stats.FilerRequestCounter.WithLabelValues("options").Inc()
|
||||
OptionsHandler(w, r, false)
|
||||
|
||||
@ -131,6 +131,9 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
|
||||
|
||||
if r.Method == "HEAD" {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
|
||||
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
|
||||
return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, true)
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
@ -150,10 +153,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
|
||||
}
|
||||
}
|
||||
|
||||
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error {
|
||||
if httpStatusCode != 0 {
|
||||
w.WriteHeader(httpStatusCode)
|
||||
}
|
||||
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
|
||||
if offset+size <= int64(len(entry.Content)) {
|
||||
_, err := writer.Write(entry.Content[offset : offset+size])
|
||||
if err != nil {
|
||||
@ -161,7 +161,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
|
||||
}
|
||||
return err
|
||||
}
|
||||
return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size)
|
||||
return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, false)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
@ -52,7 +52,7 @@ func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, u
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
|
||||
func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, contentLength int64) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
@ -66,7 +66,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
|
||||
query.Get("rack"),
|
||||
)
|
||||
|
||||
fs.autoChunk(ctx, w, r, so)
|
||||
fs.autoChunk(ctx, w, r, contentLength, so)
|
||||
util.CloseRequest(r)
|
||||
|
||||
}
|
||||
|
||||
@ -2,11 +2,8 @@ package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
@ -19,13 +16,12 @@ import (
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) {
|
||||
func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, contentLength int64, so *operation.StorageOption) {
|
||||
|
||||
// autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
|
||||
query := r.URL.Query()
|
||||
@ -51,14 +47,16 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
|
||||
if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") {
|
||||
reply, err = fs.mkdir(ctx, w, r)
|
||||
} else {
|
||||
reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, so)
|
||||
reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, contentLength, so)
|
||||
}
|
||||
} else {
|
||||
reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, so)
|
||||
reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, contentLength, so)
|
||||
}
|
||||
if err != nil {
|
||||
if strings.HasPrefix(err.Error(), "read input:") {
|
||||
writeJsonError(w, r, 499, err)
|
||||
} else if strings.HasSuffix(err.Error(), "is a file") {
|
||||
writeJsonError(w, r, http.StatusConflict, err)
|
||||
} else {
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
}
|
||||
@ -70,7 +68,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
|
||||
func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
|
||||
|
||||
multipartReader, multipartReaderErr := r.MultipartReader()
|
||||
if multipartReaderErr != nil {
|
||||
@ -91,7 +89,7 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite
|
||||
contentType = ""
|
||||
}
|
||||
|
||||
fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, so)
|
||||
fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, contentLength, so)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@ -102,7 +100,7 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
|
||||
func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
|
||||
|
||||
fileName := path.Base(r.URL.Path)
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
@ -110,7 +108,7 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter
|
||||
contentType = ""
|
||||
}
|
||||
|
||||
fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, so)
|
||||
fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, contentLength, so)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@ -212,7 +210,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
|
||||
fs.saveAmzMetaData(r, entry)
|
||||
SaveAmzMetaData(r, entry.Extended, false)
|
||||
|
||||
for k, v := range r.Header {
|
||||
if len(v) > 0 && strings.HasPrefix(k, needle.PairNamePrefix) {
|
||||
@ -229,92 +227,6 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
||||
return filerResult, replyerr
|
||||
}
|
||||
|
||||
func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error, []byte) {
|
||||
var fileChunks []*filer_pb.FileChunk
|
||||
|
||||
md5Hash := md5.New()
|
||||
var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash))
|
||||
|
||||
chunkOffset := int64(0)
|
||||
var smallContent []byte
|
||||
|
||||
for {
|
||||
limitedReader := io.LimitReader(partReader, int64(chunkSize))
|
||||
|
||||
data, err := ioutil.ReadAll(limitedReader)
|
||||
if err != nil {
|
||||
return nil, nil, 0, err, nil
|
||||
}
|
||||
if chunkOffset == 0 && !isAppend(r) {
|
||||
if len(data) < fs.option.SaveToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && len(data) < 4*1024 {
|
||||
smallContent = data
|
||||
chunkOffset += int64(len(data))
|
||||
break
|
||||
}
|
||||
}
|
||||
dataReader := util.NewBytesReader(data)
|
||||
|
||||
// retry to assign a different file id
|
||||
var fileId, urlLocation string
|
||||
var auth security.EncodedJwt
|
||||
var assignErr, uploadErr error
|
||||
var uploadResult *operation.UploadResult
|
||||
for i := 0; i < 3; i++ {
|
||||
// assign one file id for one chunk
|
||||
fileId, urlLocation, auth, assignErr = fs.assignNewFileInfo(so)
|
||||
if assignErr != nil {
|
||||
return nil, nil, 0, assignErr, nil
|
||||
}
|
||||
|
||||
// upload the chunk to the volume server
|
||||
uploadResult, uploadErr, _ = fs.doUpload(urlLocation, w, r, dataReader, fileName, contentType, nil, auth)
|
||||
if uploadErr != nil {
|
||||
time.Sleep(251 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if uploadErr != nil {
|
||||
return nil, nil, 0, uploadErr, nil
|
||||
}
|
||||
|
||||
// if last chunk exhausted the reader exactly at the border
|
||||
if uploadResult.Size == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Save to chunk manifest structure
|
||||
fileChunks = append(fileChunks, uploadResult.ToPbFileChunk(fileId, chunkOffset))
|
||||
|
||||
glog.V(4).Infof("uploaded %s chunk %d to %s [%d,%d)", fileName, len(fileChunks), fileId, chunkOffset, chunkOffset+int64(uploadResult.Size))
|
||||
|
||||
// reset variables for the next chunk
|
||||
chunkOffset = chunkOffset + int64(uploadResult.Size)
|
||||
|
||||
// if last chunk was not at full chunk size, but already exhausted the reader
|
||||
if int64(uploadResult.Size) < int64(chunkSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return fileChunks, md5Hash, chunkOffset, nil, smallContent
|
||||
}
|
||||
|
||||
func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {
|
||||
|
||||
stats.FilerRequestCounter.WithLabelValues("chunkUpload").Inc()
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
|
||||
}()
|
||||
|
||||
uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
|
||||
if uploadResult != nil && uploadResult.RetryCount > 0 {
|
||||
stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
|
||||
}
|
||||
return uploadResult, err, data
|
||||
}
|
||||
|
||||
func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType {
|
||||
|
||||
return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) {
|
||||
@ -383,17 +295,24 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http
|
||||
return filerResult, replyerr
|
||||
}
|
||||
|
||||
func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
|
||||
func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool) (metadata map[string][]byte) {
|
||||
|
||||
metadata = make(map[string][]byte)
|
||||
if !isReplace {
|
||||
for k, v := range existing {
|
||||
metadata[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
|
||||
entry.Extended[xhttp.AmzStorageClass] = []byte(sc)
|
||||
metadata[xhttp.AmzStorageClass] = []byte(sc)
|
||||
}
|
||||
|
||||
if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
|
||||
for _, v := range strings.Split(tags, "&") {
|
||||
tag := strings.Split(v, "=")
|
||||
if len(tag) == 2 {
|
||||
entry.Extended[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
|
||||
metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -401,8 +320,11 @@ func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
|
||||
for header, values := range r.Header {
|
||||
if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
|
||||
for _, value := range values {
|
||||
entry.Extended[header] = []byte(value)
|
||||
metadata[header] = []byte(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
168
weed/server/filer_server_handlers_write_upload.go
Normal file
168
weed/server/filer_server_handlers_write_upload.go
Normal file
@ -0,0 +1,168 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
limitedUploadProcessor = util.NewLimitedOutOfOrderProcessor(int32(runtime.NumCPU()))
|
||||
)
|
||||
|
||||
func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, contentLength int64, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, dataSize int64, err error, smallContent []byte) {
|
||||
|
||||
md5Hash = md5.New()
|
||||
var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash))
|
||||
|
||||
// save small content directly
|
||||
if !isAppend(r) && ((0 < contentLength && contentLength < fs.option.SaveToFilerLimit) || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && contentLength < 4*1024) {
|
||||
smallContent, err = ioutil.ReadAll(partReader)
|
||||
dataSize = int64(len(smallContent))
|
||||
return
|
||||
}
|
||||
|
||||
resultsChan := make(chan *ChunkCreationResult, 2)
|
||||
|
||||
var waitForAllData sync.WaitGroup
|
||||
waitForAllData.Add(1)
|
||||
go func() {
|
||||
// process upload results
|
||||
defer waitForAllData.Done()
|
||||
for result := range resultsChan {
|
||||
if result.err != nil {
|
||||
err = result.err
|
||||
continue
|
||||
}
|
||||
|
||||
// Save to chunk manifest structure
|
||||
fileChunks = append(fileChunks, result.chunk)
|
||||
}
|
||||
}()
|
||||
|
||||
var lock sync.Mutex
|
||||
readOffset := int64(0)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for err == nil {
|
||||
|
||||
wg.Add(1)
|
||||
request := func() {
|
||||
defer wg.Done()
|
||||
|
||||
var localOffset int64
|
||||
// read from the input
|
||||
lock.Lock()
|
||||
localOffset = readOffset
|
||||
limitedReader := io.LimitReader(partReader, int64(chunkSize))
|
||||
data, readErr := ioutil.ReadAll(limitedReader)
|
||||
readOffset += int64(len(data))
|
||||
lock.Unlock()
|
||||
// handle read errors
|
||||
if readErr != nil {
|
||||
if readErr != io.EOF {
|
||||
if err == nil {
|
||||
err = readErr
|
||||
}
|
||||
resultsChan <- &ChunkCreationResult{
|
||||
err: readErr,
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(data) == 0 {
|
||||
readErr = io.EOF
|
||||
return
|
||||
}
|
||||
|
||||
// upload
|
||||
dataReader := util.NewBytesReader(data)
|
||||
fileId, uploadResult, uploadErr := fs.doCreateChunk(w, r, so, dataReader, fileName, contentType)
|
||||
if uploadErr != nil {
|
||||
if err == nil {
|
||||
err = uploadErr
|
||||
}
|
||||
resultsChan <- &ChunkCreationResult{
|
||||
err: uploadErr,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
glog.V(4).Infof("uploaded %s to %s [%d,%d)", fileName, fileId, localOffset, localOffset+int64(uploadResult.Size))
|
||||
|
||||
// send back uploaded file chunk
|
||||
resultsChan <- &ChunkCreationResult{
|
||||
chunk: uploadResult.ToPbFileChunk(fileId, localOffset),
|
||||
}
|
||||
|
||||
}
|
||||
limitedUploadProcessor.Execute(request)
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(resultsChan)
|
||||
}()
|
||||
|
||||
waitForAllData.Wait()
|
||||
|
||||
return fileChunks, md5Hash, readOffset, err, nil
|
||||
}
|
||||
|
||||
type ChunkCreationResult struct {
|
||||
chunk *filer_pb.FileChunk
|
||||
err error
|
||||
}
|
||||
|
||||
func (fs *FilerServer) doCreateChunk(w http.ResponseWriter, r *http.Request, so *operation.StorageOption, dataReader *util.BytesReader, fileName string, contentType string) (string, *operation.UploadResult, error) {
|
||||
// retry to assign a different file id
|
||||
var fileId, urlLocation string
|
||||
var auth security.EncodedJwt
|
||||
var assignErr, uploadErr error
|
||||
var uploadResult *operation.UploadResult
|
||||
for i := 0; i < 3; i++ {
|
||||
// assign one file id for one chunk
|
||||
fileId, urlLocation, auth, assignErr = fs.assignNewFileInfo(so)
|
||||
if assignErr != nil {
|
||||
return "", nil, assignErr
|
||||
}
|
||||
|
||||
// upload the chunk to the volume server
|
||||
uploadResult, uploadErr, _ = fs.doUpload(urlLocation, w, r, dataReader, fileName, contentType, nil, auth)
|
||||
if uploadErr != nil {
|
||||
time.Sleep(251 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
return fileId, uploadResult, uploadErr
|
||||
}
|
||||
|
||||
func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {
|
||||
|
||||
stats.FilerRequestCounter.WithLabelValues("chunkUpload").Inc()
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
|
||||
}()
|
||||
|
||||
uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
|
||||
if uploadResult != nil && uploadResult.RetryCount > 0 {
|
||||
stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
|
||||
}
|
||||
return uploadResult, err, data
|
||||
}
|
||||
@ -80,10 +80,14 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
|
||||
dn.AdjustMaxVolumeCounts(heartbeat.MaxVolumeCounts)
|
||||
|
||||
glog.V(4).Infof("master received heartbeat %s", heartbeat.String())
|
||||
var dataCenter string
|
||||
if dc := dn.GetDataCenter(); dc != nil {
|
||||
dataCenter = string(dc.Id())
|
||||
}
|
||||
message := &master_pb.VolumeLocation{
|
||||
Url: dn.Url(),
|
||||
PublicUrl: dn.PublicUrl,
|
||||
DataCenter: string(dn.GetDataCenter().Id()),
|
||||
DataCenter: dataCenter,
|
||||
}
|
||||
if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 {
|
||||
// process delta volume ids if exists for fast volume id updates
|
||||
|
||||
@ -277,6 +277,13 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer
|
||||
glog.Error(err)
|
||||
seq = nil
|
||||
}
|
||||
case "snowflake":
|
||||
var err error
|
||||
seq, err = sequence.NewSnowflakeSequencer(fmt.Sprintf("%s:%d", option.Host, option.Port))
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
seq = nil
|
||||
}
|
||||
default:
|
||||
seq = sequence.NewMemorySequencer()
|
||||
}
|
||||
|
||||
38
weed/server/volume_grpc_read_write.go
Normal file
38
weed/server/volume_grpc_read_write.go
Normal file
@ -0,0 +1,38 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
func (vs *VolumeServer) ReadNeedleBlob(ctx context.Context, req *volume_server_pb.ReadNeedleBlobRequest) (resp *volume_server_pb.ReadNeedleBlobResponse, err error) {
|
||||
resp = &volume_server_pb.ReadNeedleBlobResponse{}
|
||||
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
|
||||
if v == nil {
|
||||
return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
|
||||
}
|
||||
|
||||
resp.NeedleBlob, err = v.ReadNeedleBlob(req.Offset, types.Size(req.Size))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read needle blob offset %d size %d: %v", req.Offset, req.Size, err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (vs *VolumeServer) WriteNeedleBlob(ctx context.Context, req *volume_server_pb.WriteNeedleBlobRequest) (resp *volume_server_pb.WriteNeedleBlobResponse, err error) {
|
||||
resp = &volume_server_pb.WriteNeedleBlobResponse{}
|
||||
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
|
||||
if v == nil {
|
||||
return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
|
||||
}
|
||||
|
||||
if err = v.WriteNeedleBlob(types.NeedleId(req.NeedleId), req.NeedleBlob, types.Size(req.Size)); err != nil {
|
||||
return nil, fmt.Errorf("write blob needle %d size %d: %v", req.NeedleId, req.Size, err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
@ -34,6 +35,10 @@ type VolumeServer struct {
|
||||
fileSizeLimitBytes int64
|
||||
isHeartbeating bool
|
||||
stopChan chan bool
|
||||
|
||||
inFlightDataSize int64
|
||||
inFlightDataLimitCond *sync.Cond
|
||||
concurrentUploadLimit int64
|
||||
}
|
||||
|
||||
func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
|
||||
@ -48,6 +53,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
|
||||
readRedirect bool,
|
||||
compactionMBPerSecond int,
|
||||
fileSizeLimitMB int,
|
||||
concurrentUploadLimit int64,
|
||||
) *VolumeServer {
|
||||
|
||||
v := util.GetViper()
|
||||
@ -72,6 +78,8 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
|
||||
fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024,
|
||||
isHeartbeating: true,
|
||||
stopChan: make(chan bool),
|
||||
inFlightDataLimitCond: sync.NewCond(new(sync.Mutex)),
|
||||
concurrentUploadLimit: concurrentUploadLimit,
|
||||
}
|
||||
vs.SeedMasterNodes = masterNodes
|
||||
|
||||
|
||||
@ -2,7 +2,9 @@ package weed_server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
|
||||
@ -40,8 +42,24 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
|
||||
stats.DeleteRequest()
|
||||
vs.guard.WhiteList(vs.DeleteHandler)(w, r)
|
||||
case "PUT", "POST":
|
||||
|
||||
// wait until in flight data is less than the limit
|
||||
contentLength := getContentLength(r)
|
||||
vs.inFlightDataLimitCond.L.Lock()
|
||||
for atomic.LoadInt64(&vs.inFlightDataSize) > vs.concurrentUploadLimit {
|
||||
vs.inFlightDataLimitCond.Wait()
|
||||
}
|
||||
atomic.AddInt64(&vs.inFlightDataSize, contentLength)
|
||||
vs.inFlightDataLimitCond.L.Unlock()
|
||||
defer func() {
|
||||
atomic.AddInt64(&vs.inFlightDataSize, -contentLength)
|
||||
vs.inFlightDataLimitCond.Signal()
|
||||
}()
|
||||
|
||||
// processs uploads
|
||||
stats.WriteRequest()
|
||||
vs.guard.WhiteList(vs.PostHandler)(w, r)
|
||||
|
||||
case "OPTIONS":
|
||||
stats.ReadRequest()
|
||||
w.Header().Add("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
|
||||
@ -49,6 +67,18 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
|
||||
func getContentLength(r *http.Request) int64 {
|
||||
contentLength := r.Header.Get("Content-Length")
|
||||
if contentLength != "" {
|
||||
length, err := strconv.ParseInt(contentLength, 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return length
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
|
||||
if r.Header.Get("Origin") != "" {
|
||||
|
||||
@ -27,7 +27,7 @@ var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
|
||||
|
||||
func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// println(r.Method + " " + r.URL.Path)
|
||||
glog.V(9).Info(r.Method + " " + r.URL.Path + " " + r.Header.Get("Range"))
|
||||
|
||||
stats.VolumeServerRequestCounter.WithLabelValues("get").Inc()
|
||||
start := time.Now()
|
||||
@ -261,13 +261,10 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re
|
||||
return nil
|
||||
}
|
||||
|
||||
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error {
|
||||
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
|
||||
if _, e = rs.Seek(offset, 0); e != nil {
|
||||
return e
|
||||
}
|
||||
if httpStatusCode != 0 {
|
||||
w.WriteHeader(httpStatusCode)
|
||||
}
|
||||
_, e = io.CopyN(writer, rs, size)
|
||||
return e
|
||||
})
|
||||
|
||||
@ -52,7 +52,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write
|
||||
return err
|
||||
}
|
||||
|
||||
return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
|
||||
return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
|
||||
|
||||
})
|
||||
|
||||
|
||||
@ -36,6 +36,10 @@ func (c *commandFsMv) Help() string {
|
||||
|
||||
func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
|
||||
|
||||
if len(args) != 2 {
|
||||
return fmt.Errorf("need to have 2 arguments")
|
||||
}
|
||||
|
||||
sourcePath, err := commandEnv.parseUrl(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@ -54,7 +54,7 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer
|
||||
return fmt.Errorf("list buckets under %v: %v", filerBucketsPath, err)
|
||||
}
|
||||
|
||||
for _, bucket:= range buckets {
|
||||
for _, bucket := range buckets {
|
||||
c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo)
|
||||
}
|
||||
|
||||
@ -63,7 +63,7 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer
|
||||
}
|
||||
|
||||
func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration) error {
|
||||
uploadsDir := filerBucketsPath+"/"+bucket+"/.uploads"
|
||||
uploadsDir := filerBucketsPath + "/" + bucket + "/.uploads"
|
||||
var staleUploads []string
|
||||
now := time.Now()
|
||||
err := filer_pb.List(commandEnv, uploadsDir, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
@ -77,8 +77,8 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io
|
||||
return fmt.Errorf("list uploads under %v: %v", uploadsDir, err)
|
||||
}
|
||||
|
||||
for _, staleUpload:= range staleUploads {
|
||||
deleteUrl := fmt.Sprintf("http://%s:%d%s/%s?recursive=true&ignoreRecursiveError=true",commandEnv.option.FilerHost, commandEnv.option.FilerPort,uploadsDir, staleUpload)
|
||||
for _, staleUpload := range staleUploads {
|
||||
deleteUrl := fmt.Sprintf("http://%s:%d%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerHost, commandEnv.option.FilerPort, uploadsDir, staleUpload)
|
||||
fmt.Fprintf(writer, "purge %s\n", deleteUrl)
|
||||
|
||||
err = util.Delete(deleteUrl, "")
|
||||
@ -89,4 +89,4 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@ -393,9 +393,8 @@ func adjustAfterMove(v *master_pb.VolumeInformationMessage, volumeReplicas map[u
|
||||
if replica.location.dataNode.Id == fullNode.info.Id &&
|
||||
replica.location.rack == fullNode.rack &&
|
||||
replica.location.dc == fullNode.dc {
|
||||
replica.location.dc = emptyNode.dc
|
||||
replica.location.rack = emptyNode.rack
|
||||
replica.location.dataNode = emptyNode.info
|
||||
loc := newLocation(emptyNode.dc, emptyNode.rack, emptyNode.info)
|
||||
replica.location = &loc
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@ -169,3 +169,15 @@ func TestIsGoodMove(t *testing.T) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestBalance(t *testing.T) {
|
||||
topologyInfo := parseOutput(topoData)
|
||||
volumeServers := collectVolumeServersByDc(topologyInfo, "")
|
||||
volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
|
||||
diskTypes := collectVolumeDiskTypes(topologyInfo)
|
||||
|
||||
if err := balanceVolumeServers(nil, diskTypes, volumeReplicas, volumeServers, 30*1024*1024*1024, "ALL_COLLECTIONS", false); err != nil {
|
||||
t.Errorf("balance: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
257
weed/shell/command_volume_check_disk.go
Normal file
257
weed/shell/command_volume_check_disk.go
Normal file
@ -0,0 +1,257 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
|
||||
"io"
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Commands = append(Commands, &commandVolumeCheckDisk{})
|
||||
}
|
||||
|
||||
type commandVolumeCheckDisk struct {
|
||||
env *CommandEnv
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDisk) Name() string {
|
||||
return "volume.check.disk"
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDisk) Help() string {
|
||||
return `check all replicated volumes to find and fix inconsistencies
|
||||
|
||||
How it works:
|
||||
|
||||
find all volumes that are replicated
|
||||
for each volume id, if there are more than 2 replicas, find one pair with the largest 2 in file count.
|
||||
for the pair volume A and B
|
||||
append entries in A and not in B to B
|
||||
append entries in B and not in A to A
|
||||
|
||||
`
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
|
||||
|
||||
if err = commandEnv.confirmIsLocked(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
||||
slowMode := fsckCommand.Bool("slow", false, "slow mode checks all replicas even file counts are the same")
|
||||
verbose := fsckCommand.Bool("v", false, "verbose mode")
|
||||
applyChanges := fsckCommand.Bool("force", false, "apply the fix")
|
||||
nonRepairThreshold := fsckCommand.Float64("nonRepairThreshold", 0.3, "repair when missing keys is not more than this limit")
|
||||
if err = fsckCommand.Parse(args); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
c.env = commandEnv
|
||||
|
||||
// collect topology information
|
||||
topologyInfo, _, err := collectTopologyInfo(commandEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
|
||||
|
||||
// pick 1 pairs of volume replica
|
||||
fileCount := func(replica *VolumeReplica) uint64 {
|
||||
return replica.info.FileCount - replica.info.DeleteCount
|
||||
}
|
||||
aDB, bDB := needle_map.NewMemDb(), needle_map.NewMemDb()
|
||||
defer aDB.Close()
|
||||
defer bDB.Close()
|
||||
|
||||
for _, replicas := range volumeReplicas {
|
||||
sort.Slice(replicas, func(i, j int) bool {
|
||||
return fileCount(replicas[i]) > fileCount(replicas[j])
|
||||
})
|
||||
for len(replicas) >= 2 {
|
||||
a, b := replicas[0], replicas[1]
|
||||
if !*slowMode {
|
||||
if fileCount(a) == fileCount(b) {
|
||||
replicas = replicas[1:]
|
||||
continue
|
||||
}
|
||||
}
|
||||
if a.info.ReadOnly || b.info.ReadOnly {
|
||||
fmt.Fprintf(writer, "skipping readonly volume %d on %s and %s\n", a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id)
|
||||
continue
|
||||
}
|
||||
|
||||
// reset index db
|
||||
aDB.Close()
|
||||
bDB.Close()
|
||||
aDB, bDB = needle_map.NewMemDb(), needle_map.NewMemDb()
|
||||
|
||||
// read index db
|
||||
if err := c.readIndexDatabase(aDB, a.info.Collection, a.info.Id, a.location.dataNode.Id, *verbose, writer); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.readIndexDatabase(bDB, b.info.Collection, b.info.Id, b.location.dataNode.Id, *verbose, writer); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// find and make up the differnces
|
||||
if err := c.doVolumeCheckDisk(aDB, bDB, a, b, *verbose, writer, *applyChanges, *nonRepairThreshold); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.doVolumeCheckDisk(bDB, aDB, b, a, *verbose, writer, *applyChanges, *nonRepairThreshold); err != nil {
|
||||
return err
|
||||
}
|
||||
replicas = replicas[1:]
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDisk) doVolumeCheckDisk(subtrahend, minuend *needle_map.MemDb, source, target *VolumeReplica, verbose bool, writer io.Writer, applyChanges bool, nonRepairThreshold float64) error {
|
||||
|
||||
// find missing keys
|
||||
// hash join, can be more efficient
|
||||
var missingNeedles []needle_map.NeedleValue
|
||||
var counter int
|
||||
subtrahend.AscendingVisit(func(value needle_map.NeedleValue) error {
|
||||
counter++
|
||||
if _, found := minuend.Get(value.Key); !found {
|
||||
missingNeedles = append(missingNeedles, value)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
fmt.Fprintf(writer, "volume %d %s has %d entries, %s missed %d entries\n", source.info.Id, source.location.dataNode.Id, counter, target.location.dataNode.Id, len(missingNeedles))
|
||||
|
||||
if counter == 0 || len(missingNeedles) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
missingNeedlesFraction := float64(len(missingNeedles)) / float64(counter)
|
||||
if missingNeedlesFraction > nonRepairThreshold {
|
||||
return fmt.Errorf(
|
||||
"failed to start repair volume %d, percentage of missing keys is greater than the threshold: %.2f > %.2f",
|
||||
source.info.Id, missingNeedlesFraction, nonRepairThreshold)
|
||||
}
|
||||
|
||||
for _, needleValue := range missingNeedles {
|
||||
|
||||
needleBlob, err := c.readSourceNeedleBlob(source.location.dataNode.Id, source.info.Id, needleValue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !applyChanges {
|
||||
continue
|
||||
}
|
||||
|
||||
if verbose {
|
||||
fmt.Fprintf(writer, "read %d,%x %s => %s \n", source.info.Id, needleValue.Key, source.location.dataNode.Id, target.location.dataNode.Id)
|
||||
}
|
||||
|
||||
if err := c.writeNeedleBlobToTarget(target.location.dataNode.Id, source.info.Id, needleValue, needleBlob); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDisk) readSourceNeedleBlob(sourceVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue) (needleBlob []byte, err error) {
|
||||
|
||||
err = operation.WithVolumeServerClient(sourceVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
resp, err := client.ReadNeedleBlob(context.Background(), &volume_server_pb.ReadNeedleBlobRequest{
|
||||
VolumeId: volumeId,
|
||||
NeedleId: uint64(needleValue.Key),
|
||||
Offset: needleValue.Offset.ToActualOffset(),
|
||||
Size: int32(needleValue.Size),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
needleBlob = resp.NeedleBlob
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDisk) writeNeedleBlobToTarget(targetVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue, needleBlob []byte) error {
|
||||
|
||||
return operation.WithVolumeServerClient(targetVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
_, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{
|
||||
VolumeId: volumeId,
|
||||
NeedleId: uint64(needleValue.Key),
|
||||
Size: int32(needleValue.Size),
|
||||
NeedleBlob: needleBlob,
|
||||
})
|
||||
return err
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDisk) readIndexDatabase(db *needle_map.MemDb, collection string, volumeId uint32, volumeServer string, verbose bool, writer io.Writer) error {
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := c.copyVolumeIndexFile(collection, volumeId, volumeServer, &buf, verbose, writer); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if verbose {
|
||||
fmt.Fprintf(writer, "load collection %s volume %d index size %d from %s ...\n", collection, volumeId, buf.Len(), volumeServer)
|
||||
}
|
||||
|
||||
return db.LoadFromReaderAt(bytes.NewReader(buf.Bytes()))
|
||||
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDisk) copyVolumeIndexFile(collection string, volumeId uint32, volumeServer string, buf *bytes.Buffer, verbose bool, writer io.Writer) error {
|
||||
|
||||
return operation.WithVolumeServerClient(volumeServer, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
||||
|
||||
ext := ".idx"
|
||||
|
||||
copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
|
||||
VolumeId: volumeId,
|
||||
Ext: ".idx",
|
||||
CompactionRevision: math.MaxUint32,
|
||||
StopOffset: math.MaxInt64,
|
||||
Collection: collection,
|
||||
IsEcVolume: false,
|
||||
IgnoreSourceFileNotFound: false,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err)
|
||||
}
|
||||
|
||||
err = writeToBuffer(copyFileClient, buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, volumeServer, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
func writeToBuffer(client volume_server_pb.VolumeServer_CopyFileClient, buf *bytes.Buffer) error {
|
||||
for {
|
||||
resp, receiveErr := client.Recv()
|
||||
if receiveErr == io.EOF {
|
||||
break
|
||||
}
|
||||
if receiveErr != nil {
|
||||
return fmt.Errorf("receiving: %v", receiveErr)
|
||||
}
|
||||
buf.Write(resp.FileContent)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
893
weed/shell/command_volume_list_test.go
Normal file
893
weed/shell/command_volume_list_test.go
Normal file
@ -0,0 +1,893 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
func TestParsing(t *testing.T) {
|
||||
topo := parseOutput(topoData)
|
||||
|
||||
assert.Equal(t, 5, len(topo.DataCenterInfos))
|
||||
|
||||
}
|
||||
|
||||
func parseOutput(output string) *master_pb.TopologyInfo {
|
||||
lines := strings.Split(output, "\n")
|
||||
var topo *master_pb.TopologyInfo
|
||||
var dc *master_pb.DataCenterInfo
|
||||
var rack *master_pb.RackInfo
|
||||
var dn *master_pb.DataNodeInfo
|
||||
var disk *master_pb.DiskInfo
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
parts := strings.Split(line, " ")
|
||||
switch parts[0] {
|
||||
case "Topology":
|
||||
if topo == nil {
|
||||
topo = &master_pb.TopologyInfo{}
|
||||
}
|
||||
case "DataCenter":
|
||||
if dc == nil {
|
||||
dc = &master_pb.DataCenterInfo{
|
||||
Id: parts[1],
|
||||
}
|
||||
topo.DataCenterInfos = append(topo.DataCenterInfos, dc)
|
||||
} else {
|
||||
dc = nil
|
||||
}
|
||||
case "Rack":
|
||||
if rack == nil {
|
||||
rack = &master_pb.RackInfo{
|
||||
Id: parts[1],
|
||||
}
|
||||
dc.RackInfos = append(dc.RackInfos, rack)
|
||||
} else {
|
||||
rack = nil
|
||||
}
|
||||
case "DataNode":
|
||||
if dn == nil {
|
||||
dn = &master_pb.DataNodeInfo{
|
||||
Id: parts[1],
|
||||
DiskInfos: make(map[string]*master_pb.DiskInfo),
|
||||
}
|
||||
rack.DataNodeInfos = append(rack.DataNodeInfos, dn)
|
||||
} else {
|
||||
dn = nil
|
||||
}
|
||||
case "Disk":
|
||||
if disk == nil {
|
||||
diskType := parts[1][:strings.Index(parts[1], "(")]
|
||||
maxVolumeCountStr := parts[1][strings.Index(parts[1], "/")+1:]
|
||||
maxVolumeCount, _ := strconv.Atoi(maxVolumeCountStr)
|
||||
disk = &master_pb.DiskInfo{
|
||||
Type: diskType,
|
||||
MaxVolumeCount: uint64(maxVolumeCount),
|
||||
}
|
||||
dn.DiskInfos[types.ToDiskType(diskType).String()] = disk
|
||||
} else {
|
||||
disk = nil
|
||||
}
|
||||
case "volume":
|
||||
volumeLine := line[len("volume "):]
|
||||
volume := &master_pb.VolumeInformationMessage{}
|
||||
proto.UnmarshalText(volumeLine, volume)
|
||||
disk.VolumeInfos = append(disk.VolumeInfos, volume)
|
||||
}
|
||||
}
|
||||
|
||||
return topo
|
||||
}
|
||||
|
||||
const topoData = `
|
||||
Topology volumeSizeLimit:1024 MB hdd(volume:760/7280 active:760 free:6520 remote:0)
|
||||
DataCenter dc1 hdd(volume:0/0 active:0 free:0 remote:0)
|
||||
Rack DefaultRack hdd(volume:0/0 active:0 free:0 remote:0)
|
||||
Rack DefaultRack total size:0 file_count:0
|
||||
DataCenter dc1 total size:0 file_count:0
|
||||
DataCenter dc2 hdd(volume:86/430 active:86 free:344 remote:0)
|
||||
Rack rack1 hdd(volume:50/240 active:50 free:190 remote:0)
|
||||
DataNode 192.168.1.4:8080 hdd(volume:50/240 active:50 free:190 remote:0)
|
||||
Disk hdd(volume:50/240 active:50 free:190 remote:0)
|
||||
volume id:15 size:1115965064 collection:"collection0" file_count:83 replica_placement:100 version:3 modified_at_second:1609923671
|
||||
volume id:21 size:1097631536 collection:"collection0" file_count:82 delete_count:7 deleted_byte_count:68975485 replica_placement:100 version:3 modified_at_second:1609929578
|
||||
volume id:22 size:1086828272 collection:"collection0" file_count:75 replica_placement:100 version:3 modified_at_second:1609930001
|
||||
volume id:23 size:1076380216 collection:"collection0" file_count:68 replica_placement:100 version:3 modified_at_second:1609930434
|
||||
volume id:24 size:1074139776 collection:"collection0" file_count:90 replica_placement:100 version:3 modified_at_second:1609930909
|
||||
volume id:25 size:690757512 collection:"collection0" file_count:38 replica_placement:100 version:3 modified_at_second:1611144216
|
||||
volume id:27 size:298886792 file_count:1608 replica_placement:100 version:3 modified_at_second:1615632482
|
||||
volume id:28 size:308919192 file_count:1591 delete_count:1 deleted_byte_count:125280 replica_placement:100 version:3 modified_at_second:1615631762
|
||||
volume id:29 size:281582680 file_count:1537 replica_placement:100 version:3 modified_at_second:1615629422
|
||||
volume id:30 size:289466144 file_count:1566 delete_count:1 deleted_byte_count:124972 replica_placement:100 version:3 modified_at_second:1615632422
|
||||
volume id:31 size:273363256 file_count:1498 replica_placement:100 version:3 modified_at_second:1615631642
|
||||
volume id:33 size:1130226400 collection:"collection1" file_count:7322 delete_count:172 deleted_byte_count:45199399 replica_placement:100 version:3 modified_at_second:1615618789
|
||||
volume id:38 size:1075545744 collection:"collection1" file_count:13324 delete_count:100 deleted_byte_count:25223906 replica_placement:100 version:3 modified_at_second:1615569830
|
||||
volume id:51 size:1076796120 collection:"collection1" file_count:10550 delete_count:39 deleted_byte_count:12723654 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615547786
|
||||
volume id:52 size:1083529728 collection:"collection1" file_count:10128 delete_count:32 deleted_byte_count:10608391 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615599195
|
||||
volume id:54 size:1045022344 collection:"collection1" file_count:9408 delete_count:30 deleted_byte_count:15132106 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812
|
||||
volume id:63 size:956941112 collection:"collection1" file_count:8271 delete_count:32 deleted_byte_count:15876189 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632036
|
||||
volume id:69 size:869213648 collection:"collection1" file_count:7293 delete_count:102 deleted_byte_count:30643207 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630534
|
||||
volume id:74 size:957046128 collection:"collection1" file_count:6982 delete_count:258 deleted_byte_count:73054259 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631460
|
||||
volume id:80 size:827912928 collection:"collection1" file_count:6914 delete_count:17 deleted_byte_count:5689635 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631157
|
||||
volume id:84 size:873121856 collection:"collection1" file_count:8200 delete_count:13 deleted_byte_count:3131676 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631161
|
||||
volume id:85 size:1023869320 collection:"collection1" file_count:7788 delete_count:234 deleted_byte_count:78037967 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631723
|
||||
volume id:97 size:1053112992 collection:"collection1" file_count:6789 delete_count:50 deleted_byte_count:38894001 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631193
|
||||
volume id:98 size:1077836440 collection:"collection1" file_count:7605 delete_count:202 deleted_byte_count:73180379 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615523691
|
||||
volume id:105 size:1073996824 collection:"collection1" file_count:6872 delete_count:20 deleted_byte_count:14482293 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615499757
|
||||
volume id:106 size:1075458664 collection:"collection1" file_count:7182 delete_count:307 deleted_byte_count:69349053 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615598137
|
||||
volume id:112 size:1076392512 collection:"collection1" file_count:8291 delete_count:156 deleted_byte_count:74120183 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569823
|
||||
volume id:116 size:1074489504 collection:"collection1" file_count:9981 delete_count:174 deleted_byte_count:53998777 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615611565
|
||||
volume id:119 size:1075940104 collection:"collection1" file_count:9003 delete_count:12 deleted_byte_count:9128155 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573878
|
||||
volume id:128 size:1074874632 collection:"collection1" file_count:9821 delete_count:148 deleted_byte_count:43633334 replica_placement:100 version:3 modified_at_second:1615602670
|
||||
volume id:133 size:1075952760 collection:"collection1" file_count:9538 delete_count:74 deleted_byte_count:19558008 replica_placement:100 version:3 modified_at_second:1615584779
|
||||
volume id:136 size:1074433552 collection:"collection1" file_count:9593 delete_count:72 deleted_byte_count:26912512 replica_placement:100 version:3 modified_at_second:1615376036
|
||||
volume id:138 size:1074465744 collection:"collection1" file_count:10120 delete_count:55 deleted_byte_count:15875438 replica_placement:100 version:3 modified_at_second:1615572231
|
||||
volume id:140 size:1076203744 collection:"collection1" file_count:11219 delete_count:57 deleted_byte_count:19864498 replica_placement:100 version:3 modified_at_second:1615571947
|
||||
volume id:144 size:1074549720 collection:"collection1" file_count:8780 delete_count:50 deleted_byte_count:52475146 replica_placement:100 version:3 modified_at_second:1615573451
|
||||
volume id:161 size:1077397192 collection:"collection1" file_count:9988 delete_count:28 deleted_byte_count:12509164 replica_placement:100 version:3 modified_at_second:1615631452
|
||||
volume id:173 size:1074154704 collection:"collection1" file_count:30884 delete_count:34 deleted_byte_count:2578509 replica_placement:100 version:3 modified_at_second:1615591904
|
||||
volume id:174 size:1073824232 collection:"collection1" file_count:30689 delete_count:36 deleted_byte_count:2160116 replica_placement:100 version:3 modified_at_second:1615598914
|
||||
volume id:197 size:1075423240 collection:"collection1" file_count:16473 delete_count:15 deleted_byte_count:12552442 replica_placement:100 version:3 modified_at_second:1615485254
|
||||
volume id:219 size:1092298904 collection:"collection1" file_count:3193 delete_count:17 deleted_byte_count:2047576 replica_placement:100 version:3 modified_at_second:1615579316
|
||||
volume id:263 size:1077167352 collection:"collection2" file_count:20227 delete_count:4 deleted_byte_count:97887 replica_placement:100 version:3 modified_at_second:1614871567
|
||||
volume id:272 size:1076146040 collection:"collection2" file_count:21034 delete_count:2 deleted_byte_count:216564 replica_placement:100 version:3 modified_at_second:1614884139
|
||||
volume id:291 size:1076256760 collection:"collection2" file_count:28301 delete_count:5 deleted_byte_count:116027 replica_placement:100 version:3 modified_at_second:1614904924
|
||||
volume id:299 size:1075147824 collection:"collection2" file_count:22927 delete_count:4 deleted_byte_count:345569 replica_placement:100 version:3 modified_at_second:1614918454
|
||||
volume id:301 size:1074655600 collection:"collection2" file_count:22543 delete_count:6 deleted_byte_count:136968 replica_placement:100 version:3 modified_at_second:1614918378
|
||||
volume id:302 size:1077559792 collection:"collection2" file_count:23124 delete_count:7 deleted_byte_count:293111 replica_placement:100 version:3 modified_at_second:1614925500
|
||||
volume id:339 size:1078402392 collection:"collection2" file_count:22309 replica_placement:100 version:3 modified_at_second:1614969996
|
||||
volume id:345 size:1074560760 collection:"collection2" file_count:22117 delete_count:2 deleted_byte_count:373286 replica_placement:100 version:3 modified_at_second:1614977458
|
||||
volume id:355 size:1075239792 collection:"collection2" file_count:22244 delete_count:1 deleted_byte_count:23282 replica_placement:100 version:3 modified_at_second:1614992157
|
||||
volume id:373 size:1080928000 collection:"collection2" file_count:22617 delete_count:4 deleted_byte_count:91849 replica_placement:100 version:3 modified_at_second:1615016877
|
||||
Disk hdd total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253
|
||||
DataNode 192.168.1.4:8080 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253
|
||||
Rack rack1 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253
|
||||
Rack rack2 hdd(volume:36/190 active:36 free:154 remote:0)
|
||||
DataNode 192.168.1.2:8080 hdd(volume:36/190 active:36 free:154 remote:0)
|
||||
Disk hdd(volume:36/190 active:36 free:154 remote:0)
|
||||
volume id:2 size:289228560 file_count:1640 delete_count:4 deleted_byte_count:480564 replica_placement:100 version:3 compact_revision:6 modified_at_second:1615630622
|
||||
volume id:3 size:308743136 file_count:1638 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632242
|
||||
volume id:4 size:285986968 file_count:1641 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632302
|
||||
volume id:6 size:302411024 file_count:1604 delete_count:2 deleted_byte_count:274587 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631402
|
||||
volume id:7 size:1924728 collection:"collection4" file_count:15 replica_placement:100 version:3 modified_at_second:1609331040
|
||||
volume id:9 size:77337416 collection:"collection3" file_count:58 replica_placement:100 version:3 ttl:772 modified_at_second:1615513762
|
||||
volume id:10 size:1212784656 collection:"collection0" file_count:58 replica_placement:100 version:3 modified_at_second:1609814550
|
||||
volume id:12 size:1110923848 collection:"collection0" file_count:45 replica_placement:100 version:3 modified_at_second:1609819732
|
||||
volume id:13 size:1184910656 collection:"collection0" file_count:47 replica_placement:100 version:3 modified_at_second:1609827837
|
||||
volume id:14 size:1107475720 collection:"collection0" file_count:80 delete_count:3 deleted_byte_count:6870 replica_placement:100 version:3 modified_at_second:1612956980
|
||||
volume id:16 size:1113666104 collection:"collection0" file_count:73 delete_count:5 deleted_byte_count:6318 replica_placement:100 version:3 modified_at_second:1612957007
|
||||
volume id:17 size:1095115800 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:7099 replica_placement:100 version:3 modified_at_second:1612957000
|
||||
volume id:21 size:1097631664 collection:"collection0" file_count:82 delete_count:11 deleted_byte_count:68985100 replica_placement:100 version:3 modified_at_second:1612957007
|
||||
volume id:56 size:1001897616 collection:"collection1" file_count:8762 delete_count:37 deleted_byte_count:65375405 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632014
|
||||
volume id:81 size:880693104 collection:"collection1" file_count:7481 delete_count:236 deleted_byte_count:80386421 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631396
|
||||
volume id:104 size:1076383624 collection:"collection1" file_count:7663 delete_count:184 deleted_byte_count:100728071 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615602658
|
||||
volume id:107 size:1073811840 collection:"collection1" file_count:7436 delete_count:168 deleted_byte_count:57747484 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615293569
|
||||
volume id:113 size:1076709184 collection:"collection1" file_count:9355 delete_count:177 deleted_byte_count:59796765 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569822
|
||||
volume id:139 size:1074163936 collection:"collection1" file_count:9315 delete_count:42 deleted_byte_count:10630966 replica_placement:100 version:3 modified_at_second:1615571946
|
||||
volume id:151 size:1098659752 collection:"collection1" file_count:10808 delete_count:24 deleted_byte_count:7088102 replica_placement:100 version:3 modified_at_second:1615586389
|
||||
volume id:155 size:1075140688 collection:"collection1" file_count:10882 delete_count:32 deleted_byte_count:9076141 replica_placement:100 version:3 modified_at_second:1615606614
|
||||
volume id:167 size:1073958176 collection:"collection1" file_count:25229 delete_count:48 deleted_byte_count:25871565 replica_placement:100 version:3 modified_at_second:1615602669
|
||||
volume id:177 size:1074120216 collection:"collection1" file_count:22293 delete_count:16 deleted_byte_count:3803952 replica_placement:100 version:3 modified_at_second:1615516892
|
||||
volume id:179 size:1074313920 collection:"collection1" file_count:21829 delete_count:24 deleted_byte_count:45552859 replica_placement:100 version:3 modified_at_second:1615580308
|
||||
volume id:182 size:1076131280 collection:"collection1" file_count:31987 delete_count:21 deleted_byte_count:1452346 replica_placement:100 version:3 modified_at_second:1615568922
|
||||
volume id:215 size:1068268216 collection:"collection1" file_count:2813 delete_count:10 deleted_byte_count:5676795 replica_placement:100 version:3 modified_at_second:1615586386
|
||||
volume id:217 size:1075381872 collection:"collection1" file_count:3331 delete_count:14 deleted_byte_count:2009141 replica_placement:100 version:3 modified_at_second:1615401638
|
||||
volume id:283 size:1080178944 collection:"collection2" file_count:19462 delete_count:7 deleted_byte_count:660407 replica_placement:100 version:3 modified_at_second:1614896626
|
||||
volume id:303 size:1075944504 collection:"collection2" file_count:22541 delete_count:2 deleted_byte_count:13617 replica_placement:100 version:3 modified_at_second:1614925431
|
||||
volume id:309 size:1075178624 collection:"collection2" file_count:22692 delete_count:3 deleted_byte_count:171124 replica_placement:100 version:3 modified_at_second:1614931409
|
||||
volume id:323 size:1074608200 collection:"collection2" file_count:21605 delete_count:4 deleted_byte_count:172090 replica_placement:100 version:3 modified_at_second:1614950526
|
||||
volume id:344 size:1075035448 collection:"collection2" file_count:21765 delete_count:1 deleted_byte_count:24623 replica_placement:100 version:3 modified_at_second:1614977465
|
||||
volume id:347 size:1075145496 collection:"collection2" file_count:22178 delete_count:1 deleted_byte_count:79392 replica_placement:100 version:3 modified_at_second:1614984727
|
||||
volume id:357 size:1074276208 collection:"collection2" file_count:23137 delete_count:4 deleted_byte_count:188487 replica_placement:100 version:3 modified_at_second:1614998792
|
||||
volume id:380 size:1010760456 collection:"collection2" file_count:14921 delete_count:6 deleted_byte_count:65678 replica_placement:100 version:3 modified_at_second:1615632322
|
||||
volume id:381 size:939292792 collection:"collection2" file_count:14619 delete_count:2 deleted_byte_count:5119 replica_placement:100 version:3 modified_at_second:1615632324
|
||||
Disk hdd total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088
|
||||
DataNode 192.168.1.2:8080 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088
|
||||
Rack rack2 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088
|
||||
DataCenter dc2 total size:82098209920 file_count:907048 deleted_file:3671 deleted_bytes:1475897341
|
||||
DataCenter dc3 hdd(volume:108/1850 active:108 free:1742 remote:0)
|
||||
Rack rack3 hdd(volume:108/1850 active:108 free:1742 remote:0)
|
||||
DataNode 192.168.1.6:8080 hdd(volume:108/1850 active:108 free:1742 remote:0)
|
||||
Disk hdd(volume:108/1850 active:108 free:1742 remote:0)
|
||||
volume id:1 size:284685936 file_count:1557 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632062
|
||||
volume id:32 size:281390512 file_count:1496 delete_count:6 deleted_byte_count:546403 replica_placement:100 version:3 modified_at_second:1615632362
|
||||
volume id:47 size:444599784 collection:"collection1" file_count:709 delete_count:19 deleted_byte_count:11913451 replica_placement:100 version:3 modified_at_second:1615632397
|
||||
volume id:49 size:1078775288 collection:"collection1" file_count:9636 delete_count:22 deleted_byte_count:5625976 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630446
|
||||
volume id:68 size:898630584 collection:"collection1" file_count:6934 delete_count:95 deleted_byte_count:27460707 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284
|
||||
volume id:88 size:1073767976 collection:"collection1" file_count:14995 delete_count:206 deleted_byte_count:81222360 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615629897
|
||||
volume id:202 size:1077533160 collection:"collection1" file_count:2847 delete_count:67 deleted_byte_count:65172985 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615588497
|
||||
volume id:203 size:1027316272 collection:"collection1" file_count:3040 delete_count:11 deleted_byte_count:3993230 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631728
|
||||
volume id:205 size:1078485304 collection:"collection1" file_count:2869 delete_count:43 deleted_byte_count:18290259 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615579314
|
||||
volume id:206 size:1082045848 collection:"collection1" file_count:2979 delete_count:225 deleted_byte_count:88220074 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615564274
|
||||
volume id:209 size:1074083592 collection:"collection1" file_count:3238 delete_count:4 deleted_byte_count:1494244 replica_placement:100 version:3 modified_at_second:1615419954
|
||||
volume id:211 size:1080610712 collection:"collection1" file_count:3247 delete_count:7 deleted_byte_count:1891456 replica_placement:100 version:3 modified_at_second:1615269124
|
||||
volume id:212 size:1078293360 collection:"collection1" file_count:3106 delete_count:6 deleted_byte_count:2085755 replica_placement:100 version:3 modified_at_second:1615586387
|
||||
volume id:213 size:1093587976 collection:"collection1" file_count:3681 delete_count:12 deleted_byte_count:3138791 replica_placement:100 version:3 modified_at_second:1615586387
|
||||
volume id:214 size:1074486992 collection:"collection1" file_count:3217 delete_count:10 deleted_byte_count:6392871 replica_placement:100 version:3 modified_at_second:1615586383
|
||||
volume id:216 size:1080073496 collection:"collection1" file_count:3316 delete_count:4 deleted_byte_count:179819 replica_placement:100 version:3 modified_at_second:1615586387
|
||||
volume id:222 size:1106623104 collection:"collection1" file_count:3273 delete_count:11 deleted_byte_count:2114627 replica_placement:100 version:3 modified_at_second:1615586243
|
||||
volume id:223 size:1075233064 collection:"collection1" file_count:2966 delete_count:9 deleted_byte_count:744001 replica_placement:100 version:3 modified_at_second:1615586244
|
||||
volume id:227 size:1106699896 collection:"collection1" file_count:2827 delete_count:20 deleted_byte_count:5496790 replica_placement:100 version:3 modified_at_second:1615609989
|
||||
volume id:229 size:1109855312 collection:"collection1" file_count:2857 delete_count:22 deleted_byte_count:2839883 replica_placement:100 version:3 modified_at_second:1615609988
|
||||
volume id:230 size:1080722984 collection:"collection1" file_count:2898 delete_count:15 deleted_byte_count:3929261 replica_placement:100 version:3 modified_at_second:1615610537
|
||||
volume id:231 size:1112917696 collection:"collection1" file_count:3151 delete_count:20 deleted_byte_count:2989828 replica_placement:100 version:3 modified_at_second:1615611350
|
||||
volume id:233 size:1080526464 collection:"collection1" file_count:3136 delete_count:61 deleted_byte_count:17991717 replica_placement:100 version:3 modified_at_second:1615611352
|
||||
volume id:234 size:1073835280 collection:"collection1" file_count:2965 delete_count:41 deleted_byte_count:4960354 replica_placement:100 version:3 modified_at_second:1615611351
|
||||
volume id:235 size:1075586104 collection:"collection1" file_count:2767 delete_count:33 deleted_byte_count:3216540 replica_placement:100 version:3 modified_at_second:1615611354
|
||||
volume id:237 size:375722792 collection:"collection1" file_count:736 delete_count:16 deleted_byte_count:4464870 replica_placement:100 version:3 modified_at_second:1615631727
|
||||
volume id:239 size:426569024 collection:"collection1" file_count:693 delete_count:19 deleted_byte_count:13020783 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630838
|
||||
volume id:241 size:380217424 collection:"collection1" file_count:633 delete_count:6 deleted_byte_count:1715768 replica_placement:100 version:3 modified_at_second:1615632006
|
||||
volume id:244 size:1080295352 collection:"collection2" file_count:10812 delete_count:1 deleted_byte_count:795 replica_placement:100 version:3 modified_at_second:1614852171
|
||||
volume id:245 size:1074597056 collection:"collection2" file_count:10371 delete_count:3 deleted_byte_count:209701 replica_placement:100 version:3 modified_at_second:1614852093
|
||||
volume id:246 size:1075998648 collection:"collection2" file_count:10365 delete_count:1 deleted_byte_count:13112 replica_placement:100 version:3 modified_at_second:1614852105
|
||||
volume id:248 size:1084301184 collection:"collection2" file_count:11217 delete_count:4 deleted_byte_count:746488 replica_placement:100 version:3 modified_at_second:1614856285
|
||||
volume id:249 size:1074819136 collection:"collection2" file_count:10763 delete_count:2 deleted_byte_count:271699 replica_placement:100 version:3 modified_at_second:1614856230
|
||||
volume id:251 size:1075684488 collection:"collection2" file_count:10847 replica_placement:100 version:3 modified_at_second:1614856270
|
||||
volume id:252 size:1075065208 collection:"collection2" file_count:14622 delete_count:2 deleted_byte_count:5228 replica_placement:100 version:3 modified_at_second:1614861196
|
||||
volume id:253 size:1087328816 collection:"collection2" file_count:14920 delete_count:3 deleted_byte_count:522994 replica_placement:100 version:3 modified_at_second:1614861255
|
||||
volume id:255 size:1079581640 collection:"collection2" file_count:14877 delete_count:3 deleted_byte_count:101223 replica_placement:100 version:3 modified_at_second:1614861233
|
||||
volume id:256 size:1074283592 collection:"collection2" file_count:14157 delete_count:1 deleted_byte_count:18156 replica_placement:100 version:3 modified_at_second:1614861100
|
||||
volume id:258 size:1075527216 collection:"collection2" file_count:18421 delete_count:4 deleted_byte_count:267833 replica_placement:100 version:3 modified_at_second:1614866420
|
||||
volume id:259 size:1075507776 collection:"collection2" file_count:18079 delete_count:2 deleted_byte_count:71992 replica_placement:100 version:3 modified_at_second:1614866381
|
||||
volume id:264 size:1081624192 collection:"collection2" file_count:21151 replica_placement:100 version:3 modified_at_second:1614871629
|
||||
volume id:265 size:1076401104 collection:"collection2" file_count:19932 delete_count:2 deleted_byte_count:160823 replica_placement:100 version:3 modified_at_second:1615629130
|
||||
volume id:266 size:1075617464 collection:"collection2" file_count:20075 delete_count:1 deleted_byte_count:1039 replica_placement:100 version:3 modified_at_second:1614871526
|
||||
volume id:267 size:1075699544 collection:"collection2" file_count:21039 delete_count:3 deleted_byte_count:59956 replica_placement:100 version:3 modified_at_second:1614877294
|
||||
volume id:268 size:1074490592 collection:"collection2" file_count:21698 delete_count:1 deleted_byte_count:33968 replica_placement:100 version:3 modified_at_second:1614877434
|
||||
volume id:269 size:1077552872 collection:"collection2" file_count:21875 delete_count:4 deleted_byte_count:347272 replica_placement:100 version:3 modified_at_second:1614877481
|
||||
volume id:270 size:1076876568 collection:"collection2" file_count:22057 delete_count:1 deleted_byte_count:43916 replica_placement:100 version:3 modified_at_second:1614877469
|
||||
volume id:275 size:1078349024 collection:"collection2" file_count:20808 delete_count:1 deleted_byte_count:1118 replica_placement:100 version:3 modified_at_second:1614884147
|
||||
volume id:277 size:1074956288 collection:"collection2" file_count:19260 delete_count:2 deleted_byte_count:172356 replica_placement:100 version:3 modified_at_second:1614889988
|
||||
volume id:278 size:1078798640 collection:"collection2" file_count:20597 delete_count:5 deleted_byte_count:400060 replica_placement:100 version:3 modified_at_second:1614890292
|
||||
volume id:279 size:1077325040 collection:"collection2" file_count:19671 delete_count:6 deleted_byte_count:379116 replica_placement:100 version:3 modified_at_second:1614890229
|
||||
volume id:280 size:1077432216 collection:"collection2" file_count:20286 delete_count:1 deleted_byte_count:879 replica_placement:100 version:3 modified_at_second:1614890262
|
||||
volume id:281 size:1077581096 collection:"collection2" file_count:20206 delete_count:3 deleted_byte_count:143964 replica_placement:100 version:3 modified_at_second:1614890237
|
||||
volume id:284 size:1074533384 collection:"collection2" file_count:22196 delete_count:4 deleted_byte_count:154683 replica_placement:100 version:3 modified_at_second:1614897231
|
||||
volume id:285 size:1082128688 collection:"collection2" file_count:21804 delete_count:1 deleted_byte_count:1064 replica_placement:100 version:3 modified_at_second:1614897165
|
||||
volume id:289 size:1075284256 collection:"collection2" file_count:29342 delete_count:5 deleted_byte_count:100454 replica_placement:100 version:3 modified_at_second:1614904977
|
||||
volume id:290 size:1074723792 collection:"collection2" file_count:28340 delete_count:4 deleted_byte_count:199064 replica_placement:100 version:3 modified_at_second:1614904924
|
||||
volume id:291 size:1076256768 collection:"collection2" file_count:28301 delete_count:5 deleted_byte_count:116027 replica_placement:100 version:3 modified_at_second:1614904924
|
||||
volume id:293 size:1075409792 collection:"collection2" file_count:26063 delete_count:4 deleted_byte_count:183834 replica_placement:100 version:3 modified_at_second:1614912235
|
||||
volume id:294 size:1075444048 collection:"collection2" file_count:26076 delete_count:4 deleted_byte_count:194914 replica_placement:100 version:3 modified_at_second:1614912220
|
||||
volume id:296 size:1077824032 collection:"collection2" file_count:26741 delete_count:4 deleted_byte_count:199906 replica_placement:100 version:3 modified_at_second:1614912301
|
||||
volume id:297 size:1080229136 collection:"collection2" file_count:23409 delete_count:5 deleted_byte_count:46268 replica_placement:100 version:3 modified_at_second:1614918481
|
||||
volume id:298 size:1075410136 collection:"collection2" file_count:23222 delete_count:2 deleted_byte_count:46110 replica_placement:100 version:3 modified_at_second:1614918474
|
||||
volume id:299 size:1075147936 collection:"collection2" file_count:22927 delete_count:4 deleted_byte_count:345569 replica_placement:100 version:3 modified_at_second:1614918455
|
||||
volume id:300 size:1076212392 collection:"collection2" file_count:22892 delete_count:2 deleted_byte_count:61320 replica_placement:100 version:3 modified_at_second:1614918464
|
||||
volume id:301 size:1074655600 collection:"collection2" file_count:22543 delete_count:6 deleted_byte_count:136968 replica_placement:100 version:3 modified_at_second:1614918378
|
||||
volume id:303 size:1075944480 collection:"collection2" file_count:22541 delete_count:2 deleted_byte_count:13617 replica_placement:100 version:3 modified_at_second:1614925431
|
||||
volume id:306 size:1074764016 collection:"collection2" file_count:22939 replica_placement:100 version:3 modified_at_second:1614925462
|
||||
volume id:307 size:1076568000 collection:"collection2" file_count:23377 delete_count:2 deleted_byte_count:25453 replica_placement:100 version:3 modified_at_second:1614931448
|
||||
volume id:308 size:1074022392 collection:"collection2" file_count:23086 delete_count:2 deleted_byte_count:2127 replica_placement:100 version:3 modified_at_second:1614931401
|
||||
volume id:309 size:1075178664 collection:"collection2" file_count:22692 delete_count:3 deleted_byte_count:171124 replica_placement:100 version:3 modified_at_second:1614931409
|
||||
volume id:310 size:1074761528 collection:"collection2" file_count:21441 delete_count:3 deleted_byte_count:13934 replica_placement:100 version:3 modified_at_second:1614931077
|
||||
volume id:314 size:1074670840 collection:"collection2" file_count:20964 delete_count:4 deleted_byte_count:304291 replica_placement:100 version:3 modified_at_second:1614937441
|
||||
volume id:315 size:1084153544 collection:"collection2" file_count:23638 delete_count:2 deleted_byte_count:53956 replica_placement:100 version:3 modified_at_second:1614937885
|
||||
volume id:317 size:1076215096 collection:"collection2" file_count:23572 delete_count:2 deleted_byte_count:1441356 replica_placement:100 version:3 modified_at_second:1614943965
|
||||
volume id:318 size:1075965168 collection:"collection2" file_count:22459 delete_count:2 deleted_byte_count:37778 replica_placement:100 version:3 modified_at_second:1614943862
|
||||
volume id:319 size:1073952880 collection:"collection2" file_count:22286 delete_count:2 deleted_byte_count:43421 replica_placement:100 version:3 modified_at_second:1614943810
|
||||
volume id:320 size:1082437792 collection:"collection2" file_count:21544 delete_count:3 deleted_byte_count:16712 replica_placement:100 version:3 modified_at_second:1614943599
|
||||
volume id:321 size:1081477904 collection:"collection2" file_count:23531 delete_count:5 deleted_byte_count:262564 replica_placement:100 version:3 modified_at_second:1614943982
|
||||
volume id:324 size:1075606680 collection:"collection2" file_count:20799 delete_count:1 deleted_byte_count:251210 replica_placement:100 version:3 modified_at_second:1614950310
|
||||
volume id:325 size:1080701144 collection:"collection2" file_count:21735 replica_placement:100 version:3 modified_at_second:1614950525
|
||||
volume id:330 size:1080825832 collection:"collection2" file_count:22464 delete_count:2 deleted_byte_count:15771 replica_placement:100 version:3 modified_at_second:1614956477
|
||||
volume id:332 size:1075569928 collection:"collection2" file_count:22097 delete_count:3 deleted_byte_count:98273 replica_placement:100 version:3 modified_at_second:1614962869
|
||||
volume id:334 size:1075607880 collection:"collection2" file_count:22546 delete_count:6 deleted_byte_count:101538 replica_placement:100 version:3 modified_at_second:1614962978
|
||||
volume id:336 size:1087853056 collection:"collection2" file_count:22801 delete_count:2 deleted_byte_count:26394 replica_placement:100 version:3 modified_at_second:1614963005
|
||||
volume id:337 size:1075646784 collection:"collection2" file_count:21934 delete_count:1 deleted_byte_count:3397 replica_placement:100 version:3 modified_at_second:1614969937
|
||||
volume id:338 size:1076118304 collection:"collection2" file_count:21680 replica_placement:100 version:3 modified_at_second:1614969850
|
||||
volume id:340 size:1079462184 collection:"collection2" file_count:22319 delete_count:4 deleted_byte_count:93620 replica_placement:100 version:3 modified_at_second:1614969978
|
||||
volume id:341 size:1074448400 collection:"collection2" file_count:21590 delete_count:5 deleted_byte_count:160085 replica_placement:100 version:3 modified_at_second:1614969858
|
||||
volume id:342 size:1080186424 collection:"collection2" file_count:22405 delete_count:4 deleted_byte_count:64819 replica_placement:100 version:3 modified_at_second:1614977521
|
||||
volume id:344 size:1075035416 collection:"collection2" file_count:21765 delete_count:1 deleted_byte_count:24623 replica_placement:100 version:3 modified_at_second:1614977465
|
||||
volume id:345 size:1074560760 collection:"collection2" file_count:22117 delete_count:2 deleted_byte_count:373286 replica_placement:100 version:3 modified_at_second:1614977457
|
||||
volume id:346 size:1076464112 collection:"collection2" file_count:22320 delete_count:4 deleted_byte_count:798258 replica_placement:100 version:3 modified_at_second:1615631322
|
||||
volume id:348 size:1080623640 collection:"collection2" file_count:21667 delete_count:1 deleted_byte_count:2443 replica_placement:100 version:3 modified_at_second:1614984606
|
||||
volume id:350 size:1074756688 collection:"collection2" file_count:21990 delete_count:3 deleted_byte_count:233881 replica_placement:100 version:3 modified_at_second:1614984682
|
||||
volume id:351 size:1078795112 collection:"collection2" file_count:23660 delete_count:3 deleted_byte_count:102141 replica_placement:100 version:3 modified_at_second:1614984816
|
||||
volume id:352 size:1077145936 collection:"collection2" file_count:22066 delete_count:1 deleted_byte_count:1018 replica_placement:100 version:3 modified_at_second:1614992130
|
||||
volume id:353 size:1074897496 collection:"collection2" file_count:21266 delete_count:2 deleted_byte_count:3105374 replica_placement:100 version:3 modified_at_second:1614991951
|
||||
volume id:355 size:1075239728 collection:"collection2" file_count:22244 delete_count:1 deleted_byte_count:23282 replica_placement:100 version:3 modified_at_second:1614992157
|
||||
volume id:356 size:1083305048 collection:"collection2" file_count:21552 delete_count:4 deleted_byte_count:14472 replica_placement:100 version:3 modified_at_second:1614992028
|
||||
volume id:358 size:1085152368 collection:"collection2" file_count:23756 delete_count:3 deleted_byte_count:44531 replica_placement:100 version:3 modified_at_second:1614998824
|
||||
volume id:360 size:1075532456 collection:"collection2" file_count:22574 delete_count:3 deleted_byte_count:1774776 replica_placement:100 version:3 modified_at_second:1614998770
|
||||
volume id:361 size:1075362744 collection:"collection2" file_count:22272 delete_count:1 deleted_byte_count:3497 replica_placement:100 version:3 modified_at_second:1614998668
|
||||
volume id:375 size:1076140568 collection:"collection2" file_count:21880 delete_count:2 deleted_byte_count:51103 replica_placement:100 version:3 modified_at_second:1615016787
|
||||
volume id:376 size:1074845944 collection:"collection2" file_count:22908 delete_count:4 deleted_byte_count:432305 replica_placement:100 version:3 modified_at_second:1615016916
|
||||
volume id:377 size:957284144 collection:"collection2" file_count:14923 delete_count:1 deleted_byte_count:1797 replica_placement:100 version:3 modified_at_second:1615632323
|
||||
volume id:378 size:959273936 collection:"collection2" file_count:15027 delete_count:4 deleted_byte_count:231414 replica_placement:100 version:3 modified_at_second:1615632323
|
||||
volume id:381 size:939261032 collection:"collection2" file_count:14615 delete_count:5 deleted_byte_count:1192272 replica_placement:100 version:3 modified_at_second:1615632324
|
||||
Disk hdd total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585
|
||||
DataNode 192.168.1.6:8080 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585
|
||||
Rack rack3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585
|
||||
DataCenter dc3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585
|
||||
DataCenter dc4 hdd(volume:267/2000 active:267 free:1733 remote:0)
|
||||
Rack DefaultRack hdd(volume:267/2000 active:267 free:1733 remote:0)
|
||||
DataNode 192.168.1.1:8080 hdd(volume:267/2000 active:267 free:1733 remote:0)
|
||||
Disk hdd(volume:267/2000 active:267 free:1733 remote:0)
|
||||
volume id:1 size:284693256 file_count:1558 delete_count:2 deleted_byte_count:4818 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632062
|
||||
volume id:2 size:289228560 file_count:1640 delete_count:4 deleted_byte_count:464508 replica_placement:100 version:3 compact_revision:6 modified_at_second:1615630622
|
||||
volume id:3 size:308741952 file_count:1637 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632242
|
||||
volume id:4 size:285986968 file_count:1640 delete_count:1 deleted_byte_count:145095 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632302
|
||||
volume id:5 size:293806008 file_count:1669 delete_count:2 deleted_byte_count:274334 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631342
|
||||
volume id:6 size:302411024 file_count:1604 delete_count:2 deleted_byte_count:274587 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631402
|
||||
volume id:7 size:1924728 collection:"collection4" file_count:15 replica_placement:100 version:3 modified_at_second:1609331040
|
||||
volume id:9 size:77337416 collection:"collection3" file_count:58 replica_placement:100 version:3 ttl:772 modified_at_second:1615513762
|
||||
volume id:10 size:1212784656 collection:"collection0" file_count:58 replica_placement:100 version:3 modified_at_second:1609814543
|
||||
volume id:11 size:1109224552 collection:"collection0" file_count:44 replica_placement:100 version:3 modified_at_second:1609815123
|
||||
volume id:12 size:1110923848 collection:"collection0" file_count:45 replica_placement:100 version:3 modified_at_second:1609819726
|
||||
volume id:13 size:1184910656 collection:"collection0" file_count:47 replica_placement:100 version:3 modified_at_second:1609827832
|
||||
volume id:14 size:1107475720 collection:"collection0" file_count:80 delete_count:3 deleted_byte_count:6870 replica_placement:100 version:3 modified_at_second:1612956983
|
||||
volume id:15 size:1115965160 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:4956 replica_placement:100 version:3 modified_at_second:1612957001
|
||||
volume id:16 size:1113666048 collection:"collection0" file_count:73 delete_count:5 deleted_byte_count:6318 replica_placement:100 version:3 modified_at_second:1612957007
|
||||
volume id:17 size:1095115800 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:7099 replica_placement:100 version:3 modified_at_second:1612957000
|
||||
volume id:18 size:1096678688 collection:"collection0" file_count:88 delete_count:4 deleted_byte_count:8633 replica_placement:100 version:3 modified_at_second:1612957000
|
||||
volume id:19 size:1096923792 collection:"collection0" file_count:100 delete_count:10 deleted_byte_count:75779917 replica_placement:100 version:3 compact_revision:4 modified_at_second:1612957011
|
||||
volume id:20 size:1074760432 collection:"collection0" file_count:82 delete_count:5 deleted_byte_count:12156 replica_placement:100 version:3 compact_revision:2 modified_at_second:1612957011
|
||||
volume id:22 size:1086828368 collection:"collection0" file_count:75 delete_count:3 deleted_byte_count:5551 replica_placement:100 version:3 modified_at_second:1612957007
|
||||
volume id:23 size:1076380280 collection:"collection0" file_count:68 delete_count:2 deleted_byte_count:2910 replica_placement:100 version:3 modified_at_second:1612957011
|
||||
volume id:24 size:1074139808 collection:"collection0" file_count:90 delete_count:1 deleted_byte_count:1977 replica_placement:100 version:3 modified_at_second:1612957011
|
||||
volume id:25 size:690757544 collection:"collection0" file_count:38 delete_count:1 deleted_byte_count:1944 replica_placement:100 version:3 modified_at_second:1612956995
|
||||
volume id:26 size:532657632 collection:"collection0" file_count:100 delete_count:4 deleted_byte_count:9081 replica_placement:100 version:3 modified_at_second:1614170023
|
||||
volume id:34 size:1077111136 collection:"collection1" file_count:9781 delete_count:110 deleted_byte_count:20894827 replica_placement:100 version:3 modified_at_second:1615619366
|
||||
volume id:35 size:1075241656 collection:"collection1" file_count:10523 delete_count:96 deleted_byte_count:46618989 replica_placement:100 version:3 modified_at_second:1615618790
|
||||
volume id:36 size:1075118360 collection:"collection1" file_count:10342 delete_count:116 deleted_byte_count:25493106 replica_placement:100 version:3 modified_at_second:1615606148
|
||||
volume id:37 size:1075895584 collection:"collection1" file_count:12013 delete_count:98 deleted_byte_count:50747932 replica_placement:100 version:3 modified_at_second:1615594777
|
||||
volume id:39 size:1076606536 collection:"collection1" file_count:12612 delete_count:78 deleted_byte_count:17462730 replica_placement:100 version:3 modified_at_second:1615611959
|
||||
volume id:40 size:1075358552 collection:"collection1" file_count:12597 delete_count:62 deleted_byte_count:11657901 replica_placement:100 version:3 modified_at_second:1615612994
|
||||
volume id:41 size:1076283528 collection:"collection1" file_count:12088 delete_count:84 deleted_byte_count:19319268 replica_placement:100 version:3 modified_at_second:1615596736
|
||||
volume id:42 size:1093948352 collection:"collection1" file_count:7889 delete_count:47 deleted_byte_count:5697275 replica_placement:100 version:3 modified_at_second:1615548908
|
||||
volume id:43 size:1116445864 collection:"collection1" file_count:7358 delete_count:54 deleted_byte_count:9534379 replica_placement:100 version:3 modified_at_second:1615566170
|
||||
volume id:44 size:1077582560 collection:"collection1" file_count:7295 delete_count:50 deleted_byte_count:12618414 replica_placement:100 version:3 modified_at_second:1615566170
|
||||
volume id:45 size:1075254640 collection:"collection1" file_count:10772 delete_count:76 deleted_byte_count:22426345 replica_placement:100 version:3 modified_at_second:1615573499
|
||||
volume id:46 size:1075286056 collection:"collection1" file_count:9947 delete_count:309 deleted_byte_count:105601163 replica_placement:100 version:3 modified_at_second:1615569826
|
||||
volume id:48 size:1076778720 collection:"collection1" file_count:9850 delete_count:77 deleted_byte_count:16641907 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630690
|
||||
volume id:50 size:1076688224 collection:"collection1" file_count:7921 delete_count:26 deleted_byte_count:5162032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615610879
|
||||
volume id:52 size:1083529704 collection:"collection1" file_count:10128 delete_count:32 deleted_byte_count:10608391 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615599195
|
||||
volume id:53 size:1063089216 collection:"collection1" file_count:9832 delete_count:31 deleted_byte_count:9273066 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156
|
||||
volume id:55 size:1012890016 collection:"collection1" file_count:8651 delete_count:27 deleted_byte_count:9418841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631452
|
||||
volume id:57 size:839849792 collection:"collection1" file_count:7514 delete_count:24 deleted_byte_count:6228543 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631774
|
||||
volume id:58 size:908064200 collection:"collection1" file_count:8128 delete_count:21 deleted_byte_count:6113731 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632342
|
||||
volume id:59 size:988302272 collection:"collection1" file_count:8098 delete_count:20 deleted_byte_count:3947615 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632238
|
||||
volume id:60 size:1010702480 collection:"collection1" file_count:8969 delete_count:79 deleted_byte_count:24782814 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632439
|
||||
volume id:61 size:975604488 collection:"collection1" file_count:8683 delete_count:20 deleted_byte_count:10276072 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631176
|
||||
volume id:62 size:873845936 collection:"collection1" file_count:7897 delete_count:23 deleted_byte_count:10920170 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631133
|
||||
volume id:64 size:965638488 collection:"collection1" file_count:8218 delete_count:27 deleted_byte_count:6922489 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631031
|
||||
volume id:65 size:823283552 collection:"collection1" file_count:7834 delete_count:29 deleted_byte_count:5950610 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632306
|
||||
volume id:66 size:821343440 collection:"collection1" file_count:7383 delete_count:29 deleted_byte_count:12010343 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631968
|
||||
volume id:67 size:878713872 collection:"collection1" file_count:7299 delete_count:117 deleted_byte_count:24857326 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156
|
||||
volume id:68 size:898630584 collection:"collection1" file_count:6934 delete_count:95 deleted_byte_count:27460707 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284
|
||||
volume id:70 size:886695472 collection:"collection1" file_count:7769 delete_count:164 deleted_byte_count:45162513 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632398
|
||||
volume id:71 size:907608392 collection:"collection1" file_count:7658 delete_count:122 deleted_byte_count:27622941 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307
|
||||
volume id:72 size:903990720 collection:"collection1" file_count:6996 delete_count:240 deleted_byte_count:74147727 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630982
|
||||
volume id:73 size:929047664 collection:"collection1" file_count:7038 delete_count:227 deleted_byte_count:65336664 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630707
|
||||
volume id:74 size:957046128 collection:"collection1" file_count:6981 delete_count:259 deleted_byte_count:73080838 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631460
|
||||
volume id:75 size:908044992 collection:"collection1" file_count:6911 delete_count:268 deleted_byte_count:73934373 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632430
|
||||
volume id:76 size:985296744 collection:"collection1" file_count:6566 delete_count:61 deleted_byte_count:44464430 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284
|
||||
volume id:77 size:929398296 collection:"collection1" file_count:7427 delete_count:238 deleted_byte_count:59581579 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632013
|
||||
volume id:78 size:1075671512 collection:"collection1" file_count:7540 delete_count:258 deleted_byte_count:71726846 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615582829
|
||||
volume id:79 size:948225472 collection:"collection1" file_count:6997 delete_count:227 deleted_byte_count:60625763 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631326
|
||||
volume id:82 size:1041661800 collection:"collection1" file_count:7043 delete_count:207 deleted_byte_count:52275724 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632430
|
||||
volume id:83 size:936195856 collection:"collection1" file_count:7593 delete_count:13 deleted_byte_count:4633917 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632029
|
||||
volume id:85 size:1023867520 collection:"collection1" file_count:7787 delete_count:240 deleted_byte_count:82091742 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631723
|
||||
volume id:86 size:1009437488 collection:"collection1" file_count:8474 delete_count:236 deleted_byte_count:64543674 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812
|
||||
volume id:87 size:922276640 collection:"collection1" file_count:12902 delete_count:13 deleted_byte_count:3412959 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632438
|
||||
volume id:89 size:1044401976 collection:"collection1" file_count:14943 delete_count:243 deleted_byte_count:58543159 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632208
|
||||
volume id:90 size:891145784 collection:"collection1" file_count:14608 delete_count:10 deleted_byte_count:2564369 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615629390
|
||||
volume id:91 size:936572832 collection:"collection1" file_count:14686 delete_count:11 deleted_byte_count:4717727 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631851
|
||||
volume id:92 size:992440712 collection:"collection1" file_count:7061 delete_count:195 deleted_byte_count:60649573 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630566
|
||||
volume id:93 size:1079603768 collection:"collection1" file_count:7878 delete_count:270 deleted_byte_count:74150048 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615556015
|
||||
volume id:94 size:1030685824 collection:"collection1" file_count:7660 delete_count:207 deleted_byte_count:70150733 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631616
|
||||
volume id:95 size:990879168 collection:"collection1" file_count:6620 delete_count:206 deleted_byte_count:60363604 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631866
|
||||
volume id:96 size:989296136 collection:"collection1" file_count:7544 delete_count:229 deleted_byte_count:59931853 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630778
|
||||
volume id:97 size:1053112992 collection:"collection1" file_count:6789 delete_count:50 deleted_byte_count:38894001 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631194
|
||||
volume id:99 size:1071718504 collection:"collection1" file_count:7470 delete_count:8 deleted_byte_count:9624950 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631175
|
||||
volume id:100 size:1083617440 collection:"collection1" file_count:7018 delete_count:187 deleted_byte_count:61304236 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615505917
|
||||
volume id:101 size:1077109520 collection:"collection1" file_count:7706 delete_count:226 deleted_byte_count:77864841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630994
|
||||
volume id:102 size:1074359920 collection:"collection1" file_count:7338 delete_count:7 deleted_byte_count:6499151 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615626683
|
||||
volume id:103 size:1075863904 collection:"collection1" file_count:7184 delete_count:186 deleted_byte_count:58872238 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615628417
|
||||
volume id:104 size:1076383768 collection:"collection1" file_count:7663 delete_count:184 deleted_byte_count:100578087 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615602661
|
||||
volume id:105 size:1073996824 collection:"collection1" file_count:6873 delete_count:19 deleted_byte_count:14271533 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615499756
|
||||
volume id:108 size:1074648024 collection:"collection1" file_count:7472 delete_count:194 deleted_byte_count:70864699 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593232
|
||||
volume id:109 size:1075254560 collection:"collection1" file_count:7556 delete_count:263 deleted_byte_count:55155265 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615502487
|
||||
volume id:110 size:1076575744 collection:"collection1" file_count:6996 delete_count:163 deleted_byte_count:52954032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615590786
|
||||
volume id:111 size:1073826232 collection:"collection1" file_count:7355 delete_count:155 deleted_byte_count:50083578 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593233
|
||||
volume id:114 size:1074762784 collection:"collection1" file_count:8802 delete_count:156 deleted_byte_count:38470055 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615591826
|
||||
volume id:115 size:1076192240 collection:"collection1" file_count:7690 delete_count:154 deleted_byte_count:32267193 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615285295
|
||||
volume id:116 size:1074489504 collection:"collection1" file_count:9981 delete_count:174 deleted_byte_count:53998777 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615611567
|
||||
volume id:117 size:1073917192 collection:"collection1" file_count:9520 delete_count:114 deleted_byte_count:21835126 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573714
|
||||
volume id:118 size:1074064400 collection:"collection1" file_count:8738 delete_count:15 deleted_byte_count:3460697 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615516265
|
||||
volume id:119 size:1075940104 collection:"collection1" file_count:9003 delete_count:12 deleted_byte_count:9128155 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573880
|
||||
volume id:120 size:1076115928 collection:"collection1" file_count:9639 delete_count:118 deleted_byte_count:33357871 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615482567
|
||||
volume id:121 size:1078803248 collection:"collection1" file_count:10113 delete_count:441 deleted_byte_count:94128627 replica_placement:100 version:3 modified_at_second:1615506629
|
||||
volume id:122 size:1076235312 collection:"collection1" file_count:9106 delete_count:252 deleted_byte_count:93041272 replica_placement:100 version:3 modified_at_second:1615585913
|
||||
volume id:123 size:1080491112 collection:"collection1" file_count:10623 delete_count:302 deleted_byte_count:83956998 replica_placement:100 version:3 modified_at_second:1615585916
|
||||
volume id:124 size:1074519360 collection:"collection1" file_count:9457 delete_count:286 deleted_byte_count:74752459 replica_placement:100 version:3 modified_at_second:1615585916
|
||||
volume id:125 size:1088687040 collection:"collection1" file_count:9518 delete_count:281 deleted_byte_count:76037905 replica_placement:100 version:3 modified_at_second:1615585913
|
||||
volume id:126 size:1073867464 collection:"collection1" file_count:9320 delete_count:278 deleted_byte_count:94547424 replica_placement:100 version:3 modified_at_second:1615585912
|
||||
volume id:127 size:1074907336 collection:"collection1" file_count:9900 delete_count:133 deleted_byte_count:48570820 replica_placement:100 version:3 modified_at_second:1615612991
|
||||
volume id:129 size:1074704272 collection:"collection1" file_count:10012 delete_count:150 deleted_byte_count:64491721 replica_placement:100 version:3 modified_at_second:1615627566
|
||||
volume id:130 size:1075000632 collection:"collection1" file_count:10633 delete_count:161 deleted_byte_count:34768201 replica_placement:100 version:3 modified_at_second:1615582330
|
||||
volume id:131 size:1075279584 collection:"collection1" file_count:10075 delete_count:135 deleted_byte_count:29795712 replica_placement:100 version:3 modified_at_second:1615523898
|
||||
volume id:132 size:1088539496 collection:"collection1" file_count:11051 delete_count:71 deleted_byte_count:17178322 replica_placement:100 version:3 modified_at_second:1615619584
|
||||
volume id:133 size:1075952760 collection:"collection1" file_count:9538 delete_count:74 deleted_byte_count:19558008 replica_placement:100 version:3 modified_at_second:1615584780
|
||||
volume id:134 size:1074367304 collection:"collection1" file_count:10662 delete_count:69 deleted_byte_count:25530139 replica_placement:100 version:3 modified_at_second:1615555876
|
||||
volume id:135 size:1073906720 collection:"collection1" file_count:10446 delete_count:71 deleted_byte_count:28599975 replica_placement:100 version:3 modified_at_second:1615569816
|
||||
volume id:137 size:1074309264 collection:"collection1" file_count:9633 delete_count:50 deleted_byte_count:27487972 replica_placement:100 version:3 modified_at_second:1615572231
|
||||
volume id:139 size:1074163936 collection:"collection1" file_count:9314 delete_count:43 deleted_byte_count:10631353 replica_placement:100 version:3 modified_at_second:1615571946
|
||||
volume id:141 size:1074619488 collection:"collection1" file_count:9840 delete_count:45 deleted_byte_count:40890181 replica_placement:100 version:3 modified_at_second:1615630994
|
||||
volume id:142 size:1075732992 collection:"collection1" file_count:9009 delete_count:48 deleted_byte_count:9912854 replica_placement:100 version:3 modified_at_second:1615598914
|
||||
volume id:143 size:1075011280 collection:"collection1" file_count:9608 delete_count:51 deleted_byte_count:37282460 replica_placement:100 version:3 modified_at_second:1615488586
|
||||
volume id:145 size:1074394928 collection:"collection1" file_count:9255 delete_count:34 deleted_byte_count:38011392 replica_placement:100 version:3 modified_at_second:1615591825
|
||||
volume id:146 size:1076337520 collection:"collection1" file_count:10492 delete_count:50 deleted_byte_count:17071505 replica_placement:100 version:3 modified_at_second:1615632005
|
||||
volume id:147 size:1077130544 collection:"collection1" file_count:10451 delete_count:27 deleted_byte_count:8290907 replica_placement:100 version:3 modified_at_second:1615604117
|
||||
volume id:148 size:1076066568 collection:"collection1" file_count:9547 delete_count:33 deleted_byte_count:7034089 replica_placement:100 version:3 modified_at_second:1615586393
|
||||
volume id:149 size:1074989016 collection:"collection1" file_count:8352 delete_count:35 deleted_byte_count:7179742 replica_placement:100 version:3 modified_at_second:1615494496
|
||||
volume id:150 size:1076290408 collection:"collection1" file_count:9328 delete_count:33 deleted_byte_count:43417791 replica_placement:100 version:3 modified_at_second:1615611569
|
||||
volume id:151 size:1098659752 collection:"collection1" file_count:10805 delete_count:27 deleted_byte_count:7209106 replica_placement:100 version:3 modified_at_second:1615586390
|
||||
volume id:152 size:1075941376 collection:"collection1" file_count:9951 delete_count:36 deleted_byte_count:25348335 replica_placement:100 version:3 modified_at_second:1615606614
|
||||
volume id:153 size:1078539784 collection:"collection1" file_count:10924 delete_count:34 deleted_byte_count:12603081 replica_placement:100 version:3 modified_at_second:1615606614
|
||||
volume id:154 size:1081244752 collection:"collection1" file_count:11002 delete_count:31 deleted_byte_count:8467560 replica_placement:100 version:3 modified_at_second:1615478471
|
||||
volume id:156 size:1074975832 collection:"collection1" file_count:9535 delete_count:40 deleted_byte_count:11426621 replica_placement:100 version:3 modified_at_second:1615628342
|
||||
volume id:157 size:1076758536 collection:"collection1" file_count:10012 delete_count:19 deleted_byte_count:11688737 replica_placement:100 version:3 modified_at_second:1615597782
|
||||
volume id:158 size:1087251976 collection:"collection1" file_count:9972 delete_count:20 deleted_byte_count:10328429 replica_placement:100 version:3 modified_at_second:1615588879
|
||||
volume id:159 size:1074132336 collection:"collection1" file_count:9382 delete_count:27 deleted_byte_count:11474574 replica_placement:100 version:3 modified_at_second:1615593593
|
||||
volume id:160 size:1075680976 collection:"collection1" file_count:9772 delete_count:22 deleted_byte_count:4981968 replica_placement:100 version:3 modified_at_second:1615597782
|
||||
volume id:161 size:1077397136 collection:"collection1" file_count:9988 delete_count:28 deleted_byte_count:12509164 replica_placement:100 version:3 modified_at_second:1615631452
|
||||
volume id:162 size:1074286880 collection:"collection1" file_count:11220 delete_count:17 deleted_byte_count:1815547 replica_placement:100 version:3 modified_at_second:1615478127
|
||||
volume id:163 size:1074457224 collection:"collection1" file_count:12524 delete_count:27 deleted_byte_count:6359619 replica_placement:100 version:3 modified_at_second:1615579313
|
||||
volume id:164 size:1074261256 collection:"collection1" file_count:11922 delete_count:25 deleted_byte_count:2923288 replica_placement:100 version:3 modified_at_second:1615620085
|
||||
volume id:165 size:1073891080 collection:"collection1" file_count:9152 delete_count:12 deleted_byte_count:19164659 replica_placement:100 version:3 modified_at_second:1615471910
|
||||
volume id:166 size:1075637536 collection:"collection1" file_count:14211 delete_count:24 deleted_byte_count:20415490 replica_placement:100 version:3 modified_at_second:1615491021
|
||||
volume id:167 size:1073958280 collection:"collection1" file_count:25231 delete_count:48 deleted_byte_count:26022344 replica_placement:100 version:3 modified_at_second:1615620014
|
||||
volume id:168 size:1074718864 collection:"collection1" file_count:25702 delete_count:40 deleted_byte_count:4024775 replica_placement:100 version:3 modified_at_second:1615585664
|
||||
volume id:169 size:1073863032 collection:"collection1" file_count:25248 delete_count:43 deleted_byte_count:3013817 replica_placement:100 version:3 modified_at_second:1615569832
|
||||
volume id:170 size:1075747088 collection:"collection1" file_count:24596 delete_count:41 deleted_byte_count:3494711 replica_placement:100 version:3 modified_at_second:1615579207
|
||||
volume id:171 size:1081881400 collection:"collection1" file_count:24215 delete_count:36 deleted_byte_count:3191335 replica_placement:100 version:3 modified_at_second:1615596486
|
||||
volume id:172 size:1074787304 collection:"collection1" file_count:31236 delete_count:50 deleted_byte_count:3316482 replica_placement:100 version:3 modified_at_second:1615612385
|
||||
volume id:174 size:1073824160 collection:"collection1" file_count:30689 delete_count:36 deleted_byte_count:2160116 replica_placement:100 version:3 modified_at_second:1615598914
|
||||
volume id:175 size:1077742472 collection:"collection1" file_count:32353 delete_count:33 deleted_byte_count:1861403 replica_placement:100 version:3 modified_at_second:1615559516
|
||||
volume id:176 size:1073854800 collection:"collection1" file_count:30582 delete_count:34 deleted_byte_count:7701976 replica_placement:100 version:3 modified_at_second:1615626169
|
||||
volume id:178 size:1087560112 collection:"collection1" file_count:23482 delete_count:22 deleted_byte_count:18810492 replica_placement:100 version:3 modified_at_second:1615541369
|
||||
volume id:179 size:1074313920 collection:"collection1" file_count:21829 delete_count:24 deleted_byte_count:45574435 replica_placement:100 version:3 modified_at_second:1615580308
|
||||
volume id:180 size:1078438448 collection:"collection1" file_count:23614 delete_count:12 deleted_byte_count:4496474 replica_placement:100 version:3 modified_at_second:1614773243
|
||||
volume id:181 size:1074571672 collection:"collection1" file_count:22898 delete_count:19 deleted_byte_count:6628413 replica_placement:100 version:3 modified_at_second:1614745117
|
||||
volume id:183 size:1076361616 collection:"collection1" file_count:31293 delete_count:16 deleted_byte_count:468841 replica_placement:100 version:3 modified_at_second:1615572985
|
||||
volume id:184 size:1074594216 collection:"collection1" file_count:31368 delete_count:22 deleted_byte_count:857453 replica_placement:100 version:3 modified_at_second:1615586578
|
||||
volume id:185 size:1074099592 collection:"collection1" file_count:30612 delete_count:17 deleted_byte_count:2610847 replica_placement:100 version:3 modified_at_second:1615506835
|
||||
volume id:186 size:1074220664 collection:"collection1" file_count:31450 delete_count:15 deleted_byte_count:391855 replica_placement:100 version:3 modified_at_second:1615614934
|
||||
volume id:187 size:1074396112 collection:"collection1" file_count:31853 delete_count:17 deleted_byte_count:454283 replica_placement:100 version:3 modified_at_second:1615590491
|
||||
volume id:188 size:1074732632 collection:"collection1" file_count:31867 delete_count:19 deleted_byte_count:393743 replica_placement:100 version:3 modified_at_second:1615487645
|
||||
volume id:189 size:1074847824 collection:"collection1" file_count:31450 delete_count:16 deleted_byte_count:1040552 replica_placement:100 version:3 modified_at_second:1615335661
|
||||
volume id:190 size:1074008968 collection:"collection1" file_count:31987 delete_count:11 deleted_byte_count:685125 replica_placement:100 version:3 modified_at_second:1615447162
|
||||
volume id:191 size:1075492960 collection:"collection1" file_count:31301 delete_count:19 deleted_byte_count:708401 replica_placement:100 version:3 modified_at_second:1615357457
|
||||
volume id:192 size:1075857384 collection:"collection1" file_count:31490 delete_count:25 deleted_byte_count:720617 replica_placement:100 version:3 modified_at_second:1615621632
|
||||
volume id:193 size:1076616760 collection:"collection1" file_count:31907 delete_count:16 deleted_byte_count:464900 replica_placement:100 version:3 modified_at_second:1615507877
|
||||
volume id:194 size:1073985792 collection:"collection1" file_count:31434 delete_count:18 deleted_byte_count:391432 replica_placement:100 version:3 modified_at_second:1615559502
|
||||
volume id:195 size:1074158304 collection:"collection1" file_count:31453 delete_count:15 deleted_byte_count:718266 replica_placement:100 version:3 modified_at_second:1615559331
|
||||
volume id:196 size:1074594640 collection:"collection1" file_count:31665 delete_count:18 deleted_byte_count:3468922 replica_placement:100 version:3 modified_at_second:1615501690
|
||||
volume id:198 size:1075104624 collection:"collection1" file_count:16577 delete_count:18 deleted_byte_count:6583181 replica_placement:100 version:3 modified_at_second:1615623371
|
||||
volume id:199 size:1078117688 collection:"collection1" file_count:16497 delete_count:14 deleted_byte_count:1514286 replica_placement:100 version:3 modified_at_second:1615585987
|
||||
volume id:200 size:1075630464 collection:"collection1" file_count:16380 delete_count:18 deleted_byte_count:1103109 replica_placement:100 version:3 modified_at_second:1615485252
|
||||
volume id:201 size:1091460440 collection:"collection1" file_count:16684 delete_count:26 deleted_byte_count:5590335 replica_placement:100 version:3 modified_at_second:1615585987
|
||||
volume id:204 size:1079766904 collection:"collection1" file_count:3233 delete_count:255 deleted_byte_count:104707641 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615565702
|
||||
volume id:207 size:1081939960 collection:"collection1" file_count:3010 delete_count:4 deleted_byte_count:692350 replica_placement:100 version:3 modified_at_second:1615269061
|
||||
volume id:208 size:1077863624 collection:"collection1" file_count:3147 delete_count:6 deleted_byte_count:858726 replica_placement:100 version:3 modified_at_second:1615495515
|
||||
volume id:209 size:1074083592 collection:"collection1" file_count:3238 delete_count:4 deleted_byte_count:1494244 replica_placement:100 version:3 modified_at_second:1615419954
|
||||
volume id:210 size:1094311304 collection:"collection1" file_count:3468 delete_count:4 deleted_byte_count:466433 replica_placement:100 version:3 modified_at_second:1615495515
|
||||
volume id:211 size:1080610712 collection:"collection1" file_count:3247 delete_count:7 deleted_byte_count:1891456 replica_placement:100 version:3 modified_at_second:1615269124
|
||||
volume id:216 size:1080073496 collection:"collection1" file_count:3316 delete_count:4 deleted_byte_count:179819 replica_placement:100 version:3 modified_at_second:1615586387
|
||||
volume id:218 size:1081263944 collection:"collection1" file_count:3433 delete_count:14 deleted_byte_count:3454237 replica_placement:100 version:3 modified_at_second:1615603637
|
||||
volume id:220 size:1081928312 collection:"collection1" file_count:3166 delete_count:13 deleted_byte_count:4127709 replica_placement:100 version:3 modified_at_second:1615579317
|
||||
volume id:221 size:1106545536 collection:"collection1" file_count:3153 delete_count:11 deleted_byte_count:1496835 replica_placement:100 version:3 modified_at_second:1615269138
|
||||
volume id:224 size:1093691520 collection:"collection1" file_count:3463 delete_count:10 deleted_byte_count:1128328 replica_placement:100 version:3 modified_at_second:1615601870
|
||||
volume id:225 size:1080698928 collection:"collection1" file_count:3115 delete_count:7 deleted_byte_count:18170416 replica_placement:100 version:3 modified_at_second:1615434685
|
||||
volume id:226 size:1103504792 collection:"collection1" file_count:2965 delete_count:10 deleted_byte_count:2639254 replica_placement:100 version:3 modified_at_second:1615601870
|
||||
volume id:227 size:1106699864 collection:"collection1" file_count:2827 delete_count:19 deleted_byte_count:5393310 replica_placement:100 version:3 modified_at_second:1615609989
|
||||
volume id:228 size:1109784072 collection:"collection1" file_count:2504 delete_count:24 deleted_byte_count:5458950 replica_placement:100 version:3 modified_at_second:1615610489
|
||||
volume id:229 size:1109855256 collection:"collection1" file_count:2857 delete_count:22 deleted_byte_count:2839883 replica_placement:100 version:3 modified_at_second:1615609989
|
||||
volume id:231 size:1112917664 collection:"collection1" file_count:3151 delete_count:19 deleted_byte_count:2852517 replica_placement:100 version:3 modified_at_second:1615611350
|
||||
volume id:232 size:1073901520 collection:"collection1" file_count:3004 delete_count:54 deleted_byte_count:10273081 replica_placement:100 version:3 modified_at_second:1615611352
|
||||
volume id:233 size:1080526464 collection:"collection1" file_count:3136 delete_count:61 deleted_byte_count:17991717 replica_placement:100 version:3 modified_at_second:1615611354
|
||||
volume id:236 size:1089476200 collection:"collection1" file_count:3231 delete_count:53 deleted_byte_count:11625921 replica_placement:100 version:3 modified_at_second:1615611351
|
||||
volume id:238 size:354320000 collection:"collection1" file_count:701 delete_count:17 deleted_byte_count:5940420 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632030
|
||||
volume id:240 size:424791528 collection:"collection1" file_count:734 delete_count:12 deleted_byte_count:7353071 replica_placement:100 version:3 modified_at_second:1615631669
|
||||
volume id:242 size:1075383304 collection:"collection2" file_count:10470 replica_placement:100 version:3 modified_at_second:1614852115
|
||||
volume id:243 size:1088174560 collection:"collection2" file_count:11109 delete_count:1 deleted_byte_count:938 replica_placement:100 version:3 modified_at_second:1614852202
|
||||
volume id:245 size:1074597056 collection:"collection2" file_count:10371 delete_count:3 deleted_byte_count:209701 replica_placement:100 version:3 modified_at_second:1614852093
|
||||
volume id:247 size:1075859784 collection:"collection2" file_count:10443 delete_count:2 deleted_byte_count:564486 replica_placement:100 version:3 modified_at_second:1614856152
|
||||
volume id:249 size:1074819168 collection:"collection2" file_count:10763 delete_count:2 deleted_byte_count:271699 replica_placement:100 version:3 modified_at_second:1614856231
|
||||
volume id:250 size:1080572256 collection:"collection2" file_count:10220 replica_placement:100 version:3 modified_at_second:1614856129
|
||||
volume id:251 size:1075684408 collection:"collection2" file_count:10847 replica_placement:100 version:3 modified_at_second:1614856270
|
||||
volume id:254 size:1074830800 collection:"collection2" file_count:14140 delete_count:2 deleted_byte_count:105892 replica_placement:100 version:3 modified_at_second:1614861115
|
||||
volume id:257 size:1082621664 collection:"collection2" file_count:18172 delete_count:2 deleted_byte_count:25125 replica_placement:100 version:3 modified_at_second:1614866395
|
||||
volume id:260 size:1075105664 collection:"collection2" file_count:17316 delete_count:4 deleted_byte_count:2015310 replica_placement:100 version:3 modified_at_second:1614866226
|
||||
volume id:261 size:1076628592 collection:"collection2" file_count:18355 delete_count:1 deleted_byte_count:1155 replica_placement:100 version:3 modified_at_second:1614866420
|
||||
volume id:262 size:1078492464 collection:"collection2" file_count:20390 delete_count:3 deleted_byte_count:287601 replica_placement:100 version:3 modified_at_second:1614871601
|
||||
volume id:263 size:1077167440 collection:"collection2" file_count:20227 delete_count:4 deleted_byte_count:97887 replica_placement:100 version:3 modified_at_second:1614871567
|
||||
volume id:268 size:1074490592 collection:"collection2" file_count:21698 delete_count:1 deleted_byte_count:33968 replica_placement:100 version:3 modified_at_second:1614877435
|
||||
volume id:269 size:1077552720 collection:"collection2" file_count:21875 delete_count:4 deleted_byte_count:347272 replica_placement:100 version:3 modified_at_second:1614877481
|
||||
volume id:271 size:1076992648 collection:"collection2" file_count:22640 delete_count:1 deleted_byte_count:30645 replica_placement:100 version:3 modified_at_second:1614877504
|
||||
volume id:273 size:1074873432 collection:"collection2" file_count:20511 delete_count:3 deleted_byte_count:46076 replica_placement:100 version:3 modified_at_second:1614884046
|
||||
volume id:274 size:1075994128 collection:"collection2" file_count:20997 replica_placement:100 version:3 modified_at_second:1614884113
|
||||
volume id:276 size:1076899888 collection:"collection2" file_count:20190 delete_count:1 deleted_byte_count:8798 replica_placement:100 version:3 modified_at_second:1614884003
|
||||
volume id:277 size:1074956160 collection:"collection2" file_count:19260 delete_count:2 deleted_byte_count:172356 replica_placement:100 version:3 modified_at_second:1614889988
|
||||
volume id:279 size:1077325096 collection:"collection2" file_count:19671 delete_count:6 deleted_byte_count:379116 replica_placement:100 version:3 modified_at_second:1614890230
|
||||
volume id:282 size:1075232240 collection:"collection2" file_count:22659 delete_count:4 deleted_byte_count:67915 replica_placement:100 version:3 modified_at_second:1614897304
|
||||
volume id:284 size:1074533384 collection:"collection2" file_count:22196 delete_count:4 deleted_byte_count:154683 replica_placement:100 version:3 modified_at_second:1614897231
|
||||
volume id:285 size:1082128576 collection:"collection2" file_count:21804 delete_count:1 deleted_byte_count:1064 replica_placement:100 version:3 modified_at_second:1614897165
|
||||
volume id:286 size:1077464824 collection:"collection2" file_count:23905 delete_count:6 deleted_byte_count:630577 replica_placement:100 version:3 modified_at_second:1614897401
|
||||
volume id:287 size:1074590528 collection:"collection2" file_count:28163 delete_count:5 deleted_byte_count:35727 replica_placement:100 version:3 modified_at_second:1614904874
|
||||
volume id:288 size:1075406800 collection:"collection2" file_count:27243 delete_count:2 deleted_byte_count:51519 replica_placement:100 version:3 modified_at_second:1614904738
|
||||
volume id:292 size:1092010744 collection:"collection2" file_count:26781 delete_count:5 deleted_byte_count:508910 replica_placement:100 version:3 modified_at_second:1614912327
|
||||
volume id:293 size:1075409776 collection:"collection2" file_count:26063 delete_count:4 deleted_byte_count:183834 replica_placement:100 version:3 modified_at_second:1614912235
|
||||
volume id:294 size:1075443992 collection:"collection2" file_count:26076 delete_count:4 deleted_byte_count:194914 replica_placement:100 version:3 modified_at_second:1614912220
|
||||
volume id:295 size:1074702376 collection:"collection2" file_count:24488 delete_count:3 deleted_byte_count:48555 replica_placement:100 version:3 modified_at_second:1614911929
|
||||
volume id:300 size:1076212424 collection:"collection2" file_count:22892 delete_count:2 deleted_byte_count:61320 replica_placement:100 version:3 modified_at_second:1614918464
|
||||
volume id:304 size:1081038888 collection:"collection2" file_count:24505 delete_count:2 deleted_byte_count:124447 replica_placement:100 version:3 modified_at_second:1614925567
|
||||
volume id:305 size:1074185552 collection:"collection2" file_count:22074 delete_count:5 deleted_byte_count:20221 replica_placement:100 version:3 modified_at_second:1614925312
|
||||
volume id:310 size:1074761520 collection:"collection2" file_count:21441 delete_count:3 deleted_byte_count:13934 replica_placement:100 version:3 modified_at_second:1614931077
|
||||
volume id:311 size:1088248208 collection:"collection2" file_count:23553 delete_count:6 deleted_byte_count:191716 replica_placement:100 version:3 modified_at_second:1614931460
|
||||
volume id:312 size:1075037808 collection:"collection2" file_count:22524 replica_placement:100 version:3 modified_at_second:1614937832
|
||||
volume id:313 size:1074876016 collection:"collection2" file_count:22404 delete_count:4 deleted_byte_count:51728 replica_placement:100 version:3 modified_at_second:1614937755
|
||||
volume id:314 size:1074670840 collection:"collection2" file_count:20964 delete_count:4 deleted_byte_count:304291 replica_placement:100 version:3 modified_at_second:1614937441
|
||||
volume id:315 size:1084153456 collection:"collection2" file_count:23638 delete_count:2 deleted_byte_count:53956 replica_placement:100 version:3 modified_at_second:1614937884
|
||||
volume id:316 size:1077720784 collection:"collection2" file_count:22605 delete_count:1 deleted_byte_count:8503 replica_placement:100 version:3 modified_at_second:1614937838
|
||||
volume id:317 size:1076215040 collection:"collection2" file_count:23572 delete_count:2 deleted_byte_count:1441356 replica_placement:100 version:3 modified_at_second:1614943965
|
||||
volume id:319 size:1073952744 collection:"collection2" file_count:22286 delete_count:2 deleted_byte_count:43421 replica_placement:100 version:3 modified_at_second:1614943810
|
||||
volume id:320 size:1082437736 collection:"collection2" file_count:21544 delete_count:3 deleted_byte_count:16712 replica_placement:100 version:3 modified_at_second:1614943591
|
||||
volume id:321 size:1081477960 collection:"collection2" file_count:23531 delete_count:5 deleted_byte_count:262564 replica_placement:100 version:3 modified_at_second:1614943982
|
||||
volume id:322 size:1078471600 collection:"collection2" file_count:21905 delete_count:3 deleted_byte_count:145002 replica_placement:100 version:3 modified_at_second:1614950574
|
||||
volume id:324 size:1075606712 collection:"collection2" file_count:20799 delete_count:1 deleted_byte_count:251210 replica_placement:100 version:3 modified_at_second:1614950310
|
||||
volume id:326 size:1076059936 collection:"collection2" file_count:22564 delete_count:2 deleted_byte_count:192886 replica_placement:100 version:3 modified_at_second:1614950619
|
||||
volume id:327 size:1076121224 collection:"collection2" file_count:22007 delete_count:3 deleted_byte_count:60358 replica_placement:100 version:3 modified_at_second:1614956487
|
||||
volume id:328 size:1074767928 collection:"collection2" file_count:21720 delete_count:3 deleted_byte_count:56429 replica_placement:100 version:3 modified_at_second:1614956362
|
||||
volume id:329 size:1076691776 collection:"collection2" file_count:22411 delete_count:5 deleted_byte_count:214092 replica_placement:100 version:3 modified_at_second:1614956485
|
||||
volume id:331 size:1074957192 collection:"collection2" file_count:21230 delete_count:4 deleted_byte_count:62145 replica_placement:100 version:3 modified_at_second:1614956259
|
||||
volume id:333 size:1074270192 collection:"collection2" file_count:21271 delete_count:2 deleted_byte_count:168122 replica_placement:100 version:3 modified_at_second:1614962697
|
||||
volume id:335 size:1076235176 collection:"collection2" file_count:22391 delete_count:3 deleted_byte_count:8838 replica_placement:100 version:3 modified_at_second:1614962970
|
||||
volume id:336 size:1087853032 collection:"collection2" file_count:22801 delete_count:2 deleted_byte_count:26394 replica_placement:100 version:3 modified_at_second:1614963003
|
||||
volume id:338 size:1076118360 collection:"collection2" file_count:21680 replica_placement:100 version:3 modified_at_second:1614969850
|
||||
volume id:342 size:1080186296 collection:"collection2" file_count:22405 delete_count:4 deleted_byte_count:64819 replica_placement:100 version:3 modified_at_second:1614977518
|
||||
volume id:343 size:1075345184 collection:"collection2" file_count:21095 delete_count:2 deleted_byte_count:20581 replica_placement:100 version:3 modified_at_second:1614977148
|
||||
volume id:349 size:1075957824 collection:"collection2" file_count:22395 delete_count:2 deleted_byte_count:61565 replica_placement:100 version:3 modified_at_second:1614984748
|
||||
volume id:350 size:1074756688 collection:"collection2" file_count:21990 delete_count:3 deleted_byte_count:233881 replica_placement:100 version:3 modified_at_second:1614984682
|
||||
volume id:354 size:1085213992 collection:"collection2" file_count:23150 delete_count:4 deleted_byte_count:82391 replica_placement:100 version:3 modified_at_second:1614992207
|
||||
volume id:356 size:1083304992 collection:"collection2" file_count:21552 delete_count:4 deleted_byte_count:14472 replica_placement:100 version:3 modified_at_second:1614992027
|
||||
volume id:358 size:1085152312 collection:"collection2" file_count:23756 delete_count:3 deleted_byte_count:44531 replica_placement:100 version:3 modified_at_second:1614998824
|
||||
volume id:359 size:1074211240 collection:"collection2" file_count:22437 delete_count:2 deleted_byte_count:187953 replica_placement:100 version:3 modified_at_second:1614998711
|
||||
volume id:362 size:1074074120 collection:"collection2" file_count:20595 delete_count:1 deleted_byte_count:112145 replica_placement:100 version:3 modified_at_second:1615004407
|
||||
volume id:363 size:1078859496 collection:"collection2" file_count:23177 delete_count:4 deleted_byte_count:9601 replica_placement:100 version:3 modified_at_second:1615004822
|
||||
volume id:364 size:1081280816 collection:"collection2" file_count:22686 delete_count:1 deleted_byte_count:84375 replica_placement:100 version:3 modified_at_second:1615004813
|
||||
volume id:365 size:1075736632 collection:"collection2" file_count:22193 delete_count:5 deleted_byte_count:259033 replica_placement:100 version:3 modified_at_second:1615004776
|
||||
volume id:366 size:1075267272 collection:"collection2" file_count:21856 delete_count:5 deleted_byte_count:138363 replica_placement:100 version:3 modified_at_second:1615004703
|
||||
volume id:367 size:1076403648 collection:"collection2" file_count:22995 delete_count:2 deleted_byte_count:36955 replica_placement:100 version:3 modified_at_second:1615010985
|
||||
volume id:368 size:1074822016 collection:"collection2" file_count:22252 delete_count:4 deleted_byte_count:3291946 replica_placement:100 version:3 modified_at_second:1615010878
|
||||
volume id:369 size:1091472040 collection:"collection2" file_count:23709 delete_count:4 deleted_byte_count:400876 replica_placement:100 version:3 modified_at_second:1615011019
|
||||
volume id:370 size:1076040480 collection:"collection2" file_count:22092 delete_count:2 deleted_byte_count:115388 replica_placement:100 version:3 modified_at_second:1615010877
|
||||
volume id:371 size:1078806160 collection:"collection2" file_count:22685 delete_count:2 deleted_byte_count:68905 replica_placement:100 version:3 modified_at_second:1615010994
|
||||
volume id:372 size:1076193312 collection:"collection2" file_count:22774 delete_count:1 deleted_byte_count:3495 replica_placement:100 version:3 modified_at_second:1615016911
|
||||
volume id:374 size:1085011080 collection:"collection2" file_count:23054 delete_count:2 deleted_byte_count:89034 replica_placement:100 version:3 modified_at_second:1615016917
|
||||
volume id:375 size:1076140688 collection:"collection2" file_count:21880 delete_count:2 deleted_byte_count:51103 replica_placement:100 version:3 modified_at_second:1615016787
|
||||
volume id:378 size:959273824 collection:"collection2" file_count:15031 replica_placement:100 version:3 modified_at_second:1615632323
|
||||
volume id:379 size:1014108592 collection:"collection2" file_count:15360 delete_count:8 deleted_byte_count:2524591 replica_placement:100 version:3 modified_at_second:1615632323
|
||||
volume id:380 size:1010760464 collection:"collection2" file_count:14920 delete_count:7 deleted_byte_count:134859 replica_placement:100 version:3 modified_at_second:1615632322
|
||||
Disk hdd total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457
|
||||
DataNode 192.168.1.1:8080 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457
|
||||
Rack DefaultRack total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457
|
||||
DataCenter dc4 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457
|
||||
DataCenter dc5 hdd(volume:299/3000 active:299 free:2701 remote:0)
|
||||
Rack DefaultRack hdd(volume:299/3000 active:299 free:2701 remote:0)
|
||||
DataNode 192.168.1.5:8080 hdd(volume:299/3000 active:299 free:2701 remote:0)
|
||||
Disk hdd(volume:299/3000 active:299 free:2701 remote:0)
|
||||
volume id:5 size:293806008 file_count:1669 delete_count:2 deleted_byte_count:274334 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631342
|
||||
volume id:11 size:1109224552 collection:"collection0" file_count:44 replica_placement:100 version:3 modified_at_second:1615629606
|
||||
volume id:18 size:1096678688 collection:"collection0" file_count:88 delete_count:4 deleted_byte_count:8633 replica_placement:100 version:3 modified_at_second:1615631673
|
||||
volume id:19 size:1096923792 collection:"collection0" file_count:100 delete_count:10 deleted_byte_count:75779917 replica_placement:100 version:3 compact_revision:4 modified_at_second:1615630117
|
||||
volume id:20 size:1074760432 collection:"collection0" file_count:82 delete_count:5 deleted_byte_count:12156 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615629340
|
||||
volume id:26 size:532657632 collection:"collection0" file_count:100 delete_count:4 deleted_byte_count:9081 replica_placement:100 version:3 modified_at_second:1614170024
|
||||
volume id:27 size:298886792 file_count:1608 replica_placement:100 version:3 modified_at_second:1615632482
|
||||
volume id:28 size:308919192 file_count:1591 delete_count:1 deleted_byte_count:125280 replica_placement:100 version:3 modified_at_second:1615631762
|
||||
volume id:29 size:281582688 file_count:1537 replica_placement:100 version:3 modified_at_second:1615629422
|
||||
volume id:30 size:289466144 file_count:1566 delete_count:1 deleted_byte_count:124972 replica_placement:100 version:3 modified_at_second:1615632422
|
||||
volume id:31 size:273363256 file_count:1498 replica_placement:100 version:3 modified_at_second:1615631642
|
||||
volume id:32 size:281343360 file_count:1497 replica_placement:100 version:3 modified_at_second:1615632362
|
||||
volume id:33 size:1130226344 collection:"collection1" file_count:7322 delete_count:172 deleted_byte_count:45199399 replica_placement:100 version:3 modified_at_second:1615618789
|
||||
volume id:34 size:1077111136 collection:"collection1" file_count:9781 delete_count:110 deleted_byte_count:20894827 replica_placement:100 version:3 modified_at_second:1615619366
|
||||
volume id:35 size:1075241744 collection:"collection1" file_count:10523 delete_count:97 deleted_byte_count:46586217 replica_placement:100 version:3 modified_at_second:1615618790
|
||||
volume id:36 size:1075118336 collection:"collection1" file_count:10341 delete_count:118 deleted_byte_count:24753278 replica_placement:100 version:3 modified_at_second:1615606148
|
||||
volume id:37 size:1075895576 collection:"collection1" file_count:12013 delete_count:98 deleted_byte_count:50747932 replica_placement:100 version:3 modified_at_second:1615594776
|
||||
volume id:38 size:1075545744 collection:"collection1" file_count:13324 delete_count:100 deleted_byte_count:25223906 replica_placement:100 version:3 modified_at_second:1615569830
|
||||
volume id:39 size:1076606536 collection:"collection1" file_count:12612 delete_count:78 deleted_byte_count:17462730 replica_placement:100 version:3 modified_at_second:1615611959
|
||||
volume id:40 size:1075358552 collection:"collection1" file_count:12597 delete_count:62 deleted_byte_count:11657901 replica_placement:100 version:3 modified_at_second:1615612994
|
||||
volume id:41 size:1076283592 collection:"collection1" file_count:12088 delete_count:84 deleted_byte_count:19311237 replica_placement:100 version:3 modified_at_second:1615596736
|
||||
volume id:42 size:1093948352 collection:"collection1" file_count:7889 delete_count:47 deleted_byte_count:5697275 replica_placement:100 version:3 modified_at_second:1615548906
|
||||
volume id:43 size:1116445864 collection:"collection1" file_count:7355 delete_count:57 deleted_byte_count:9727158 replica_placement:100 version:3 modified_at_second:1615566167
|
||||
volume id:44 size:1077582560 collection:"collection1" file_count:7295 delete_count:50 deleted_byte_count:12618414 replica_placement:100 version:3 modified_at_second:1615566170
|
||||
volume id:45 size:1075254640 collection:"collection1" file_count:10772 delete_count:76 deleted_byte_count:22426345 replica_placement:100 version:3 modified_at_second:1615573498
|
||||
volume id:46 size:1075286056 collection:"collection1" file_count:9947 delete_count:309 deleted_byte_count:105601163 replica_placement:100 version:3 modified_at_second:1615569825
|
||||
volume id:47 size:444599784 collection:"collection1" file_count:709 delete_count:19 deleted_byte_count:11913451 replica_placement:100 version:3 modified_at_second:1615632397
|
||||
volume id:48 size:1076778664 collection:"collection1" file_count:9850 delete_count:77 deleted_byte_count:16641907 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630690
|
||||
volume id:49 size:1078775288 collection:"collection1" file_count:9631 delete_count:27 deleted_byte_count:5985628 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615575823
|
||||
volume id:50 size:1076688288 collection:"collection1" file_count:7921 delete_count:26 deleted_byte_count:5162032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615610876
|
||||
volume id:51 size:1076796120 collection:"collection1" file_count:10550 delete_count:39 deleted_byte_count:12723654 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615547786
|
||||
volume id:53 size:1063089216 collection:"collection1" file_count:9832 delete_count:31 deleted_byte_count:9273066 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156
|
||||
volume id:54 size:1045022288 collection:"collection1" file_count:9409 delete_count:29 deleted_byte_count:15102818 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630813
|
||||
volume id:55 size:1012890016 collection:"collection1" file_count:8651 delete_count:27 deleted_byte_count:9418841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631453
|
||||
volume id:56 size:1002412240 collection:"collection1" file_count:8762 delete_count:40 deleted_byte_count:65885831 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632014
|
||||
volume id:57 size:839849792 collection:"collection1" file_count:7514 delete_count:24 deleted_byte_count:6228543 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631775
|
||||
volume id:58 size:908064192 collection:"collection1" file_count:8128 delete_count:21 deleted_byte_count:6113731 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632343
|
||||
volume id:59 size:988302272 collection:"collection1" file_count:8098 delete_count:20 deleted_byte_count:3947615 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632238
|
||||
volume id:60 size:1010702480 collection:"collection1" file_count:8969 delete_count:79 deleted_byte_count:24782814 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632439
|
||||
volume id:61 size:975604544 collection:"collection1" file_count:8683 delete_count:20 deleted_byte_count:10276072 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631176
|
||||
volume id:62 size:873845904 collection:"collection1" file_count:7897 delete_count:23 deleted_byte_count:10920170 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631133
|
||||
volume id:63 size:956941176 collection:"collection1" file_count:8271 delete_count:32 deleted_byte_count:15876189 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632036
|
||||
volume id:64 size:965638424 collection:"collection1" file_count:8218 delete_count:27 deleted_byte_count:6922489 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631032
|
||||
volume id:65 size:823283608 collection:"collection1" file_count:7834 delete_count:29 deleted_byte_count:5950610 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307
|
||||
volume id:66 size:821343440 collection:"collection1" file_count:7383 delete_count:29 deleted_byte_count:12010343 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631968
|
||||
volume id:67 size:878713880 collection:"collection1" file_count:7299 delete_count:117 deleted_byte_count:24857326 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156
|
||||
volume id:69 size:863913896 collection:"collection1" file_count:7291 delete_count:100 deleted_byte_count:25335024 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630534
|
||||
volume id:70 size:886695472 collection:"collection1" file_count:7769 delete_count:164 deleted_byte_count:45162513 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632398
|
||||
volume id:71 size:907608392 collection:"collection1" file_count:7658 delete_count:122 deleted_byte_count:27622941 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307
|
||||
volume id:72 size:903990720 collection:"collection1" file_count:6996 delete_count:240 deleted_byte_count:74147727 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630985
|
||||
volume id:73 size:929047544 collection:"collection1" file_count:7038 delete_count:227 deleted_byte_count:65336664 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630707
|
||||
volume id:75 size:908045000 collection:"collection1" file_count:6911 delete_count:268 deleted_byte_count:73934373 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632430
|
||||
volume id:76 size:985296744 collection:"collection1" file_count:6566 delete_count:61 deleted_byte_count:44464430 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284
|
||||
volume id:77 size:929398296 collection:"collection1" file_count:7427 delete_count:238 deleted_byte_count:59581579 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632014
|
||||
volume id:78 size:1075671512 collection:"collection1" file_count:7540 delete_count:258 deleted_byte_count:71726846 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615582829
|
||||
volume id:79 size:948225472 collection:"collection1" file_count:6997 delete_count:227 deleted_byte_count:60625763 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631326
|
||||
volume id:80 size:827912928 collection:"collection1" file_count:6916 delete_count:15 deleted_byte_count:5611604 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631159
|
||||
volume id:81 size:880693168 collection:"collection1" file_count:7481 delete_count:238 deleted_byte_count:80880878 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631395
|
||||
volume id:82 size:1041660512 collection:"collection1" file_count:7043 delete_count:207 deleted_byte_count:52275724 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632430
|
||||
volume id:83 size:936194288 collection:"collection1" file_count:7593 delete_count:13 deleted_byte_count:4633917 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632029
|
||||
volume id:84 size:871262320 collection:"collection1" file_count:8190 delete_count:14 deleted_byte_count:3150948 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631161
|
||||
volume id:86 size:1009434632 collection:"collection1" file_count:8474 delete_count:236 deleted_byte_count:64543674 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812
|
||||
volume id:87 size:922274624 collection:"collection1" file_count:12902 delete_count:13 deleted_byte_count:3412959 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632438
|
||||
volume id:88 size:1073767976 collection:"collection1" file_count:14994 delete_count:207 deleted_byte_count:82380696 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615541383
|
||||
volume id:89 size:1044421824 collection:"collection1" file_count:14943 delete_count:243 deleted_byte_count:58543159 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632208
|
||||
volume id:90 size:891163760 collection:"collection1" file_count:14608 delete_count:10 deleted_byte_count:2564369 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615629392
|
||||
volume id:91 size:936573952 collection:"collection1" file_count:14686 delete_count:11 deleted_byte_count:4717727 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631851
|
||||
volume id:92 size:992439144 collection:"collection1" file_count:7061 delete_count:195 deleted_byte_count:60649573 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630566
|
||||
volume id:93 size:1079602592 collection:"collection1" file_count:7878 delete_count:270 deleted_byte_count:74150048 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615556013
|
||||
volume id:94 size:1030684704 collection:"collection1" file_count:7660 delete_count:207 deleted_byte_count:70150733 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631616
|
||||
volume id:95 size:990877824 collection:"collection1" file_count:6620 delete_count:206 deleted_byte_count:60363604 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631867
|
||||
volume id:96 size:989294848 collection:"collection1" file_count:7544 delete_count:229 deleted_byte_count:59931853 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630778
|
||||
volume id:98 size:1077836472 collection:"collection1" file_count:7605 delete_count:202 deleted_byte_count:73180379 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615523691
|
||||
volume id:99 size:1071718496 collection:"collection1" file_count:7470 delete_count:8 deleted_byte_count:9624950 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631175
|
||||
volume id:100 size:1083617472 collection:"collection1" file_count:7018 delete_count:187 deleted_byte_count:61304236 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615505914
|
||||
volume id:101 size:1077109408 collection:"collection1" file_count:7706 delete_count:226 deleted_byte_count:77864780 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630994
|
||||
volume id:102 size:1074359920 collection:"collection1" file_count:7338 delete_count:7 deleted_byte_count:6499151 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615626682
|
||||
volume id:103 size:1075863904 collection:"collection1" file_count:7184 delete_count:186 deleted_byte_count:58872238 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615628417
|
||||
volume id:106 size:1075458680 collection:"collection1" file_count:7182 delete_count:307 deleted_byte_count:69349053 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615598137
|
||||
volume id:107 size:1073811776 collection:"collection1" file_count:7436 delete_count:168 deleted_byte_count:57747428 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615293569
|
||||
volume id:108 size:1074648024 collection:"collection1" file_count:7472 delete_count:194 deleted_byte_count:70864699 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593231
|
||||
volume id:109 size:1075254560 collection:"collection1" file_count:7556 delete_count:263 deleted_byte_count:55155265 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615502487
|
||||
volume id:110 size:1076575744 collection:"collection1" file_count:6996 delete_count:163 deleted_byte_count:52954032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615590786
|
||||
volume id:111 size:1073826176 collection:"collection1" file_count:7355 delete_count:155 deleted_byte_count:50083578 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593232
|
||||
volume id:112 size:1076392512 collection:"collection1" file_count:8291 delete_count:156 deleted_byte_count:74120183 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569823
|
||||
volume id:113 size:1076709184 collection:"collection1" file_count:9355 delete_count:177 deleted_byte_count:59796765 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569822
|
||||
volume id:114 size:1074762792 collection:"collection1" file_count:8802 delete_count:156 deleted_byte_count:38470055 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615591826
|
||||
volume id:115 size:1076192296 collection:"collection1" file_count:7690 delete_count:154 deleted_byte_count:32267193 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615285296
|
||||
volume id:117 size:1073917192 collection:"collection1" file_count:9520 delete_count:114 deleted_byte_count:21835126 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573712
|
||||
volume id:118 size:1074064344 collection:"collection1" file_count:8738 delete_count:15 deleted_byte_count:3460697 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615516264
|
||||
volume id:120 size:1076115928 collection:"collection1" file_count:9639 delete_count:118 deleted_byte_count:33357871 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615482567
|
||||
volume id:121 size:1078803320 collection:"collection1" file_count:10113 delete_count:441 deleted_byte_count:94128627 replica_placement:100 version:3 modified_at_second:1615506626
|
||||
volume id:122 size:1076235312 collection:"collection1" file_count:9106 delete_count:252 deleted_byte_count:93041272 replica_placement:100 version:3 modified_at_second:1615585912
|
||||
volume id:123 size:1080491112 collection:"collection1" file_count:10623 delete_count:302 deleted_byte_count:83956998 replica_placement:100 version:3 modified_at_second:1615585916
|
||||
volume id:124 size:1074519360 collection:"collection1" file_count:9457 delete_count:286 deleted_byte_count:74752459 replica_placement:100 version:3 modified_at_second:1615585913
|
||||
volume id:125 size:1088687040 collection:"collection1" file_count:9518 delete_count:281 deleted_byte_count:76037905 replica_placement:100 version:3 modified_at_second:1615585913
|
||||
volume id:126 size:1073867408 collection:"collection1" file_count:9320 delete_count:278 deleted_byte_count:94547424 replica_placement:100 version:3 modified_at_second:1615585911
|
||||
volume id:127 size:1074907336 collection:"collection1" file_count:9900 delete_count:133 deleted_byte_count:48570820 replica_placement:100 version:3 modified_at_second:1615612990
|
||||
volume id:128 size:1074874632 collection:"collection1" file_count:9821 delete_count:148 deleted_byte_count:43633334 replica_placement:100 version:3 modified_at_second:1615602670
|
||||
volume id:129 size:1074704328 collection:"collection1" file_count:10012 delete_count:150 deleted_byte_count:64491721 replica_placement:100 version:3 modified_at_second:1615627566
|
||||
volume id:130 size:1075000632 collection:"collection1" file_count:10633 delete_count:161 deleted_byte_count:34768201 replica_placement:100 version:3 modified_at_second:1615582327
|
||||
volume id:131 size:1075279584 collection:"collection1" file_count:10075 delete_count:135 deleted_byte_count:29795712 replica_placement:100 version:3 modified_at_second:1615523898
|
||||
volume id:132 size:1088539552 collection:"collection1" file_count:11051 delete_count:71 deleted_byte_count:17178322 replica_placement:100 version:3 modified_at_second:1615619581
|
||||
volume id:134 size:1074367304 collection:"collection1" file_count:10662 delete_count:69 deleted_byte_count:25530139 replica_placement:100 version:3 modified_at_second:1615555873
|
||||
volume id:135 size:1073906776 collection:"collection1" file_count:10446 delete_count:71 deleted_byte_count:28599975 replica_placement:100 version:3 modified_at_second:1615569816
|
||||
volume id:136 size:1074433552 collection:"collection1" file_count:9593 delete_count:72 deleted_byte_count:26912512 replica_placement:100 version:3 modified_at_second:1615376036
|
||||
volume id:137 size:1074309264 collection:"collection1" file_count:9633 delete_count:50 deleted_byte_count:27487972 replica_placement:100 version:3 modified_at_second:1615572231
|
||||
volume id:138 size:1074465744 collection:"collection1" file_count:10120 delete_count:55 deleted_byte_count:15875438 replica_placement:100 version:3 modified_at_second:1615572231
|
||||
volume id:140 size:1076203744 collection:"collection1" file_count:11219 delete_count:57 deleted_byte_count:19864498 replica_placement:100 version:3 modified_at_second:1615571947
|
||||
volume id:141 size:1074619488 collection:"collection1" file_count:9840 delete_count:45 deleted_byte_count:40890181 replica_placement:100 version:3 modified_at_second:1615630994
|
||||
volume id:142 size:1075733064 collection:"collection1" file_count:9009 delete_count:48 deleted_byte_count:9912854 replica_placement:100 version:3 modified_at_second:1615598913
|
||||
volume id:143 size:1075011280 collection:"collection1" file_count:9608 delete_count:51 deleted_byte_count:37282460 replica_placement:100 version:3 modified_at_second:1615488584
|
||||
volume id:144 size:1074549720 collection:"collection1" file_count:8780 delete_count:50 deleted_byte_count:52475146 replica_placement:100 version:3 modified_at_second:1615573451
|
||||
volume id:145 size:1074394928 collection:"collection1" file_count:9255 delete_count:34 deleted_byte_count:38011392 replica_placement:100 version:3 modified_at_second:1615591825
|
||||
volume id:146 size:1076337576 collection:"collection1" file_count:10492 delete_count:50 deleted_byte_count:17071505 replica_placement:100 version:3 modified_at_second:1615632005
|
||||
volume id:147 size:1077130576 collection:"collection1" file_count:10451 delete_count:27 deleted_byte_count:8290907 replica_placement:100 version:3 modified_at_second:1615604115
|
||||
volume id:148 size:1076066568 collection:"collection1" file_count:9547 delete_count:33 deleted_byte_count:7034089 replica_placement:100 version:3 modified_at_second:1615586390
|
||||
volume id:149 size:1074989016 collection:"collection1" file_count:8352 delete_count:35 deleted_byte_count:7179742 replica_placement:100 version:3 modified_at_second:1615494494
|
||||
volume id:150 size:1076290328 collection:"collection1" file_count:9328 delete_count:33 deleted_byte_count:43417791 replica_placement:100 version:3 modified_at_second:1615611567
|
||||
volume id:152 size:1075941400 collection:"collection1" file_count:9951 delete_count:36 deleted_byte_count:25348335 replica_placement:100 version:3 modified_at_second:1615606614
|
||||
volume id:153 size:1078539784 collection:"collection1" file_count:10924 delete_count:34 deleted_byte_count:12603081 replica_placement:100 version:3 modified_at_second:1615606614
|
||||
volume id:154 size:1081244696 collection:"collection1" file_count:11002 delete_count:31 deleted_byte_count:8467560 replica_placement:100 version:3 modified_at_second:1615478469
|
||||
volume id:155 size:1075140688 collection:"collection1" file_count:10882 delete_count:32 deleted_byte_count:10076804 replica_placement:100 version:3 modified_at_second:1615606614
|
||||
volume id:156 size:1074975832 collection:"collection1" file_count:9535 delete_count:40 deleted_byte_count:11426621 replica_placement:100 version:3 modified_at_second:1615628341
|
||||
volume id:157 size:1076758536 collection:"collection1" file_count:10012 delete_count:19 deleted_byte_count:11688737 replica_placement:100 version:3 modified_at_second:1615597782
|
||||
volume id:158 size:1087251976 collection:"collection1" file_count:9972 delete_count:20 deleted_byte_count:10328429 replica_placement:100 version:3 modified_at_second:1615588879
|
||||
volume id:159 size:1074132368 collection:"collection1" file_count:9382 delete_count:27 deleted_byte_count:11474574 replica_placement:100 version:3 modified_at_second:1615593593
|
||||
volume id:160 size:1075680952 collection:"collection1" file_count:9772 delete_count:22 deleted_byte_count:4981968 replica_placement:100 version:3 modified_at_second:1615597780
|
||||
volume id:162 size:1074286880 collection:"collection1" file_count:11220 delete_count:17 deleted_byte_count:1815547 replica_placement:100 version:3 modified_at_second:1615478126
|
||||
volume id:163 size:1074457192 collection:"collection1" file_count:12524 delete_count:27 deleted_byte_count:6359619 replica_placement:100 version:3 modified_at_second:1615579313
|
||||
volume id:164 size:1074261248 collection:"collection1" file_count:11922 delete_count:25 deleted_byte_count:2923288 replica_placement:100 version:3 modified_at_second:1615620084
|
||||
volume id:165 size:1073891016 collection:"collection1" file_count:9152 delete_count:12 deleted_byte_count:19164659 replica_placement:100 version:3 modified_at_second:1615471907
|
||||
volume id:166 size:1075637536 collection:"collection1" file_count:14211 delete_count:24 deleted_byte_count:20415490 replica_placement:100 version:3 modified_at_second:1615491019
|
||||
volume id:168 size:1074718808 collection:"collection1" file_count:25702 delete_count:40 deleted_byte_count:4024775 replica_placement:100 version:3 modified_at_second:1615585664
|
||||
volume id:169 size:1073863128 collection:"collection1" file_count:25248 delete_count:43 deleted_byte_count:3013817 replica_placement:100 version:3 modified_at_second:1615569832
|
||||
volume id:170 size:1075747096 collection:"collection1" file_count:24596 delete_count:41 deleted_byte_count:3494711 replica_placement:100 version:3 modified_at_second:1615579204
|
||||
volume id:171 size:1081881312 collection:"collection1" file_count:24215 delete_count:36 deleted_byte_count:3191335 replica_placement:100 version:3 modified_at_second:1615596485
|
||||
volume id:172 size:1074787312 collection:"collection1" file_count:31236 delete_count:50 deleted_byte_count:3316482 replica_placement:100 version:3 modified_at_second:1615612385
|
||||
volume id:173 size:1074154648 collection:"collection1" file_count:30884 delete_count:34 deleted_byte_count:2430948 replica_placement:100 version:3 modified_at_second:1615591904
|
||||
volume id:175 size:1077742504 collection:"collection1" file_count:32353 delete_count:33 deleted_byte_count:1861403 replica_placement:100 version:3 modified_at_second:1615559515
|
||||
volume id:176 size:1073854800 collection:"collection1" file_count:30582 delete_count:34 deleted_byte_count:7701976 replica_placement:100 version:3 modified_at_second:1615626169
|
||||
volume id:177 size:1074120120 collection:"collection1" file_count:22293 delete_count:16 deleted_byte_count:3719562 replica_placement:100 version:3 modified_at_second:1615516891
|
||||
volume id:178 size:1087560112 collection:"collection1" file_count:23482 delete_count:22 deleted_byte_count:18810492 replica_placement:100 version:3 modified_at_second:1615541369
|
||||
volume id:180 size:1078438536 collection:"collection1" file_count:23614 delete_count:12 deleted_byte_count:4496474 replica_placement:100 version:3 modified_at_second:1614773242
|
||||
volume id:181 size:1074571768 collection:"collection1" file_count:22898 delete_count:19 deleted_byte_count:6628413 replica_placement:100 version:3 modified_at_second:1614745116
|
||||
volume id:182 size:1076131280 collection:"collection1" file_count:31987 delete_count:21 deleted_byte_count:1416142 replica_placement:100 version:3 modified_at_second:1615568922
|
||||
volume id:183 size:1076361448 collection:"collection1" file_count:31293 delete_count:16 deleted_byte_count:468841 replica_placement:100 version:3 modified_at_second:1615572982
|
||||
volume id:184 size:1074594160 collection:"collection1" file_count:31368 delete_count:22 deleted_byte_count:857453 replica_placement:100 version:3 modified_at_second:1615586578
|
||||
volume id:185 size:1074099624 collection:"collection1" file_count:30612 delete_count:17 deleted_byte_count:2610847 replica_placement:100 version:3 modified_at_second:1615506832
|
||||
volume id:186 size:1074220864 collection:"collection1" file_count:31450 delete_count:15 deleted_byte_count:391855 replica_placement:100 version:3 modified_at_second:1615614933
|
||||
volume id:187 size:1074395944 collection:"collection1" file_count:31853 delete_count:17 deleted_byte_count:454283 replica_placement:100 version:3 modified_at_second:1615590490
|
||||
volume id:188 size:1074732792 collection:"collection1" file_count:31867 delete_count:19 deleted_byte_count:393743 replica_placement:100 version:3 modified_at_second:1615487645
|
||||
volume id:189 size:1074847896 collection:"collection1" file_count:31450 delete_count:16 deleted_byte_count:1040552 replica_placement:100 version:3 modified_at_second:1615335661
|
||||
volume id:190 size:1074008912 collection:"collection1" file_count:31987 delete_count:11 deleted_byte_count:685125 replica_placement:100 version:3 modified_at_second:1615447161
|
||||
volume id:191 size:1075493024 collection:"collection1" file_count:31301 delete_count:19 deleted_byte_count:708401 replica_placement:100 version:3 modified_at_second:1615357456
|
||||
volume id:192 size:1075857400 collection:"collection1" file_count:31490 delete_count:25 deleted_byte_count:720617 replica_placement:100 version:3 modified_at_second:1615621632
|
||||
volume id:193 size:1076616768 collection:"collection1" file_count:31907 delete_count:16 deleted_byte_count:464900 replica_placement:100 version:3 modified_at_second:1615507875
|
||||
volume id:194 size:1073985624 collection:"collection1" file_count:31434 delete_count:18 deleted_byte_count:391432 replica_placement:100 version:3 modified_at_second:1615559499
|
||||
volume id:195 size:1074158312 collection:"collection1" file_count:31453 delete_count:15 deleted_byte_count:718266 replica_placement:100 version:3 modified_at_second:1615559331
|
||||
volume id:196 size:1074594784 collection:"collection1" file_count:31665 delete_count:18 deleted_byte_count:3468922 replica_placement:100 version:3 modified_at_second:1615501688
|
||||
volume id:197 size:1075423296 collection:"collection1" file_count:16473 delete_count:15 deleted_byte_count:12552442 replica_placement:100 version:3 modified_at_second:1615485253
|
||||
volume id:198 size:1075104712 collection:"collection1" file_count:16577 delete_count:18 deleted_byte_count:6583181 replica_placement:100 version:3 modified_at_second:1615623369
|
||||
volume id:199 size:1078117688 collection:"collection1" file_count:16497 delete_count:14 deleted_byte_count:1514286 replica_placement:100 version:3 modified_at_second:1615585984
|
||||
volume id:200 size:1075630536 collection:"collection1" file_count:16380 delete_count:18 deleted_byte_count:1103109 replica_placement:100 version:3 modified_at_second:1615485252
|
||||
volume id:201 size:1091460440 collection:"collection1" file_count:16684 delete_count:26 deleted_byte_count:5590335 replica_placement:100 version:3 modified_at_second:1615585987
|
||||
volume id:202 size:1077533160 collection:"collection1" file_count:2847 delete_count:67 deleted_byte_count:65172985 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615588497
|
||||
volume id:203 size:1027316272 collection:"collection1" file_count:3040 delete_count:11 deleted_byte_count:3993230 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631728
|
||||
volume id:204 size:1079766872 collection:"collection1" file_count:3233 delete_count:255 deleted_byte_count:104707641 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615565701
|
||||
volume id:205 size:1078485304 collection:"collection1" file_count:2869 delete_count:43 deleted_byte_count:18290259 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615579314
|
||||
volume id:206 size:1082045848 collection:"collection1" file_count:2979 delete_count:225 deleted_byte_count:88220074 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630989
|
||||
volume id:207 size:1081939960 collection:"collection1" file_count:3010 delete_count:4 deleted_byte_count:692350 replica_placement:100 version:3 modified_at_second:1615269061
|
||||
volume id:208 size:1077863624 collection:"collection1" file_count:3147 delete_count:6 deleted_byte_count:858726 replica_placement:100 version:3 modified_at_second:1615495515
|
||||
volume id:210 size:1094311304 collection:"collection1" file_count:3468 delete_count:4 deleted_byte_count:466433 replica_placement:100 version:3 modified_at_second:1615495515
|
||||
volume id:212 size:1078293448 collection:"collection1" file_count:3106 delete_count:6 deleted_byte_count:2085755 replica_placement:100 version:3 modified_at_second:1615586387
|
||||
volume id:213 size:1093588072 collection:"collection1" file_count:3681 delete_count:12 deleted_byte_count:3138791 replica_placement:100 version:3 modified_at_second:1615586387
|
||||
volume id:214 size:1074486992 collection:"collection1" file_count:3217 delete_count:10 deleted_byte_count:6392871 replica_placement:100 version:3 modified_at_second:1615586383
|
||||
volume id:215 size:1074798704 collection:"collection1" file_count:2819 delete_count:31 deleted_byte_count:10873569 replica_placement:100 version:3 modified_at_second:1615586386
|
||||
volume id:217 size:1075381872 collection:"collection1" file_count:3331 delete_count:14 deleted_byte_count:2009141 replica_placement:100 version:3 modified_at_second:1615401638
|
||||
volume id:218 size:1081263944 collection:"collection1" file_count:3433 delete_count:14 deleted_byte_count:3454237 replica_placement:100 version:3 modified_at_second:1615603637
|
||||
volume id:219 size:1092298816 collection:"collection1" file_count:3193 delete_count:17 deleted_byte_count:2047576 replica_placement:100 version:3 modified_at_second:1615579316
|
||||
volume id:220 size:1081928312 collection:"collection1" file_count:3166 delete_count:13 deleted_byte_count:4127709 replica_placement:100 version:3 modified_at_second:1615579317
|
||||
volume id:221 size:1106545456 collection:"collection1" file_count:3153 delete_count:11 deleted_byte_count:1496835 replica_placement:100 version:3 modified_at_second:1615269138
|
||||
volume id:222 size:1106623104 collection:"collection1" file_count:3273 delete_count:11 deleted_byte_count:2114627 replica_placement:100 version:3 modified_at_second:1615586243
|
||||
volume id:223 size:1075233064 collection:"collection1" file_count:2966 delete_count:9 deleted_byte_count:744001 replica_placement:100 version:3 modified_at_second:1615586244
|
||||
volume id:224 size:1093691520 collection:"collection1" file_count:3463 delete_count:10 deleted_byte_count:1128328 replica_placement:100 version:3 modified_at_second:1615601870
|
||||
volume id:225 size:1080698928 collection:"collection1" file_count:3115 delete_count:7 deleted_byte_count:18170416 replica_placement:100 version:3 modified_at_second:1615434684
|
||||
volume id:226 size:1103504768 collection:"collection1" file_count:2965 delete_count:10 deleted_byte_count:2639254 replica_placement:100 version:3 modified_at_second:1615601867
|
||||
volume id:228 size:1109784072 collection:"collection1" file_count:2504 delete_count:24 deleted_byte_count:5458950 replica_placement:100 version:3 modified_at_second:1615610489
|
||||
volume id:230 size:1080722984 collection:"collection1" file_count:2898 delete_count:15 deleted_byte_count:3929261 replica_placement:100 version:3 modified_at_second:1615610537
|
||||
volume id:232 size:1073901520 collection:"collection1" file_count:3004 delete_count:54 deleted_byte_count:10273081 replica_placement:100 version:3 modified_at_second:1615611351
|
||||
volume id:234 size:1073835280 collection:"collection1" file_count:2965 delete_count:41 deleted_byte_count:4960354 replica_placement:100 version:3 modified_at_second:1615611351
|
||||
volume id:235 size:1075586104 collection:"collection1" file_count:2767 delete_count:33 deleted_byte_count:3216540 replica_placement:100 version:3 modified_at_second:1615611354
|
||||
volume id:236 size:1089476136 collection:"collection1" file_count:3231 delete_count:53 deleted_byte_count:11625921 replica_placement:100 version:3 modified_at_second:1615611351
|
||||
volume id:237 size:375722792 collection:"collection1" file_count:736 delete_count:16 deleted_byte_count:4464870 replica_placement:100 version:3 modified_at_second:1615631727
|
||||
volume id:238 size:354320000 collection:"collection1" file_count:701 delete_count:17 deleted_byte_count:5940420 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632030
|
||||
volume id:239 size:426569024 collection:"collection1" file_count:693 delete_count:19 deleted_byte_count:13020783 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630841
|
||||
volume id:240 size:424791528 collection:"collection1" file_count:733 delete_count:13 deleted_byte_count:7515220 replica_placement:100 version:3 modified_at_second:1615631670
|
||||
volume id:241 size:380217424 collection:"collection1" file_count:633 delete_count:6 deleted_byte_count:1715768 replica_placement:100 version:3 modified_at_second:1615632006
|
||||
volume id:242 size:1075383392 collection:"collection2" file_count:10470 replica_placement:100 version:3 modified_at_second:1614852116
|
||||
volume id:243 size:1088174704 collection:"collection2" file_count:11109 delete_count:1 deleted_byte_count:938 replica_placement:100 version:3 modified_at_second:1614852203
|
||||
volume id:244 size:1080295352 collection:"collection2" file_count:10812 delete_count:1 deleted_byte_count:795 replica_placement:100 version:3 modified_at_second:1615628825
|
||||
volume id:246 size:1075998672 collection:"collection2" file_count:10365 delete_count:1 deleted_byte_count:13112 replica_placement:100 version:3 modified_at_second:1614852106
|
||||
volume id:247 size:1075859808 collection:"collection2" file_count:10443 delete_count:2 deleted_byte_count:564486 replica_placement:100 version:3 modified_at_second:1614856152
|
||||
volume id:248 size:1084301208 collection:"collection2" file_count:11217 delete_count:4 deleted_byte_count:746488 replica_placement:100 version:3 modified_at_second:1614856285
|
||||
volume id:250 size:1080572168 collection:"collection2" file_count:10220 replica_placement:100 version:3 modified_at_second:1614856129
|
||||
volume id:252 size:1075065264 collection:"collection2" file_count:14622 delete_count:2 deleted_byte_count:5228 replica_placement:100 version:3 modified_at_second:1614861200
|
||||
volume id:253 size:1087328880 collection:"collection2" file_count:14920 delete_count:3 deleted_byte_count:522994 replica_placement:100 version:3 modified_at_second:1614861258
|
||||
volume id:254 size:1074830736 collection:"collection2" file_count:14140 delete_count:2 deleted_byte_count:105892 replica_placement:100 version:3 modified_at_second:1614861115
|
||||
volume id:255 size:1079581640 collection:"collection2" file_count:14877 delete_count:3 deleted_byte_count:101223 replica_placement:100 version:3 modified_at_second:1614861233
|
||||
volume id:256 size:1074283592 collection:"collection2" file_count:14157 delete_count:1 deleted_byte_count:18156 replica_placement:100 version:3 modified_at_second:1614861100
|
||||
volume id:257 size:1082621720 collection:"collection2" file_count:18172 delete_count:2 deleted_byte_count:25125 replica_placement:100 version:3 modified_at_second:1614866402
|
||||
volume id:258 size:1075527216 collection:"collection2" file_count:18421 delete_count:4 deleted_byte_count:267833 replica_placement:100 version:3 modified_at_second:1614866420
|
||||
volume id:259 size:1075507848 collection:"collection2" file_count:18079 delete_count:2 deleted_byte_count:71992 replica_placement:100 version:3 modified_at_second:1614866381
|
||||
volume id:260 size:1075105664 collection:"collection2" file_count:17316 delete_count:4 deleted_byte_count:2015310 replica_placement:100 version:3 modified_at_second:1614866226
|
||||
volume id:261 size:1076628592 collection:"collection2" file_count:18355 delete_count:1 deleted_byte_count:1155 replica_placement:100 version:3 modified_at_second:1614866420
|
||||
volume id:262 size:1078492584 collection:"collection2" file_count:20390 delete_count:3 deleted_byte_count:287601 replica_placement:100 version:3 modified_at_second:1614871601
|
||||
volume id:264 size:1081624192 collection:"collection2" file_count:21151 replica_placement:100 version:3 modified_at_second:1614871629
|
||||
volume id:265 size:1076401104 collection:"collection2" file_count:19932 delete_count:2 deleted_byte_count:160823 replica_placement:100 version:3 modified_at_second:1614871543
|
||||
volume id:266 size:1075617552 collection:"collection2" file_count:20075 delete_count:1 deleted_byte_count:1039 replica_placement:100 version:3 modified_at_second:1614871526
|
||||
volume id:267 size:1075699376 collection:"collection2" file_count:21039 delete_count:3 deleted_byte_count:59956 replica_placement:100 version:3 modified_at_second:1614877294
|
||||
volume id:270 size:1076876424 collection:"collection2" file_count:22057 delete_count:1 deleted_byte_count:43916 replica_placement:100 version:3 modified_at_second:1614877469
|
||||
volume id:271 size:1076992704 collection:"collection2" file_count:22640 delete_count:1 deleted_byte_count:30645 replica_placement:100 version:3 modified_at_second:1614877504
|
||||
volume id:272 size:1076145912 collection:"collection2" file_count:21034 delete_count:2 deleted_byte_count:216564 replica_placement:100 version:3 modified_at_second:1614884139
|
||||
volume id:273 size:1074873432 collection:"collection2" file_count:20511 delete_count:3 deleted_byte_count:46076 replica_placement:100 version:3 modified_at_second:1614884046
|
||||
volume id:274 size:1075994184 collection:"collection2" file_count:20997 replica_placement:100 version:3 modified_at_second:1614884113
|
||||
volume id:275 size:1078349024 collection:"collection2" file_count:20808 delete_count:1 deleted_byte_count:1118 replica_placement:100 version:3 modified_at_second:1614884147
|
||||
volume id:276 size:1076899880 collection:"collection2" file_count:20190 delete_count:1 deleted_byte_count:8798 replica_placement:100 version:3 modified_at_second:1614884003
|
||||
volume id:278 size:1078798632 collection:"collection2" file_count:20597 delete_count:5 deleted_byte_count:400060 replica_placement:100 version:3 modified_at_second:1614890292
|
||||
volume id:280 size:1077432160 collection:"collection2" file_count:20286 delete_count:1 deleted_byte_count:879 replica_placement:100 version:3 modified_at_second:1614890262
|
||||
volume id:281 size:1077581064 collection:"collection2" file_count:20206 delete_count:3 deleted_byte_count:143964 replica_placement:100 version:3 modified_at_second:1614890237
|
||||
volume id:282 size:1075232184 collection:"collection2" file_count:22659 delete_count:4 deleted_byte_count:67915 replica_placement:100 version:3 modified_at_second:1614897304
|
||||
volume id:283 size:1080178880 collection:"collection2" file_count:19462 delete_count:7 deleted_byte_count:660407 replica_placement:100 version:3 modified_at_second:1614896623
|
||||
volume id:286 size:1077464816 collection:"collection2" file_count:23905 delete_count:6 deleted_byte_count:630577 replica_placement:100 version:3 modified_at_second:1614897401
|
||||
volume id:287 size:1074590536 collection:"collection2" file_count:28163 delete_count:5 deleted_byte_count:35727 replica_placement:100 version:3 modified_at_second:1614904875
|
||||
volume id:288 size:1075406920 collection:"collection2" file_count:27243 delete_count:2 deleted_byte_count:51519 replica_placement:100 version:3 modified_at_second:1614904738
|
||||
volume id:289 size:1075284312 collection:"collection2" file_count:29342 delete_count:5 deleted_byte_count:100454 replica_placement:100 version:3 modified_at_second:1614904977
|
||||
volume id:290 size:1074723800 collection:"collection2" file_count:28340 delete_count:4 deleted_byte_count:199064 replica_placement:100 version:3 modified_at_second:1614904924
|
||||
volume id:292 size:1092010672 collection:"collection2" file_count:26781 delete_count:5 deleted_byte_count:508910 replica_placement:100 version:3 modified_at_second:1614912325
|
||||
volume id:295 size:1074702320 collection:"collection2" file_count:24488 delete_count:3 deleted_byte_count:48555 replica_placement:100 version:3 modified_at_second:1614911929
|
||||
volume id:296 size:1077824056 collection:"collection2" file_count:26741 delete_count:4 deleted_byte_count:199906 replica_placement:100 version:3 modified_at_second:1614912301
|
||||
volume id:297 size:1080229176 collection:"collection2" file_count:23409 delete_count:5 deleted_byte_count:46268 replica_placement:100 version:3 modified_at_second:1614918481
|
||||
volume id:298 size:1075410024 collection:"collection2" file_count:23222 delete_count:2 deleted_byte_count:46110 replica_placement:100 version:3 modified_at_second:1614918474
|
||||
volume id:302 size:1077559640 collection:"collection2" file_count:23124 delete_count:7 deleted_byte_count:293111 replica_placement:100 version:3 modified_at_second:1614925500
|
||||
volume id:304 size:1081038944 collection:"collection2" file_count:24505 delete_count:2 deleted_byte_count:124447 replica_placement:100 version:3 modified_at_second:1614925569
|
||||
volume id:305 size:1074185376 collection:"collection2" file_count:22074 delete_count:5 deleted_byte_count:20221 replica_placement:100 version:3 modified_at_second:1614925312
|
||||
volume id:306 size:1074763952 collection:"collection2" file_count:22939 replica_placement:100 version:3 modified_at_second:1614925462
|
||||
volume id:307 size:1076567912 collection:"collection2" file_count:23377 delete_count:2 deleted_byte_count:25453 replica_placement:100 version:3 modified_at_second:1614931448
|
||||
volume id:308 size:1074022336 collection:"collection2" file_count:23086 delete_count:2 deleted_byte_count:2127 replica_placement:100 version:3 modified_at_second:1614931401
|
||||
volume id:311 size:1088248344 collection:"collection2" file_count:23553 delete_count:6 deleted_byte_count:191716 replica_placement:100 version:3 modified_at_second:1614931463
|
||||
volume id:312 size:1075037528 collection:"collection2" file_count:22524 replica_placement:100 version:3 modified_at_second:1614937831
|
||||
volume id:313 size:1074875960 collection:"collection2" file_count:22404 delete_count:4 deleted_byte_count:51728 replica_placement:100 version:3 modified_at_second:1614937755
|
||||
volume id:316 size:1077720776 collection:"collection2" file_count:22605 delete_count:1 deleted_byte_count:8503 replica_placement:100 version:3 modified_at_second:1614937838
|
||||
volume id:318 size:1075965168 collection:"collection2" file_count:22459 delete_count:2 deleted_byte_count:37778 replica_placement:100 version:3 modified_at_second:1614943862
|
||||
volume id:322 size:1078471536 collection:"collection2" file_count:21905 delete_count:3 deleted_byte_count:145002 replica_placement:100 version:3 modified_at_second:1614950572
|
||||
volume id:323 size:1074608056 collection:"collection2" file_count:21605 delete_count:4 deleted_byte_count:172090 replica_placement:100 version:3 modified_at_second:1614950526
|
||||
volume id:325 size:1080701232 collection:"collection2" file_count:21735 replica_placement:100 version:3 modified_at_second:1614950525
|
||||
volume id:326 size:1076059920 collection:"collection2" file_count:22564 delete_count:2 deleted_byte_count:192886 replica_placement:100 version:3 modified_at_second:1614950619
|
||||
volume id:327 size:1076121304 collection:"collection2" file_count:22007 delete_count:3 deleted_byte_count:60358 replica_placement:100 version:3 modified_at_second:1614956487
|
||||
volume id:328 size:1074767816 collection:"collection2" file_count:21720 delete_count:3 deleted_byte_count:56429 replica_placement:100 version:3 modified_at_second:1614956362
|
||||
volume id:329 size:1076691960 collection:"collection2" file_count:22411 delete_count:5 deleted_byte_count:214092 replica_placement:100 version:3 modified_at_second:1614956485
|
||||
volume id:330 size:1080825760 collection:"collection2" file_count:22464 delete_count:2 deleted_byte_count:15771 replica_placement:100 version:3 modified_at_second:1614956476
|
||||
volume id:331 size:1074957256 collection:"collection2" file_count:21230 delete_count:4 deleted_byte_count:62145 replica_placement:100 version:3 modified_at_second:1614956259
|
||||
volume id:332 size:1075569928 collection:"collection2" file_count:22097 delete_count:3 deleted_byte_count:98273 replica_placement:100 version:3 modified_at_second:1614962869
|
||||
volume id:333 size:1074270160 collection:"collection2" file_count:21271 delete_count:2 deleted_byte_count:168122 replica_placement:100 version:3 modified_at_second:1614962697
|
||||
volume id:334 size:1075607880 collection:"collection2" file_count:22546 delete_count:6 deleted_byte_count:101538 replica_placement:100 version:3 modified_at_second:1614962978
|
||||
volume id:335 size:1076235136 collection:"collection2" file_count:22391 delete_count:3 deleted_byte_count:8838 replica_placement:100 version:3 modified_at_second:1614962970
|
||||
volume id:337 size:1075646896 collection:"collection2" file_count:21934 delete_count:1 deleted_byte_count:3397 replica_placement:100 version:3 modified_at_second:1614969937
|
||||
volume id:339 size:1078402392 collection:"collection2" file_count:22309 replica_placement:100 version:3 modified_at_second:1614969995
|
||||
volume id:340 size:1079462152 collection:"collection2" file_count:22319 delete_count:4 deleted_byte_count:93620 replica_placement:100 version:3 modified_at_second:1614969977
|
||||
volume id:341 size:1074448360 collection:"collection2" file_count:21590 delete_count:5 deleted_byte_count:160085 replica_placement:100 version:3 modified_at_second:1614969858
|
||||
volume id:343 size:1075345072 collection:"collection2" file_count:21095 delete_count:2 deleted_byte_count:20581 replica_placement:100 version:3 modified_at_second:1614977148
|
||||
volume id:346 size:1076464112 collection:"collection2" file_count:22320 delete_count:4 deleted_byte_count:798258 replica_placement:100 version:3 modified_at_second:1614977511
|
||||
volume id:347 size:1075145248 collection:"collection2" file_count:22178 delete_count:1 deleted_byte_count:79392 replica_placement:100 version:3 modified_at_second:1614984727
|
||||
volume id:348 size:1080623544 collection:"collection2" file_count:21667 delete_count:1 deleted_byte_count:2443 replica_placement:100 version:3 modified_at_second:1614984604
|
||||
volume id:349 size:1075957672 collection:"collection2" file_count:22395 delete_count:2 deleted_byte_count:61565 replica_placement:100 version:3 modified_at_second:1614984748
|
||||
volume id:351 size:1078795120 collection:"collection2" file_count:23660 delete_count:3 deleted_byte_count:102141 replica_placement:100 version:3 modified_at_second:1614984816
|
||||
volume id:352 size:1077145936 collection:"collection2" file_count:22066 delete_count:1 deleted_byte_count:1018 replica_placement:100 version:3 modified_at_second:1614992130
|
||||
volume id:353 size:1074897496 collection:"collection2" file_count:21266 delete_count:2 deleted_byte_count:3105374 replica_placement:100 version:3 modified_at_second:1614991951
|
||||
volume id:354 size:1085214104 collection:"collection2" file_count:23150 delete_count:4 deleted_byte_count:82391 replica_placement:100 version:3 modified_at_second:1614992208
|
||||
volume id:357 size:1074276152 collection:"collection2" file_count:23137 delete_count:4 deleted_byte_count:188487 replica_placement:100 version:3 modified_at_second:1614998792
|
||||
volume id:359 size:1074211296 collection:"collection2" file_count:22437 delete_count:2 deleted_byte_count:187953 replica_placement:100 version:3 modified_at_second:1614998711
|
||||
volume id:360 size:1075532512 collection:"collection2" file_count:22574 delete_count:3 deleted_byte_count:1774776 replica_placement:100 version:3 modified_at_second:1614998770
|
||||
volume id:361 size:1075362744 collection:"collection2" file_count:22272 delete_count:1 deleted_byte_count:3497 replica_placement:100 version:3 modified_at_second:1614998668
|
||||
volume id:362 size:1074074176 collection:"collection2" file_count:20595 delete_count:1 deleted_byte_count:112145 replica_placement:100 version:3 modified_at_second:1615004407
|
||||
volume id:363 size:1078859640 collection:"collection2" file_count:23177 delete_count:4 deleted_byte_count:9601 replica_placement:100 version:3 modified_at_second:1615004823
|
||||
volume id:364 size:1081280880 collection:"collection2" file_count:22686 delete_count:1 deleted_byte_count:84375 replica_placement:100 version:3 modified_at_second:1615004813
|
||||
volume id:365 size:1075736632 collection:"collection2" file_count:22193 delete_count:5 deleted_byte_count:259033 replica_placement:100 version:3 modified_at_second:1615004776
|
||||
volume id:366 size:1075267272 collection:"collection2" file_count:21856 delete_count:5 deleted_byte_count:138363 replica_placement:100 version:3 modified_at_second:1615004703
|
||||
volume id:367 size:1076403648 collection:"collection2" file_count:22995 delete_count:2 deleted_byte_count:36955 replica_placement:100 version:3 modified_at_second:1615010985
|
||||
volume id:368 size:1074821960 collection:"collection2" file_count:22252 delete_count:4 deleted_byte_count:3291946 replica_placement:100 version:3 modified_at_second:1615010877
|
||||
volume id:369 size:1091472040 collection:"collection2" file_count:23709 delete_count:4 deleted_byte_count:400876 replica_placement:100 version:3 modified_at_second:1615011021
|
||||
volume id:370 size:1076040544 collection:"collection2" file_count:22092 delete_count:2 deleted_byte_count:115388 replica_placement:100 version:3 modified_at_second:1615010877
|
||||
volume id:371 size:1078806216 collection:"collection2" file_count:22685 delete_count:2 deleted_byte_count:68905 replica_placement:100 version:3 modified_at_second:1615010995
|
||||
volume id:372 size:1076193344 collection:"collection2" file_count:22774 delete_count:1 deleted_byte_count:3495 replica_placement:100 version:3 modified_at_second:1615016911
|
||||
volume id:373 size:1080928088 collection:"collection2" file_count:22617 delete_count:4 deleted_byte_count:91849 replica_placement:100 version:3 modified_at_second:1615016878
|
||||
volume id:374 size:1085011176 collection:"collection2" file_count:23054 delete_count:2 deleted_byte_count:89034 replica_placement:100 version:3 modified_at_second:1615016917
|
||||
volume id:376 size:1074845832 collection:"collection2" file_count:22908 delete_count:4 deleted_byte_count:432305 replica_placement:100 version:3 modified_at_second:1615016916
|
||||
volume id:377 size:957434264 collection:"collection2" file_count:14929 delete_count:1 deleted_byte_count:43099 replica_placement:100 version:3 modified_at_second:1615632323
|
||||
volume id:379 size:1014108528 collection:"collection2" file_count:15362 delete_count:6 deleted_byte_count:2481613 replica_placement:100 version:3 modified_at_second:1615632323
|
||||
Disk hdd total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660
|
||||
DataNode 192.168.1.5:8080 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660
|
||||
Rack DefaultRack total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660
|
||||
DataCenter dc5 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660
|
||||
total size:775256653592 file_count:10478712 deleted_file:33754 deleted_bytes:10839266043
|
||||
`
|
||||
17
weed/shell/command_volume_server_evacuate_test.go
Normal file
17
weed/shell/command_volume_server_evacuate_test.go
Normal file
@ -0,0 +1,17 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestVolumeServerEvacuate(t *testing.T) {
|
||||
topologyInfo := parseOutput(topoData)
|
||||
|
||||
volumeServer := "192.168.1.4:8080"
|
||||
|
||||
if err := evacuateNormalVolumes(nil, topologyInfo, volumeServer, true, false, os.Stdout); err != nil {
|
||||
t.Errorf("evacuate: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
@ -19,6 +19,10 @@ var (
|
||||
|
||||
func RunShell(options ShellOptions) {
|
||||
|
||||
sort.Slice(Commands, func(i, j int) bool {
|
||||
return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0
|
||||
})
|
||||
|
||||
line = liner.NewLiner()
|
||||
defer line.Close()
|
||||
|
||||
@ -96,9 +100,6 @@ func printGenericHelp() {
|
||||
`
|
||||
fmt.Print(msg)
|
||||
|
||||
sort.Slice(Commands, func(i, j int) bool {
|
||||
return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0
|
||||
})
|
||||
for _, c := range Commands {
|
||||
helpTexts := strings.SplitN(c.Help(), "\n", 2)
|
||||
fmt.Printf(" %-30s\t# %s \n", c.Name(), helpTexts[0])
|
||||
@ -114,10 +115,6 @@ func printHelp(cmds []string) {
|
||||
} else {
|
||||
cmd := strings.ToLower(args[0])
|
||||
|
||||
sort.Slice(Commands, func(i, j int) bool {
|
||||
return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0
|
||||
})
|
||||
|
||||
for _, c := range Commands {
|
||||
if c.Name() == cmd {
|
||||
fmt.Printf(" %s\t# %s\n", c.Name(), c.Help())
|
||||
|
||||
@ -35,7 +35,7 @@ func NewCRCwriter(w io.Writer) *CRCwriter {
|
||||
|
||||
return &CRCwriter{
|
||||
crc: CRC(0),
|
||||
w: w,
|
||||
w: w,
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -3,13 +3,12 @@ package needle
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/backend"
|
||||
. "github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"io"
|
||||
"math"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -156,6 +155,35 @@ func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset u
|
||||
return offset, size, actualSize, err
|
||||
}
|
||||
|
||||
func WriteNeedleBlob(w backend.BackendStorageFile, dataSlice []byte, size Size, appendAtNs uint64, version Version) (offset uint64, err error) {
|
||||
|
||||
if end, _, e := w.GetStat(); e == nil {
|
||||
defer func(w backend.BackendStorageFile, off int64) {
|
||||
if err != nil {
|
||||
if te := w.Truncate(end); te != nil {
|
||||
glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te)
|
||||
}
|
||||
}
|
||||
}(w, end)
|
||||
offset = uint64(end)
|
||||
} else {
|
||||
err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
|
||||
return
|
||||
}
|
||||
|
||||
if version == Version3 {
|
||||
tsOffset := NeedleHeaderSize + size + NeedleChecksumSize
|
||||
util.Uint64toBytes(dataSlice[tsOffset:tsOffset+TimestampSize], appendAtNs)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
_, err = w.WriteAt(dataSlice, int64(offset))
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, version Version) (dataSlice []byte, err error) {
|
||||
|
||||
dataSize := GetActualSize(size, version)
|
||||
|
||||
@ -2,6 +2,7 @@ package needle_map
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
@ -104,7 +105,13 @@ func (cm *MemDb) LoadFromIdx(idxName string) (ret error) {
|
||||
}
|
||||
defer idxFile.Close()
|
||||
|
||||
return idx.WalkIndexFile(idxFile, func(key NeedleId, offset Offset, size Size) error {
|
||||
return cm.LoadFromReaderAt(idxFile)
|
||||
|
||||
}
|
||||
|
||||
func (cm *MemDb) LoadFromReaderAt(readerAt io.ReaderAt) (ret error) {
|
||||
|
||||
return idx.WalkIndexFile(readerAt, func(key NeedleId, offset Offset, size Size) error {
|
||||
if offset.IsZero() || size.IsDeleted() {
|
||||
return cm.Delete(key)
|
||||
}
|
||||
|
||||
@ -217,6 +217,9 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
|
||||
location.volumesLock.RLock()
|
||||
for _, v := range location.volumes {
|
||||
curMaxFileKey, volumeMessage := v.ToVolumeInformationMessage()
|
||||
if volumeMessage == nil {
|
||||
continue
|
||||
}
|
||||
if maxFileKey < curMaxFileKey {
|
||||
maxFileKey = curMaxFileKey
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user